GlobOpt.cpp 665 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761107621076310764107651076610767107681076910770107711077210773107741077510776107771077810779107801078110782107831078410785107861078710788107891079010791107921079310794107951079610797107981079910800108011080210803108041080510806108071080810809108101081110812108131081410815108161081710818108191082010821108221082310824108251082610827108281082910830108311083210833108341083510836108371083810839108401084110842108431084410845108461084710848108491085010851108521085310854108551085610857108581085910860108611086210863108641086510866108671086810869108701087110872108731087410875108761087710878108791088010881108821088310884108851088610887108881088910890108911089210893108941089510896108971089810899109001090110902109031090410905109061090710908109091091010911109121091310914109151091610917109181091910920109211092210923109241092510926109271092810929109301093110932109331093410935109361093710938109391094010941109421094310944109451094610947109481094910950109511095210953109541095510956109571095810959109601096110962109631096410965109661096710968109691097010971109721097310974109751097610977109781097910980109811098210983109841098510986109871098810989109901099110992109931099410995109961099710998109991100011001110021100311004110051100611007110081100911010110111101211013110141101511016110171101811019110201102111022110231102411025110261102711028110291103011031110321103311034110351103611037110381103911040110411104211043110441104511046110471104811049110501105111052110531105411055110561105711058110591106011061110621106311064110651106611067110681106911070110711107211073110741107511076110771107811079110801108111082110831108411085110861108711088110891109011091110921109311094110951109611097110981109911100111011110211103111041110511106111071110811109111101111111112111131111411115111161111711118111191112011121111221112311124111251112611127111281112911130111311113211133111341113511136111371113811139111401114111142111431114411145111461114711148111491115011151111521115311154111551115611157111581115911160111611116211163111641116511166111671116811169111701117111172111731117411175111761117711178111791118011181111821118311184111851118611187111881118911190111911119211193111941119511196111971119811199112001120111202112031120411205112061120711208112091121011211112121121311214112151121611217112181121911220112211122211223112241122511226112271122811229112301123111232112331123411235112361123711238112391124011241112421124311244112451124611247112481124911250112511125211253112541125511256112571125811259112601126111262112631126411265112661126711268112691127011271112721127311274112751127611277112781127911280112811128211283112841128511286112871128811289112901129111292112931129411295112961129711298112991130011301113021130311304113051130611307113081130911310113111131211313113141131511316113171131811319113201132111322113231132411325113261132711328113291133011331113321133311334113351133611337113381133911340113411134211343113441134511346113471134811349113501135111352113531135411355113561135711358113591136011361113621136311364113651136611367113681136911370113711137211373113741137511376113771137811379113801138111382113831138411385113861138711388113891139011391113921139311394113951139611397113981139911400114011140211403114041140511406114071140811409114101141111412114131141411415114161141711418114191142011421114221142311424114251142611427114281142911430114311143211433114341143511436114371143811439114401144111442114431144411445114461144711448114491145011451114521145311454114551145611457114581145911460114611146211463114641146511466114671146811469114701147111472114731147411475114761147711478114791148011481114821148311484114851148611487114881148911490114911149211493114941149511496114971149811499115001150111502115031150411505115061150711508115091151011511115121151311514115151151611517115181151911520115211152211523115241152511526115271152811529115301153111532115331153411535115361153711538115391154011541115421154311544115451154611547115481154911550115511155211553115541155511556115571155811559115601156111562115631156411565115661156711568115691157011571115721157311574115751157611577115781157911580115811158211583115841158511586115871158811589115901159111592115931159411595115961159711598115991160011601116021160311604116051160611607116081160911610116111161211613116141161511616116171161811619116201162111622116231162411625116261162711628116291163011631116321163311634116351163611637116381163911640116411164211643116441164511646116471164811649116501165111652116531165411655116561165711658116591166011661116621166311664116651166611667116681166911670116711167211673116741167511676116771167811679116801168111682116831168411685116861168711688116891169011691116921169311694116951169611697116981169911700117011170211703117041170511706117071170811709117101171111712117131171411715117161171711718117191172011721117221172311724117251172611727117281172911730117311173211733117341173511736117371173811739117401174111742117431174411745117461174711748117491175011751117521175311754117551175611757117581175911760117611176211763117641176511766117671176811769117701177111772117731177411775117761177711778117791178011781117821178311784117851178611787117881178911790117911179211793117941179511796117971179811799118001180111802118031180411805118061180711808118091181011811118121181311814118151181611817118181181911820118211182211823118241182511826118271182811829118301183111832118331183411835118361183711838118391184011841118421184311844118451184611847118481184911850118511185211853118541185511856118571185811859118601186111862118631186411865118661186711868118691187011871118721187311874118751187611877118781187911880118811188211883118841188511886118871188811889118901189111892118931189411895118961189711898118991190011901119021190311904119051190611907119081190911910119111191211913119141191511916119171191811919119201192111922119231192411925119261192711928119291193011931119321193311934119351193611937119381193911940119411194211943119441194511946119471194811949119501195111952119531195411955119561195711958119591196011961119621196311964119651196611967119681196911970119711197211973119741197511976119771197811979119801198111982119831198411985119861198711988119891199011991119921199311994119951199611997119981199912000120011200212003120041200512006120071200812009120101201112012120131201412015120161201712018120191202012021120221202312024120251202612027120281202912030120311203212033120341203512036120371203812039120401204112042120431204412045120461204712048120491205012051120521205312054120551205612057120581205912060120611206212063120641206512066120671206812069120701207112072120731207412075120761207712078120791208012081120821208312084120851208612087120881208912090120911209212093120941209512096120971209812099121001210112102121031210412105121061210712108121091211012111121121211312114121151211612117121181211912120121211212212123121241212512126121271212812129121301213112132121331213412135121361213712138121391214012141121421214312144121451214612147121481214912150121511215212153121541215512156121571215812159121601216112162121631216412165121661216712168121691217012171121721217312174121751217612177121781217912180121811218212183121841218512186121871218812189121901219112192121931219412195121961219712198121991220012201122021220312204122051220612207122081220912210122111221212213122141221512216122171221812219122201222112222122231222412225122261222712228122291223012231122321223312234122351223612237122381223912240122411224212243122441224512246122471224812249122501225112252122531225412255122561225712258122591226012261122621226312264122651226612267122681226912270122711227212273122741227512276122771227812279122801228112282122831228412285122861228712288122891229012291122921229312294122951229612297122981229912300123011230212303123041230512306123071230812309123101231112312123131231412315123161231712318123191232012321123221232312324123251232612327123281232912330123311233212333123341233512336123371233812339123401234112342123431234412345123461234712348123491235012351123521235312354123551235612357123581235912360123611236212363123641236512366123671236812369123701237112372123731237412375123761237712378123791238012381123821238312384123851238612387123881238912390123911239212393123941239512396123971239812399124001240112402124031240412405124061240712408124091241012411124121241312414124151241612417124181241912420124211242212423124241242512426124271242812429124301243112432124331243412435124361243712438124391244012441124421244312444124451244612447124481244912450124511245212453124541245512456124571245812459124601246112462124631246412465124661246712468124691247012471124721247312474124751247612477124781247912480124811248212483124841248512486124871248812489124901249112492124931249412495124961249712498124991250012501125021250312504125051250612507125081250912510125111251212513125141251512516125171251812519125201252112522125231252412525125261252712528125291253012531125321253312534125351253612537125381253912540125411254212543125441254512546125471254812549125501255112552125531255412555125561255712558125591256012561125621256312564125651256612567125681256912570125711257212573125741257512576125771257812579125801258112582125831258412585125861258712588125891259012591125921259312594125951259612597125981259912600126011260212603126041260512606126071260812609126101261112612126131261412615126161261712618126191262012621126221262312624126251262612627126281262912630126311263212633126341263512636126371263812639126401264112642126431264412645126461264712648126491265012651126521265312654126551265612657126581265912660126611266212663126641266512666126671266812669126701267112672126731267412675126761267712678126791268012681126821268312684126851268612687126881268912690126911269212693126941269512696126971269812699127001270112702127031270412705127061270712708127091271012711127121271312714127151271612717127181271912720127211272212723127241272512726127271272812729127301273112732127331273412735127361273712738127391274012741127421274312744127451274612747127481274912750127511275212753127541275512756127571275812759127601276112762127631276412765127661276712768127691277012771127721277312774127751277612777127781277912780127811278212783127841278512786127871278812789127901279112792127931279412795127961279712798127991280012801128021280312804128051280612807128081280912810128111281212813128141281512816128171281812819128201282112822128231282412825128261282712828128291283012831128321283312834128351283612837128381283912840128411284212843128441284512846128471284812849128501285112852128531285412855128561285712858128591286012861128621286312864128651286612867128681286912870128711287212873128741287512876128771287812879128801288112882128831288412885128861288712888128891289012891128921289312894128951289612897128981289912900129011290212903129041290512906129071290812909129101291112912129131291412915129161291712918129191292012921129221292312924129251292612927129281292912930129311293212933129341293512936129371293812939129401294112942129431294412945129461294712948129491295012951129521295312954129551295612957129581295912960129611296212963129641296512966129671296812969129701297112972129731297412975129761297712978129791298012981129821298312984129851298612987129881298912990129911299212993129941299512996129971299812999130001300113002130031300413005130061300713008130091301013011130121301313014130151301613017130181301913020130211302213023130241302513026130271302813029130301303113032130331303413035130361303713038130391304013041130421304313044130451304613047130481304913050130511305213053130541305513056130571305813059130601306113062130631306413065130661306713068130691307013071130721307313074130751307613077130781307913080130811308213083130841308513086130871308813089130901309113092130931309413095130961309713098130991310013101131021310313104131051310613107131081310913110131111311213113131141311513116131171311813119131201312113122131231312413125131261312713128131291313013131131321313313134131351313613137131381313913140131411314213143131441314513146131471314813149131501315113152131531315413155131561315713158131591316013161131621316313164131651316613167131681316913170131711317213173131741317513176131771317813179131801318113182131831318413185131861318713188131891319013191131921319313194131951319613197131981319913200132011320213203132041320513206132071320813209132101321113212132131321413215132161321713218132191322013221132221322313224132251322613227132281322913230132311323213233132341323513236132371323813239132401324113242132431324413245132461324713248132491325013251132521325313254132551325613257132581325913260132611326213263132641326513266132671326813269132701327113272132731327413275132761327713278132791328013281132821328313284132851328613287132881328913290132911329213293132941329513296132971329813299133001330113302133031330413305133061330713308133091331013311133121331313314133151331613317133181331913320133211332213323133241332513326133271332813329133301333113332133331333413335133361333713338133391334013341133421334313344133451334613347133481334913350133511335213353133541335513356133571335813359133601336113362133631336413365133661336713368133691337013371133721337313374133751337613377133781337913380133811338213383133841338513386133871338813389133901339113392133931339413395133961339713398133991340013401134021340313404134051340613407134081340913410134111341213413134141341513416134171341813419134201342113422134231342413425134261342713428134291343013431134321343313434134351343613437134381343913440134411344213443134441344513446134471344813449134501345113452134531345413455134561345713458134591346013461134621346313464134651346613467134681346913470134711347213473134741347513476134771347813479134801348113482134831348413485134861348713488134891349013491134921349313494134951349613497134981349913500135011350213503135041350513506135071350813509135101351113512135131351413515135161351713518135191352013521135221352313524135251352613527135281352913530135311353213533135341353513536135371353813539135401354113542135431354413545135461354713548135491355013551135521355313554135551355613557135581355913560135611356213563135641356513566135671356813569135701357113572135731357413575135761357713578135791358013581135821358313584135851358613587135881358913590135911359213593135941359513596135971359813599136001360113602136031360413605136061360713608136091361013611136121361313614136151361613617136181361913620136211362213623136241362513626136271362813629136301363113632136331363413635136361363713638136391364013641136421364313644136451364613647136481364913650136511365213653136541365513656136571365813659136601366113662136631366413665136661366713668136691367013671136721367313674136751367613677136781367913680136811368213683136841368513686136871368813689136901369113692136931369413695136961369713698136991370013701137021370313704137051370613707137081370913710137111371213713137141371513716137171371813719137201372113722137231372413725137261372713728137291373013731137321373313734137351373613737137381373913740137411374213743137441374513746137471374813749137501375113752137531375413755137561375713758137591376013761137621376313764137651376613767137681376913770137711377213773137741377513776137771377813779137801378113782137831378413785137861378713788137891379013791137921379313794137951379613797137981379913800138011380213803138041380513806138071380813809138101381113812138131381413815138161381713818138191382013821138221382313824138251382613827138281382913830138311383213833138341383513836138371383813839138401384113842138431384413845138461384713848138491385013851138521385313854138551385613857138581385913860138611386213863138641386513866138671386813869138701387113872138731387413875138761387713878138791388013881138821388313884138851388613887138881388913890138911389213893138941389513896138971389813899139001390113902139031390413905139061390713908139091391013911139121391313914139151391613917139181391913920139211392213923139241392513926139271392813929139301393113932139331393413935139361393713938139391394013941139421394313944139451394613947139481394913950139511395213953139541395513956139571395813959139601396113962139631396413965139661396713968139691397013971139721397313974139751397613977139781397913980139811398213983139841398513986139871398813989139901399113992139931399413995139961399713998139991400014001140021400314004140051400614007140081400914010140111401214013140141401514016140171401814019140201402114022140231402414025140261402714028140291403014031140321403314034140351403614037140381403914040140411404214043140441404514046140471404814049140501405114052140531405414055140561405714058140591406014061140621406314064140651406614067140681406914070140711407214073140741407514076140771407814079140801408114082140831408414085140861408714088140891409014091140921409314094140951409614097140981409914100141011410214103141041410514106141071410814109141101411114112141131411414115141161411714118141191412014121141221412314124141251412614127141281412914130141311413214133141341413514136141371413814139141401414114142141431414414145141461414714148141491415014151141521415314154141551415614157141581415914160141611416214163141641416514166141671416814169141701417114172141731417414175141761417714178141791418014181141821418314184141851418614187141881418914190141911419214193141941419514196141971419814199142001420114202142031420414205142061420714208142091421014211142121421314214142151421614217142181421914220142211422214223142241422514226142271422814229142301423114232142331423414235142361423714238142391424014241142421424314244142451424614247142481424914250142511425214253142541425514256142571425814259142601426114262142631426414265142661426714268142691427014271142721427314274142751427614277142781427914280142811428214283142841428514286142871428814289142901429114292142931429414295142961429714298142991430014301143021430314304143051430614307143081430914310143111431214313143141431514316143171431814319143201432114322143231432414325143261432714328143291433014331143321433314334143351433614337143381433914340143411434214343143441434514346143471434814349143501435114352143531435414355143561435714358143591436014361143621436314364143651436614367143681436914370143711437214373143741437514376143771437814379143801438114382143831438414385143861438714388143891439014391143921439314394143951439614397143981439914400144011440214403144041440514406144071440814409144101441114412144131441414415144161441714418144191442014421144221442314424144251442614427144281442914430144311443214433144341443514436144371443814439144401444114442144431444414445144461444714448144491445014451144521445314454144551445614457144581445914460144611446214463144641446514466144671446814469144701447114472144731447414475144761447714478144791448014481144821448314484144851448614487144881448914490144911449214493144941449514496144971449814499145001450114502145031450414505145061450714508145091451014511145121451314514145151451614517145181451914520145211452214523145241452514526145271452814529145301453114532145331453414535145361453714538145391454014541145421454314544145451454614547145481454914550145511455214553145541455514556145571455814559145601456114562145631456414565145661456714568145691457014571145721457314574145751457614577145781457914580145811458214583145841458514586145871458814589145901459114592145931459414595145961459714598145991460014601146021460314604146051460614607146081460914610146111461214613146141461514616146171461814619146201462114622146231462414625146261462714628146291463014631146321463314634146351463614637146381463914640146411464214643146441464514646146471464814649146501465114652146531465414655146561465714658146591466014661146621466314664146651466614667146681466914670146711467214673146741467514676146771467814679146801468114682146831468414685146861468714688146891469014691146921469314694146951469614697146981469914700147011470214703147041470514706147071470814709147101471114712147131471414715147161471714718147191472014721147221472314724147251472614727147281472914730147311473214733147341473514736147371473814739147401474114742147431474414745147461474714748147491475014751147521475314754147551475614757147581475914760147611476214763147641476514766147671476814769147701477114772147731477414775147761477714778147791478014781147821478314784147851478614787147881478914790147911479214793147941479514796147971479814799148001480114802148031480414805148061480714808148091481014811148121481314814148151481614817148181481914820148211482214823148241482514826148271482814829148301483114832148331483414835148361483714838148391484014841148421484314844148451484614847148481484914850148511485214853148541485514856148571485814859148601486114862148631486414865148661486714868148691487014871148721487314874148751487614877148781487914880148811488214883148841488514886148871488814889148901489114892148931489414895148961489714898148991490014901149021490314904149051490614907149081490914910149111491214913149141491514916149171491814919149201492114922149231492414925149261492714928149291493014931149321493314934149351493614937149381493914940149411494214943149441494514946149471494814949149501495114952149531495414955149561495714958149591496014961149621496314964149651496614967149681496914970149711497214973149741497514976149771497814979149801498114982149831498414985149861498714988149891499014991149921499314994149951499614997149981499915000150011500215003150041500515006150071500815009150101501115012150131501415015150161501715018150191502015021150221502315024150251502615027150281502915030150311503215033150341503515036150371503815039150401504115042150431504415045150461504715048150491505015051150521505315054150551505615057150581505915060150611506215063150641506515066150671506815069150701507115072150731507415075150761507715078150791508015081150821508315084150851508615087150881508915090150911509215093150941509515096150971509815099151001510115102151031510415105151061510715108151091511015111151121511315114151151511615117151181511915120151211512215123151241512515126151271512815129151301513115132151331513415135151361513715138151391514015141151421514315144151451514615147151481514915150151511515215153151541515515156151571515815159151601516115162151631516415165151661516715168151691517015171151721517315174151751517615177151781517915180151811518215183151841518515186151871518815189151901519115192151931519415195151961519715198151991520015201152021520315204152051520615207152081520915210152111521215213152141521515216152171521815219152201522115222152231522415225152261522715228152291523015231152321523315234152351523615237152381523915240152411524215243152441524515246152471524815249152501525115252152531525415255152561525715258152591526015261152621526315264152651526615267152681526915270152711527215273152741527515276152771527815279152801528115282152831528415285152861528715288152891529015291152921529315294152951529615297152981529915300153011530215303153041530515306153071530815309153101531115312153131531415315153161531715318153191532015321153221532315324153251532615327153281532915330153311533215333153341533515336153371533815339153401534115342153431534415345153461534715348153491535015351153521535315354153551535615357153581535915360153611536215363153641536515366153671536815369153701537115372153731537415375153761537715378153791538015381153821538315384153851538615387153881538915390153911539215393153941539515396153971539815399154001540115402154031540415405154061540715408154091541015411154121541315414154151541615417154181541915420154211542215423154241542515426154271542815429154301543115432154331543415435154361543715438154391544015441154421544315444154451544615447154481544915450154511545215453154541545515456154571545815459154601546115462154631546415465154661546715468154691547015471154721547315474154751547615477154781547915480154811548215483154841548515486154871548815489154901549115492154931549415495154961549715498154991550015501155021550315504155051550615507155081550915510155111551215513155141551515516155171551815519155201552115522155231552415525155261552715528155291553015531155321553315534155351553615537155381553915540155411554215543155441554515546155471554815549155501555115552155531555415555155561555715558155591556015561155621556315564155651556615567155681556915570155711557215573155741557515576155771557815579155801558115582155831558415585155861558715588155891559015591155921559315594155951559615597155981559915600156011560215603156041560515606156071560815609156101561115612156131561415615156161561715618156191562015621156221562315624156251562615627156281562915630156311563215633156341563515636156371563815639156401564115642156431564415645156461564715648156491565015651156521565315654156551565615657156581565915660156611566215663156641566515666156671566815669156701567115672156731567415675156761567715678156791568015681156821568315684156851568615687156881568915690156911569215693156941569515696156971569815699157001570115702157031570415705157061570715708157091571015711157121571315714157151571615717157181571915720157211572215723157241572515726157271572815729157301573115732157331573415735157361573715738157391574015741157421574315744157451574615747157481574915750157511575215753157541575515756157571575815759157601576115762157631576415765157661576715768157691577015771157721577315774157751577615777157781577915780157811578215783157841578515786157871578815789157901579115792157931579415795157961579715798157991580015801158021580315804158051580615807158081580915810158111581215813158141581515816158171581815819158201582115822158231582415825158261582715828158291583015831158321583315834158351583615837158381583915840158411584215843158441584515846158471584815849158501585115852158531585415855158561585715858158591586015861158621586315864158651586615867158681586915870158711587215873158741587515876158771587815879158801588115882158831588415885158861588715888158891589015891158921589315894158951589615897158981589915900159011590215903159041590515906159071590815909159101591115912159131591415915159161591715918159191592015921159221592315924159251592615927159281592915930159311593215933159341593515936159371593815939159401594115942159431594415945159461594715948159491595015951159521595315954159551595615957159581595915960159611596215963159641596515966159671596815969159701597115972159731597415975159761597715978159791598015981159821598315984159851598615987159881598915990159911599215993159941599515996159971599815999160001600116002160031600416005160061600716008160091601016011160121601316014160151601616017160181601916020160211602216023160241602516026160271602816029160301603116032160331603416035160361603716038160391604016041160421604316044160451604616047160481604916050160511605216053160541605516056160571605816059160601606116062160631606416065160661606716068160691607016071160721607316074160751607616077160781607916080160811608216083160841608516086160871608816089160901609116092160931609416095160961609716098160991610016101161021610316104161051610616107161081610916110161111611216113161141611516116161171611816119161201612116122161231612416125161261612716128161291613016131161321613316134161351613616137161381613916140161411614216143161441614516146161471614816149161501615116152161531615416155161561615716158161591616016161161621616316164161651616616167161681616916170161711617216173161741617516176161771617816179161801618116182161831618416185161861618716188161891619016191161921619316194161951619616197161981619916200162011620216203162041620516206162071620816209162101621116212162131621416215162161621716218162191622016221162221622316224162251622616227162281622916230162311623216233162341623516236162371623816239162401624116242162431624416245162461624716248162491625016251162521625316254162551625616257162581625916260162611626216263162641626516266162671626816269162701627116272162731627416275162761627716278162791628016281162821628316284162851628616287162881628916290162911629216293162941629516296162971629816299163001630116302163031630416305163061630716308163091631016311163121631316314163151631616317163181631916320163211632216323163241632516326163271632816329163301633116332163331633416335163361633716338163391634016341163421634316344163451634616347163481634916350163511635216353163541635516356163571635816359163601636116362163631636416365163661636716368163691637016371163721637316374163751637616377163781637916380163811638216383163841638516386163871638816389163901639116392163931639416395163961639716398163991640016401164021640316404164051640616407164081640916410164111641216413164141641516416164171641816419164201642116422164231642416425164261642716428164291643016431164321643316434164351643616437164381643916440164411644216443164441644516446164471644816449164501645116452164531645416455164561645716458164591646016461164621646316464164651646616467164681646916470164711647216473164741647516476164771647816479164801648116482164831648416485164861648716488164891649016491164921649316494164951649616497164981649916500165011650216503165041650516506165071650816509165101651116512165131651416515165161651716518165191652016521165221652316524165251652616527165281652916530165311653216533165341653516536165371653816539165401654116542165431654416545165461654716548165491655016551165521655316554165551655616557165581655916560165611656216563165641656516566165671656816569165701657116572165731657416575165761657716578165791658016581165821658316584165851658616587165881658916590165911659216593165941659516596165971659816599166001660116602166031660416605166061660716608166091661016611166121661316614166151661616617166181661916620166211662216623166241662516626166271662816629166301663116632166331663416635166361663716638166391664016641166421664316644166451664616647166481664916650166511665216653166541665516656166571665816659166601666116662166631666416665166661666716668166691667016671166721667316674166751667616677166781667916680166811668216683166841668516686166871668816689166901669116692166931669416695166961669716698166991670016701167021670316704167051670616707167081670916710167111671216713167141671516716167171671816719167201672116722167231672416725167261672716728167291673016731167321673316734167351673616737167381673916740167411674216743167441674516746167471674816749167501675116752167531675416755167561675716758167591676016761167621676316764167651676616767167681676916770167711677216773167741677516776167771677816779167801678116782167831678416785167861678716788167891679016791167921679316794167951679616797167981679916800168011680216803168041680516806168071680816809168101681116812168131681416815168161681716818168191682016821168221682316824168251682616827168281682916830168311683216833168341683516836168371683816839168401684116842168431684416845168461684716848168491685016851168521685316854168551685616857168581685916860168611686216863168641686516866168671686816869168701687116872168731687416875168761687716878168791688016881168821688316884168851688616887168881688916890168911689216893168941689516896168971689816899169001690116902169031690416905169061690716908169091691016911169121691316914169151691616917169181691916920169211692216923169241692516926169271692816929169301693116932169331693416935169361693716938169391694016941169421694316944169451694616947169481694916950169511695216953169541695516956169571695816959169601696116962169631696416965169661696716968169691697016971169721697316974169751697616977169781697916980169811698216983169841698516986169871698816989169901699116992169931699416995169961699716998169991700017001170021700317004170051700617007170081700917010170111701217013170141701517016170171701817019170201702117022170231702417025170261702717028170291703017031170321703317034170351703617037170381703917040170411704217043170441704517046170471704817049170501705117052170531705417055170561705717058170591706017061170621706317064170651706617067170681706917070170711707217073170741707517076170771707817079170801708117082170831708417085170861708717088170891709017091170921709317094170951709617097170981709917100171011710217103171041710517106171071710817109171101711117112171131711417115171161711717118171191712017121171221712317124171251712617127171281712917130171311713217133171341713517136171371713817139171401714117142171431714417145171461714717148171491715017151171521715317154171551715617157171581715917160171611716217163171641716517166171671716817169171701717117172171731717417175171761717717178171791718017181171821718317184171851718617187171881718917190171911719217193171941719517196171971719817199172001720117202172031720417205172061720717208172091721017211172121721317214172151721617217172181721917220172211722217223172241722517226172271722817229172301723117232172331723417235172361723717238172391724017241172421724317244172451724617247172481724917250172511725217253172541725517256172571725817259172601726117262172631726417265172661726717268172691727017271172721727317274172751727617277172781727917280172811728217283172841728517286172871728817289172901729117292172931729417295172961729717298172991730017301173021730317304173051730617307173081730917310173111731217313173141731517316173171731817319173201732117322173231732417325173261732717328173291733017331173321733317334173351733617337173381733917340173411734217343173441734517346173471734817349173501735117352173531735417355173561735717358173591736017361173621736317364173651736617367173681736917370173711737217373173741737517376173771737817379173801738117382173831738417385173861738717388173891739017391173921739317394173951739617397173981739917400174011740217403174041740517406174071740817409174101741117412174131741417415174161741717418174191742017421174221742317424174251742617427174281742917430174311743217433174341743517436174371743817439174401744117442174431744417445174461744717448174491745017451174521745317454174551745617457174581745917460174611746217463174641746517466174671746817469174701747117472174731747417475174761747717478174791748017481174821748317484174851748617487174881748917490174911749217493174941749517496174971749817499175001750117502175031750417505175061750717508175091751017511175121751317514175151751617517175181751917520175211752217523175241752517526175271752817529175301753117532175331753417535175361753717538175391754017541175421754317544175451754617547175481754917550175511755217553175541755517556175571755817559175601756117562175631756417565175661756717568175691757017571175721757317574175751757617577175781757917580175811758217583175841758517586175871758817589175901759117592175931759417595175961759717598175991760017601176021760317604176051760617607176081760917610176111761217613176141761517616176171761817619176201762117622176231762417625176261762717628176291763017631176321763317634176351763617637176381763917640176411764217643176441764517646176471764817649176501765117652176531765417655176561765717658176591766017661176621766317664176651766617667176681766917670176711767217673176741767517676176771767817679176801768117682176831768417685176861768717688176891769017691176921769317694176951769617697176981769917700177011770217703177041770517706177071770817709177101771117712177131771417715177161771717718177191772017721177221772317724177251772617727177281772917730177311773217733177341773517736177371773817739177401774117742177431774417745177461774717748177491775017751177521775317754177551775617757177581775917760177611776217763177641776517766177671776817769177701777117772177731777417775177761777717778177791778017781177821778317784177851778617787177881778917790177911779217793177941779517796177971779817799178001780117802178031780417805178061780717808178091781017811178121781317814178151781617817178181781917820178211782217823178241782517826178271782817829178301783117832178331783417835178361783717838178391784017841178421784317844178451784617847178481784917850178511785217853178541785517856178571785817859178601786117862178631786417865178661786717868178691787017871178721787317874178751787617877178781787917880178811788217883178841788517886178871788817889178901789117892178931789417895178961789717898178991790017901179021790317904179051790617907179081790917910179111791217913179141791517916179171791817919179201792117922179231792417925179261792717928179291793017931179321793317934179351793617937179381793917940179411794217943179441794517946179471794817949179501795117952179531795417955179561795717958179591796017961179621796317964179651796617967179681796917970179711797217973179741797517976179771797817979179801798117982179831798417985179861798717988179891799017991179921799317994179951799617997179981799918000180011800218003180041800518006180071800818009180101801118012180131801418015180161801718018180191802018021
  1. //-------------------------------------------------------------------------------------------------------
  2. // Copyright (C) Microsoft Corporation and contributors. All rights reserved.
  3. // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
  4. //-------------------------------------------------------------------------------------------------------
  5. #include "Backend.h"
  6. #if DBG_DUMP
  7. #define DO_MEMOP_TRACE() (PHASE_TRACE(Js::MemOpPhase, this->func) ||\
  8. PHASE_TRACE(Js::MemSetPhase, this->func) ||\
  9. PHASE_TRACE(Js::MemCopyPhase, this->func))
  10. #define DO_MEMOP_TRACE_PHASE(phase) (PHASE_TRACE(Js::MemOpPhase, this->func) || PHASE_TRACE(Js::phase ## Phase, this->func))
  11. #define OUTPUT_MEMOP_TRACE(loop, instr, ...) {\
  12. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];\
  13. Output::Print(15, _u("Function: %s%s, Loop: %u: "), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), loop->GetLoopNumber());\
  14. Output::Print(__VA_ARGS__);\
  15. IR::Instr* __instr__ = instr;\
  16. if(__instr__) __instr__->DumpByteCodeOffset();\
  17. if(__instr__) Output::Print(_u(" (%s)"), Js::OpCodeUtil::GetOpCodeName(__instr__->m_opcode));\
  18. Output::Print(_u("\n"));\
  19. Output::Flush(); \
  20. }
  21. #define TRACE_MEMOP(loop, instr, ...) \
  22. if (DO_MEMOP_TRACE()) {\
  23. Output::Print(_u("TRACE MemOp:"));\
  24. OUTPUT_MEMOP_TRACE(loop, instr, __VA_ARGS__)\
  25. }
  26. #define TRACE_MEMOP_VERBOSE(loop, instr, ...) if(CONFIG_FLAG(Verbose)) {TRACE_MEMOP(loop, instr, __VA_ARGS__)}
  27. #define TRACE_MEMOP_PHASE(phase, loop, instr, ...) \
  28. if (DO_MEMOP_TRACE_PHASE(phase))\
  29. {\
  30. Output::Print(_u("TRACE ") _u(#phase) _u(":"));\
  31. OUTPUT_MEMOP_TRACE(loop, instr, __VA_ARGS__)\
  32. }
  33. #define TRACE_MEMOP_PHASE_VERBOSE(phase, loop, instr, ...) if(CONFIG_FLAG(Verbose)) {TRACE_MEMOP_PHASE(phase, loop, instr, __VA_ARGS__)}
  34. #else
  35. #define DO_MEMOP_TRACE()
  36. #define DO_MEMOP_TRACE_PHASE(phase)
  37. #define OUTPUT_MEMOP_TRACE(loop, instr, ...)
  38. #define TRACE_MEMOP(loop, instr, ...)
  39. #define TRACE_MEMOP_VERBOSE(loop, instr, ...)
  40. #define TRACE_MEMOP_PHASE(phase, loop, instr, ...)
  41. #define TRACE_MEMOP_PHASE_VERBOSE(phase, loop, instr, ...)
  42. #endif
  43. class AutoRestoreVal
  44. {
  45. private:
  46. Value *const originalValue;
  47. Value *const tempValue;
  48. Value * *const valueRef;
  49. public:
  50. AutoRestoreVal(Value *const originalValue, Value * *const tempValueRef)
  51. : originalValue(originalValue), tempValue(*tempValueRef), valueRef(tempValueRef)
  52. {
  53. }
  54. ~AutoRestoreVal()
  55. {
  56. if(*valueRef == tempValue)
  57. {
  58. *valueRef = originalValue;
  59. }
  60. }
  61. PREVENT_COPY(AutoRestoreVal);
  62. };
  63. GlobOpt::GlobOpt(Func * func)
  64. : func(func),
  65. intConstantToStackSymMap(nullptr),
  66. intConstantToValueMap(nullptr),
  67. currentValue(FirstNewValueNumber),
  68. prePassLoop(nullptr),
  69. alloc(nullptr),
  70. isCallHelper(false),
  71. inInlinedBuiltIn(false),
  72. rootLoopPrePass(nullptr),
  73. noImplicitCallUsesToInsert(nullptr),
  74. valuesCreatedForClone(nullptr),
  75. valuesCreatedForMerge(nullptr),
  76. instrCountSinceLastCleanUp(0),
  77. isRecursiveCallOnLandingPad(false),
  78. updateInductionVariableValueNumber(false),
  79. isPerformingLoopBackEdgeCompensation(false),
  80. currentRegion(nullptr),
  81. auxSlotPtrSyms(nullptr),
  82. changedSymsAfterIncBailoutCandidate(nullptr),
  83. doTypeSpec(
  84. !IsTypeSpecPhaseOff(func)),
  85. doAggressiveIntTypeSpec(
  86. doTypeSpec &&
  87. DoAggressiveIntTypeSpec(func)),
  88. doAggressiveMulIntTypeSpec(
  89. doTypeSpec &&
  90. !PHASE_OFF(Js::AggressiveMulIntTypeSpecPhase, func) &&
  91. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsAggressiveMulIntTypeSpecDisabled(func->IsLoopBody()))),
  92. doDivIntTypeSpec(
  93. doAggressiveIntTypeSpec &&
  94. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsDivIntTypeSpecDisabled(func->IsLoopBody()))),
  95. doLossyIntTypeSpec(
  96. doTypeSpec &&
  97. DoLossyIntTypeSpec(func)),
  98. doFloatTypeSpec(
  99. doTypeSpec &&
  100. DoFloatTypeSpec(func)),
  101. doArrayCheckHoist(
  102. DoArrayCheckHoist(func)),
  103. doArrayMissingValueCheckHoist(
  104. doArrayCheckHoist &&
  105. DoArrayMissingValueCheckHoist(func)),
  106. doArraySegmentHoist(
  107. doArrayCheckHoist &&
  108. DoArraySegmentHoist(ValueType::GetObject(ObjectType::Int32Array), func)),
  109. doJsArraySegmentHoist(
  110. doArraySegmentHoist &&
  111. DoArraySegmentHoist(ValueType::GetObject(ObjectType::Array), func)),
  112. doArrayLengthHoist(
  113. doArrayCheckHoist &&
  114. DoArrayLengthHoist(func)),
  115. doEliminateArrayAccessHelperCall(
  116. doArrayCheckHoist &&
  117. !PHASE_OFF(Js::EliminateArrayAccessHelperCallPhase, func)),
  118. doTrackRelativeIntBounds(
  119. doAggressiveIntTypeSpec &&
  120. DoPathDependentValues() &&
  121. !PHASE_OFF(Js::Phase::TrackRelativeIntBoundsPhase, func)),
  122. doBoundCheckElimination(
  123. doTrackRelativeIntBounds &&
  124. !PHASE_OFF(Js::Phase::BoundCheckEliminationPhase, func)),
  125. doBoundCheckHoist(
  126. doEliminateArrayAccessHelperCall &&
  127. doBoundCheckElimination &&
  128. DoConstFold() &&
  129. !PHASE_OFF(Js::Phase::BoundCheckHoistPhase, func) &&
  130. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsBoundCheckHoistDisabled(func->IsLoopBody()))),
  131. doLoopCountBasedBoundCheckHoist(
  132. doBoundCheckHoist &&
  133. !PHASE_OFF(Js::Phase::LoopCountBasedBoundCheckHoistPhase, func) &&
  134. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsLoopCountBasedBoundCheckHoistDisabled(func->IsLoopBody()))),
  135. doPowIntIntTypeSpec(
  136. doAggressiveIntTypeSpec &&
  137. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsPowIntIntTypeSpecDisabled())),
  138. doTagChecks(
  139. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsTagCheckDisabled())),
  140. isAsmJSFunc(func->GetJITFunctionBody()->IsAsmJsMode())
  141. {
  142. }
  143. void
  144. GlobOpt::BackwardPass(Js::Phase tag)
  145. {
  146. BEGIN_CODEGEN_PHASE(this->func, tag);
  147. ::BackwardPass backwardPass(this->func, this, tag);
  148. backwardPass.Optimize();
  149. END_CODEGEN_PHASE(this->func, tag);
  150. }
  151. void
  152. GlobOpt::Optimize()
  153. {
  154. this->objectTypeSyms = nullptr;
  155. this->func->argInsCount = this->func->GetInParamsCount();
  156. if (!func->GetJITFunctionBody()->IsAsmJsMode())
  157. {
  158. // Don't include "this" pointer in the count when not in AsmJs mode (AsmJS does not have "this").
  159. this->func->argInsCount--;
  160. }
  161. if (!func->DoGlobOpt())
  162. {
  163. this->lengthEquivBv = nullptr;
  164. this->argumentsEquivBv = nullptr;
  165. this->callerEquivBv = nullptr;
  166. // Still need to run the dead store phase to calculate the live reg on back edge
  167. this->BackwardPass(Js::DeadStorePhase);
  168. CannotAllocateArgumentsObjectOnStack(nullptr);
  169. return;
  170. }
  171. {
  172. this->lengthEquivBv = this->func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::length, nullptr); // Used to kill live "length" properties
  173. this->argumentsEquivBv = func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::arguments, nullptr); // Used to kill live "arguments" properties
  174. this->callerEquivBv = func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::caller, nullptr); // Used to kill live "caller" properties
  175. // The backward phase needs the glob opt's allocator to allocate the propertyTypeValueMap
  176. // in GlobOpt::EnsurePropertyTypeValue and ranges of instructions where int overflow may be ignored.
  177. // (see BackwardPass::TrackIntUsage)
  178. PageAllocator * pageAllocator = this->func->m_alloc->GetPageAllocator();
  179. NoRecoverMemoryJitArenaAllocator localAlloc(_u("BE-GlobOpt"), pageAllocator, Js::Throw::OutOfMemory);
  180. this->alloc = &localAlloc;
  181. NoRecoverMemoryJitArenaAllocator localTempAlloc(_u("BE-GlobOpt temp"), pageAllocator, Js::Throw::OutOfMemory);
  182. this->tempAlloc = &localTempAlloc;
  183. // The forward passes use info (upwardExposedUses) from the backward pass. This info
  184. // isn't available for some of the symbols created during the backward pass, or the forward pass.
  185. // Keep track of the last symbol for which we're guaranteed to have data.
  186. this->maxInitialSymID = this->func->m_symTable->GetMaxSymID();
  187. #if DBG
  188. this->BackwardPass(Js::CaptureByteCodeRegUsePhase);
  189. #endif
  190. this->BackwardPass(Js::BackwardPhase);
  191. this->ForwardPass();
  192. this->BackwardPass(Js::DeadStorePhase);
  193. }
  194. this->TailDupPass();
  195. }
  196. bool GlobOpt::ShouldExpectConventionalArrayIndexValue(IR::IndirOpnd *const indirOpnd)
  197. {
  198. Assert(indirOpnd);
  199. if(!indirOpnd->GetIndexOpnd())
  200. {
  201. return indirOpnd->GetOffset() >= 0;
  202. }
  203. IR::RegOpnd *const indexOpnd = indirOpnd->GetIndexOpnd();
  204. if(indexOpnd->m_sym->m_isNotNumber)
  205. {
  206. // Typically, single-def or any sym-specific information for type-specialized syms should not be used because all of
  207. // their defs will not have been accounted for until after the forward pass. But m_isNotNumber is only ever changed from
  208. // false to true, so it's okay in this case.
  209. return false;
  210. }
  211. StackSym *indexVarSym = indexOpnd->m_sym;
  212. if(indexVarSym->IsTypeSpec())
  213. {
  214. indexVarSym = indexVarSym->GetVarEquivSym(nullptr);
  215. Assert(indexVarSym);
  216. }
  217. else if(!IsLoopPrePass())
  218. {
  219. // Don't use single-def info or const flags for type-specialized syms, as all of their defs will not have been accounted
  220. // for until after the forward pass. Also, don't use the const flags in a loop prepass because the const flags may not
  221. // be up-to-date.
  222. if (indexOpnd->IsNotInt())
  223. {
  224. return false;
  225. }
  226. StackSym *const indexSym = indexOpnd->m_sym;
  227. if(indexSym->IsIntConst())
  228. {
  229. return indexSym->GetIntConstValue() >= 0;
  230. }
  231. }
  232. Value *const indexValue = CurrentBlockData()->FindValue(indexVarSym);
  233. if(!indexValue)
  234. {
  235. // Treat it as Uninitialized, assume it's going to be valid
  236. return true;
  237. }
  238. ValueInfo *const indexValueInfo = indexValue->GetValueInfo();
  239. int32 indexConstantValue;
  240. if(indexValueInfo->TryGetIntConstantValue(&indexConstantValue))
  241. {
  242. return indexConstantValue >= 0;
  243. }
  244. if(indexValueInfo->IsUninitialized())
  245. {
  246. // Assume it's going to be valid
  247. return true;
  248. }
  249. return indexValueInfo->HasBeenNumber() && !indexValueInfo->HasBeenFloat();
  250. }
  251. //
  252. // Either result is float or 1/x or cst1/cst2 where cst1%cst2 != 0
  253. //
  254. ValueType GlobOpt::GetDivValueType(IR::Instr* instr, Value* src1Val, Value* src2Val, bool specialize)
  255. {
  256. ValueInfo *src1ValueInfo = (src1Val ? src1Val->GetValueInfo() : nullptr);
  257. ValueInfo *src2ValueInfo = (src2Val ? src2Val->GetValueInfo() : nullptr);
  258. if (instr->IsProfiledInstr() && instr->m_func->HasProfileInfo())
  259. {
  260. ValueType resultType = instr->m_func->GetReadOnlyProfileInfo()->GetDivProfileInfo(static_cast<Js::ProfileId>(instr->AsProfiledInstr()->u.profileId));
  261. if (resultType.IsLikelyInt())
  262. {
  263. if (specialize && src1ValueInfo && src2ValueInfo
  264. && ((src1ValueInfo->IsInt() && src2ValueInfo->IsInt()) ||
  265. (this->DoDivIntTypeSpec() && src1ValueInfo->IsLikelyInt() && src2ValueInfo->IsLikelyInt())))
  266. {
  267. return ValueType::GetInt(true);
  268. }
  269. return resultType;
  270. }
  271. // Consider: Checking that the sources are numbers.
  272. if (resultType.IsLikelyFloat())
  273. {
  274. return ValueType::Float;
  275. }
  276. return resultType;
  277. }
  278. int32 src1IntConstantValue;
  279. if(!src1ValueInfo || !src1ValueInfo->TryGetIntConstantValue(&src1IntConstantValue))
  280. {
  281. return ValueType::Number;
  282. }
  283. if (src1IntConstantValue == 1)
  284. {
  285. return ValueType::Float;
  286. }
  287. int32 src2IntConstantValue;
  288. if(!src2Val || !src2ValueInfo->TryGetIntConstantValue(&src2IntConstantValue))
  289. {
  290. return ValueType::Number;
  291. }
  292. if (src2IntConstantValue // Avoid divide by zero
  293. && !(src1IntConstantValue == 0x80000000 && src2IntConstantValue == -1) // Avoid integer overflow
  294. && (src1IntConstantValue % src2IntConstantValue) != 0)
  295. {
  296. return ValueType::Float;
  297. }
  298. return ValueType::Number;
  299. }
  300. void
  301. GlobOpt::ForwardPass()
  302. {
  303. BEGIN_CODEGEN_PHASE(this->func, Js::ForwardPhase);
  304. #if DBG_DUMP
  305. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::GlobOptPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()))
  306. {
  307. this->func->DumpHeader();
  308. }
  309. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::GlobOptPhase))
  310. {
  311. this->TraceSettings();
  312. }
  313. #endif
  314. // GetConstantCount() gives us the right size to pick for the SparseArray, but we may need more if we've inlined
  315. // functions with constants. There will be a gap in the symbol numbering between the main constants and
  316. // the inlined ones, so we'll most likely need a new array chunk. Make the min size of the array chunks be 64
  317. // in case we have a main function with very few constants and a bunch of constants from inlined functions.
  318. this->byteCodeConstantValueArray = SparseArray<Value>::New(this->alloc, max(this->func->GetJITFunctionBody()->GetConstCount(), 64U));
  319. this->byteCodeConstantValueNumbersBv = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  320. this->tempBv = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  321. this->prePassCopyPropSym = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  322. this->slotSyms = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  323. this->byteCodeUses = nullptr;
  324. this->propertySymUse = nullptr;
  325. // changedSymsAfterIncBailoutCandidate helps track building incremental bailout in ForwardPass
  326. this->changedSymsAfterIncBailoutCandidate = JitAnew(alloc, BVSparse<JitArenaAllocator>, alloc);
  327. this->auxSlotPtrSyms = JitAnew(alloc, BVSparse<JitArenaAllocator>, alloc);
  328. #if DBG
  329. this->byteCodeUsesBeforeOpt = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  330. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase) && this->DoFunctionFieldCopyProp())
  331. {
  332. Output::Print(_u("TRACE: CanDoFieldCopyProp Func: "));
  333. this->func->DumpFullFunctionName();
  334. Output::Print(_u("\n"));
  335. }
  336. #endif
  337. OpndList localNoImplicitCallUsesToInsert(alloc);
  338. this->noImplicitCallUsesToInsert = &localNoImplicitCallUsesToInsert;
  339. IntConstantToStackSymMap localIntConstantToStackSymMap(alloc);
  340. this->intConstantToStackSymMap = &localIntConstantToStackSymMap;
  341. IntConstantToValueMap localIntConstantToValueMap(alloc);
  342. this->intConstantToValueMap = &localIntConstantToValueMap;
  343. Int64ConstantToValueMap localInt64ConstantToValueMap(alloc);
  344. this->int64ConstantToValueMap = &localInt64ConstantToValueMap;
  345. AddrConstantToValueMap localAddrConstantToValueMap(alloc);
  346. this->addrConstantToValueMap = &localAddrConstantToValueMap;
  347. StringConstantToValueMap localStringConstantToValueMap(alloc);
  348. this->stringConstantToValueMap = &localStringConstantToValueMap;
  349. SymIdToInstrMap localPrePassInstrMap(alloc);
  350. this->prePassInstrMap = &localPrePassInstrMap;
  351. ValueSetByValueNumber localValuesCreatedForClone(alloc, 64);
  352. this->valuesCreatedForClone = &localValuesCreatedForClone;
  353. ValueNumberPairToValueMap localValuesCreatedForMerge(alloc, 64);
  354. this->valuesCreatedForMerge = &localValuesCreatedForMerge;
  355. #if DBG
  356. BVSparse<JitArenaAllocator> localFinishedStackLiteralInitFld(alloc);
  357. this->finishedStackLiteralInitFld = &localFinishedStackLiteralInitFld;
  358. #endif
  359. FOREACH_BLOCK_IN_FUNC_EDITING(block, this->func)
  360. {
  361. this->OptBlock(block);
  362. } NEXT_BLOCK_IN_FUNC_EDITING;
  363. if (!PHASE_OFF(Js::MemOpPhase, this->func))
  364. {
  365. ProcessMemOp();
  366. }
  367. this->noImplicitCallUsesToInsert = nullptr;
  368. this->intConstantToStackSymMap = nullptr;
  369. this->intConstantToValueMap = nullptr;
  370. this->int64ConstantToValueMap = nullptr;
  371. this->addrConstantToValueMap = nullptr;
  372. this->stringConstantToValueMap = nullptr;
  373. #if DBG
  374. this->finishedStackLiteralInitFld = nullptr;
  375. uint freedCount = 0;
  376. uint spilledCount = 0;
  377. #endif
  378. FOREACH_BLOCK_IN_FUNC(block, this->func)
  379. {
  380. #if DBG
  381. if (block->GetDataUseCount() == 0)
  382. {
  383. freedCount++;
  384. }
  385. else
  386. {
  387. spilledCount++;
  388. }
  389. #endif
  390. block->SetDataUseCount(0);
  391. if (block->cloneStrCandidates)
  392. {
  393. JitAdelete(this->alloc, block->cloneStrCandidates);
  394. block->cloneStrCandidates = nullptr;
  395. }
  396. } NEXT_BLOCK_IN_FUNC;
  397. // Make sure we free most of them.
  398. Assert(freedCount >= spilledCount);
  399. // this->alloc will be freed right after return, no need to free it here
  400. this->changedSymsAfterIncBailoutCandidate = nullptr;
  401. this->auxSlotPtrSyms = nullptr;
  402. END_CODEGEN_PHASE(this->func, Js::ForwardPhase);
  403. }
  404. void
  405. GlobOpt::OptBlock(BasicBlock *block)
  406. {
  407. if (this->func->m_fg->RemoveUnreachableBlock(block, this))
  408. {
  409. GOPT_TRACE(_u("Removing unreachable block #%d\n"), block->GetBlockNum());
  410. return;
  411. }
  412. Loop * loop = block->loop;
  413. if (loop && block->isLoopHeader)
  414. {
  415. if (loop != this->prePassLoop)
  416. {
  417. OptLoops(loop);
  418. if (!IsLoopPrePass() && loop->parent)
  419. {
  420. loop->fieldPRESymStores->Or(loop->parent->fieldPRESymStores);
  421. }
  422. if (!this->IsLoopPrePass() && DoFieldPRE(loop))
  423. {
  424. // Note: !IsLoopPrePass means this was a root loop pre-pass. FieldPre() is called once per loop.
  425. this->FieldPRE(loop);
  426. // Re-optimize the landing pad
  427. BasicBlock *landingPad = loop->landingPad;
  428. this->isRecursiveCallOnLandingPad = true;
  429. this->OptBlock(landingPad);
  430. this->isRecursiveCallOnLandingPad = false;
  431. }
  432. }
  433. }
  434. this->currentBlock = block;
  435. PrepareLoopArrayCheckHoist();
  436. block->MergePredBlocksValueMaps(this);
  437. this->intOverflowCurrentlyMattersInRange = true;
  438. this->intOverflowDoesNotMatterRange = this->currentBlock->intOverflowDoesNotMatterRange;
  439. if (!DoFieldCopyProp() && !DoFieldRefOpts())
  440. {
  441. this->KillAllFields(CurrentBlockData()->liveFields);
  442. }
  443. this->tempAlloc->Reset();
  444. if(loop && block->isLoopHeader)
  445. {
  446. loop->firstValueNumberInLoop = this->currentValue;
  447. }
  448. GOPT_TRACE_BLOCK(block, true);
  449. FOREACH_INSTR_IN_BLOCK_EDITING(instr, instrNext, block)
  450. {
  451. GOPT_TRACE_INSTRTRACE(instr);
  452. BailOutInfo* oldBailOutInfo = nullptr;
  453. bool isCheckAuxBailoutNeeded = this->func->IsJitInDebugMode() && !this->IsLoopPrePass();
  454. if (isCheckAuxBailoutNeeded && instr->HasAuxBailOut() && !instr->HasBailOutInfo())
  455. {
  456. oldBailOutInfo = instr->GetBailOutInfo();
  457. Assert(oldBailOutInfo);
  458. }
  459. bool isInstrRemoved = false;
  460. instrNext = this->OptInstr(instr, &isInstrRemoved);
  461. // If we still have instrs with only aux bail out, convert aux bail out back to regular bail out and fill it.
  462. // During OptInstr some instr can be moved out to a different block, in this case bailout info is going to be replaced
  463. // with e.g. loop bailout info which is filled as part of processing that block, thus we don't need to fill it here.
  464. if (isCheckAuxBailoutNeeded && !isInstrRemoved && instr->HasAuxBailOut() && !instr->HasBailOutInfo())
  465. {
  466. if (instr->GetBailOutInfo() == oldBailOutInfo)
  467. {
  468. instr->PromoteAuxBailOut();
  469. FillBailOutInfo(block, instr);
  470. }
  471. else
  472. {
  473. AssertMsg(instr->GetBailOutInfo(), "With aux bailout, the bailout info should not be removed by OptInstr.");
  474. }
  475. }
  476. } NEXT_INSTR_IN_BLOCK_EDITING;
  477. GOPT_TRACE_BLOCK(block, false);
  478. if (block->loop)
  479. {
  480. if (IsLoopPrePass())
  481. {
  482. if (DoBoundCheckHoist())
  483. {
  484. DetectUnknownChangesToInductionVariables(&block->globOptData);
  485. }
  486. }
  487. else
  488. {
  489. isPerformingLoopBackEdgeCompensation = true;
  490. Assert(this->tempBv->IsEmpty());
  491. BVSparse<JitArenaAllocator> tempBv2(this->tempAlloc);
  492. // On loop back-edges, we need to restore the state of the type specialized
  493. // symbols to that of the loop header.
  494. FOREACH_SUCCESSOR_BLOCK(succ, block)
  495. {
  496. if (succ->isLoopHeader && succ->loop->IsDescendentOrSelf(block->loop))
  497. {
  498. BVSparse<JitArenaAllocator> *liveOnBackEdge = block->loop->regAlloc.liveOnBackEdgeSyms;
  499. liveOnBackEdge->Or(block->loop->fieldPRESymStores);
  500. this->tempBv->Minus(block->loop->varSymsOnEntry, block->globOptData.liveVarSyms);
  501. this->tempBv->And(liveOnBackEdge);
  502. this->ToVar(this->tempBv, block);
  503. // Lossy int in the loop header, and no int on the back-edge - need a lossy conversion to int
  504. this->tempBv->Minus(block->loop->lossyInt32SymsOnEntry, block->globOptData.liveInt32Syms);
  505. this->tempBv->And(liveOnBackEdge);
  506. this->ToInt32(this->tempBv, block, true /* lossy */);
  507. // Lossless int in the loop header, and no lossless int on the back-edge - need a lossless conversion to int
  508. this->tempBv->Minus(block->loop->int32SymsOnEntry, block->loop->lossyInt32SymsOnEntry);
  509. tempBv2.Minus(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms);
  510. this->tempBv->Minus(&tempBv2);
  511. this->tempBv->And(liveOnBackEdge);
  512. this->ToInt32(this->tempBv, block, false /* lossy */);
  513. this->tempBv->Minus(block->loop->float64SymsOnEntry, block->globOptData.liveFloat64Syms);
  514. this->tempBv->And(liveOnBackEdge);
  515. this->ToFloat64(this->tempBv, block);
  516. // For ints and floats, go aggressive and type specialize in the landing pad any symbol which was specialized on
  517. // entry to the loop body (in the loop header), and is still specialized on this tail, but wasn't specialized in
  518. // the landing pad.
  519. // Lossy int in the loop header and no int in the landing pad - need a lossy conversion to int
  520. // (entry.lossyInt32 - landingPad.int32)
  521. this->tempBv->Minus(block->loop->lossyInt32SymsOnEntry, block->loop->landingPad->globOptData.liveInt32Syms);
  522. this->tempBv->And(liveOnBackEdge);
  523. this->ToInt32(this->tempBv, block->loop->landingPad, true /* lossy */);
  524. // Lossless int in the loop header, and no lossless int in the landing pad - need a lossless conversion to int
  525. // ((entry.int32 - entry.lossyInt32) - (landingPad.int32 - landingPad.lossyInt32))
  526. this->tempBv->Minus(block->loop->int32SymsOnEntry, block->loop->lossyInt32SymsOnEntry);
  527. tempBv2.Minus(
  528. block->loop->landingPad->globOptData.liveInt32Syms,
  529. block->loop->landingPad->globOptData.liveLossyInt32Syms);
  530. this->tempBv->Minus(&tempBv2);
  531. this->tempBv->And(liveOnBackEdge);
  532. this->ToInt32(this->tempBv, block->loop->landingPad, false /* lossy */);
  533. // ((entry.float64 - landingPad.float64) & block.float64)
  534. this->tempBv->Minus(block->loop->float64SymsOnEntry, block->loop->landingPad->globOptData.liveFloat64Syms);
  535. this->tempBv->And(block->globOptData.liveFloat64Syms);
  536. this->tempBv->And(liveOnBackEdge);
  537. this->ToFloat64(this->tempBv, block->loop->landingPad);
  538. if (block->loop->symsRequiringCompensationToMergedValueInfoMap)
  539. {
  540. InsertValueCompensation(block, succ, block->loop->symsRequiringCompensationToMergedValueInfoMap);
  541. }
  542. // Now that we're done with the liveFields within this loop, trim the set to those syms
  543. // that the backward pass told us were live out of the loop.
  544. // This assumes we have no further need of the liveFields within the loop.
  545. if (block->loop->liveOutFields)
  546. {
  547. block->globOptData.liveFields->And(block->loop->liveOutFields);
  548. }
  549. }
  550. } NEXT_SUCCESSOR_BLOCK;
  551. this->tempBv->ClearAll();
  552. isPerformingLoopBackEdgeCompensation = false;
  553. }
  554. }
  555. block->PathDepBranchFolding(this);
  556. #if DBG
  557. // The set of live lossy int32 syms should be a subset of all live int32 syms
  558. this->tempBv->And(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms);
  559. Assert(this->tempBv->Count() == block->globOptData.liveLossyInt32Syms->Count());
  560. // The set of live lossy int32 syms should be a subset of live var or float syms (var or float sym containing the lossless
  561. // value of the sym should be live)
  562. this->tempBv->Or(block->globOptData.liveVarSyms, block->globOptData.liveFloat64Syms);
  563. this->tempBv->And(block->globOptData.liveLossyInt32Syms);
  564. Assert(this->tempBv->Count() == block->globOptData.liveLossyInt32Syms->Count());
  565. this->tempBv->ClearAll();
  566. Assert(this->currentBlock == block);
  567. #endif
  568. }
  569. void
  570. GlobOpt::OptLoops(Loop *loop)
  571. {
  572. Assert(loop != nullptr);
  573. #if DBG
  574. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase) &&
  575. !DoFunctionFieldCopyProp() && DoFieldCopyProp(loop))
  576. {
  577. Output::Print(_u("TRACE: CanDoFieldCopyProp Loop: "));
  578. this->func->DumpFullFunctionName();
  579. uint loopNumber = loop->GetLoopNumber();
  580. Assert(loopNumber != Js::LoopHeader::NoLoop);
  581. Output::Print(_u(" Loop: %d\n"), loopNumber);
  582. }
  583. #endif
  584. Loop *previousLoop = this->prePassLoop;
  585. this->prePassLoop = loop;
  586. if (previousLoop == nullptr)
  587. {
  588. Assert(this->rootLoopPrePass == nullptr);
  589. this->rootLoopPrePass = loop;
  590. this->prePassInstrMap->Clear();
  591. if (loop->parent == nullptr)
  592. {
  593. // Outer most loop...
  594. this->prePassCopyPropSym->ClearAll();
  595. }
  596. }
  597. Assert(loop->symsAssignedToInLoop != nullptr);
  598. if (loop->symsUsedBeforeDefined == nullptr)
  599. {
  600. loop->symsUsedBeforeDefined = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  601. loop->likelyIntSymsUsedBeforeDefined = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  602. loop->likelyNumberSymsUsedBeforeDefined = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  603. loop->forceFloat64SymsOnEntry = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  604. loop->symsDefInLoop = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  605. loop->fieldKilled = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  606. loop->fieldPRESymStores = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  607. loop->allFieldsKilled = false;
  608. }
  609. else
  610. {
  611. loop->symsUsedBeforeDefined->ClearAll();
  612. loop->likelyIntSymsUsedBeforeDefined->ClearAll();
  613. loop->likelyNumberSymsUsedBeforeDefined->ClearAll();
  614. loop->forceFloat64SymsOnEntry->ClearAll();
  615. loop->symsDefInLoop->ClearAll();
  616. loop->fieldKilled->ClearAll();
  617. loop->allFieldsKilled = false;
  618. loop->initialValueFieldMap.Reset();
  619. }
  620. FOREACH_BLOCK_IN_LOOP(block, loop)
  621. {
  622. block->SetDataUseCount(block->GetSuccList()->Count());
  623. OptBlock(block);
  624. } NEXT_BLOCK_IN_LOOP;
  625. if (previousLoop == nullptr)
  626. {
  627. Assert(this->rootLoopPrePass == loop);
  628. this->rootLoopPrePass = nullptr;
  629. }
  630. this->prePassLoop = previousLoop;
  631. }
  632. void
  633. GlobOpt::TailDupPass()
  634. {
  635. FOREACH_LOOP_IN_FUNC_EDITING(loop, this->func)
  636. {
  637. BasicBlock* header = loop->GetHeadBlock();
  638. BasicBlock* loopTail = nullptr;
  639. FOREACH_PREDECESSOR_BLOCK(pred, header)
  640. {
  641. if (loop->IsDescendentOrSelf(pred->loop))
  642. {
  643. loopTail = pred;
  644. break;
  645. }
  646. } NEXT_PREDECESSOR_BLOCK;
  647. if (loopTail)
  648. {
  649. AssertMsg(loopTail->GetLastInstr()->IsBranchInstr(), "LastInstr of loop should always be a branch no?");
  650. if (!loopTail->GetPredList()->HasOne())
  651. {
  652. TryTailDup(loopTail->GetLastInstr()->AsBranchInstr());
  653. }
  654. }
  655. } NEXT_LOOP_IN_FUNC_EDITING;
  656. }
  657. bool
  658. GlobOpt::TryTailDup(IR::BranchInstr *tailBranch)
  659. {
  660. if (PHASE_OFF(Js::TailDupPhase, tailBranch->m_func->GetTopFunc()))
  661. {
  662. return false;
  663. }
  664. if (tailBranch->IsConditional())
  665. {
  666. return false;
  667. }
  668. IR::Instr *instr;
  669. uint instrCount = 0;
  670. for (instr = tailBranch->GetPrevRealInstrOrLabel(); !instr->IsLabelInstr(); instr = instr->GetPrevRealInstrOrLabel())
  671. {
  672. if (instr->HasBailOutInfo())
  673. {
  674. break;
  675. }
  676. if (!OpCodeAttr::CanCSE(instr->m_opcode))
  677. {
  678. // Consider: We could be more aggressive here
  679. break;
  680. }
  681. instrCount++;
  682. if (instrCount > 1)
  683. {
  684. // Consider: If copy handled single-def tmps renaming, we could do more instrs
  685. break;
  686. }
  687. }
  688. if (!instr->IsLabelInstr())
  689. {
  690. return false;
  691. }
  692. IR::LabelInstr *mergeLabel = instr->AsLabelInstr();
  693. IR::Instr *mergeLabelPrev = mergeLabel->m_prev;
  694. // Skip unreferenced labels
  695. while (mergeLabelPrev->IsLabelInstr() && mergeLabelPrev->AsLabelInstr()->labelRefs.Empty())
  696. {
  697. mergeLabelPrev = mergeLabelPrev->m_prev;
  698. }
  699. BasicBlock* labelBlock = mergeLabel->GetBasicBlock();
  700. uint origPredCount = labelBlock->GetPredList()->Count();
  701. uint dupCount = 0;
  702. // We are good to go. Let's do the tail duplication.
  703. FOREACH_SLISTCOUNTED_ENTRY_EDITING(IR::BranchInstr*, branchEntry, &mergeLabel->labelRefs, iter)
  704. {
  705. if (branchEntry->IsUnconditional() && !branchEntry->IsMultiBranch() && branchEntry != mergeLabelPrev && branchEntry != tailBranch)
  706. {
  707. for (instr = mergeLabel->m_next; instr != tailBranch; instr = instr->m_next)
  708. {
  709. branchEntry->InsertBefore(instr->Copy());
  710. }
  711. instr = branchEntry;
  712. branchEntry->ReplaceTarget(mergeLabel, tailBranch->GetTarget());
  713. while(!instr->IsLabelInstr())
  714. {
  715. instr = instr->m_prev;
  716. }
  717. BasicBlock* branchBlock = instr->AsLabelInstr()->GetBasicBlock();
  718. labelBlock->RemovePred(branchBlock, func->m_fg);
  719. func->m_fg->AddEdge(branchBlock, tailBranch->GetTarget()->GetBasicBlock());
  720. dupCount++;
  721. }
  722. } NEXT_SLISTCOUNTED_ENTRY_EDITING;
  723. // If we've duplicated everywhere, tail block is dead and should be removed.
  724. if (dupCount == origPredCount)
  725. {
  726. AssertMsg(mergeLabel->labelRefs.Empty(), "Should not remove block with referenced label.");
  727. func->m_fg->RemoveBlock(labelBlock, nullptr, true);
  728. }
  729. return true;
  730. }
  731. void
  732. GlobOpt::ToVar(BVSparse<JitArenaAllocator> *bv, BasicBlock *block, IR::Instr* insertBeforeInstr /* = nullptr */)
  733. {
  734. FOREACH_BITSET_IN_SPARSEBV(id, bv)
  735. {
  736. StackSym *stackSym = this->func->m_symTable->FindStackSym(id);
  737. IR::RegOpnd *newOpnd = IR::RegOpnd::New(stackSym, TyVar, this->func);
  738. IR::Instr* lastInstr = block->GetLastInstr();
  739. if (insertBeforeInstr != nullptr)
  740. {
  741. this->ToVar(insertBeforeInstr, newOpnd, block, nullptr, false);
  742. }
  743. else if (lastInstr->IsBranchInstr() || lastInstr->m_opcode == Js::OpCode::BailTarget)
  744. {
  745. // If branch is using this symbol, hoist the operand as the ToVar load will get
  746. // inserted right before the branch.
  747. IR::Opnd *src1 = lastInstr->GetSrc1();
  748. if (src1)
  749. {
  750. if (src1->IsRegOpnd() && src1->AsRegOpnd()->m_sym == stackSym)
  751. {
  752. lastInstr->HoistSrc1(Js::OpCode::Ld_A);
  753. }
  754. IR::Opnd *src2 = lastInstr->GetSrc2();
  755. if (src2)
  756. {
  757. if (src2->IsRegOpnd() && src2->AsRegOpnd()->m_sym == stackSym)
  758. {
  759. lastInstr->HoistSrc2(Js::OpCode::Ld_A);
  760. }
  761. }
  762. }
  763. this->ToVar(lastInstr, newOpnd, block, nullptr, false);
  764. }
  765. else
  766. {
  767. IR::Instr *lastNextInstr = lastInstr->m_next;
  768. this->ToVar(lastNextInstr, newOpnd, block, nullptr, false);
  769. }
  770. } NEXT_BITSET_IN_SPARSEBV;
  771. }
  772. void
  773. GlobOpt::ToInt32(BVSparse<JitArenaAllocator> *bv, BasicBlock *block, bool lossy, IR::Instr *insertBeforeInstr)
  774. {
  775. return this->ToTypeSpec(bv, block, TyInt32, IR::BailOutIntOnly, lossy, insertBeforeInstr);
  776. }
  777. void
  778. GlobOpt::ToFloat64(BVSparse<JitArenaAllocator> *bv, BasicBlock *block)
  779. {
  780. return this->ToTypeSpec(bv, block, TyFloat64, IR::BailOutNumberOnly);
  781. }
  782. void
  783. GlobOpt::ToTypeSpec(BVSparse<JitArenaAllocator> *bv, BasicBlock *block, IRType toType, IR::BailOutKind bailOutKind, bool lossy, IR::Instr *insertBeforeInstr)
  784. {
  785. FOREACH_BITSET_IN_SPARSEBV(id, bv)
  786. {
  787. StackSym *stackSym = this->func->m_symTable->FindStackSym(id);
  788. IRType fromType = TyIllegal;
  789. // Win8 bug: 757126. If we are trying to type specialize the arguments object,
  790. // let's make sure stack args optimization is not enabled. This is a problem, particularly,
  791. // if the instruction comes from an unreachable block. In other cases, the pass on the
  792. // instruction itself should disable arguments object optimization.
  793. if(block->globOptData.argObjSyms && block->globOptData.IsArgumentsSymID(id))
  794. {
  795. CannotAllocateArgumentsObjectOnStack(nullptr);
  796. }
  797. if (block->globOptData.liveVarSyms->Test(id))
  798. {
  799. fromType = TyVar;
  800. }
  801. else if (block->globOptData.liveInt32Syms->Test(id) && !block->globOptData.liveLossyInt32Syms->Test(id))
  802. {
  803. fromType = TyInt32;
  804. stackSym = stackSym->GetInt32EquivSym(this->func);
  805. }
  806. else if (block->globOptData.liveFloat64Syms->Test(id))
  807. {
  808. fromType = TyFloat64;
  809. stackSym = stackSym->GetFloat64EquivSym(this->func);
  810. }
  811. else
  812. {
  813. Assert(UNREACHED);
  814. }
  815. IR::RegOpnd *newOpnd = IR::RegOpnd::New(stackSym, fromType, this->func);
  816. this->ToTypeSpecUse(nullptr, newOpnd, block, nullptr, nullptr, toType, bailOutKind, lossy, insertBeforeInstr);
  817. } NEXT_BITSET_IN_SPARSEBV;
  818. }
  819. void GlobOpt::PRE::FindPossiblePRECandidates(Loop *loop, JitArenaAllocator *alloc)
  820. {
  821. // Find the set of PRE candidates
  822. BasicBlock *loopHeader = loop->GetHeadBlock();
  823. PRECandidates *candidates = nullptr;
  824. bool firstBackEdge = true;
  825. FOREACH_PREDECESSOR_BLOCK(blockPred, loopHeader)
  826. {
  827. if (!loop->IsDescendentOrSelf(blockPred->loop))
  828. {
  829. // Not a loop back-edge
  830. continue;
  831. }
  832. if (firstBackEdge)
  833. {
  834. candidates = this->globOpt->FindBackEdgePRECandidates(blockPred, alloc);
  835. }
  836. else
  837. {
  838. blockPred->globOptData.RemoveUnavailableCandidates(candidates);
  839. }
  840. } NEXT_PREDECESSOR_BLOCK;
  841. this->candidates = candidates;
  842. }
  843. BOOL GlobOpt::PRE::PreloadPRECandidate(Loop *loop, GlobHashBucket* candidate)
  844. {
  845. // Insert a load for each field PRE candidate.
  846. PropertySym *propertySym = candidate->value->AsPropertySym();
  847. if (!candidates->candidatesToProcess->TestAndClear(propertySym->m_id))
  848. {
  849. return false;
  850. }
  851. Value * propSymValueOnBackEdge = candidate->element;
  852. StackSym *objPtrSym = propertySym->m_stackSym;
  853. Sym * objPtrCopyPropSym = nullptr;
  854. if (!loop->landingPad->globOptData.IsLive(objPtrSym))
  855. {
  856. if (PHASE_OFF(Js::MakeObjSymLiveInLandingPadPhase, this->globOpt->func))
  857. {
  858. return false;
  859. }
  860. if (objPtrSym->IsSingleDef())
  861. {
  862. // We can still try to do PRE if the object sym is single def, even if its not live in the landing pad.
  863. // We'll have to add a def instruction for the object sym in the landing pad, and then we can continue
  864. // pre-loading the current PRE candidate.
  865. // Case in point:
  866. // $L1
  867. // value|symStore
  868. // t1 = o.x (v1|t3)
  869. // t2 = t1.y (v2|t4) <-- t1 is not live in the loop landing pad
  870. // jmp $L1
  871. if (!InsertSymDefinitionInLandingPad(objPtrSym, loop, &objPtrCopyPropSym))
  872. {
  873. #if DBG_DUMP
  874. TraceFailedPreloadInLandingPad(loop, propertySym, _u("Failed to insert load of object sym in landing pad"));
  875. #endif
  876. return false;
  877. }
  878. }
  879. else
  880. {
  881. #if DBG_DUMP
  882. TraceFailedPreloadInLandingPad(loop, propertySym, _u("Object sym not live in landing pad and not single-def"));
  883. #endif
  884. return false;
  885. }
  886. }
  887. Assert(loop->landingPad->globOptData.IsLive(objPtrSym));
  888. BasicBlock *landingPad = loop->landingPad;
  889. Sym *symStore = propSymValueOnBackEdge->GetValueInfo()->GetSymStore();
  890. // The symStore can't be live into the loop
  891. // The symStore needs to still have the same value
  892. Assert(symStore && symStore->IsStackSym());
  893. if (loop->landingPad->globOptData.IsLive(symStore))
  894. {
  895. // May have already been hoisted:
  896. // o.x = t1;
  897. // o.y = t1;
  898. return false;
  899. }
  900. Value *landingPadValue = landingPad->globOptData.FindValue(propertySym);
  901. // Value should be added as initial value or already be there.
  902. Assert(landingPadValue);
  903. IR::Instr * ldInstrInLoop = this->globOpt->prePassInstrMap->Lookup(propertySym->m_id, nullptr);
  904. Assert(ldInstrInLoop);
  905. Assert(ldInstrInLoop->GetDst() == nullptr);
  906. // Create instr to put in landing pad for compensation
  907. Assert(IsPREInstrCandidateLoad(ldInstrInLoop->m_opcode));
  908. IR::Instr * ldInstr = InsertPropertySymPreloadInLandingPad(ldInstrInLoop, loop, propertySym);
  909. if (!ldInstr)
  910. {
  911. return false;
  912. }
  913. Assert(ldInstr->GetDst() == nullptr);
  914. ldInstr->SetDst(IR::RegOpnd::New(symStore->AsStackSym(), TyVar, this->globOpt->func));
  915. loop->fieldPRESymStores->Set(symStore->m_id);
  916. landingPad->globOptData.liveVarSyms->Set(symStore->m_id);
  917. Value * objPtrValue = landingPad->globOptData.FindValue(objPtrSym);
  918. objPtrCopyPropSym = objPtrCopyPropSym ? objPtrCopyPropSym : objPtrValue ? landingPad->globOptData.GetCopyPropSym(objPtrSym, objPtrValue) : nullptr;
  919. if (objPtrCopyPropSym)
  920. {
  921. // If we inserted T4 = T1.y, and T3 is the copy prop sym for T1 in the landing pad, we need T3.y
  922. // to be live on back edges to have the merge produce a value for T3.y. Having a value for T1.y
  923. // produced from the merge is not enough as the T1.y in the loop will get obj-ptr-copy-propped to
  924. // T3.y
  925. // T3.y
  926. PropertySym *newPropSym = PropertySym::FindOrCreate(
  927. objPtrCopyPropSym->m_id, propertySym->m_propertyId, propertySym->GetPropertyIdIndex(), propertySym->GetInlineCacheIndex(), propertySym->m_fieldKind, this->globOpt->func);
  928. if (!landingPad->globOptData.FindValue(newPropSym))
  929. {
  930. landingPad->globOptData.SetValue(landingPadValue, newPropSym);
  931. landingPad->globOptData.liveFields->Set(newPropSym->m_id);
  932. MakePropertySymLiveOnBackEdges(newPropSym, loop, propSymValueOnBackEdge);
  933. }
  934. }
  935. ValueType valueType(ValueType::Uninitialized);
  936. Value *initialValue = nullptr;
  937. if (loop->initialValueFieldMap.TryGetValue(propertySym, &initialValue))
  938. {
  939. if (ldInstr->IsProfiledInstr())
  940. {
  941. if (initialValue->GetValueNumber() == propSymValueOnBackEdge->GetValueNumber())
  942. {
  943. if (propSymValueOnBackEdge->GetValueInfo()->IsUninitialized())
  944. {
  945. valueType = ldInstr->AsProfiledInstr()->u.FldInfo().valueType;
  946. }
  947. else
  948. {
  949. valueType = propSymValueOnBackEdge->GetValueInfo()->Type();
  950. }
  951. }
  952. else
  953. {
  954. valueType = ValueType::Uninitialized;
  955. }
  956. ldInstr->AsProfiledInstr()->u.FldInfo().valueType = valueType;
  957. }
  958. }
  959. else
  960. {
  961. valueType = landingPadValue->GetValueInfo()->Type();
  962. }
  963. loop->symsUsedBeforeDefined->Set(symStore->m_id);
  964. if (valueType.IsLikelyNumber())
  965. {
  966. loop->likelyNumberSymsUsedBeforeDefined->Set(symStore->m_id);
  967. if (globOpt->DoAggressiveIntTypeSpec() ? valueType.IsLikelyInt() : valueType.IsInt())
  968. {
  969. // Can only force int conversions in the landing pad based on likely-int values if aggressive int type
  970. // specialization is enabled
  971. loop->likelyIntSymsUsedBeforeDefined->Set(symStore->m_id);
  972. }
  973. }
  974. #if DBG_DUMP
  975. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldPREPhase, this->globOpt->func->GetSourceContextId(), this->globOpt->func->GetLocalFunctionId()))
  976. {
  977. Output::Print(_u("** TRACE: Field PRE: field pre-loaded in landing pad of loop head #%-3d: "), loop->GetHeadBlock()->GetBlockNum());
  978. ldInstr->Dump();
  979. Output::Print(_u("\n"));
  980. Output::Flush();
  981. }
  982. #endif
  983. return true;
  984. }
  985. void GlobOpt::PRE::PreloadPRECandidates(Loop *loop)
  986. {
  987. // Insert loads in landing pad for field PRE candidates. Iterate while(changed)
  988. // for the o.x.y cases.
  989. BOOL changed = true;
  990. if (!candidates || !candidates->candidatesList)
  991. {
  992. return;
  993. }
  994. Assert(loop->landingPad->GetFirstInstr() == loop->landingPad->GetLastInstr());
  995. while (changed)
  996. {
  997. changed = false;
  998. FOREACH_SLIST_ENTRY_EDITING(GlobHashBucket*, candidate, (SList<GlobHashBucket*>*)candidates->candidatesList, iter)
  999. {
  1000. if (this->PreloadPRECandidate(loop, candidate))
  1001. {
  1002. changed = true;
  1003. iter.RemoveCurrent();
  1004. }
  1005. if (PHASE_TRACE(Js::FieldPREPhase, this->globOpt->func))
  1006. {
  1007. Output::Print(_u("============================\n"));
  1008. Output::Flush();
  1009. }
  1010. } NEXT_SLIST_ENTRY_EDITING;
  1011. }
  1012. }
  1013. void GlobOpt::FieldPRE(Loop *loop)
  1014. {
  1015. if (!DoFieldPRE(loop))
  1016. {
  1017. return;
  1018. }
  1019. GlobOpt::PRE pre(this);
  1020. pre.FieldPRE(loop);
  1021. }
  1022. void GlobOpt::InsertValueCompensation(
  1023. BasicBlock *const predecessor,
  1024. BasicBlock *const successor,
  1025. const SymToValueInfoMap *symsRequiringCompensationToMergedValueInfoMap)
  1026. {
  1027. Assert(predecessor);
  1028. Assert(successor);
  1029. AssertOrFailFast(predecessor != successor);
  1030. Assert(symsRequiringCompensationToMergedValueInfoMap->Count() != 0);
  1031. IR::Instr *insertBeforeInstr = predecessor->GetLastInstr();
  1032. Func *const func = insertBeforeInstr->m_func;
  1033. bool setLastInstrInPredecessor;
  1034. // If this is a loop back edge, and the successor has been completed, don't attempt to update its block data.
  1035. // The update is unnecessary, and the data has likely been freed.
  1036. bool updateSuccessorBlockData = !this->isPerformingLoopBackEdgeCompensation || successor->GetDataUseCount() > 0;
  1037. if(insertBeforeInstr->IsBranchInstr() || insertBeforeInstr->m_opcode == Js::OpCode::BailTarget)
  1038. {
  1039. // Don't insert code between the branch and the corresponding ByteCodeUses instructions
  1040. while(insertBeforeInstr->m_prev->m_opcode == Js::OpCode::ByteCodeUses)
  1041. {
  1042. insertBeforeInstr = insertBeforeInstr->m_prev;
  1043. }
  1044. setLastInstrInPredecessor = false;
  1045. }
  1046. else
  1047. {
  1048. // Insert at the end of the block and set the last instruction
  1049. Assert(insertBeforeInstr->m_next);
  1050. insertBeforeInstr = insertBeforeInstr->m_next; // Instruction after the last instruction in the predecessor
  1051. setLastInstrInPredecessor = true;
  1052. }
  1053. GlobOptBlockData &predecessorBlockData = predecessor->globOptData;
  1054. GlobOptBlockData &successorBlockData = successor->globOptData;
  1055. struct DelayChangeValueInfo
  1056. {
  1057. Value* predecessorValue;
  1058. ArrayValueInfo* valueInfo;
  1059. void ChangeValueInfo(BasicBlock* predecessor, GlobOpt* g)
  1060. {
  1061. g->ChangeValueInfo(
  1062. predecessor,
  1063. predecessorValue,
  1064. valueInfo,
  1065. false /*allowIncompatibleType*/,
  1066. true /*compensated*/);
  1067. }
  1068. };
  1069. JsUtil::List<DelayChangeValueInfo, ArenaAllocator> delayChangeValueInfo(alloc);
  1070. for(auto it = symsRequiringCompensationToMergedValueInfoMap->GetIterator(); it.IsValid(); it.MoveNext())
  1071. {
  1072. const auto &entry = it.Current();
  1073. Sym *const sym = entry.Key();
  1074. Value *const predecessorValue = predecessorBlockData.FindValue(sym);
  1075. Assert(predecessorValue);
  1076. ValueInfo *const predecessorValueInfo = predecessorValue->GetValueInfo();
  1077. // Currently, array value infos are the only ones that require compensation based on values
  1078. Assert(predecessorValueInfo->IsAnyOptimizedArray());
  1079. const ArrayValueInfo *const predecessorArrayValueInfo = predecessorValueInfo->AsArrayValueInfo();
  1080. StackSym *const predecessorHeadSegmentSym = predecessorArrayValueInfo->HeadSegmentSym();
  1081. StackSym *const predecessorHeadSegmentLengthSym = predecessorArrayValueInfo->HeadSegmentLengthSym();
  1082. StackSym *const predecessorLengthSym = predecessorArrayValueInfo->LengthSym();
  1083. ValueInfo *const mergedValueInfo = entry.Value();
  1084. const ArrayValueInfo *const mergedArrayValueInfo = mergedValueInfo->AsArrayValueInfo();
  1085. StackSym *const mergedHeadSegmentSym = mergedArrayValueInfo->HeadSegmentSym();
  1086. StackSym *const mergedHeadSegmentLengthSym = mergedArrayValueInfo->HeadSegmentLengthSym();
  1087. StackSym *const mergedLengthSym = mergedArrayValueInfo->LengthSym();
  1088. Assert(!mergedHeadSegmentSym || predecessorHeadSegmentSym);
  1089. Assert(!mergedHeadSegmentLengthSym || predecessorHeadSegmentLengthSym);
  1090. Assert(!mergedLengthSym || predecessorLengthSym);
  1091. bool compensated = false;
  1092. if(mergedHeadSegmentSym && predecessorHeadSegmentSym != mergedHeadSegmentSym)
  1093. {
  1094. IR::Instr *const newInstr =
  1095. IR::Instr::New(
  1096. Js::OpCode::Ld_A,
  1097. IR::RegOpnd::New(mergedHeadSegmentSym, mergedHeadSegmentSym->GetType(), func),
  1098. IR::RegOpnd::New(predecessorHeadSegmentSym, predecessorHeadSegmentSym->GetType(), func),
  1099. func);
  1100. newInstr->GetDst()->SetIsJITOptimizedReg(true);
  1101. newInstr->GetSrc1()->SetIsJITOptimizedReg(true);
  1102. newInstr->SetByteCodeOffset(insertBeforeInstr);
  1103. insertBeforeInstr->InsertBefore(newInstr);
  1104. compensated = true;
  1105. }
  1106. if(mergedHeadSegmentLengthSym && predecessorHeadSegmentLengthSym != mergedHeadSegmentLengthSym)
  1107. {
  1108. IR::Instr *const newInstr =
  1109. IR::Instr::New(
  1110. Js::OpCode::Ld_A,
  1111. IR::RegOpnd::New(mergedHeadSegmentLengthSym, mergedHeadSegmentLengthSym->GetType(), func),
  1112. IR::RegOpnd::New(predecessorHeadSegmentLengthSym, predecessorHeadSegmentLengthSym->GetType(), func),
  1113. func);
  1114. newInstr->GetDst()->SetIsJITOptimizedReg(true);
  1115. newInstr->GetSrc1()->SetIsJITOptimizedReg(true);
  1116. newInstr->SetByteCodeOffset(insertBeforeInstr);
  1117. insertBeforeInstr->InsertBefore(newInstr);
  1118. compensated = true;
  1119. // Merge the head segment length value
  1120. Assert(predecessorBlockData.liveVarSyms->Test(predecessorHeadSegmentLengthSym->m_id));
  1121. predecessorBlockData.liveVarSyms->Set(mergedHeadSegmentLengthSym->m_id);
  1122. Value *const predecessorHeadSegmentLengthValue =
  1123. predecessorBlockData.FindValue(predecessorHeadSegmentLengthSym);
  1124. Assert(predecessorHeadSegmentLengthValue);
  1125. predecessorBlockData.SetValue(predecessorHeadSegmentLengthValue, mergedHeadSegmentLengthSym);
  1126. if (updateSuccessorBlockData)
  1127. {
  1128. successorBlockData.liveVarSyms->Set(mergedHeadSegmentLengthSym->m_id);
  1129. Value *const mergedHeadSegmentLengthValue = successorBlockData.FindValue(mergedHeadSegmentLengthSym);
  1130. if(mergedHeadSegmentLengthValue)
  1131. {
  1132. Assert(mergedHeadSegmentLengthValue->GetValueNumber() != predecessorHeadSegmentLengthValue->GetValueNumber());
  1133. if(predecessorHeadSegmentLengthValue->GetValueInfo() != mergedHeadSegmentLengthValue->GetValueInfo())
  1134. {
  1135. mergedHeadSegmentLengthValue->SetValueInfo(
  1136. ValueInfo::MergeLikelyIntValueInfo(
  1137. this->alloc,
  1138. mergedHeadSegmentLengthValue,
  1139. predecessorHeadSegmentLengthValue,
  1140. mergedHeadSegmentLengthValue->GetValueInfo()->Type()
  1141. .Merge(predecessorHeadSegmentLengthValue->GetValueInfo()->Type())));
  1142. }
  1143. }
  1144. else
  1145. {
  1146. successorBlockData.SetValue(CopyValue(predecessorHeadSegmentLengthValue), mergedHeadSegmentLengthSym);
  1147. }
  1148. }
  1149. }
  1150. if(mergedLengthSym && predecessorLengthSym != mergedLengthSym)
  1151. {
  1152. IR::Instr *const newInstr =
  1153. IR::Instr::New(
  1154. Js::OpCode::Ld_I4,
  1155. IR::RegOpnd::New(mergedLengthSym, mergedLengthSym->GetType(), func),
  1156. IR::RegOpnd::New(predecessorLengthSym, predecessorLengthSym->GetType(), func),
  1157. func);
  1158. newInstr->GetDst()->SetIsJITOptimizedReg(true);
  1159. newInstr->GetSrc1()->SetIsJITOptimizedReg(true);
  1160. newInstr->SetByteCodeOffset(insertBeforeInstr);
  1161. insertBeforeInstr->InsertBefore(newInstr);
  1162. compensated = true;
  1163. // Merge the length value
  1164. Assert(predecessorBlockData.liveVarSyms->Test(predecessorLengthSym->m_id));
  1165. predecessorBlockData.liveVarSyms->Set(mergedLengthSym->m_id);
  1166. Value *const predecessorLengthValue = predecessorBlockData.FindValue(predecessorLengthSym);
  1167. Assert(predecessorLengthValue);
  1168. predecessorBlockData.SetValue(predecessorLengthValue, mergedLengthSym);
  1169. if (updateSuccessorBlockData)
  1170. {
  1171. successorBlockData.liveVarSyms->Set(mergedLengthSym->m_id);
  1172. Value *const mergedLengthValue = successorBlockData.FindValue(mergedLengthSym);
  1173. if(mergedLengthValue)
  1174. {
  1175. Assert(mergedLengthValue->GetValueNumber() != predecessorLengthValue->GetValueNumber());
  1176. if(predecessorLengthValue->GetValueInfo() != mergedLengthValue->GetValueInfo())
  1177. {
  1178. mergedLengthValue->SetValueInfo(
  1179. ValueInfo::MergeLikelyIntValueInfo(
  1180. this->alloc,
  1181. mergedLengthValue,
  1182. predecessorLengthValue,
  1183. mergedLengthValue->GetValueInfo()->Type().Merge(predecessorLengthValue->GetValueInfo()->Type())));
  1184. }
  1185. }
  1186. else
  1187. {
  1188. successorBlockData.SetValue(CopyValue(predecessorLengthValue), mergedLengthSym);
  1189. }
  1190. }
  1191. }
  1192. if(compensated)
  1193. {
  1194. // Save the new ValueInfo for later.
  1195. // We don't want other symbols needing compensation to see this new one
  1196. delayChangeValueInfo.Add({
  1197. predecessorValue,
  1198. ArrayValueInfo::New(
  1199. alloc,
  1200. predecessorValueInfo->Type(),
  1201. mergedHeadSegmentSym ? mergedHeadSegmentSym : predecessorHeadSegmentSym,
  1202. mergedHeadSegmentLengthSym ? mergedHeadSegmentLengthSym : predecessorHeadSegmentLengthSym,
  1203. mergedLengthSym ? mergedLengthSym : predecessorLengthSym,
  1204. predecessorValueInfo->GetSymStore())
  1205. });
  1206. }
  1207. }
  1208. // Once we've compensated all the symbols, update the new ValueInfo.
  1209. delayChangeValueInfo.Map([predecessor, this](int, DelayChangeValueInfo d) { d.ChangeValueInfo(predecessor, this); });
  1210. if(setLastInstrInPredecessor)
  1211. {
  1212. predecessor->SetLastInstr(insertBeforeInstr->m_prev);
  1213. }
  1214. }
  1215. bool
  1216. GlobOpt::AreFromSameBytecodeFunc(IR::RegOpnd const* src1, IR::RegOpnd const* dst) const
  1217. {
  1218. Assert(this->func->m_symTable->FindStackSym(src1->m_sym->m_id) == src1->m_sym);
  1219. Assert(this->func->m_symTable->FindStackSym(dst->m_sym->m_id) == dst->m_sym);
  1220. if (dst->m_sym->HasByteCodeRegSlot() && src1->m_sym->HasByteCodeRegSlot())
  1221. {
  1222. return src1->m_sym->GetByteCodeFunc() == dst->m_sym->GetByteCodeFunc();
  1223. }
  1224. return false;
  1225. }
  1226. /*
  1227. * This is for scope object removal along with Heap Arguments optimization.
  1228. * We track several instructions to facilitate the removal of scope object.
  1229. * - LdSlotArr - This instr is tracked to keep track of the formals array (the dest)
  1230. * - InlineeStart - To keep track of the stack syms for the formals of the inlinee.
  1231. */
  1232. void
  1233. GlobOpt::TrackInstrsForScopeObjectRemoval(IR::Instr * instr)
  1234. {
  1235. IR::Opnd* dst = instr->GetDst();
  1236. IR::Opnd* src1 = instr->GetSrc1();
  1237. if (instr->m_opcode == Js::OpCode::Ld_A && src1->IsRegOpnd())
  1238. {
  1239. AssertMsg(!instr->m_func->IsStackArgsEnabled() || !src1->IsScopeObjOpnd(instr->m_func), "There can be no aliasing for scope object.");
  1240. }
  1241. // The following is to track formals array for Stack Arguments optimization with Formals
  1242. if (instr->m_func->IsStackArgsEnabled() && !this->IsLoopPrePass())
  1243. {
  1244. if (instr->m_opcode == Js::OpCode::LdSlotArr)
  1245. {
  1246. if (instr->GetSrc1()->IsScopeObjOpnd(instr->m_func))
  1247. {
  1248. AssertMsg(!instr->m_func->GetJITFunctionBody()->HasImplicitArgIns(), "No mapping is required in this case. So it should already be generating ArgIns.");
  1249. instr->m_func->TrackFormalsArraySym(dst->GetStackSym()->m_id);
  1250. }
  1251. }
  1252. else if (instr->m_opcode == Js::OpCode::InlineeStart)
  1253. {
  1254. Assert(instr->m_func->IsInlined());
  1255. Js::ArgSlot actualsCount = instr->m_func->actualCount - 1;
  1256. Js::ArgSlot formalsCount = instr->m_func->GetJITFunctionBody()->GetInParamsCount() - 1;
  1257. Func * func = instr->m_func;
  1258. Func * inlinerFunc = func->GetParentFunc(); //Inliner's func
  1259. IR::Instr * argOutInstr = instr->GetSrc2()->GetStackSym()->GetInstrDef();
  1260. //The argout immediately before the InlineeStart will be the ArgOut for NewScObject
  1261. //So we don't want to track the stack sym for this argout.- Skipping it here.
  1262. if (instr->m_func->IsInlinedConstructor())
  1263. {
  1264. //PRE might introduce a second defintion for the Src1. So assert for the opcode only when it has single definition.
  1265. Assert(argOutInstr->GetSrc1()->GetStackSym()->GetInstrDef() == nullptr ||
  1266. argOutInstr->GetSrc1()->GetStackSym()->GetInstrDef()->m_opcode == Js::OpCode::NewScObjectNoCtor);
  1267. argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef();
  1268. }
  1269. if (formalsCount < actualsCount)
  1270. {
  1271. Js::ArgSlot extraActuals = actualsCount - formalsCount;
  1272. //Skipping extra actuals passed
  1273. for (Js::ArgSlot i = 0; i < extraActuals; i++)
  1274. {
  1275. argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef();
  1276. }
  1277. }
  1278. StackSym * undefinedSym = nullptr;
  1279. for (Js::ArgSlot param = formalsCount; param > 0; param--)
  1280. {
  1281. StackSym * argOutSym = nullptr;
  1282. if (argOutInstr->GetSrc1())
  1283. {
  1284. if (argOutInstr->GetSrc1()->IsRegOpnd())
  1285. {
  1286. argOutSym = argOutInstr->GetSrc1()->GetStackSym();
  1287. }
  1288. else
  1289. {
  1290. // We will always have ArgOut instr - so the source operand will not be removed.
  1291. argOutSym = StackSym::New(inlinerFunc);
  1292. IR::Opnd * srcOpnd = argOutInstr->GetSrc1();
  1293. IR::Opnd * dstOpnd = IR::RegOpnd::New(argOutSym, TyVar, inlinerFunc);
  1294. IR::Instr * assignInstr = IR::Instr::New(Js::OpCode::Ld_A, dstOpnd, srcOpnd, inlinerFunc);
  1295. instr->InsertBefore(assignInstr);
  1296. }
  1297. }
  1298. Assert(!func->HasStackSymForFormal(param - 1));
  1299. if (param <= actualsCount)
  1300. {
  1301. Assert(argOutSym);
  1302. func->TrackStackSymForFormalIndex(param - 1, argOutSym);
  1303. argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef();
  1304. }
  1305. else
  1306. {
  1307. /*When param is out of range of actuals count, load undefined*/
  1308. // TODO: saravind: This will insert undefined for each of the param not having an actual. - Clean up this by having a sym for undefined on func ?
  1309. Assert(formalsCount > actualsCount);
  1310. if (undefinedSym == nullptr)
  1311. {
  1312. undefinedSym = StackSym::New(inlinerFunc);
  1313. IR::Opnd * srcOpnd = IR::AddrOpnd::New(inlinerFunc->GetScriptContextInfo()->GetUndefinedAddr(), IR::AddrOpndKindDynamicMisc, inlinerFunc);
  1314. IR::Opnd * dstOpnd = IR::RegOpnd::New(undefinedSym, TyVar, inlinerFunc);
  1315. IR::Instr * assignUndefined = IR::Instr::New(Js::OpCode::Ld_A, dstOpnd, srcOpnd, inlinerFunc);
  1316. instr->InsertBefore(assignUndefined);
  1317. }
  1318. func->TrackStackSymForFormalIndex(param - 1, undefinedSym);
  1319. }
  1320. }
  1321. }
  1322. }
  1323. }
  1324. void
  1325. GlobOpt::OptArguments(IR::Instr *instr)
  1326. {
  1327. IR::Opnd* dst = instr->GetDst();
  1328. IR::Opnd* src1 = instr->GetSrc1();
  1329. IR::Opnd* src2 = instr->GetSrc2();
  1330. TrackInstrsForScopeObjectRemoval(instr);
  1331. if (!TrackArgumentsObject())
  1332. {
  1333. return;
  1334. }
  1335. if (instr->HasAnyLoadHeapArgsOpCode())
  1336. {
  1337. #ifdef ENABLE_DEBUG_CONFIG_OPTIONS
  1338. if (instr->m_func->IsStackArgsEnabled())
  1339. {
  1340. if (instr->GetSrc1()->IsRegOpnd() && instr->m_func->GetJITFunctionBody()->GetInParamsCount() > 1)
  1341. {
  1342. StackSym * scopeObjSym = instr->GetSrc1()->GetStackSym();
  1343. Assert(scopeObjSym);
  1344. Assert(scopeObjSym->GetInstrDef()->m_opcode == Js::OpCode::InitCachedScope || scopeObjSym->GetInstrDef()->m_opcode == Js::OpCode::NewScopeObject);
  1345. Assert(instr->m_func->GetScopeObjSym() == scopeObjSym);
  1346. if (PHASE_VERBOSE_TRACE1(Js::StackArgFormalsOptPhase))
  1347. {
  1348. Output::Print(_u("StackArgFormals : %s (%d) :Setting scopeObjSym in forward pass. \n"), instr->m_func->GetJITFunctionBody()->GetDisplayName(), instr->m_func->GetJITFunctionBody()->GetFunctionNumber());
  1349. Output::Flush();
  1350. }
  1351. }
  1352. }
  1353. #endif
  1354. if (instr->m_func->GetJITFunctionBody()->GetInParamsCount() != 1 && !instr->m_func->IsStackArgsEnabled())
  1355. {
  1356. CannotAllocateArgumentsObjectOnStack(instr->m_func);
  1357. }
  1358. else
  1359. {
  1360. CurrentBlockData()->TrackArgumentsSym(dst->AsRegOpnd());
  1361. }
  1362. return;
  1363. }
  1364. // Keep track of arguments objects and its aliases
  1365. // LdHeapArguments loads the arguments object and Ld_A tracks the aliases.
  1366. if ((instr->m_opcode == Js::OpCode::Ld_A || instr->m_opcode == Js::OpCode::BytecodeArgOutCapture) && (src1->IsRegOpnd() && CurrentBlockData()->IsArgumentsOpnd(src1)))
  1367. {
  1368. // In the debug mode, we don't want to optimize away the aliases. Since we may have to show them on the inspection.
  1369. if (((!AreFromSameBytecodeFunc(src1->AsRegOpnd(), dst->AsRegOpnd()) || this->currentBlock->loop) && instr->m_opcode != Js::OpCode::BytecodeArgOutCapture) || this->func->IsJitInDebugMode())
  1370. {
  1371. CannotAllocateArgumentsObjectOnStack(instr->m_func);
  1372. return;
  1373. }
  1374. // Disable stack args if we are aliasing arguments inside try block to a writethrough symbol.
  1375. // We don't have precise tracking of these symbols, so bailout couldn't know if it needs to restore arguments object or not after exception
  1376. Region* tryRegion = this->currentRegion ? this->currentRegion->GetSelfOrFirstTryAncestor() : nullptr;
  1377. if (tryRegion && tryRegion->GetType() == RegionTypeTry &&
  1378. tryRegion->writeThroughSymbolsSet &&
  1379. tryRegion->writeThroughSymbolsSet->Test(dst->AsRegOpnd()->m_sym->m_id))
  1380. {
  1381. CannotAllocateArgumentsObjectOnStack(instr->m_func);
  1382. return;
  1383. }
  1384. if(!dst->AsRegOpnd()->GetStackSym()->m_nonEscapingArgObjAlias)
  1385. {
  1386. CurrentBlockData()->TrackArgumentsSym(dst->AsRegOpnd());
  1387. }
  1388. return;
  1389. }
  1390. if (!CurrentBlockData()->TestAnyArgumentsSym())
  1391. {
  1392. // There are no syms to track yet, don't start tracking arguments sym.
  1393. return;
  1394. }
  1395. // Avoid loop prepass
  1396. if (this->currentBlock->loop && this->IsLoopPrePass())
  1397. {
  1398. return;
  1399. }
  1400. SymID id = 0;
  1401. switch(instr->m_opcode)
  1402. {
  1403. case Js::OpCode::LdElemI_A:
  1404. case Js::OpCode::TypeofElem:
  1405. {
  1406. Assert(src1->IsIndirOpnd());
  1407. IR::RegOpnd *indexOpnd = src1->AsIndirOpnd()->GetIndexOpnd();
  1408. if (indexOpnd && CurrentBlockData()->IsArgumentsSymID(indexOpnd->m_sym->m_id))
  1409. {
  1410. // Pathological test cases such as a[arguments]
  1411. CannotAllocateArgumentsObjectOnStack(instr->m_func);
  1412. return;
  1413. }
  1414. IR::RegOpnd *baseOpnd = src1->AsIndirOpnd()->GetBaseOpnd();
  1415. id = baseOpnd->m_sym->m_id;
  1416. if (CurrentBlockData()->IsArgumentsSymID(id))
  1417. {
  1418. instr->usesStackArgumentsObject = true;
  1419. }
  1420. break;
  1421. }
  1422. case Js::OpCode::LdLen_A:
  1423. {
  1424. Assert(src1->IsRegOpnd());
  1425. if(CurrentBlockData()->IsArgumentsOpnd(src1))
  1426. {
  1427. instr->usesStackArgumentsObject = true;
  1428. }
  1429. break;
  1430. }
  1431. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  1432. {
  1433. if (CurrentBlockData()->IsArgumentsOpnd(src1))
  1434. {
  1435. instr->usesStackArgumentsObject = true;
  1436. instr->m_func->unoptimizableArgumentsObjReference++;
  1437. }
  1438. if (CurrentBlockData()->IsArgumentsOpnd(src1) &&
  1439. src1->AsRegOpnd()->m_sym->GetInstrDef()->m_opcode == Js::OpCode::BytecodeArgOutCapture)
  1440. {
  1441. // Apply inlining results in such usage - this is to ignore this sym that is def'd by ByteCodeArgOutCapture
  1442. // It's needed because we do not have block level merging of arguments object and this def due to inlining can turn off stack args opt.
  1443. IR::Instr* builtinStart = instr->GetNextRealInstr();
  1444. if (builtinStart->m_opcode == Js::OpCode::InlineBuiltInStart)
  1445. {
  1446. IR::Opnd* builtinOpnd = builtinStart->GetSrc1();
  1447. if (builtinStart->GetSrc1()->IsAddrOpnd())
  1448. {
  1449. Assert(builtinOpnd->AsAddrOpnd()->m_isFunction);
  1450. Js::BuiltinFunction builtinFunction = Js::JavascriptLibrary::GetBuiltInForFuncInfo(((FixedFieldInfo*)builtinOpnd->AsAddrOpnd()->m_metadata)->GetLocalFuncId());
  1451. if (builtinFunction == Js::BuiltinFunction::JavascriptFunction_Apply)
  1452. {
  1453. CurrentBlockData()->ClearArgumentsSym(src1->AsRegOpnd());
  1454. instr->m_func->unoptimizableArgumentsObjReference--;
  1455. }
  1456. }
  1457. else if (builtinOpnd->IsRegOpnd())
  1458. {
  1459. if (builtinOpnd->AsRegOpnd()->m_sym->m_builtInIndex == Js::BuiltinFunction::JavascriptFunction_Apply)
  1460. {
  1461. CurrentBlockData()->ClearArgumentsSym(src1->AsRegOpnd());
  1462. instr->m_func->unoptimizableArgumentsObjReference--;
  1463. }
  1464. }
  1465. }
  1466. }
  1467. break;
  1468. }
  1469. case Js::OpCode::BailOnNotStackArgs:
  1470. case Js::OpCode::ArgOut_A_FromStackArgs:
  1471. case Js::OpCode::BytecodeArgOutUse:
  1472. {
  1473. if (src1 && CurrentBlockData()->IsArgumentsOpnd(src1))
  1474. {
  1475. instr->usesStackArgumentsObject = true;
  1476. }
  1477. break;
  1478. }
  1479. default:
  1480. {
  1481. // Super conservative here, if we see the arguments or any of its alias being used in any
  1482. // other opcode just don't do this optimization. Revisit this to optimize further if we see any common
  1483. // case is missed.
  1484. if (src1)
  1485. {
  1486. if (src1->IsRegOpnd() || src1->IsSymOpnd() || src1->IsIndirOpnd())
  1487. {
  1488. if (CurrentBlockData()->IsArgumentsOpnd(src1))
  1489. {
  1490. #ifdef PERF_HINT
  1491. if (PHASE_TRACE1(Js::PerfHintPhase))
  1492. {
  1493. WritePerfHint(PerfHints::HeapArgumentsCreated, instr->m_func, instr->GetByteCodeOffset());
  1494. }
  1495. #endif
  1496. CannotAllocateArgumentsObjectOnStack(instr->m_func);
  1497. return;
  1498. }
  1499. }
  1500. }
  1501. if (src2)
  1502. {
  1503. if (src2->IsRegOpnd() || src2->IsSymOpnd() || src2->IsIndirOpnd())
  1504. {
  1505. if (CurrentBlockData()->IsArgumentsOpnd(src2))
  1506. {
  1507. #ifdef PERF_HINT
  1508. if (PHASE_TRACE1(Js::PerfHintPhase))
  1509. {
  1510. WritePerfHint(PerfHints::HeapArgumentsCreated, instr->m_func, instr->GetByteCodeOffset());
  1511. }
  1512. #endif
  1513. CannotAllocateArgumentsObjectOnStack(instr->m_func);
  1514. return;
  1515. }
  1516. }
  1517. }
  1518. // We should look at dst last to correctly handle cases where it's the same as one of the src operands.
  1519. if (dst)
  1520. {
  1521. if (dst->IsIndirOpnd() || dst->IsSymOpnd())
  1522. {
  1523. if (CurrentBlockData()->IsArgumentsOpnd(dst))
  1524. {
  1525. #ifdef PERF_HINT
  1526. if (PHASE_TRACE1(Js::PerfHintPhase))
  1527. {
  1528. WritePerfHint(PerfHints::HeapArgumentsModification, instr->m_func, instr->GetByteCodeOffset());
  1529. }
  1530. #endif
  1531. CannotAllocateArgumentsObjectOnStack(instr->m_func);
  1532. return;
  1533. }
  1534. }
  1535. else if (dst->IsRegOpnd())
  1536. {
  1537. if (this->currentBlock->loop && CurrentBlockData()->IsArgumentsOpnd(dst))
  1538. {
  1539. #ifdef PERF_HINT
  1540. if (PHASE_TRACE1(Js::PerfHintPhase))
  1541. {
  1542. WritePerfHint(PerfHints::HeapArgumentsModification, instr->m_func, instr->GetByteCodeOffset());
  1543. }
  1544. #endif
  1545. CannotAllocateArgumentsObjectOnStack(instr->m_func);
  1546. return;
  1547. }
  1548. CurrentBlockData()->ClearArgumentsSym(dst->AsRegOpnd());
  1549. }
  1550. }
  1551. }
  1552. break;
  1553. }
  1554. return;
  1555. }
  1556. void
  1557. GlobOpt::MarkArgumentsUsedForBranch(IR::Instr * instr)
  1558. {
  1559. // If it's a conditional branch instruction and the operand used for branching is one of the arguments
  1560. // to the function, tag the m_argUsedForBranch of the functionBody so that it can be used later for inlining decisions.
  1561. if (instr->IsBranchInstr() && !instr->AsBranchInstr()->IsUnconditional())
  1562. {
  1563. IR::BranchInstr * bInstr = instr->AsBranchInstr();
  1564. IR::Opnd *src1 = bInstr->GetSrc1();
  1565. IR::Opnd *src2 = bInstr->GetSrc2();
  1566. // These are used because we don't want to rely on src1 or src2 to always be the register/constant
  1567. IR::RegOpnd *regOpnd = nullptr;
  1568. if (!src2 && (instr->m_opcode == Js::OpCode::BrFalse_A || instr->m_opcode == Js::OpCode::BrTrue_A) && src1->IsRegOpnd())
  1569. {
  1570. regOpnd = src1->AsRegOpnd();
  1571. }
  1572. // We need to check for (0===arg) and (arg===0); this is especially important since some minifiers
  1573. // change all instances of one to the other.
  1574. else if (src2 && src2->IsConstOpnd() && src1->IsRegOpnd())
  1575. {
  1576. regOpnd = src1->AsRegOpnd();
  1577. }
  1578. else if (src2 && src2->IsRegOpnd() && src1->IsConstOpnd())
  1579. {
  1580. regOpnd = src2->AsRegOpnd();
  1581. }
  1582. if (regOpnd != nullptr)
  1583. {
  1584. if (regOpnd->m_sym->IsSingleDef())
  1585. {
  1586. IR::Instr * defInst = regOpnd->m_sym->GetInstrDef();
  1587. IR::Opnd *defSym = defInst->GetSrc1();
  1588. if (defSym && defSym->IsSymOpnd() && defSym->AsSymOpnd()->m_sym->IsStackSym()
  1589. && defSym->AsSymOpnd()->m_sym->AsStackSym()->IsParamSlotSym())
  1590. {
  1591. uint16 param = defSym->AsSymOpnd()->m_sym->AsStackSym()->GetParamSlotNum();
  1592. // We only support functions with 13 arguments to ensure optimal size of callSiteInfo
  1593. if (param < Js::Constants::MaximumArgumentCountForConstantArgumentInlining)
  1594. {
  1595. this->func->GetJITOutput()->SetArgUsedForBranch((uint8)param);
  1596. }
  1597. }
  1598. }
  1599. }
  1600. }
  1601. }
  1602. const InductionVariable*
  1603. GlobOpt::GetInductionVariable(SymID sym, Loop *loop)
  1604. {
  1605. if (loop->inductionVariables)
  1606. {
  1607. for (auto it = loop->inductionVariables->GetIterator(); it.IsValid(); it.MoveNext())
  1608. {
  1609. InductionVariable* iv = &it.CurrentValueReference();
  1610. if (!iv->IsChangeDeterminate() || !iv->IsChangeUnidirectional())
  1611. {
  1612. continue;
  1613. }
  1614. if (iv->Sym()->m_id == sym)
  1615. {
  1616. return iv;
  1617. }
  1618. }
  1619. }
  1620. return nullptr;
  1621. }
  1622. bool
  1623. GlobOpt::IsSymIDInductionVariable(SymID sym, Loop *loop)
  1624. {
  1625. return GetInductionVariable(sym, loop) != nullptr;
  1626. }
  1627. SymID
  1628. GlobOpt::GetVarSymID(StackSym *sym)
  1629. {
  1630. if (sym && sym->m_type != TyVar)
  1631. {
  1632. sym = sym->GetVarEquivSym(nullptr);
  1633. }
  1634. if (!sym)
  1635. {
  1636. return Js::Constants::InvalidSymID;
  1637. }
  1638. return sym->m_id;
  1639. }
  1640. bool
  1641. GlobOpt::IsAllowedForMemOpt(IR::Instr* instr, bool isMemset, IR::RegOpnd *baseOpnd, IR::Opnd *indexOpnd)
  1642. {
  1643. Assert(instr);
  1644. if (!baseOpnd || !indexOpnd)
  1645. {
  1646. return false;
  1647. }
  1648. Loop* loop = this->currentBlock->loop;
  1649. const ValueType baseValueType(baseOpnd->GetValueType());
  1650. const ValueType indexValueType(indexOpnd->GetValueType());
  1651. // Validate the array and index types
  1652. if (
  1653. !indexValueType.IsInt() ||
  1654. !(
  1655. baseValueType.IsTypedIntOrFloatArray() ||
  1656. baseValueType.IsArray()
  1657. )
  1658. )
  1659. {
  1660. #if DBG_DUMP
  1661. wchar indexValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  1662. indexValueType.ToString(indexValueTypeStr);
  1663. wchar baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  1664. baseValueType.ToString(baseValueTypeStr);
  1665. TRACE_MEMOP_VERBOSE(loop, instr, _u("Index[%s] or Array[%s] value type is invalid"), indexValueTypeStr, baseValueTypeStr);
  1666. #endif
  1667. return false;
  1668. }
  1669. // The following is conservative and works around a bug in induction variable analysis.
  1670. if (baseOpnd->IsArrayRegOpnd())
  1671. {
  1672. IR::ArrayRegOpnd *baseArrayOp = baseOpnd->AsArrayRegOpnd();
  1673. bool hasBoundChecksRemoved = (
  1674. baseArrayOp->EliminatedLowerBoundCheck() &&
  1675. baseArrayOp->EliminatedUpperBoundCheck() &&
  1676. !instr->extractedUpperBoundCheckWithoutHoisting &&
  1677. !instr->loadedArrayHeadSegment &&
  1678. !instr->loadedArrayHeadSegmentLength
  1679. );
  1680. if (!hasBoundChecksRemoved)
  1681. {
  1682. TRACE_MEMOP_VERBOSE(loop, instr, _u("Missing bounds check optimization"));
  1683. return false;
  1684. }
  1685. }
  1686. else
  1687. {
  1688. return false;
  1689. }
  1690. if (!baseValueType.IsTypedArray())
  1691. {
  1692. // Check if the instr can kill the value type of the array
  1693. JsArrayKills arrayKills = CheckJsArrayKills(instr);
  1694. if (arrayKills.KillsValueType(baseValueType))
  1695. {
  1696. TRACE_MEMOP_VERBOSE(loop, instr, _u("The array (s%d) can lose its value type"), GetVarSymID(baseOpnd->GetStackSym()));
  1697. return false;
  1698. }
  1699. }
  1700. // Process the Index Operand
  1701. if (!this->OptIsInvariant(baseOpnd, this->currentBlock, loop, CurrentBlockData()->FindValue(baseOpnd->m_sym), false, true))
  1702. {
  1703. TRACE_MEMOP_VERBOSE(loop, instr, _u("Base (s%d) is not invariant"), GetVarSymID(baseOpnd->GetStackSym()));
  1704. return false;
  1705. }
  1706. // Validate the index
  1707. Assert(indexOpnd->GetStackSym());
  1708. SymID indexSymID = GetVarSymID(indexOpnd->GetStackSym());
  1709. const InductionVariable* iv = GetInductionVariable(indexSymID, loop);
  1710. if (!iv)
  1711. {
  1712. // If the index is not an induction variable return
  1713. TRACE_MEMOP_VERBOSE(loop, instr, _u("Index (s%d) is not an induction variable"), indexSymID);
  1714. return false;
  1715. }
  1716. Assert(iv->IsChangeDeterminate() && iv->IsChangeUnidirectional());
  1717. const IntConstantBounds & bounds = iv->ChangeBounds();
  1718. if (loop->memOpInfo)
  1719. {
  1720. // Only accept induction variables that increments by 1
  1721. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 };
  1722. inductionVariableChangeInfo = loop->memOpInfo->inductionVariableChangeInfoMap->Lookup(indexSymID, inductionVariableChangeInfo);
  1723. if (
  1724. (bounds.LowerBound() != 1 && bounds.LowerBound() != -1) ||
  1725. (bounds.UpperBound() != bounds.LowerBound()) ||
  1726. inductionVariableChangeInfo.unroll > 1 // Must be 0 (not seen yet) or 1 (already seen)
  1727. )
  1728. {
  1729. TRACE_MEMOP_VERBOSE(loop, instr, _u("The index does not change by 1: %d><%d, unroll=%d"), bounds.LowerBound(), bounds.UpperBound(), inductionVariableChangeInfo.unroll);
  1730. return false;
  1731. }
  1732. // Check if the index is the same in all MemOp optimization in this loop
  1733. if (!loop->memOpInfo->candidates->Empty())
  1734. {
  1735. Loop::MemOpCandidate* previousCandidate = loop->memOpInfo->candidates->Head();
  1736. // All MemOp operations within the same loop must use the same index
  1737. if (previousCandidate->index != indexSymID)
  1738. {
  1739. TRACE_MEMOP_VERBOSE(loop, instr, _u("The index is not the same as other MemOp in the loop"));
  1740. return false;
  1741. }
  1742. }
  1743. }
  1744. return true;
  1745. }
  1746. bool
  1747. GlobOpt::CollectMemcopyLdElementI(IR::Instr *instr, Loop *loop)
  1748. {
  1749. Assert(instr->GetSrc1()->IsIndirOpnd());
  1750. IR::IndirOpnd *src1 = instr->GetSrc1()->AsIndirOpnd();
  1751. IR::Opnd *indexOpnd = src1->GetIndexOpnd();
  1752. IR::RegOpnd *baseOpnd = src1->GetBaseOpnd()->AsRegOpnd();
  1753. SymID baseSymID = GetVarSymID(baseOpnd->GetStackSym());
  1754. if (!IsAllowedForMemOpt(instr, false, baseOpnd, indexOpnd))
  1755. {
  1756. return false;
  1757. }
  1758. SymID inductionSymID = GetVarSymID(indexOpnd->GetStackSym());
  1759. Assert(IsSymIDInductionVariable(inductionSymID, loop));
  1760. loop->EnsureMemOpVariablesInitialized();
  1761. bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID);
  1762. IR::Opnd * dst = instr->GetDst();
  1763. if (!dst->IsRegOpnd() || !dst->AsRegOpnd()->GetStackSym()->IsSingleDef())
  1764. {
  1765. return false;
  1766. }
  1767. Loop::MemCopyCandidate* memcopyInfo = memcopyInfo = JitAnewStruct(this->func->GetTopFunc()->m_fg->alloc, Loop::MemCopyCandidate);
  1768. memcopyInfo->ldBase = baseSymID;
  1769. memcopyInfo->ldCount = 1;
  1770. memcopyInfo->count = 0;
  1771. memcopyInfo->bIndexAlreadyChanged = isIndexPreIncr;
  1772. memcopyInfo->base = Js::Constants::InvalidSymID; //need to find the stElem first
  1773. memcopyInfo->index = inductionSymID;
  1774. memcopyInfo->transferSym = dst->AsRegOpnd()->GetStackSym();
  1775. loop->memOpInfo->candidates->Prepend(memcopyInfo);
  1776. return true;
  1777. }
  1778. bool
  1779. GlobOpt::CollectMemsetStElementI(IR::Instr *instr, Loop *loop)
  1780. {
  1781. Assert(instr->GetDst()->IsIndirOpnd());
  1782. IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd();
  1783. IR::Opnd *indexOp = dst->GetIndexOpnd();
  1784. IR::RegOpnd *baseOp = dst->GetBaseOpnd()->AsRegOpnd();
  1785. if (!IsAllowedForMemOpt(instr, true, baseOp, indexOp))
  1786. {
  1787. return false;
  1788. }
  1789. SymID baseSymID = GetVarSymID(baseOp->GetStackSym());
  1790. IR::Opnd *srcDef = instr->GetSrc1();
  1791. StackSym *srcSym = nullptr;
  1792. if (srcDef->IsRegOpnd())
  1793. {
  1794. IR::RegOpnd* opnd = srcDef->AsRegOpnd();
  1795. if (this->OptIsInvariant(opnd, this->currentBlock, loop, CurrentBlockData()->FindValue(opnd->m_sym), true, true))
  1796. {
  1797. srcSym = opnd->GetStackSym();
  1798. }
  1799. }
  1800. BailoutConstantValue constant = {TyIllegal, 0};
  1801. if (srcDef->IsFloatConstOpnd())
  1802. {
  1803. constant.InitFloatConstValue(srcDef->AsFloatConstOpnd()->m_value);
  1804. }
  1805. else if (srcDef->IsIntConstOpnd())
  1806. {
  1807. constant.InitIntConstValue(srcDef->AsIntConstOpnd()->GetValue(), srcDef->AsIntConstOpnd()->GetType());
  1808. }
  1809. else if (srcDef->IsAddrOpnd())
  1810. {
  1811. constant.InitVarConstValue(srcDef->AsAddrOpnd()->m_address);
  1812. }
  1813. else if(!srcSym)
  1814. {
  1815. TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Source is not an invariant"));
  1816. return false;
  1817. }
  1818. // Process the Index Operand
  1819. Assert(indexOp->GetStackSym());
  1820. SymID inductionSymID = GetVarSymID(indexOp->GetStackSym());
  1821. Assert(IsSymIDInductionVariable(inductionSymID, loop));
  1822. loop->EnsureMemOpVariablesInitialized();
  1823. bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID);
  1824. Loop::MemSetCandidate* memsetInfo = JitAnewStruct(this->func->GetTopFunc()->m_fg->alloc, Loop::MemSetCandidate);
  1825. memsetInfo->base = baseSymID;
  1826. memsetInfo->index = inductionSymID;
  1827. memsetInfo->constant = constant;
  1828. memsetInfo->srcSym = srcSym;
  1829. memsetInfo->count = 1;
  1830. memsetInfo->bIndexAlreadyChanged = isIndexPreIncr;
  1831. loop->memOpInfo->candidates->Prepend(memsetInfo);
  1832. return true;
  1833. }
  1834. bool GlobOpt::CollectMemcopyStElementI(IR::Instr *instr, Loop *loop)
  1835. {
  1836. if (!loop->memOpInfo || loop->memOpInfo->candidates->Empty())
  1837. {
  1838. // There is no ldElem matching this stElem
  1839. return false;
  1840. }
  1841. Assert(instr->GetDst()->IsIndirOpnd());
  1842. IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd();
  1843. IR::Opnd *indexOp = dst->GetIndexOpnd();
  1844. IR::RegOpnd *baseOp = dst->GetBaseOpnd()->AsRegOpnd();
  1845. SymID baseSymID = GetVarSymID(baseOp->GetStackSym());
  1846. if (!instr->GetSrc1()->IsRegOpnd())
  1847. {
  1848. return false;
  1849. }
  1850. IR::RegOpnd* src1 = instr->GetSrc1()->AsRegOpnd();
  1851. if (!src1->GetIsDead())
  1852. {
  1853. // This must be the last use of the register.
  1854. // It will invalidate `var m = a[i]; b[i] = m;` but this is not a very interesting case.
  1855. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Source (s%d) is still alive after StElemI"), baseSymID);
  1856. return false;
  1857. }
  1858. if (!IsAllowedForMemOpt(instr, false, baseOp, indexOp))
  1859. {
  1860. return false;
  1861. }
  1862. SymID srcSymID = GetVarSymID(src1->GetStackSym());
  1863. // Prepare the memcopyCandidate entry
  1864. Loop::MemOpCandidate* previousCandidate = loop->memOpInfo->candidates->Head();
  1865. if (!previousCandidate->IsMemCopy())
  1866. {
  1867. return false;
  1868. }
  1869. Loop::MemCopyCandidate* memcopyInfo = previousCandidate->AsMemCopy();
  1870. // The previous candidate has to have been created by the matching ldElem
  1871. if (
  1872. memcopyInfo->base != Js::Constants::InvalidSymID ||
  1873. GetVarSymID(memcopyInfo->transferSym) != srcSymID
  1874. )
  1875. {
  1876. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("No matching LdElem found (s%d)"), baseSymID);
  1877. return false;
  1878. }
  1879. Assert(indexOp->GetStackSym());
  1880. SymID inductionSymID = GetVarSymID(indexOp->GetStackSym());
  1881. Assert(IsSymIDInductionVariable(inductionSymID, loop));
  1882. bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID);
  1883. if (isIndexPreIncr != memcopyInfo->bIndexAlreadyChanged)
  1884. {
  1885. // The index changed between the load and the store
  1886. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Index value changed between ldElem and stElem"));
  1887. return false;
  1888. }
  1889. // Consider: Can we remove the count field?
  1890. memcopyInfo->count++;
  1891. AssertOrFailFast(memcopyInfo->count <= 1);
  1892. memcopyInfo->base = baseSymID;
  1893. return true;
  1894. }
  1895. bool
  1896. GlobOpt::CollectMemOpLdElementI(IR::Instr *instr, Loop *loop)
  1897. {
  1898. Assert(instr->m_opcode == Js::OpCode::LdElemI_A);
  1899. return (!PHASE_OFF(Js::MemCopyPhase, this->func) && CollectMemcopyLdElementI(instr, loop));
  1900. }
  1901. bool
  1902. GlobOpt::CollectMemOpStElementI(IR::Instr *instr, Loop *loop)
  1903. {
  1904. Assert(instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict);
  1905. Assert(instr->GetSrc1());
  1906. return (!PHASE_OFF(Js::MemSetPhase, this->func) && CollectMemsetStElementI(instr, loop)) ||
  1907. (!PHASE_OFF(Js::MemCopyPhase, this->func) && CollectMemcopyStElementI(instr, loop));
  1908. }
  1909. bool
  1910. GlobOpt::CollectMemOpInfo(IR::Instr *instrBegin, IR::Instr *instr, Value *src1Val, Value *src2Val)
  1911. {
  1912. Assert(this->currentBlock->loop);
  1913. Loop *loop = this->currentBlock->loop;
  1914. if (!loop->blockList.HasTwo())
  1915. {
  1916. // We support memcopy and memset for loops which have only two blocks.
  1917. return false;
  1918. }
  1919. if (loop->GetLoopFlags().isInterpreted && !loop->GetLoopFlags().memopMinCountReached)
  1920. {
  1921. TRACE_MEMOP_VERBOSE(loop, instr, _u("minimum loop count not reached"))
  1922. loop->doMemOp = false;
  1923. return false;
  1924. }
  1925. Assert(loop->doMemOp);
  1926. bool isIncr = true, isChangedByOne = false;
  1927. switch (instr->m_opcode)
  1928. {
  1929. case Js::OpCode::StElemI_A:
  1930. case Js::OpCode::StElemI_A_Strict:
  1931. if (!CollectMemOpStElementI(instr, loop))
  1932. {
  1933. loop->doMemOp = false;
  1934. return false;
  1935. }
  1936. break;
  1937. case Js::OpCode::LdElemI_A:
  1938. if (!CollectMemOpLdElementI(instr, loop))
  1939. {
  1940. loop->doMemOp = false;
  1941. return false;
  1942. }
  1943. break;
  1944. case Js::OpCode::Sub_I4:
  1945. isIncr = false;
  1946. case Js::OpCode::Add_I4:
  1947. {
  1948. // The only case in which these OpCodes can contribute to an inductionVariableChangeInfo
  1949. // is when the induction variable is being modified and overwritten aswell (ex: j = j + 1)
  1950. // and not when the induction variable is modified but not overwritten (ex: k = j + 1).
  1951. // This can either be detected in IR as
  1952. // s1 = Add_I4 s1 1 // Case #1, can be seen with "j++".
  1953. // or as
  1954. // s4(s2) = Add_I4 s3(s1) 1 // Case #2, can be see with "j = j + 1".
  1955. // s1 = Ld_A s2
  1956. bool isInductionVar = false;
  1957. IR::Instr* nextInstr = instr->m_next;
  1958. if (
  1959. // Checks for Case #1 and Case #2
  1960. instr->GetDst()->GetStackSym() != nullptr &&
  1961. instr->GetDst()->IsRegOpnd() &&
  1962. (
  1963. // Checks for Case #1
  1964. (instr->GetDst()->GetStackSym() == instr->GetSrc1()->GetStackSym()) ||
  1965. // Checks for Case #2
  1966. (nextInstr&& nextInstr->m_opcode == Js::OpCode::Ld_A &&
  1967. nextInstr->GetSrc1()->IsRegOpnd() &&
  1968. nextInstr->GetDst()->IsRegOpnd() &&
  1969. GetVarSymID(instr->GetDst()->GetStackSym()) == nextInstr->GetSrc1()->GetStackSym()->m_id &&
  1970. GetVarSymID(instr->GetSrc1()->GetStackSym()) == nextInstr->GetDst()->GetStackSym()->m_id)
  1971. )
  1972. )
  1973. {
  1974. isInductionVar = true;
  1975. }
  1976. // Even if dstIsInductionVar then dst == src1 so it's safe to use src1 as the induction sym always.
  1977. StackSym* sym = instr->GetSrc1()->GetStackSym();
  1978. SymID inductionSymID = GetVarSymID(sym);
  1979. if (isInductionVar && IsSymIDInductionVariable(inductionSymID, this->currentBlock->loop))
  1980. {
  1981. if (!isChangedByOne)
  1982. {
  1983. IR::Opnd *src1, *src2;
  1984. src1 = instr->GetSrc1();
  1985. src2 = instr->GetSrc2();
  1986. if (src2->IsRegOpnd())
  1987. {
  1988. Value *val = CurrentBlockData()->FindValue(src2->AsRegOpnd()->m_sym);
  1989. if (val)
  1990. {
  1991. ValueInfo *vi = val->GetValueInfo();
  1992. int constValue;
  1993. if (vi && vi->TryGetIntConstantValue(&constValue))
  1994. {
  1995. if (constValue == 1)
  1996. {
  1997. isChangedByOne = true;
  1998. }
  1999. }
  2000. }
  2001. }
  2002. else if (src2->IsIntConstOpnd())
  2003. {
  2004. if (src2->AsIntConstOpnd()->GetValue() == 1)
  2005. {
  2006. isChangedByOne = true;
  2007. }
  2008. }
  2009. }
  2010. loop->EnsureMemOpVariablesInitialized();
  2011. if (!isChangedByOne)
  2012. {
  2013. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { Js::Constants::InvalidLoopUnrollFactor, 0 };
  2014. if (!loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID))
  2015. {
  2016. loop->memOpInfo->inductionVariableChangeInfoMap->Add(inductionSymID, inductionVariableChangeInfo);
  2017. if (sym->m_id != inductionSymID)
  2018. {
  2019. // Backwards pass uses this bit-vector to lookup upwardExposedUsed/bytecodeUpwardExposedUsed symbols, which are not necessarily vars. Just add both.
  2020. loop->memOpInfo->inductionVariableChangeInfoMap->Add(sym->m_id, inductionVariableChangeInfo);
  2021. }
  2022. }
  2023. else
  2024. {
  2025. loop->memOpInfo->inductionVariableChangeInfoMap->Item(inductionSymID, inductionVariableChangeInfo);
  2026. if (sym->m_id != inductionSymID)
  2027. {
  2028. // Backwards pass uses this bit-vector to lookup upwardExposedUsed/bytecodeUpwardExposedUsed symbols, which are not necessarily vars. Just add both.
  2029. loop->memOpInfo->inductionVariableChangeInfoMap->Item(sym->m_id, inductionVariableChangeInfo);
  2030. }
  2031. }
  2032. }
  2033. else
  2034. {
  2035. if (!loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID))
  2036. {
  2037. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 1, isIncr };
  2038. loop->memOpInfo->inductionVariableChangeInfoMap->Add(inductionSymID, inductionVariableChangeInfo);
  2039. if (sym->m_id != inductionSymID)
  2040. {
  2041. // Backwards pass uses this bit-vector to lookup upwardExposedUsed/bytecodeUpwardExposedUsed symbols, which are not necessarily vars. Just add both.
  2042. loop->memOpInfo->inductionVariableChangeInfoMap->Add(sym->m_id, inductionVariableChangeInfo);
  2043. }
  2044. }
  2045. else
  2046. {
  2047. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 };
  2048. inductionVariableChangeInfo = loop->memOpInfo->inductionVariableChangeInfoMap->Lookup(inductionSymID, inductionVariableChangeInfo);
  2049. // If inductionVariableChangeInfo.unroll has been invalidated, do
  2050. // not modify the Js::Constants::InvalidLoopUnrollFactor value
  2051. if (inductionVariableChangeInfo.unroll != Js::Constants::InvalidLoopUnrollFactor)
  2052. {
  2053. inductionVariableChangeInfo.unroll++;
  2054. }
  2055. inductionVariableChangeInfo.isIncremental = isIncr;
  2056. loop->memOpInfo->inductionVariableChangeInfoMap->Item(inductionSymID, inductionVariableChangeInfo);
  2057. if (sym->m_id != inductionSymID)
  2058. {
  2059. // Backwards pass uses this bit-vector to lookup upwardExposedUsed/bytecodeUpwardExposedUsed symbols, which are not necessarily vars. Just add both.
  2060. loop->memOpInfo->inductionVariableChangeInfoMap->Item(sym->m_id, inductionVariableChangeInfo);
  2061. }
  2062. }
  2063. }
  2064. break;
  2065. }
  2066. // Fallthrough if not an induction variable
  2067. }
  2068. default:
  2069. FOREACH_INSTR_IN_RANGE(chkInstr, instrBegin->m_next, instr)
  2070. {
  2071. if (IsInstrInvalidForMemOp(chkInstr, loop, src1Val, src2Val))
  2072. {
  2073. loop->doMemOp = false;
  2074. return false;
  2075. }
  2076. // Make sure this instruction doesn't use the memcopy transfer sym before it is checked by StElemI
  2077. if (loop->memOpInfo && !loop->memOpInfo->candidates->Empty())
  2078. {
  2079. Loop::MemOpCandidate* prevCandidate = loop->memOpInfo->candidates->Head();
  2080. if (prevCandidate->IsMemCopy())
  2081. {
  2082. Loop::MemCopyCandidate* memcopyCandidate = prevCandidate->AsMemCopy();
  2083. if (memcopyCandidate->base == Js::Constants::InvalidSymID)
  2084. {
  2085. if (chkInstr->HasSymUse(memcopyCandidate->transferSym))
  2086. {
  2087. loop->doMemOp = false;
  2088. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, chkInstr, _u("Found illegal use of LdElemI value(s%d)"), GetVarSymID(memcopyCandidate->transferSym));
  2089. return false;
  2090. }
  2091. }
  2092. }
  2093. }
  2094. }
  2095. NEXT_INSTR_IN_RANGE;
  2096. IR::Instr* prevInstr = instr->m_prev;
  2097. // If an instr where the dst is an induction variable (and thus is being written to) is not caught by a case in the above
  2098. // switch statement (which implies that this instr does not contributes to a inductionVariableChangeInfo) and in the default
  2099. // case does not set doMemOp to false (which implies that this instr does not invalidate this MemOp), then FailFast as we
  2100. // should not be performing a MemOp under these conditions.
  2101. AssertOrFailFast(!instr->GetDst() || instr->m_opcode == Js::OpCode::IncrLoopBodyCount || !loop->memOpInfo ||
  2102. // Refer to "Case #2" described above in this function. For the following IR:
  2103. // Line #1: s4(s2) = Add_I4 s3(s1) 1
  2104. // Line #2: s3(s1) = Ld_A s4(s2)
  2105. // do not consider line #2 as a violating instr
  2106. (instr->m_opcode == Js::OpCode::Ld_I4 &&
  2107. prevInstr && (prevInstr->m_opcode == Js::OpCode::Add_I4 || prevInstr->m_opcode == Js::OpCode::Sub_I4) &&
  2108. instr->GetSrc1()->IsRegOpnd() &&
  2109. instr->GetDst()->IsRegOpnd() &&
  2110. prevInstr->GetDst()->IsRegOpnd() &&
  2111. instr->GetDst()->GetStackSym() == prevInstr->GetSrc1()->GetStackSym() &&
  2112. instr->GetSrc1()->GetStackSym() == prevInstr->GetDst()->GetStackSym()) ||
  2113. !loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(GetVarSymID(instr->GetDst()->GetStackSym())));
  2114. }
  2115. return true;
  2116. }
  2117. bool
  2118. GlobOpt::IsInstrInvalidForMemOp(IR::Instr *instr, Loop *loop, Value *src1Val, Value *src2Val)
  2119. {
  2120. // List of instruction that are valid with memop (ie: instr that gets removed if memop is emitted)
  2121. if (
  2122. this->currentBlock != loop->GetHeadBlock() &&
  2123. !instr->IsLabelInstr() &&
  2124. instr->IsRealInstr() &&
  2125. instr->m_opcode != Js::OpCode::IncrLoopBodyCount &&
  2126. instr->m_opcode != Js::OpCode::StLoopBodyCount &&
  2127. instr->m_opcode != Js::OpCode::Ld_A &&
  2128. instr->m_opcode != Js::OpCode::Ld_I4 &&
  2129. !(instr->IsBranchInstr() && instr->AsBranchInstr()->IsUnconditional())
  2130. )
  2131. {
  2132. TRACE_MEMOP_VERBOSE(loop, instr, _u("Instruction not accepted for memop"));
  2133. return true;
  2134. }
  2135. // Check prev instr because it could have been added by an optimization and we won't see it here.
  2136. if (OpCodeAttr::FastFldInstr(instr->m_opcode) || (instr->m_prev && OpCodeAttr::FastFldInstr(instr->m_prev->m_opcode)))
  2137. {
  2138. // Refuse any operations interacting with Fields
  2139. TRACE_MEMOP_VERBOSE(loop, instr, _u("Field interaction detected"));
  2140. return true;
  2141. }
  2142. if (Js::OpCodeUtil::GetOpCodeLayout(instr->m_opcode) == Js::OpLayoutType::ElementSlot)
  2143. {
  2144. // Refuse any operations interacting with slots
  2145. TRACE_MEMOP_VERBOSE(loop, instr, _u("Slot interaction detected"));
  2146. return true;
  2147. }
  2148. if (this->MayNeedBailOnImplicitCall(instr, src1Val, src2Val))
  2149. {
  2150. TRACE_MEMOP_VERBOSE(loop, instr, _u("Implicit call bailout detected"));
  2151. return true;
  2152. }
  2153. return false;
  2154. }
  2155. void
  2156. GlobOpt::TryReplaceLdLen(IR::Instr *& instr)
  2157. {
  2158. // Change LdLen on objects other than arrays, strings, and 'arguments' to LdFld. Otherwise, convert the SymOpnd to a RegOpnd here.
  2159. if (instr->m_opcode == Js::OpCode::LdLen_A && instr->GetSrc1() && instr->GetSrc1()->IsSymOpnd())
  2160. {
  2161. IR::SymOpnd * opnd = instr->GetSrc1()->AsSymOpnd();
  2162. Sym *sym = opnd->m_sym;
  2163. Assert(sym->IsPropertySym());
  2164. PropertySym *originalPropertySym = sym->AsPropertySym();
  2165. IR::RegOpnd* newopnd = IR::RegOpnd::New(originalPropertySym->m_stackSym, IRType::TyVar, instr->m_func);
  2166. ValueInfo *const objectValueInfo = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym)->GetValueInfo();
  2167. // things we'd emit a fast path for
  2168. if (
  2169. objectValueInfo->IsLikelyAnyArray() ||
  2170. objectValueInfo->HasHadStringTag() ||
  2171. objectValueInfo->IsLikelyString() ||
  2172. newopnd->IsArgumentsObject() ||
  2173. (CurrentBlockData()->argObjSyms && CurrentBlockData()->IsArgumentsOpnd(newopnd))
  2174. )
  2175. {
  2176. // We need to properly transfer over the information from the old operand, which is
  2177. // a SymOpnd, to the new one, which is a RegOpnd. Unfortunately, the types mean the
  2178. // normal copy methods won't work here, so we're going to directly copy data.
  2179. newopnd->SetIsJITOptimizedReg(opnd->GetIsJITOptimizedReg());
  2180. newopnd->SetValueType(objectValueInfo->Type());
  2181. newopnd->SetIsDead(opnd->GetIsDead());
  2182. instr->ReplaceSrc1(newopnd);
  2183. }
  2184. else
  2185. {
  2186. // otherwise, change the instruction to an LdFld here.
  2187. instr->m_opcode = Js::OpCode::LdFld;
  2188. }
  2189. }
  2190. }
  2191. IR::Instr *
  2192. GlobOpt::OptInstr(IR::Instr *&instr, bool* isInstrRemoved)
  2193. {
  2194. Assert(instr->m_func->IsTopFunc() || instr->m_func->isGetterSetter || instr->m_func->callSiteIdInParentFunc != UINT16_MAX);
  2195. IR::Opnd *src1, *src2;
  2196. Value *src1Val = nullptr, *src2Val = nullptr, *dstVal = nullptr;
  2197. Value *src1IndirIndexVal = nullptr, *dstIndirIndexVal = nullptr;
  2198. IR::Instr *instrPrev = instr->m_prev;
  2199. IR::Instr *instrNext = instr->m_next;
  2200. if (instr->IsLabelInstr() && this->func->HasTry() && this->func->DoOptimizeTry())
  2201. {
  2202. this->currentRegion = instr->AsLabelInstr()->GetRegion();
  2203. Assert(this->currentRegion);
  2204. }
  2205. if(PrepareForIgnoringIntOverflow(instr))
  2206. {
  2207. if(!IsLoopPrePass())
  2208. {
  2209. *isInstrRemoved = true;
  2210. currentBlock->RemoveInstr(instr);
  2211. }
  2212. return instrNext;
  2213. }
  2214. if (instr->m_opcode == Js::OpCode::Yield)
  2215. {
  2216. // TODO[generators][ianhall]: Can this and the FillBailOutInfo call below be moved to after Src1 and Src2 so that Yield can be optimized right up to the actual yield?
  2217. this->ProcessKills(instr);
  2218. }
  2219. if (!instr->IsRealInstr() || instr->IsByteCodeUsesInstr() || instr->m_opcode == Js::OpCode::Conv_Bool)
  2220. {
  2221. return instrNext;
  2222. }
  2223. if (!IsLoopPrePass())
  2224. {
  2225. // Change LdLen on objects other than arrays, strings, and 'arguments' to LdFld.
  2226. this->TryReplaceLdLen(instr);
  2227. }
  2228. // Consider: Do we ever get post-op bailout here, and if so is the FillBailOutInfo call in the right place?
  2229. if (instr->HasBailOutInfo() && !this->IsLoopPrePass())
  2230. {
  2231. this->FillBailOutInfo(this->currentBlock, instr);
  2232. }
  2233. this->instrCountSinceLastCleanUp++;
  2234. instr = this->PreOptPeep(instr);
  2235. this->OptArguments(instr);
  2236. //StackArguments Optimization - We bail out if the index is out of range of actuals.
  2237. if ((instr->m_opcode == Js::OpCode::LdElemI_A || instr->m_opcode == Js::OpCode::TypeofElem) &&
  2238. instr->DoStackArgsOpt() && !this->IsLoopPrePass())
  2239. {
  2240. GenerateBailAtOperation(&instr, IR::BailOnStackArgsOutOfActualsRange);
  2241. }
  2242. #if DBG
  2243. PropertySym *propertySymUseBefore = nullptr;
  2244. Assert(this->byteCodeUses == nullptr);
  2245. this->byteCodeUsesBeforeOpt->ClearAll();
  2246. GlobOpt::TrackByteCodeSymUsed(instr, this->byteCodeUsesBeforeOpt, &propertySymUseBefore);
  2247. Assert(noImplicitCallUsesToInsert->Count() == 0);
  2248. #endif
  2249. this->ignoredIntOverflowForCurrentInstr = false;
  2250. this->ignoredNegativeZeroForCurrentInstr = false;
  2251. src1 = instr->GetSrc1();
  2252. src2 = instr->GetSrc2();
  2253. if (src1)
  2254. {
  2255. src1Val = this->OptSrc(src1, &instr, &src1IndirIndexVal);
  2256. GOPT_TRACE_VALUENUMBER(_u("[src1] "), instr->GetSrc1(), _u("%d"), src1Val ? src1Val->GetValueNumber() : -1);
  2257. instr = this->SetTypeCheckBailOut(instr->GetSrc1(), instr, nullptr);
  2258. if (src2)
  2259. {
  2260. src2Val = this->OptSrc(src2, &instr);
  2261. GOPT_TRACE_VALUENUMBER(_u("[src2] "), instr->GetSrc2(), _u("%d"), src2Val ? src2Val->GetValueNumber() : -1);
  2262. }
  2263. }
  2264. if(instr->GetDst() && instr->GetDst()->IsIndirOpnd())
  2265. {
  2266. this->OptSrc(instr->GetDst(), &instr, &dstIndirIndexVal);
  2267. }
  2268. MarkArgumentsUsedForBranch(instr);
  2269. CSEOptimize(this->currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal);
  2270. OptimizeChecks(instr);
  2271. OptArraySrc(&instr, &src1Val, &src2Val);
  2272. OptNewScObject(&instr, src1Val);
  2273. OptStackArgLenAndConst(instr, &src1Val);
  2274. instr = this->OptPeep(instr, src1Val, src2Val);
  2275. if (instr->m_opcode == Js::OpCode::Nop ||
  2276. (instr->m_opcode == Js::OpCode::CheckThis &&
  2277. instr->GetSrc1()->IsRegOpnd() &&
  2278. instr->GetSrc1()->AsRegOpnd()->m_sym->m_isSafeThis))
  2279. {
  2280. instrNext = instr->m_next;
  2281. InsertNoImplicitCallUses(instr);
  2282. if (this->byteCodeUses)
  2283. {
  2284. this->InsertByteCodeUses(instr);
  2285. }
  2286. *isInstrRemoved = true;
  2287. this->currentBlock->RemoveInstr(instr);
  2288. return instrNext;
  2289. }
  2290. else if (instr->m_opcode == Js::OpCode::GetNewScObject && !this->IsLoopPrePass() && src1Val->GetValueInfo()->IsPrimitive())
  2291. {
  2292. // Constructor returned (src1) a primitive value, so fold this into "dst = Ld_A src2", where src2 is the new object that
  2293. // was passed into the constructor as its 'this' parameter
  2294. instr->FreeSrc1();
  2295. instr->SetSrc1(instr->UnlinkSrc2());
  2296. instr->m_opcode = Js::OpCode::Ld_A;
  2297. src1Val = src2Val;
  2298. src2Val = nullptr;
  2299. }
  2300. else if ((instr->m_opcode == Js::OpCode::TryCatch && this->func->DoOptimizeTry()) || (instr->m_opcode == Js::OpCode::TryFinally && this->func->DoOptimizeTry()))
  2301. {
  2302. ProcessTryHandler(instr);
  2303. }
  2304. else if (instr->m_opcode == Js::OpCode::BrOnException || instr->m_opcode == Js::OpCode::BrOnNoException)
  2305. {
  2306. if (this->ProcessExceptionHandlingEdges(instr))
  2307. {
  2308. *isInstrRemoved = true;
  2309. return instrNext;
  2310. }
  2311. }
  2312. bool isAlreadyTypeSpecialized = false;
  2313. if (!IsLoopPrePass() && instr->HasBailOutInfo())
  2314. {
  2315. if (instr->GetBailOutKind() == IR::BailOutExpectingInteger)
  2316. {
  2317. isAlreadyTypeSpecialized = TypeSpecializeBailoutExpectedInteger(instr, src1Val, &dstVal);
  2318. }
  2319. else if (instr->GetBailOutKind() == IR::BailOutExpectingString)
  2320. {
  2321. if (instr->GetSrc1()->IsRegOpnd())
  2322. {
  2323. if (!src1Val || !src1Val->GetValueInfo()->IsLikelyString())
  2324. {
  2325. // Disable SwitchOpt if the source is definitely not a string - This may be realized only in Globopt
  2326. Assert(IsSwitchOptEnabled());
  2327. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingString);
  2328. }
  2329. }
  2330. }
  2331. }
  2332. bool forceInvariantHoisting = false;
  2333. const bool ignoreIntOverflowInRangeForInstr = instr->ignoreIntOverflowInRange; // Save it since the instr can change
  2334. if (!isAlreadyTypeSpecialized)
  2335. {
  2336. bool redoTypeSpec;
  2337. instr = this->TypeSpecialization(instr, &src1Val, &src2Val, &dstVal, &redoTypeSpec, &forceInvariantHoisting);
  2338. if(redoTypeSpec && instr->m_opcode != Js::OpCode::Nop)
  2339. {
  2340. forceInvariantHoisting = false;
  2341. instr = this->TypeSpecialization(instr, &src1Val, &src2Val, &dstVal, &redoTypeSpec, &forceInvariantHoisting);
  2342. Assert(!redoTypeSpec);
  2343. }
  2344. if (instr->m_opcode == Js::OpCode::Nop)
  2345. {
  2346. InsertNoImplicitCallUses(instr);
  2347. if (this->byteCodeUses)
  2348. {
  2349. this->InsertByteCodeUses(instr);
  2350. }
  2351. instrNext = instr->m_next;
  2352. *isInstrRemoved = true;
  2353. this->currentBlock->RemoveInstr(instr);
  2354. return instrNext;
  2355. }
  2356. }
  2357. if (ignoreIntOverflowInRangeForInstr)
  2358. {
  2359. VerifyIntSpecForIgnoringIntOverflow(instr);
  2360. }
  2361. // Track calls after any pre-op bailouts have been inserted before the call, because they will need to restore out params.
  2362. this->TrackCalls(instr);
  2363. if (instr->GetSrc1())
  2364. {
  2365. this->UpdateObjPtrValueType(instr->GetSrc1(), instr);
  2366. }
  2367. IR::Opnd *dst = instr->GetDst();
  2368. if (dst)
  2369. {
  2370. // Copy prop dst uses and mark live/available type syms before tracking kills.
  2371. CopyPropDstUses(dst, instr, src1Val);
  2372. }
  2373. // Track mark temp object before we process the dst so we can generate pre-op bailout
  2374. instr = this->TrackMarkTempObject(instrPrev->m_next, instr);
  2375. bool removed = OptTagChecks(instr);
  2376. if (removed)
  2377. {
  2378. *isInstrRemoved = true;
  2379. return instrNext;
  2380. }
  2381. dstVal = this->OptDst(&instr, dstVal, src1Val, src2Val, dstIndirIndexVal, src1IndirIndexVal);
  2382. if (dst)
  2383. {
  2384. GOPT_TRACE_VALUENUMBER(_u("[dst] "), instr->GetDst(), _u("%d\n"), dstVal ? dstVal->GetValueNumber() : -1);
  2385. }
  2386. dst = instr->GetDst();
  2387. instrNext = instr->m_next;
  2388. if (dst)
  2389. {
  2390. if (this->func->HasTry() && this->func->DoOptimizeTry())
  2391. {
  2392. this->InsertToVarAtDefInTryRegion(instr, dst);
  2393. }
  2394. instr = this->SetTypeCheckBailOut(dst, instr, nullptr);
  2395. this->UpdateObjPtrValueType(dst, instr);
  2396. }
  2397. BVSparse<JitArenaAllocator> instrByteCodeStackSymUsedAfter(this->alloc);
  2398. PropertySym *propertySymUseAfter = nullptr;
  2399. if (this->byteCodeUses != nullptr)
  2400. {
  2401. GlobOpt::TrackByteCodeSymUsed(instr, &instrByteCodeStackSymUsedAfter, &propertySymUseAfter);
  2402. }
  2403. #if DBG
  2404. else
  2405. {
  2406. GlobOpt::TrackByteCodeSymUsed(instr, &instrByteCodeStackSymUsedAfter, &propertySymUseAfter);
  2407. instrByteCodeStackSymUsedAfter.Equal(this->byteCodeUsesBeforeOpt);
  2408. Assert(propertySymUseAfter == propertySymUseBefore);
  2409. }
  2410. #endif
  2411. bool isHoisted = false;
  2412. if (this->currentBlock->loop && !this->IsLoopPrePass())
  2413. {
  2414. isHoisted = this->TryHoistInvariant(instr, this->currentBlock, dstVal, src1Val, src2Val, true, false, forceInvariantHoisting);
  2415. }
  2416. src1 = instr->GetSrc1();
  2417. if (!this->IsLoopPrePass() && src1)
  2418. {
  2419. // instr const, nonConst => canonicalize by swapping operands
  2420. // This simplifies lowering. (somewhat machine dependent)
  2421. // Note that because of Var overflows, src1 may not have been constant prop'd to an IntConst
  2422. this->PreLowerCanonicalize(instr, &src1Val, &src2Val);
  2423. }
  2424. if (!PHASE_OFF(Js::MemOpPhase, this->func) &&
  2425. !isHoisted &&
  2426. !(instr->IsJitProfilingInstr()) &&
  2427. this->currentBlock->loop && !IsLoopPrePass() &&
  2428. !func->IsJitInDebugMode() &&
  2429. !func->IsMemOpDisabled() &&
  2430. this->currentBlock->loop->doMemOp)
  2431. {
  2432. CollectMemOpInfo(instrPrev, instr, src1Val, src2Val);
  2433. }
  2434. InsertNoImplicitCallUses(instr);
  2435. if (this->byteCodeUses != nullptr)
  2436. {
  2437. // Optimization removed some uses from the instruction.
  2438. // Need to insert fake uses so we can get the correct live register to restore in bailout.
  2439. this->byteCodeUses->Minus(&instrByteCodeStackSymUsedAfter);
  2440. if (this->propertySymUse == propertySymUseAfter)
  2441. {
  2442. this->propertySymUse = nullptr;
  2443. }
  2444. this->InsertByteCodeUses(instr);
  2445. }
  2446. if (!this->IsLoopPrePass() && !isHoisted && this->IsImplicitCallBailOutCurrentlyNeeded(instr, src1Val, src2Val))
  2447. {
  2448. IR::BailOutKind kind = IR::BailOutOnImplicitCalls;
  2449. if(instr->HasBailOutInfo())
  2450. {
  2451. Assert(instr->GetBailOutInfo()->bailOutOffset == instr->GetByteCodeOffset());
  2452. const IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  2453. if((bailOutKind & ~IR::BailOutKindBits) != IR::BailOutOnImplicitCallsPreOp)
  2454. {
  2455. Assert(!(bailOutKind & ~IR::BailOutKindBits));
  2456. instr->SetBailOutKind(bailOutKind + IR::BailOutOnImplicitCallsPreOp);
  2457. }
  2458. }
  2459. else if (instr->forcePreOpBailOutIfNeeded || this->isRecursiveCallOnLandingPad)
  2460. {
  2461. // We can't have a byte code reg slot as dst to generate a
  2462. // pre-op implicit call after we have processed the dst.
  2463. // Consider: This might miss an opportunity to use a copy prop sym to restore
  2464. // some other byte code reg if the dst is that copy prop that we already killed.
  2465. Assert(!instr->GetDst()
  2466. || !instr->GetDst()->IsRegOpnd()
  2467. || instr->GetDst()->AsRegOpnd()->GetIsJITOptimizedReg()
  2468. || !instr->GetDst()->AsRegOpnd()->m_sym->HasByteCodeRegSlot());
  2469. this->GenerateBailAtOperation(&instr, IR::BailOutOnImplicitCallsPreOp);
  2470. }
  2471. else
  2472. {
  2473. // Capture value of the bailout after the operation is done.
  2474. this->GenerateBailAfterOperation(&instr, kind);
  2475. }
  2476. }
  2477. if (this->IsLazyBailOutCurrentlyNeeded(instr, src1Val, src2Val, isHoisted))
  2478. {
  2479. this->GenerateLazyBailOut(instr);
  2480. }
  2481. if (CurrentBlockData()->capturedValuesCandidate && !this->IsLoopPrePass())
  2482. {
  2483. this->CommitCapturedValuesCandidate();
  2484. }
  2485. #if DBG
  2486. if (CONFIG_FLAG(ValidateIntRanges) && !IsLoopPrePass())
  2487. {
  2488. if (instr->ShouldEmitIntRangeCheck())
  2489. {
  2490. this->EmitIntRangeChecks(instr);
  2491. }
  2492. }
  2493. #endif
  2494. return instrNext;
  2495. }
  2496. bool
  2497. GlobOpt::IsNonNumericRegOpnd(IR::RegOpnd* opnd, bool inGlobOpt, bool* isSafeToTransferInPrepass /*=nullptr*/) const
  2498. {
  2499. if (opnd == nullptr)
  2500. {
  2501. return false;
  2502. }
  2503. if (opnd->m_sym->m_isNotNumber)
  2504. {
  2505. return true;
  2506. }
  2507. if (!inGlobOpt)
  2508. {
  2509. return false;
  2510. }
  2511. if (opnd->GetValueType().IsNumber() || currentBlock->globOptData.IsTypeSpecialized(opnd->m_sym))
  2512. {
  2513. if (!this->IsLoopPrePass())
  2514. {
  2515. return false;
  2516. }
  2517. Value * opndValue = this->currentBlock->globOptData.FindValue(opnd->m_sym);
  2518. ValueInfo * opndValueInfo = opndValue ? opndValue->GetValueInfo() : nullptr;
  2519. if (!opndValueInfo)
  2520. {
  2521. return true;
  2522. }
  2523. bool isSafeToTransfer = this->IsSafeToTransferInPrepass(opnd->m_sym, opndValueInfo);
  2524. if (isSafeToTransferInPrepass != nullptr)
  2525. {
  2526. *isSafeToTransferInPrepass = isSafeToTransfer;
  2527. }
  2528. if (this->prePassLoop->preservesNumberValue->Test(opnd->m_sym->m_id))
  2529. {
  2530. return false;
  2531. }
  2532. return !isSafeToTransfer;
  2533. }
  2534. return true;
  2535. }
  2536. bool
  2537. GlobOpt::OptTagChecks(IR::Instr *instr)
  2538. {
  2539. if (PHASE_OFF(Js::OptTagChecksPhase, this->func) || !this->DoTagChecks())
  2540. {
  2541. return false;
  2542. }
  2543. StackSym *stackSym = nullptr;
  2544. IR::SymOpnd *symOpnd = nullptr;
  2545. IR::RegOpnd *regOpnd = nullptr;
  2546. switch(instr->m_opcode)
  2547. {
  2548. case Js::OpCode::LdFld:
  2549. case Js::OpCode::LdMethodFld:
  2550. case Js::OpCode::CheckFixedFld:
  2551. case Js::OpCode::CheckPropertyGuardAndLoadType:
  2552. symOpnd = instr->GetSrc1()->AsSymOpnd();
  2553. stackSym = symOpnd->m_sym->AsPropertySym()->m_stackSym;
  2554. break;
  2555. case Js::OpCode::BailOnNotObject:
  2556. case Js::OpCode::BailOnNotArray:
  2557. if (instr->GetSrc1()->IsRegOpnd())
  2558. {
  2559. regOpnd = instr->GetSrc1()->AsRegOpnd();
  2560. stackSym = regOpnd->m_sym;
  2561. }
  2562. break;
  2563. case Js::OpCode::StFld:
  2564. symOpnd = instr->GetDst()->AsSymOpnd();
  2565. stackSym = symOpnd->m_sym->AsPropertySym()->m_stackSym;
  2566. break;
  2567. }
  2568. if (stackSym)
  2569. {
  2570. Value *value = CurrentBlockData()->FindValue(stackSym);
  2571. if (value)
  2572. {
  2573. ValueInfo *valInfo = value->GetValueInfo();
  2574. if (valInfo->GetSymStore() && valInfo->GetSymStore()->IsStackSym() && valInfo->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable())
  2575. {
  2576. return false;
  2577. }
  2578. ValueType valueType = value->GetValueInfo()->Type();
  2579. if (instr->m_opcode == Js::OpCode::BailOnNotObject)
  2580. {
  2581. if (valueType.CanBeTaggedValue())
  2582. {
  2583. // We're not adding new information to the value other than changing the value type. Preserve any existing
  2584. // information and just change the value type.
  2585. ChangeValueType(nullptr, value, valueType.SetCanBeTaggedValue(false), true /*preserveSubClassInfo*/);
  2586. return false;
  2587. }
  2588. if (!this->IsLoopPrePass())
  2589. {
  2590. if (this->byteCodeUses)
  2591. {
  2592. this->InsertByteCodeUses(instr);
  2593. }
  2594. this->currentBlock->RemoveInstr(instr);
  2595. }
  2596. return true;
  2597. }
  2598. if (valueType.CanBeTaggedValue() &&
  2599. !valueType.HasBeenNumber() &&
  2600. !this->IsLoopPrePass())
  2601. {
  2602. ValueType newValueType = valueType.SetCanBeTaggedValue(false);
  2603. // Split out the tag check as a separate instruction.
  2604. IR::Instr *bailOutInstr;
  2605. bailOutInstr = IR::BailOutInstr::New(Js::OpCode::BailOnNotObject, IR::BailOutOnTaggedValue, instr, instr->m_func);
  2606. if (!this->IsLoopPrePass())
  2607. {
  2608. FillBailOutInfo(this->currentBlock, bailOutInstr);
  2609. }
  2610. IR::RegOpnd *srcOpnd = regOpnd;
  2611. if (!srcOpnd)
  2612. {
  2613. srcOpnd = IR::RegOpnd::New(stackSym, stackSym->GetType(), instr->m_func);
  2614. AnalysisAssert(symOpnd);
  2615. if (symOpnd->GetIsJITOptimizedReg())
  2616. {
  2617. srcOpnd->SetIsJITOptimizedReg(true);
  2618. }
  2619. }
  2620. bailOutInstr->SetSrc1(srcOpnd);
  2621. bailOutInstr->GetSrc1()->SetValueType(valueType);
  2622. bailOutInstr->SetByteCodeOffset(instr);
  2623. instr->InsertBefore(bailOutInstr);
  2624. if (this->currentBlock->loop)
  2625. {
  2626. // Try hoisting the BailOnNotObject instr.
  2627. // But since this isn't the current instr being optimized, we need to play tricks with
  2628. // the byteCodeUse fields...
  2629. TrackByteCodeUsesForInstrAddedInOptInstr(bailOutInstr, [&]()
  2630. {
  2631. if (TryHoistInvariant(bailOutInstr, this->currentBlock, nullptr, value, nullptr, true, false, false, IR::BailOutOnTaggedValue))
  2632. {
  2633. Value* landingPadValue = this->currentBlock->loop->landingPad->globOptData.FindValue(stackSym);
  2634. ValueType newLandingPadValueType = landingPadValue->GetValueInfo()->Type().SetCanBeTaggedValue(false);
  2635. ChangeValueType(nullptr, landingPadValue, newLandingPadValueType, false);
  2636. }
  2637. });
  2638. }
  2639. if (symOpnd)
  2640. {
  2641. symOpnd->SetPropertyOwnerValueType(newValueType);
  2642. }
  2643. else
  2644. {
  2645. regOpnd->SetValueType(newValueType);
  2646. }
  2647. ChangeValueType(nullptr, value, newValueType, false);
  2648. }
  2649. }
  2650. }
  2651. return false;
  2652. }
  2653. bool
  2654. GlobOpt::TypeSpecializeBailoutExpectedInteger(IR::Instr* instr, Value* src1Val, Value** dstVal)
  2655. {
  2656. bool isAlreadyTypeSpecialized = false;
  2657. if(instr->GetSrc1()->IsRegOpnd())
  2658. {
  2659. if (!src1Val || !src1Val->GetValueInfo()->IsLikelyInt() || instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber)
  2660. {
  2661. Assert(IsSwitchOptEnabledForIntTypeSpec());
  2662. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingInteger);
  2663. }
  2664. // Attach the BailOutExpectingInteger to FromVar and Remove the bail out info on the Ld_A (Begin Switch) instr.
  2665. this->ToTypeSpecUse(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, TyInt32, IR::BailOutExpectingInteger, false, instr);
  2666. //TypeSpecialize the dst of Ld_A
  2667. TypeSpecializeIntDst(instr, instr->m_opcode, src1Val, src1Val, nullptr, IR::BailOutInvalid, INT32_MIN, INT32_MAX, dstVal);
  2668. isAlreadyTypeSpecialized = true;
  2669. }
  2670. instr->ClearBailOutInfo();
  2671. return isAlreadyTypeSpecialized;
  2672. }
  2673. Value*
  2674. GlobOpt::OptDst(
  2675. IR::Instr ** pInstr,
  2676. Value *dstVal,
  2677. Value *src1Val,
  2678. Value *src2Val,
  2679. Value *dstIndirIndexVal,
  2680. Value *src1IndirIndexVal)
  2681. {
  2682. IR::Instr *&instr = *pInstr;
  2683. IR::Opnd *opnd = instr->GetDst();
  2684. if (opnd)
  2685. {
  2686. if (opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd())
  2687. {
  2688. this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd());
  2689. }
  2690. if (opnd->IsIndirOpnd() && !this->IsLoopPrePass())
  2691. {
  2692. IR::RegOpnd *baseOpnd = opnd->AsIndirOpnd()->GetBaseOpnd();
  2693. const ValueType baseValueType(baseOpnd->GetValueType());
  2694. if ((
  2695. baseValueType.IsLikelyNativeArray() ||
  2696. #ifdef _M_IX86
  2697. (
  2698. !AutoSystemInfo::Data.SSE2Available() &&
  2699. baseValueType.IsLikelyObject() &&
  2700. (
  2701. baseValueType.GetObjectType() == ObjectType::Float32Array ||
  2702. baseValueType.GetObjectType() == ObjectType::Float64Array
  2703. )
  2704. )
  2705. #else
  2706. false
  2707. #endif
  2708. ) &&
  2709. instr->GetSrc1()->IsVar())
  2710. {
  2711. if(instr->m_opcode == Js::OpCode::StElemC)
  2712. {
  2713. // StElemC has different code that handles native array conversion or missing value stores. Add a bailout
  2714. // for those cases.
  2715. Assert(baseValueType.IsLikelyNativeArray());
  2716. Assert(!instr->HasBailOutInfo());
  2717. GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly);
  2718. }
  2719. else if(instr->HasBailOutInfo())
  2720. {
  2721. // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast
  2722. // path. Note that the removed bailouts should not be necessary for correctness. Bailout on native array
  2723. // conversion will be handled automatically as normal.
  2724. IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  2725. if(bailOutKind & IR::BailOutOnArrayAccessHelperCall)
  2726. {
  2727. bailOutKind -= IR::BailOutOnArrayAccessHelperCall;
  2728. }
  2729. if(bailOutKind == IR::BailOutOnImplicitCallsPreOp)
  2730. {
  2731. bailOutKind -= IR::BailOutOnImplicitCallsPreOp;
  2732. }
  2733. if(bailOutKind)
  2734. {
  2735. instr->SetBailOutKind(bailOutKind);
  2736. }
  2737. else
  2738. {
  2739. instr->ClearBailOutInfo();
  2740. }
  2741. }
  2742. }
  2743. }
  2744. }
  2745. this->ProcessKills(instr);
  2746. if (opnd)
  2747. {
  2748. if (dstVal == nullptr)
  2749. {
  2750. dstVal = ValueNumberDst(pInstr, src1Val, src2Val);
  2751. }
  2752. if (this->IsLoopPrePass())
  2753. {
  2754. // Keep track of symbols defined in the loop.
  2755. if (opnd->IsRegOpnd())
  2756. {
  2757. StackSym *symDst = opnd->AsRegOpnd()->m_sym;
  2758. rootLoopPrePass->symsDefInLoop->Set(symDst->m_id);
  2759. }
  2760. }
  2761. else if (dstVal)
  2762. {
  2763. opnd->SetValueType(dstVal->GetValueInfo()->Type());
  2764. if (currentBlock->loop &&
  2765. !IsLoopPrePass() &&
  2766. (instr->m_opcode == Js::OpCode::Ld_A || instr->m_opcode == Js::OpCode::Ld_I4) &&
  2767. instr->GetSrc1()->IsRegOpnd() &&
  2768. !func->IsJitInDebugMode())
  2769. {
  2770. // Look for the following patterns:
  2771. //
  2772. // Pattern 1:
  2773. // s1[liveOnBackEdge] = s3[dead]
  2774. //
  2775. // Pattern 2:
  2776. // s3 = operation(s1[liveOnBackEdge], s2)
  2777. // s1[liveOnBackEdge] = s3
  2778. //
  2779. // In both patterns, s1 and s3 have the same value by the end. Prefer to use s1 as the sym store instead of s3
  2780. // since s1 is live on back-edge, as otherwise, their lifetimes overlap, requiring two registers to hold the
  2781. // value instead of one.
  2782. do
  2783. {
  2784. IR::RegOpnd *const src = instr->GetSrc1()->AsRegOpnd();
  2785. StackSym *srcVarSym = src->m_sym;
  2786. if(srcVarSym->IsTypeSpec())
  2787. {
  2788. srcVarSym = srcVarSym->GetVarEquivSym(nullptr);
  2789. Assert(srcVarSym);
  2790. }
  2791. if(dstVal->GetValueInfo()->GetSymStore() != srcVarSym)
  2792. {
  2793. break;
  2794. }
  2795. IR::RegOpnd *const dst = opnd->AsRegOpnd();
  2796. StackSym *dstVarSym = dst->m_sym;
  2797. if(dstVarSym->IsTypeSpec())
  2798. {
  2799. dstVarSym = dstVarSym->GetVarEquivSym(nullptr);
  2800. Assert(dstVarSym);
  2801. }
  2802. if(!currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(dstVarSym->m_id))
  2803. {
  2804. break;
  2805. }
  2806. Value *const srcValue = CurrentBlockData()->FindValue(srcVarSym);
  2807. if(srcValue->GetValueNumber() != dstVal->GetValueNumber())
  2808. {
  2809. break;
  2810. }
  2811. if(!src->GetIsDead())
  2812. {
  2813. IR::Instr *const prevInstr = instr->GetPrevRealInstrOrLabel();
  2814. IR::Opnd *const prevDst = prevInstr->GetDst();
  2815. if(!prevDst ||
  2816. !src->IsEqualInternal(prevDst) ||
  2817. !(
  2818. (prevInstr->GetSrc1() && dst->IsEqual(prevInstr->GetSrc1())) ||
  2819. (prevInstr->GetSrc2() && dst->IsEqual(prevInstr->GetSrc2()))
  2820. ))
  2821. {
  2822. break;
  2823. }
  2824. }
  2825. this->SetSymStoreDirect(dstVal->GetValueInfo(), dstVarSym);
  2826. } while(false);
  2827. }
  2828. }
  2829. this->ValueNumberObjectType(opnd, instr);
  2830. }
  2831. this->CSEAddInstr(this->currentBlock, *pInstr, dstVal, src1Val, src2Val, dstIndirIndexVal, src1IndirIndexVal);
  2832. return dstVal;
  2833. }
  2834. void
  2835. GlobOpt::CopyPropDstUses(IR::Opnd *opnd, IR::Instr *instr, Value *src1Val)
  2836. {
  2837. if (opnd->IsSymOpnd())
  2838. {
  2839. IR::SymOpnd *symOpnd = opnd->AsSymOpnd();
  2840. if (symOpnd->m_sym->IsPropertySym())
  2841. {
  2842. PropertySym * originalPropertySym = symOpnd->m_sym->AsPropertySym();
  2843. Value *const objectValue = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym);
  2844. symOpnd->SetPropertyOwnerValueType(objectValue ? objectValue->GetValueInfo()->Type() : ValueType::Uninitialized);
  2845. this->CopyPropPropertySymObj(symOpnd, instr);
  2846. }
  2847. }
  2848. }
  2849. void
  2850. GlobOpt::SetLoopFieldInitialValue(Loop *loop, IR::Instr *instr, PropertySym *propertySym, PropertySym *originalPropertySym)
  2851. {
  2852. Value *initialValue = nullptr;
  2853. StackSym *symStore;
  2854. if (loop->allFieldsKilled || loop->fieldKilled->Test(originalPropertySym->m_id) || loop->fieldKilled->Test(propertySym->m_id))
  2855. {
  2856. return;
  2857. }
  2858. // Value already exists
  2859. if (CurrentBlockData()->FindValue(propertySym))
  2860. {
  2861. return;
  2862. }
  2863. // If this initial value was already added, we would find in the current value table.
  2864. Assert(!loop->initialValueFieldMap.TryGetValue(propertySym, &initialValue));
  2865. // If propertySym is live in landingPad, we don't need an initial value.
  2866. if (loop->landingPad->globOptData.liveFields->Test(propertySym->m_id))
  2867. {
  2868. return;
  2869. }
  2870. StackSym * objectSym = propertySym->m_stackSym;
  2871. Value *landingPadObjPtrVal, *currentObjPtrVal;
  2872. landingPadObjPtrVal = loop->landingPad->globOptData.FindValue(objectSym);
  2873. currentObjPtrVal = CurrentBlockData()->FindValue(objectSym);
  2874. auto CanSetInitialValue = [&]() -> bool {
  2875. if (!currentObjPtrVal)
  2876. {
  2877. return false;
  2878. }
  2879. if (landingPadObjPtrVal)
  2880. {
  2881. return currentObjPtrVal->GetValueNumber() == landingPadObjPtrVal->GetValueNumber();
  2882. }
  2883. else
  2884. {
  2885. if (!objectSym->IsSingleDef())
  2886. {
  2887. return false;
  2888. }
  2889. IR::Instr * defInstr = objectSym->GetInstrDef();
  2890. IR::Opnd * src1 = defInstr->GetSrc1();
  2891. while (!(src1 && src1->IsSymOpnd() && src1->AsSymOpnd()->m_sym->IsPropertySym()))
  2892. {
  2893. if (src1 && src1->IsRegOpnd() && src1->AsRegOpnd()->GetStackSym()->IsSingleDef())
  2894. {
  2895. defInstr = src1->AsRegOpnd()->GetStackSym()->GetInstrDef();
  2896. src1 = defInstr->GetSrc1();
  2897. }
  2898. else
  2899. {
  2900. return false;
  2901. }
  2902. }
  2903. return true;
  2904. // Todo: allow other kinds of operands as src1 of instr def of the object sym of the current propertySym
  2905. // SymOpnd, but not PropertySymOpnd - LdSlotArr, some LdSlots (?)
  2906. // nullptr - NewScObject
  2907. }
  2908. };
  2909. if (!CanSetInitialValue())
  2910. {
  2911. // objPtr has a different value in the landing pad.
  2912. return;
  2913. }
  2914. // The opnd's value type has not yet been initialized. Since the property sym doesn't have a value, it effectively has an
  2915. // Uninitialized value type. Use the profiled value type from the instruction.
  2916. const ValueType profiledValueType =
  2917. instr->IsProfiledInstr() ? instr->AsProfiledInstr()->u.FldInfo().valueType : ValueType::Uninitialized;
  2918. Assert(!profiledValueType.IsDefinite()); // Hence the values created here don't need to be tracked for kills
  2919. initialValue = this->NewGenericValue(profiledValueType, propertySym);
  2920. symStore = StackSym::New(this->func);
  2921. initialValue->GetValueInfo()->SetSymStore(symStore);
  2922. loop->initialValueFieldMap.Add(propertySym, initialValue->Copy(this->alloc, initialValue->GetValueNumber()));
  2923. // Copy the initial value into the landing pad, but without a symStore
  2924. Value *landingPadInitialValue = Value::New(this->alloc, initialValue->GetValueNumber(),
  2925. ValueInfo::New(this->alloc, initialValue->GetValueInfo()->Type()));
  2926. loop->landingPad->globOptData.SetValue(landingPadInitialValue, propertySym);
  2927. loop->landingPad->globOptData.liveFields->Set(propertySym->m_id);
  2928. #if DBG_DUMP
  2929. if (PHASE_TRACE(Js::FieldPREPhase, this->func))
  2930. {
  2931. Output::Print(_u("** TRACE: Field PRE initial value for loop head #%d. Val:%d symStore:"),
  2932. loop->GetHeadBlock()->GetBlockNum(), initialValue->GetValueNumber());
  2933. symStore->Dump();
  2934. Output::Print(_u("\n Instr: "));
  2935. instr->Dump();
  2936. Output::Flush();
  2937. }
  2938. #endif
  2939. // Add initial value to all the previous blocks in the loop.
  2940. FOREACH_BLOCK_BACKWARD_IN_RANGE(block, this->currentBlock->GetPrev(), loop->GetHeadBlock())
  2941. {
  2942. if (block->GetDataUseCount() == 0)
  2943. {
  2944. // All successor blocks have been processed, no point in adding the value.
  2945. continue;
  2946. }
  2947. Value *newValue = initialValue->Copy(this->alloc, initialValue->GetValueNumber());
  2948. block->globOptData.SetValue(newValue, propertySym);
  2949. block->globOptData.liveFields->Set(propertySym->m_id);
  2950. block->globOptData.SetValue(newValue, symStore);
  2951. block->globOptData.liveVarSyms->Set(symStore->m_id);
  2952. } NEXT_BLOCK_BACKWARD_IN_RANGE;
  2953. CurrentBlockData()->SetValue(initialValue, symStore);
  2954. CurrentBlockData()->liveVarSyms->Set(symStore->m_id);
  2955. CurrentBlockData()->liveFields->Set(propertySym->m_id);
  2956. }
  2957. // Examine src, apply copy prop and value number it
  2958. Value*
  2959. GlobOpt::OptSrc(IR::Opnd *opnd, IR::Instr * *pInstr, Value **indirIndexValRef, IR::IndirOpnd *parentIndirOpnd)
  2960. {
  2961. IR::Instr * &instr = *pInstr;
  2962. Assert(!indirIndexValRef || !*indirIndexValRef);
  2963. Assert(
  2964. parentIndirOpnd
  2965. ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd()
  2966. : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd());
  2967. Sym *sym;
  2968. Value *val;
  2969. PropertySym *originalPropertySym = nullptr;
  2970. switch(opnd->GetKind())
  2971. {
  2972. case IR::OpndKindIntConst:
  2973. val = this->GetIntConstantValue(opnd->AsIntConstOpnd()->AsInt32(), instr);
  2974. opnd->SetValueType(val->GetValueInfo()->Type());
  2975. return val;
  2976. case IR::OpndKindInt64Const:
  2977. val = this->GetIntConstantValue(opnd->AsInt64ConstOpnd()->GetValue(), instr);
  2978. opnd->SetValueType(val->GetValueInfo()->Type());
  2979. return val;
  2980. case IR::OpndKindFloatConst:
  2981. {
  2982. const FloatConstType floatValue = opnd->AsFloatConstOpnd()->m_value;
  2983. int32 int32Value;
  2984. if(Js::JavascriptNumber::TryGetInt32Value(floatValue, &int32Value))
  2985. {
  2986. val = GetIntConstantValue(int32Value, instr);
  2987. }
  2988. else
  2989. {
  2990. val = NewFloatConstantValue(floatValue);
  2991. }
  2992. opnd->SetValueType(val->GetValueInfo()->Type());
  2993. return val;
  2994. }
  2995. case IR::OpndKindAddr:
  2996. {
  2997. IR::AddrOpnd *addrOpnd = opnd->AsAddrOpnd();
  2998. if (addrOpnd->m_isFunction)
  2999. {
  3000. AssertMsg(!PHASE_OFF(Js::FixedMethodsPhase, instr->m_func), "Fixed function address operand with fixed method calls phase disabled?");
  3001. val = NewFixedFunctionValue((Js::JavascriptFunction *)addrOpnd->m_address, addrOpnd);
  3002. opnd->SetValueType(val->GetValueInfo()->Type());
  3003. return val;
  3004. }
  3005. else if (addrOpnd->IsVar() && Js::TaggedInt::Is(addrOpnd->m_address))
  3006. {
  3007. val = this->GetIntConstantValue(Js::TaggedInt::ToInt32(addrOpnd->m_address), instr);
  3008. opnd->SetValueType(val->GetValueInfo()->Type());
  3009. return val;
  3010. }
  3011. val = this->GetVarConstantValue(addrOpnd);
  3012. return val;
  3013. }
  3014. case IR::OpndKindSym:
  3015. {
  3016. // Clear the opnd's value type up-front, so that this code cannot accidentally use the value type set from a previous
  3017. // OptSrc on the same instruction (for instance, from an earlier loop prepass). The value type will be set from the
  3018. // value if available, before returning from this function.
  3019. opnd->SetValueType(ValueType::Uninitialized);
  3020. sym = opnd->AsSymOpnd()->m_sym;
  3021. // Don't create a new value for ArgSlots and don't copy prop them away.
  3022. if (sym->IsStackSym() && sym->AsStackSym()->IsArgSlotSym())
  3023. {
  3024. return nullptr;
  3025. }
  3026. // Unless we have profile info, don't create a new value for ArgSlots and don't copy prop them away.
  3027. if (sym->IsStackSym() && sym->AsStackSym()->IsParamSlotSym())
  3028. {
  3029. if (!instr->m_func->IsLoopBody() && instr->m_func->HasProfileInfo())
  3030. {
  3031. // Skip "this" pointer.
  3032. int paramSlotNum = sym->AsStackSym()->GetParamSlotNum() - 2;
  3033. if (paramSlotNum >= 0)
  3034. {
  3035. const auto parameterType = instr->m_func->GetReadOnlyProfileInfo()->GetParameterInfo(static_cast<Js::ArgSlot>(paramSlotNum));
  3036. val = NewGenericValue(parameterType);
  3037. opnd->SetValueType(val->GetValueInfo()->Type());
  3038. return val;
  3039. }
  3040. }
  3041. return nullptr;
  3042. }
  3043. if (!sym->IsPropertySym())
  3044. {
  3045. break;
  3046. }
  3047. originalPropertySym = sym->AsPropertySym();
  3048. // Don't give a value to 'arguments' property sym to prevent field copy prop of 'arguments'
  3049. if (originalPropertySym->AsPropertySym()->m_propertyId == Js::PropertyIds::arguments &&
  3050. originalPropertySym->AsPropertySym()->m_fieldKind == PropertyKindData)
  3051. {
  3052. if (opnd->AsSymOpnd()->IsPropertySymOpnd())
  3053. {
  3054. this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd());
  3055. }
  3056. return nullptr;
  3057. }
  3058. Value *const objectValue = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym);
  3059. opnd->AsSymOpnd()->SetPropertyOwnerValueType(
  3060. objectValue ? objectValue->GetValueInfo()->Type() : ValueType::Uninitialized);
  3061. sym = this->CopyPropPropertySymObj(opnd->AsSymOpnd(), instr);
  3062. if (!DoFieldCopyProp())
  3063. {
  3064. if (opnd->AsSymOpnd()->IsPropertySymOpnd())
  3065. {
  3066. this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd());
  3067. }
  3068. return nullptr;
  3069. }
  3070. switch (instr->m_opcode)
  3071. {
  3072. // These need the symbolic reference to the field, don't copy prop the value of the field
  3073. case Js::OpCode::DeleteFld:
  3074. case Js::OpCode::DeleteRootFld:
  3075. case Js::OpCode::DeleteFldStrict:
  3076. case Js::OpCode::DeleteRootFldStrict:
  3077. case Js::OpCode::ScopedDeleteFld:
  3078. case Js::OpCode::ScopedDeleteFldStrict:
  3079. case Js::OpCode::LdMethodFromFlags:
  3080. case Js::OpCode::BrOnNoProperty:
  3081. case Js::OpCode::BrOnNoLocalProperty:
  3082. case Js::OpCode::BrOnHasProperty:
  3083. case Js::OpCode::BrOnHasLocalProperty:
  3084. case Js::OpCode::LdMethodFldPolyInlineMiss:
  3085. case Js::OpCode::StSlotChkUndecl:
  3086. case Js::OpCode::ScopedLdInst:
  3087. return nullptr;
  3088. };
  3089. if (instr->CallsGetter())
  3090. {
  3091. return nullptr;
  3092. }
  3093. if (this->IsLoopPrePass() && this->DoFieldPRE(this->rootLoopPrePass))
  3094. {
  3095. if (!this->prePassLoop->allFieldsKilled && !this->prePassLoop->fieldKilled->Test(sym->m_id))
  3096. {
  3097. this->SetLoopFieldInitialValue(this->rootLoopPrePass, instr, sym->AsPropertySym(), originalPropertySym);
  3098. }
  3099. if (this->IsPREInstrCandidateLoad(instr->m_opcode))
  3100. {
  3101. // Foreach property sym, remember the first instruction that loads it.
  3102. // Can this be done in one call?
  3103. if (!this->prePassInstrMap->ContainsKey(sym->m_id))
  3104. {
  3105. this->prePassInstrMap->AddNew(sym->m_id, instr->CopyWithoutDst());
  3106. }
  3107. }
  3108. }
  3109. break;
  3110. }
  3111. case IR::OpndKindReg:
  3112. // Clear the opnd's value type up-front, so that this code cannot accidentally use the value type set from a previous
  3113. // OptSrc on the same instruction (for instance, from an earlier loop prepass). The value type will be set from the
  3114. // value if available, before returning from this function.
  3115. opnd->SetValueType(ValueType::Uninitialized);
  3116. sym = opnd->AsRegOpnd()->m_sym;
  3117. CurrentBlockData()->MarkTempLastUse(instr, opnd->AsRegOpnd());
  3118. if (sym->AsStackSym()->IsTypeSpec())
  3119. {
  3120. sym = sym->AsStackSym()->GetVarEquivSym(this->func);
  3121. }
  3122. break;
  3123. case IR::OpndKindIndir:
  3124. this->OptimizeIndirUses(opnd->AsIndirOpnd(), &instr, indirIndexValRef);
  3125. return nullptr;
  3126. default:
  3127. return nullptr;
  3128. }
  3129. val = CurrentBlockData()->FindValue(sym);
  3130. if (val)
  3131. {
  3132. Assert(CurrentBlockData()->IsLive(sym) || (sym->IsPropertySym()));
  3133. if (instr)
  3134. {
  3135. opnd = this->CopyProp(opnd, instr, val, parentIndirOpnd);
  3136. }
  3137. // Check if we freed the operand.
  3138. if (opnd == nullptr)
  3139. {
  3140. return nullptr;
  3141. }
  3142. // In a loop prepass, determine stack syms that are used before they are defined in the root loop for which the prepass
  3143. // is being done. This information is used to do type specialization conversions in the landing pad where appropriate.
  3144. if(IsLoopPrePass() &&
  3145. sym->IsStackSym() &&
  3146. !rootLoopPrePass->symsUsedBeforeDefined->Test(sym->m_id) &&
  3147. rootLoopPrePass->landingPad->globOptData.IsLive(sym) && !isAsmJSFunc) // no typespec in asmjs and hence skipping this
  3148. {
  3149. Value *const landingPadValue = rootLoopPrePass->landingPad->globOptData.FindValue(sym);
  3150. if(landingPadValue && val->GetValueNumber() == landingPadValue->GetValueNumber())
  3151. {
  3152. rootLoopPrePass->symsUsedBeforeDefined->Set(sym->m_id);
  3153. ValueInfo *landingPadValueInfo = landingPadValue->GetValueInfo();
  3154. if(landingPadValueInfo->IsLikelyNumber())
  3155. {
  3156. rootLoopPrePass->likelyNumberSymsUsedBeforeDefined->Set(sym->m_id);
  3157. if(DoAggressiveIntTypeSpec() ? landingPadValueInfo->IsLikelyInt() : landingPadValueInfo->IsInt())
  3158. {
  3159. // Can only force int conversions in the landing pad based on likely-int values if aggressive int type
  3160. // specialization is enabled.
  3161. rootLoopPrePass->likelyIntSymsUsedBeforeDefined->Set(sym->m_id);
  3162. }
  3163. }
  3164. }
  3165. }
  3166. }
  3167. else if ((instr->TransfersSrcValue() || OpCodeAttr::CanCSE(instr->m_opcode)) && (opnd == instr->GetSrc1() || opnd == instr->GetSrc2()))
  3168. {
  3169. if (sym->IsPropertySym())
  3170. {
  3171. val = this->CreateFieldSrcValue(sym->AsPropertySym(), originalPropertySym, &opnd, instr);
  3172. }
  3173. else
  3174. {
  3175. val = this->NewGenericValue(ValueType::Uninitialized, opnd);
  3176. }
  3177. }
  3178. if (opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd())
  3179. {
  3180. TryOptimizeInstrWithFixedDataProperty(&instr);
  3181. this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd());
  3182. }
  3183. if (val)
  3184. {
  3185. ValueType valueType(val->GetValueInfo()->Type());
  3186. // This block uses per-instruction profile information on array types to optimize using the best available profile
  3187. // information and to prevent infinite bailouts by ensuring array type information is updated on bailouts.
  3188. if (valueType.IsLikelyArray() && !valueType.IsDefinite() && !valueType.IsObject() && instr->IsProfiledInstr())
  3189. {
  3190. // See if we have profile data for the array type
  3191. IR::ProfiledInstr *const profiledInstr = instr->AsProfiledInstr();
  3192. ValueType profiledArrayType;
  3193. bool useAggressiveSpecialization = true;
  3194. switch(instr->m_opcode)
  3195. {
  3196. case Js::OpCode::LdElemI_A:
  3197. if(instr->GetSrc1()->IsIndirOpnd() && opnd == instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd())
  3198. {
  3199. profiledArrayType = profiledInstr->u.ldElemInfo->GetArrayType();
  3200. useAggressiveSpecialization = !profiledInstr->u.ldElemInfo->IsAggressiveSpecializationDisabled();
  3201. }
  3202. break;
  3203. case Js::OpCode::StElemI_A:
  3204. case Js::OpCode::StElemI_A_Strict:
  3205. case Js::OpCode::StElemC:
  3206. if(instr->GetDst()->IsIndirOpnd() && opnd == instr->GetDst()->AsIndirOpnd()->GetBaseOpnd())
  3207. {
  3208. profiledArrayType = profiledInstr->u.stElemInfo->GetArrayType();
  3209. useAggressiveSpecialization = !profiledInstr->u.stElemInfo->IsAggressiveSpecializationDisabled();
  3210. }
  3211. break;
  3212. case Js::OpCode::LdLen_A:
  3213. if(instr->GetSrc1()->IsRegOpnd() && opnd == instr->GetSrc1())
  3214. {
  3215. profiledArrayType = profiledInstr->u.LdLenInfo().GetArrayType();
  3216. useAggressiveSpecialization = !profiledInstr->u.LdLenInfo().IsAggressiveSpecializationDisabled();
  3217. }
  3218. break;
  3219. case Js::OpCode::IsIn:
  3220. if (instr->GetSrc2()->IsRegOpnd() && opnd == instr->GetSrc2())
  3221. {
  3222. profiledArrayType = profiledInstr->u.ldElemInfo->GetArrayType();
  3223. useAggressiveSpecialization = !profiledInstr->u.ldElemInfo->IsAggressiveSpecializationDisabled();
  3224. }
  3225. break;
  3226. }
  3227. if (profiledArrayType.IsLikelyObject())
  3228. {
  3229. // Ideally we want to use the most specialized type seen by this path, but when that causes bailouts use the least specialized type instead.
  3230. if (useAggressiveSpecialization &&
  3231. profiledArrayType.GetObjectType() == valueType.GetObjectType() &&
  3232. !valueType.IsLikelyNativeIntArray() &&
  3233. (
  3234. profiledArrayType.HasIntElements() || (valueType.HasVarElements() && profiledArrayType.HasFloatElements())
  3235. ))
  3236. {
  3237. // use the more specialized type profiled by the instruction.
  3238. valueType = profiledArrayType.SetHasNoMissingValues(valueType.HasNoMissingValues());
  3239. ChangeValueType(this->currentBlock, CurrentBlockData()->FindValue(opnd->AsRegOpnd()->m_sym), valueType, false);
  3240. }
  3241. else if (!useAggressiveSpecialization &&
  3242. (profiledArrayType.GetObjectType() != valueType.GetObjectType() ||
  3243. (
  3244. valueType.IsLikelyNativeArray() &&
  3245. (
  3246. profiledArrayType.HasVarElements() || (valueType.HasIntElements() && profiledArrayType.HasFloatElements())
  3247. )
  3248. )
  3249. ))
  3250. {
  3251. // Merge array type we pulled from profile with type propagated by dataflow.
  3252. if (profiledArrayType.IsLikelyArray())
  3253. {
  3254. valueType = valueType.Merge(profiledArrayType).SetHasNoMissingValues(valueType.HasNoMissingValues());
  3255. }
  3256. else
  3257. {
  3258. valueType = valueType.Merge(profiledArrayType);
  3259. }
  3260. ChangeValueType(this->currentBlock, CurrentBlockData()->FindValue(opnd->AsRegOpnd()->m_sym), valueType, false, true);
  3261. }
  3262. }
  3263. }
  3264. opnd->SetValueType(valueType);
  3265. if(!IsLoopPrePass() && opnd->IsSymOpnd() && (valueType.IsDefinite() || valueType.IsNotTaggedValue()))
  3266. {
  3267. if (opnd->AsSymOpnd()->m_sym->IsPropertySym())
  3268. {
  3269. // A property sym can only be guaranteed to have a definite value type when implicit calls are disabled from the
  3270. // point where the sym was defined with the definite value type. Insert an instruction to indicate to the
  3271. // dead-store pass that implicit calls need to be kept disabled until after this instruction.
  3272. Assert(DoFieldCopyProp());
  3273. CaptureNoImplicitCallUses(opnd, false, instr);
  3274. }
  3275. }
  3276. }
  3277. else
  3278. {
  3279. opnd->SetValueType(ValueType::Uninitialized);
  3280. }
  3281. return val;
  3282. }
  3283. /*
  3284. * GlobOpt::TryOptimizeInstrWithFixedDataProperty
  3285. * Converts Ld[Root]Fld instr to
  3286. * * CheckFixedFld
  3287. * * Dst = Ld_A <int Constant value>
  3288. * This API assumes that the source operand is a Sym/PropertySym kind.
  3289. */
  3290. void
  3291. GlobOpt::TryOptimizeInstrWithFixedDataProperty(IR::Instr ** const pInstr)
  3292. {
  3293. Assert(pInstr);
  3294. IR::Instr * &instr = *pInstr;
  3295. IR::Opnd * src1 = instr->GetSrc1();
  3296. Assert(src1 && src1->IsSymOpnd() && src1->AsSymOpnd()->IsPropertySymOpnd());
  3297. if(PHASE_OFF(Js::UseFixedDataPropsPhase, instr->m_func))
  3298. {
  3299. return;
  3300. }
  3301. if (!this->IsLoopPrePass() && !this->isRecursiveCallOnLandingPad &&
  3302. OpCodeAttr::CanLoadFixedFields(instr->m_opcode))
  3303. {
  3304. instr->TryOptimizeInstrWithFixedDataProperty(&instr, this);
  3305. }
  3306. }
  3307. // Constant prop if possible, otherwise if this value already resides in another
  3308. // symbol, reuse this previous symbol. This should help register allocation.
  3309. IR::Opnd *
  3310. GlobOpt::CopyProp(IR::Opnd *opnd, IR::Instr *instr, Value *val, IR::IndirOpnd *parentIndirOpnd)
  3311. {
  3312. Assert(
  3313. parentIndirOpnd
  3314. ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd()
  3315. : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd());
  3316. if (this->IsLoopPrePass())
  3317. {
  3318. // Transformations are not legal in prepass...
  3319. return opnd;
  3320. }
  3321. if (instr->m_opcode == Js::OpCode::CheckFixedFld || instr->m_opcode == Js::OpCode::CheckPropertyGuardAndLoadType)
  3322. {
  3323. // Don't copy prop into CheckFixedFld or CheckPropertyGuardAndLoadType
  3324. return opnd;
  3325. }
  3326. // Don't copy-prop link operands of ExtendedArgs
  3327. if (instr->m_opcode == Js::OpCode::ExtendArg_A && opnd == instr->GetSrc2())
  3328. {
  3329. return opnd;
  3330. }
  3331. // Don't copy-prop operand of SIMD instr with ExtendedArg operands. Each instr should have its exclusive EA sequence.
  3332. if (
  3333. Js::IsSimd128Opcode(instr->m_opcode) &&
  3334. instr->GetSrc1() != nullptr &&
  3335. instr->GetSrc1()->IsRegOpnd() &&
  3336. instr->GetSrc2() == nullptr
  3337. )
  3338. {
  3339. StackSym *sym = instr->GetSrc1()->GetStackSym();
  3340. if (sym && sym->IsSingleDef() && sym->GetInstrDef()->m_opcode == Js::OpCode::ExtendArg_A)
  3341. {
  3342. return opnd;
  3343. }
  3344. }
  3345. ValueInfo *valueInfo = val->GetValueInfo();
  3346. if (this->func->HasFinally())
  3347. {
  3348. // s0 = undefined was added on functions with early exit in try-finally functions, that can get copy-proped and case incorrect results
  3349. if (instr->m_opcode == Js::OpCode::ArgOut_A_Inline && valueInfo->GetSymStore() &&
  3350. valueInfo->GetSymStore()->m_id == 0)
  3351. {
  3352. // We don't want to copy-prop s0 (return symbol) into inlinee code
  3353. return opnd;
  3354. }
  3355. }
  3356. // Constant prop?
  3357. int32 intConstantValue;
  3358. int64 int64ConstantValue;
  3359. if (valueInfo->TryGetIntConstantValue(&intConstantValue))
  3360. {
  3361. if (PHASE_OFF(Js::ConstPropPhase, this->func))
  3362. {
  3363. return opnd;
  3364. }
  3365. if ((
  3366. instr->m_opcode == Js::OpCode::StElemI_A ||
  3367. instr->m_opcode == Js::OpCode::StElemI_A_Strict ||
  3368. instr->m_opcode == Js::OpCode::StElemC
  3369. ) && instr->GetSrc1() == opnd)
  3370. {
  3371. // Disabling prop to src of native array store, because we were losing the chance to type specialize.
  3372. // Is it possible to type specialize this src if we allow constants, etc., to be prop'd here?
  3373. if (instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyNativeArray())
  3374. {
  3375. return opnd;
  3376. }
  3377. }
  3378. if(opnd != instr->GetSrc1() && opnd != instr->GetSrc2())
  3379. {
  3380. if(PHASE_OFF(Js::IndirCopyPropPhase, instr->m_func))
  3381. {
  3382. return opnd;
  3383. }
  3384. // Const-prop an indir opnd's constant index into its offset
  3385. IR::Opnd *srcs[] = { instr->GetSrc1(), instr->GetSrc2(), instr->GetDst() };
  3386. for(int i = 0; i < sizeof(srcs) / sizeof(srcs[0]); ++i)
  3387. {
  3388. const auto src = srcs[i];
  3389. if(!src || !src->IsIndirOpnd())
  3390. {
  3391. continue;
  3392. }
  3393. const auto indir = src->AsIndirOpnd();
  3394. if ((int64)indir->GetOffset() + intConstantValue > INT32_MAX)
  3395. {
  3396. continue;
  3397. }
  3398. if(opnd == indir->GetIndexOpnd())
  3399. {
  3400. Assert(indir->GetScale() == 0);
  3401. GOPT_TRACE_OPND(opnd, _u("Constant prop indir index into offset (value: %d)\n"), intConstantValue);
  3402. this->CaptureByteCodeSymUses(instr);
  3403. indir->SetOffset(indir->GetOffset() + intConstantValue);
  3404. indir->SetIndexOpnd(nullptr);
  3405. }
  3406. }
  3407. return opnd;
  3408. }
  3409. if (Js::TaggedInt::IsOverflow(intConstantValue))
  3410. {
  3411. return opnd;
  3412. }
  3413. IR::Opnd *constOpnd;
  3414. if (opnd->IsVar())
  3415. {
  3416. IR::AddrOpnd *addrOpnd = IR::AddrOpnd::New(Js::TaggedInt::ToVarUnchecked((int)intConstantValue), IR::AddrOpndKindConstantVar, instr->m_func);
  3417. GOPT_TRACE_OPND(opnd, _u("Constant prop %d (value:%d)\n"), addrOpnd->m_address, intConstantValue);
  3418. constOpnd = addrOpnd;
  3419. }
  3420. else
  3421. {
  3422. // Note: Jit loop body generates some i32 operands...
  3423. Assert(opnd->IsInt32() || opnd->IsInt64() || opnd->IsUInt32());
  3424. IRType opndType;
  3425. IntConstType constVal;
  3426. if (opnd->IsUInt32())
  3427. {
  3428. // avoid sign extension
  3429. constVal = (uint32)intConstantValue;
  3430. opndType = TyUint32;
  3431. }
  3432. else
  3433. {
  3434. constVal = intConstantValue;
  3435. opndType = TyInt32;
  3436. }
  3437. IR::IntConstOpnd *intOpnd = IR::IntConstOpnd::New(constVal, opndType, instr->m_func);
  3438. GOPT_TRACE_OPND(opnd, _u("Constant prop %d (value:%d)\n"), intOpnd->GetImmediateValue(instr->m_func), intConstantValue);
  3439. constOpnd = intOpnd;
  3440. }
  3441. #if ENABLE_DEBUG_CONFIG_OPTIONS
  3442. //Need to update DumpFieldCopyPropTestTrace for every new opcode that is added for fieldcopyprop
  3443. if(Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FieldCopyPropPhase))
  3444. {
  3445. instr->DumpFieldCopyPropTestTrace(this->isRecursiveCallOnLandingPad);
  3446. }
  3447. #endif
  3448. this->CaptureByteCodeSymUses(instr);
  3449. opnd = instr->ReplaceSrc(opnd, constOpnd);
  3450. switch (instr->m_opcode)
  3451. {
  3452. case Js::OpCode::LdSlot:
  3453. case Js::OpCode::LdSlotArr:
  3454. case Js::OpCode::LdFld:
  3455. case Js::OpCode::LdFldForTypeOf:
  3456. case Js::OpCode::LdRootFldForTypeOf:
  3457. case Js::OpCode::LdFldForCallApplyTarget:
  3458. case Js::OpCode::LdRootFld:
  3459. case Js::OpCode::LdMethodFld:
  3460. case Js::OpCode::LdRootMethodFld:
  3461. case Js::OpCode::LdMethodFromFlags:
  3462. case Js::OpCode::ScopedLdMethodFld:
  3463. case Js::OpCode::ScopedLdFld:
  3464. case Js::OpCode::ScopedLdFldForTypeOf:
  3465. instr->m_opcode = Js::OpCode::Ld_A;
  3466. case Js::OpCode::Ld_A:
  3467. {
  3468. IR::Opnd * dst = instr->GetDst();
  3469. if (dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->IsSingleDef())
  3470. {
  3471. dst->AsRegOpnd()->m_sym->SetIsIntConst((int)intConstantValue);
  3472. }
  3473. break;
  3474. }
  3475. case Js::OpCode::ArgOut_A:
  3476. case Js::OpCode::ArgOut_A_Inline:
  3477. case Js::OpCode::ArgOut_A_FixupForStackArgs:
  3478. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  3479. if (instr->GetDst()->IsRegOpnd())
  3480. {
  3481. Assert(instr->GetDst()->AsRegOpnd()->m_sym->m_isSingleDef);
  3482. instr->GetDst()->AsRegOpnd()->m_sym->AsStackSym()->SetIsIntConst((int)intConstantValue);
  3483. }
  3484. else
  3485. {
  3486. instr->GetDst()->AsSymOpnd()->m_sym->AsStackSym()->SetIsIntConst((int)intConstantValue);
  3487. }
  3488. break;
  3489. case Js::OpCode::TypeofElem:
  3490. instr->m_opcode = Js::OpCode::Typeof;
  3491. break;
  3492. case Js::OpCode::StSlotChkUndecl:
  3493. if (instr->GetSrc2() == opnd)
  3494. {
  3495. // Src2 here should refer to the same location as the Dst operand, which we need to keep live
  3496. // due to the implicit read for ChkUndecl.
  3497. instr->m_opcode = Js::OpCode::StSlot;
  3498. instr->FreeSrc2();
  3499. opnd = nullptr;
  3500. }
  3501. break;
  3502. }
  3503. return opnd;
  3504. }
  3505. else if (valueInfo->TryGetIntConstantValue(&int64ConstantValue, false))
  3506. {
  3507. if (PHASE_OFF(Js::ConstPropPhase, this->func) || !PHASE_ON(Js::Int64ConstPropPhase, this->func))
  3508. {
  3509. return opnd;
  3510. }
  3511. Assert(this->func->GetJITFunctionBody()->IsWasmFunction());
  3512. if (this->func->GetJITFunctionBody()->IsWasmFunction() && opnd->IsInt64())
  3513. {
  3514. IR::Int64ConstOpnd *intOpnd = IR::Int64ConstOpnd::New(int64ConstantValue, opnd->GetType(), instr->m_func);
  3515. GOPT_TRACE_OPND(opnd, _u("Constant prop %lld (value:%lld)\n"), intOpnd->GetImmediateValue(instr->m_func), int64ConstantValue);
  3516. this->CaptureByteCodeSymUses(instr);
  3517. opnd = instr->ReplaceSrc(opnd, intOpnd);
  3518. }
  3519. return opnd;
  3520. }
  3521. Sym *opndSym = nullptr;
  3522. if (opnd->IsRegOpnd())
  3523. {
  3524. IR::RegOpnd *regOpnd = opnd->AsRegOpnd();
  3525. opndSym = regOpnd->m_sym;
  3526. }
  3527. else if (opnd->IsSymOpnd())
  3528. {
  3529. IR::SymOpnd *symOpnd = opnd->AsSymOpnd();
  3530. opndSym = symOpnd->m_sym;
  3531. }
  3532. if (!opndSym)
  3533. {
  3534. return opnd;
  3535. }
  3536. if (PHASE_OFF(Js::CopyPropPhase, this->func))
  3537. {
  3538. this->SetSymStoreDirect(valueInfo, opndSym);
  3539. return opnd;
  3540. }
  3541. StackSym *copySym = CurrentBlockData()->GetCopyPropSym(opndSym, val);
  3542. if (copySym != nullptr)
  3543. {
  3544. Assert(!opndSym->IsStackSym() || copySym->GetSymSize() == opndSym->AsStackSym()->GetSymSize());
  3545. // Copy prop.
  3546. return CopyPropReplaceOpnd(instr, opnd, copySym, parentIndirOpnd);
  3547. }
  3548. else
  3549. {
  3550. if (valueInfo->GetSymStore() && instr->m_opcode == Js::OpCode::Ld_A && instr->GetDst()->IsRegOpnd()
  3551. && valueInfo->GetSymStore() == instr->GetDst()->AsRegOpnd()->m_sym)
  3552. {
  3553. // Avoid resetting symStore after fieldHoisting:
  3554. // t1 = LdFld field <- set symStore to fieldHoistSym
  3555. // fieldHoistSym = Ld_A t1 <- we're looking at t1 now, but want to copy-prop fieldHoistSym forward
  3556. return opnd;
  3557. }
  3558. this->SetSymStoreDirect(valueInfo, opndSym);
  3559. }
  3560. return opnd;
  3561. }
  3562. IR::Opnd *
  3563. GlobOpt::CopyPropReplaceOpnd(IR::Instr * instr, IR::Opnd * opnd, StackSym * copySym, IR::IndirOpnd *parentIndirOpnd)
  3564. {
  3565. Assert(
  3566. parentIndirOpnd
  3567. ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd()
  3568. : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd());
  3569. Assert(CurrentBlockData()->IsLive(copySym));
  3570. IR::RegOpnd *regOpnd;
  3571. StackSym *newSym = copySym;
  3572. GOPT_TRACE_OPND(opnd, _u("Copy prop s%d\n"), newSym->m_id);
  3573. #if ENABLE_DEBUG_CONFIG_OPTIONS
  3574. //Need to update DumpFieldCopyPropTestTrace for every new opcode that is added for fieldcopyprop
  3575. if(Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FieldCopyPropPhase))
  3576. {
  3577. instr->DumpFieldCopyPropTestTrace(this->isRecursiveCallOnLandingPad);
  3578. }
  3579. #endif
  3580. this->CaptureByteCodeSymUses(instr);
  3581. if (opnd->IsRegOpnd())
  3582. {
  3583. regOpnd = opnd->AsRegOpnd();
  3584. regOpnd->m_sym = newSym;
  3585. regOpnd->SetIsJITOptimizedReg(true);
  3586. // The dead bit on the opnd is specific to the sym it is referencing. Since we replaced the sym, the bit is reset.
  3587. regOpnd->SetIsDead(false);
  3588. if(parentIndirOpnd)
  3589. {
  3590. return regOpnd;
  3591. }
  3592. }
  3593. else
  3594. {
  3595. // If this is an object type specialized field load inside a loop, and it produces a type value which wasn't live
  3596. // before, make sure the type check is left in the loop, because it may be the last type check in the loop protecting
  3597. // other fields which are not hoistable and are lexically upstream in the loop. If the check is not ultimately
  3598. // needed, the dead store pass will remove it.
  3599. if (this->currentBlock->loop != nullptr && opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd())
  3600. {
  3601. IR::PropertySymOpnd* propertySymOpnd = opnd->AsPropertySymOpnd();
  3602. if (CheckIfPropOpEmitsTypeCheck(instr, propertySymOpnd))
  3603. {
  3604. // We only set guarded properties in the dead store pass, so they shouldn't be set here yet. If they were
  3605. // we would need to move them from this operand to the operand which is being copy propagated.
  3606. Assert(propertySymOpnd->GetGuardedPropOps() == nullptr);
  3607. // We're creating a copy of this operand to be reused in the same spot in the flow, so we can copy all
  3608. // flow sensitive fields. However, we will do only a type check here (no property access) and only for
  3609. // the sake of downstream instructions, so the flags pertaining to this property access are irrelevant.
  3610. IR::PropertySymOpnd* checkObjTypeOpnd = CreateOpndForTypeCheckOnly(propertySymOpnd, instr->m_func);
  3611. IR::Instr* checkObjTypeInstr = IR::Instr::New(Js::OpCode::CheckObjType, instr->m_func);
  3612. checkObjTypeInstr->SetSrc1(checkObjTypeOpnd);
  3613. checkObjTypeInstr->SetByteCodeOffset(instr);
  3614. instr->InsertBefore(checkObjTypeInstr);
  3615. // Since we inserted this instruction before the one that is being processed in natural flow, we must process
  3616. // it for object type spec explicitly here.
  3617. FinishOptPropOp(checkObjTypeInstr, checkObjTypeOpnd);
  3618. Assert(!propertySymOpnd->IsTypeChecked());
  3619. checkObjTypeInstr = this->SetTypeCheckBailOut(checkObjTypeOpnd, checkObjTypeInstr, nullptr);
  3620. Assert(checkObjTypeInstr->HasBailOutInfo());
  3621. if (this->currentBlock->loop && !this->IsLoopPrePass())
  3622. {
  3623. // Try hoisting this checkObjType.
  3624. // But since this isn't the current instr being optimized, we need to play tricks with
  3625. // the byteCodeUse fields...
  3626. TrackByteCodeUsesForInstrAddedInOptInstr(checkObjTypeInstr, [&]()
  3627. {
  3628. TryHoistInvariant(checkObjTypeInstr, this->currentBlock, NULL, CurrentBlockData()->FindValue(copySym), NULL, true);
  3629. });
  3630. }
  3631. }
  3632. }
  3633. if (opnd->IsSymOpnd() && opnd->GetIsDead())
  3634. {
  3635. // Take the property sym out of the live fields set
  3636. this->EndFieldLifetime(opnd->AsSymOpnd());
  3637. }
  3638. regOpnd = IR::RegOpnd::New(newSym, opnd->GetType(), instr->m_func);
  3639. regOpnd->SetIsJITOptimizedReg(true);
  3640. instr->ReplaceSrc(opnd, regOpnd);
  3641. }
  3642. switch (instr->m_opcode)
  3643. {
  3644. case Js::OpCode::Ld_A:
  3645. if (instr->GetDst()->IsRegOpnd() && instr->GetSrc1()->IsRegOpnd() &&
  3646. instr->GetDst()->AsRegOpnd()->GetStackSym() == instr->GetSrc1()->AsRegOpnd()->GetStackSym())
  3647. {
  3648. this->InsertByteCodeUses(instr, true);
  3649. instr->m_opcode = Js::OpCode::Nop;
  3650. }
  3651. break;
  3652. case Js::OpCode::LdSlot:
  3653. case Js::OpCode::LdSlotArr:
  3654. if (instr->GetDst()->IsRegOpnd() && instr->GetSrc1()->IsRegOpnd() &&
  3655. instr->GetDst()->AsRegOpnd()->GetStackSym() == instr->GetSrc1()->AsRegOpnd()->GetStackSym())
  3656. {
  3657. this->InsertByteCodeUses(instr, true);
  3658. instr->m_opcode = Js::OpCode::Nop;
  3659. }
  3660. else
  3661. {
  3662. instr->m_opcode = Js::OpCode::Ld_A;
  3663. }
  3664. break;
  3665. case Js::OpCode::StSlotChkUndecl:
  3666. if (instr->GetSrc2()->IsRegOpnd())
  3667. {
  3668. // Src2 here should refer to the same location as the Dst operand, which we need to keep live
  3669. // due to the implicit read for ChkUndecl.
  3670. instr->m_opcode = Js::OpCode::StSlot;
  3671. instr->FreeSrc2();
  3672. return nullptr;
  3673. }
  3674. break;
  3675. case Js::OpCode::LdFld:
  3676. case Js::OpCode::LdFldForTypeOf:
  3677. case Js::OpCode::LdRootFldForTypeOf:
  3678. case Js::OpCode::LdFldForCallApplyTarget:
  3679. case Js::OpCode::LdRootFld:
  3680. case Js::OpCode::LdMethodFld:
  3681. case Js::OpCode::LdRootMethodFld:
  3682. case Js::OpCode::ScopedLdMethodFld:
  3683. case Js::OpCode::ScopedLdFld:
  3684. case Js::OpCode::ScopedLdFldForTypeOf:
  3685. instr->m_opcode = Js::OpCode::Ld_A;
  3686. break;
  3687. case Js::OpCode::LdMethodFromFlags:
  3688. // The bailout is checked on the loop top and we don't need to check bailout again in loop.
  3689. instr->m_opcode = Js::OpCode::Ld_A;
  3690. instr->ClearBailOutInfo();
  3691. break;
  3692. case Js::OpCode::TypeofElem:
  3693. instr->m_opcode = Js::OpCode::Typeof;
  3694. break;
  3695. }
  3696. CurrentBlockData()->MarkTempLastUse(instr, regOpnd);
  3697. return regOpnd;
  3698. }
  3699. ValueNumber
  3700. GlobOpt::NewValueNumber()
  3701. {
  3702. ValueNumber valueNumber = this->currentValue++;
  3703. if (valueNumber == 0)
  3704. {
  3705. Js::Throw::OutOfMemory();
  3706. }
  3707. return valueNumber;
  3708. }
  3709. Value *GlobOpt::NewValue(ValueInfo *const valueInfo)
  3710. {
  3711. return NewValue(NewValueNumber(), valueInfo);
  3712. }
  3713. Value *GlobOpt::NewValue(const ValueNumber valueNumber, ValueInfo *const valueInfo)
  3714. {
  3715. Assert(valueInfo);
  3716. return Value::New(alloc, valueNumber, valueInfo);
  3717. }
  3718. Value *GlobOpt::CopyValue(Value const *const value)
  3719. {
  3720. return CopyValue(value, NewValueNumber());
  3721. }
  3722. Value *GlobOpt::CopyValue(Value const *const value, const ValueNumber valueNumber)
  3723. {
  3724. Assert(value);
  3725. return value->Copy(alloc, valueNumber);
  3726. }
  3727. Value *
  3728. GlobOpt::NewGenericValue(const ValueType valueType)
  3729. {
  3730. return NewGenericValue(valueType, static_cast<IR::Opnd *>(nullptr));
  3731. }
  3732. Value *
  3733. GlobOpt::NewGenericValue(const ValueType valueType, IR::Opnd *const opnd)
  3734. {
  3735. // Shouldn't assign a likely-int value to something that is definitely not an int
  3736. Assert(!(valueType.IsLikelyInt() && opnd && opnd->IsNotInt()));
  3737. ValueInfo *valueInfo = ValueInfo::New(this->alloc, valueType);
  3738. Value *val = NewValue(valueInfo);
  3739. TrackNewValueForKills(val);
  3740. CurrentBlockData()->InsertNewValue(val, opnd);
  3741. return val;
  3742. }
  3743. Value *
  3744. GlobOpt::NewGenericValue(const ValueType valueType, Sym *const sym)
  3745. {
  3746. ValueInfo *valueInfo = ValueInfo::New(this->alloc, valueType);
  3747. Value *val = NewValue(valueInfo);
  3748. TrackNewValueForKills(val);
  3749. CurrentBlockData()->SetValue(val, sym);
  3750. return val;
  3751. }
  3752. Value *
  3753. GlobOpt::GetIntConstantValue(const int32 intConst, IR::Instr * instr, IR::Opnd *const opnd)
  3754. {
  3755. Value *value = nullptr;
  3756. Value *const cachedValue = this->intConstantToValueMap->Lookup(intConst, nullptr);
  3757. if(cachedValue)
  3758. {
  3759. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  3760. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  3761. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  3762. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  3763. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  3764. // Otherwise, we have to create a new value with a new value number.
  3765. Sym *const symStore = cachedValue->GetValueInfo()->GetSymStore();
  3766. if (symStore && CurrentBlockData()->IsLive(symStore))
  3767. {
  3768. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  3769. int32 symStoreIntConstantValue;
  3770. if (symStoreValue &&
  3771. symStoreValue->GetValueNumber() == cachedValue->GetValueNumber() &&
  3772. symStoreValue->GetValueInfo()->TryGetIntConstantValue(&symStoreIntConstantValue) &&
  3773. symStoreIntConstantValue == intConst)
  3774. {
  3775. value = symStoreValue;
  3776. }
  3777. }
  3778. }
  3779. if (!value)
  3780. {
  3781. value = NewIntConstantValue(intConst, instr, !Js::TaggedInt::IsOverflow(intConst));
  3782. }
  3783. return CurrentBlockData()->InsertNewValue(value, opnd);
  3784. }
  3785. Value *
  3786. GlobOpt::GetIntConstantValue(const int64 intConst, IR::Instr * instr, IR::Opnd *const opnd)
  3787. {
  3788. Assert(instr->m_func->GetJITFunctionBody()->IsWasmFunction());
  3789. Value *value = nullptr;
  3790. Value *const cachedValue = this->int64ConstantToValueMap->Lookup(intConst, nullptr);
  3791. if (cachedValue)
  3792. {
  3793. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  3794. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  3795. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  3796. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  3797. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  3798. // Otherwise, we have to create a new value with a new value number.
  3799. Sym *const symStore = cachedValue->GetValueInfo()->GetSymStore();
  3800. if (symStore && this->currentBlock->globOptData.IsLive(symStore))
  3801. {
  3802. Value *const symStoreValue = this->currentBlock->globOptData.FindValue(symStore);
  3803. int64 symStoreIntConstantValue;
  3804. if (symStoreValue &&
  3805. symStoreValue->GetValueNumber() == cachedValue->GetValueNumber() &&
  3806. symStoreValue->GetValueInfo()->TryGetInt64ConstantValue(&symStoreIntConstantValue, false) &&
  3807. symStoreIntConstantValue == intConst)
  3808. {
  3809. value = symStoreValue;
  3810. }
  3811. }
  3812. }
  3813. if (!value)
  3814. {
  3815. value = NewInt64ConstantValue(intConst, instr);
  3816. }
  3817. return this->currentBlock->globOptData.InsertNewValue(value, opnd);
  3818. }
  3819. Value *
  3820. GlobOpt::NewInt64ConstantValue(const int64 intConst, IR::Instr* instr)
  3821. {
  3822. Value * value = NewValue(Int64ConstantValueInfo::New(this->alloc, intConst));
  3823. this->int64ConstantToValueMap->Item(intConst, value);
  3824. if (!value->GetValueInfo()->GetSymStore() &&
  3825. (instr->m_opcode == Js::OpCode::LdC_A_I4 || instr->m_opcode == Js::OpCode::Ld_I4))
  3826. {
  3827. StackSym * sym = instr->GetDst()->GetStackSym();
  3828. Assert(sym && !sym->IsTypeSpec());
  3829. this->currentBlock->globOptData.SetValue(value, sym);
  3830. this->currentBlock->globOptData.liveVarSyms->Set(sym->m_id);
  3831. }
  3832. return value;
  3833. }
  3834. Value *
  3835. GlobOpt::NewIntConstantValue(const int32 intConst, IR::Instr * instr, bool isTaggable)
  3836. {
  3837. Value * value = NewValue(IntConstantValueInfo::New(this->alloc, intConst));
  3838. this->intConstantToValueMap->Item(intConst, value);
  3839. if (isTaggable &&
  3840. !PHASE_OFF(Js::HoistConstIntPhase, this->func))
  3841. {
  3842. // When creating a new int constant value, make sure it gets a symstore. If the int const doesn't have a symstore,
  3843. // any downstream instruction using the same int will have to create a new value (object) for the int.
  3844. // This gets in the way of CSE.
  3845. value = HoistConstantLoadAndPropagateValueBackward(Js::TaggedInt::ToVarUnchecked(intConst), instr, value);
  3846. if (!value->GetValueInfo()->GetSymStore() &&
  3847. (instr->m_opcode == Js::OpCode::LdC_A_I4 || instr->m_opcode == Js::OpCode::Ld_I4))
  3848. {
  3849. StackSym * sym = instr->GetDst()->GetStackSym();
  3850. Assert(sym);
  3851. if (sym->IsTypeSpec())
  3852. {
  3853. Assert(sym->IsInt32());
  3854. StackSym * varSym = sym->GetVarEquivSym(instr->m_func);
  3855. CurrentBlockData()->SetValue(value, varSym);
  3856. CurrentBlockData()->liveInt32Syms->Set(varSym->m_id);
  3857. }
  3858. else
  3859. {
  3860. CurrentBlockData()->SetValue(value, sym);
  3861. CurrentBlockData()->liveVarSyms->Set(sym->m_id);
  3862. }
  3863. }
  3864. }
  3865. return value;
  3866. }
  3867. ValueInfo *
  3868. GlobOpt::NewIntRangeValueInfo(const int32 min, const int32 max, const bool wasNegativeZeroPreventedByBailout)
  3869. {
  3870. return ValueInfo::NewIntRangeValueInfo(this->alloc, min, max, wasNegativeZeroPreventedByBailout);
  3871. }
  3872. ValueInfo *GlobOpt::NewIntRangeValueInfo(
  3873. const ValueInfo *const originalValueInfo,
  3874. const int32 min,
  3875. const int32 max) const
  3876. {
  3877. Assert(originalValueInfo);
  3878. ValueInfo *valueInfo;
  3879. if(min == max)
  3880. {
  3881. // Since int constant values are const-propped, negative zero tracking does not track them, and so it's okay to ignore
  3882. // 'wasNegativeZeroPreventedByBailout'
  3883. valueInfo = IntConstantValueInfo::New(alloc, min);
  3884. }
  3885. else
  3886. {
  3887. valueInfo =
  3888. IntRangeValueInfo::New(
  3889. alloc,
  3890. min,
  3891. max,
  3892. min <= 0 && max >= 0 && originalValueInfo->WasNegativeZeroPreventedByBailout());
  3893. }
  3894. valueInfo->SetSymStore(originalValueInfo->GetSymStore());
  3895. return valueInfo;
  3896. }
  3897. Value *
  3898. GlobOpt::NewIntRangeValue(
  3899. const int32 min,
  3900. const int32 max,
  3901. const bool wasNegativeZeroPreventedByBailout,
  3902. IR::Opnd *const opnd)
  3903. {
  3904. ValueInfo *valueInfo = this->NewIntRangeValueInfo(min, max, wasNegativeZeroPreventedByBailout);
  3905. Value *val = NewValue(valueInfo);
  3906. if (opnd)
  3907. {
  3908. GOPT_TRACE_OPND(opnd, _u("Range %d (0x%X) to %d (0x%X)\n"), min, min, max, max);
  3909. }
  3910. CurrentBlockData()->InsertNewValue(val, opnd);
  3911. return val;
  3912. }
  3913. IntBoundedValueInfo *GlobOpt::NewIntBoundedValueInfo(
  3914. const ValueInfo *const originalValueInfo,
  3915. const IntBounds *const bounds) const
  3916. {
  3917. Assert(originalValueInfo);
  3918. bounds->Verify();
  3919. IntBoundedValueInfo *const valueInfo =
  3920. IntBoundedValueInfo::New(
  3921. originalValueInfo->Type(),
  3922. bounds,
  3923. (
  3924. bounds->ConstantLowerBound() <= 0 &&
  3925. bounds->ConstantUpperBound() >= 0 &&
  3926. originalValueInfo->WasNegativeZeroPreventedByBailout()
  3927. ),
  3928. alloc);
  3929. valueInfo->SetSymStore(originalValueInfo->GetSymStore());
  3930. return valueInfo;
  3931. }
  3932. Value *GlobOpt::NewIntBoundedValue(
  3933. const ValueType valueType,
  3934. const IntBounds *const bounds,
  3935. const bool wasNegativeZeroPreventedByBailout,
  3936. IR::Opnd *const opnd)
  3937. {
  3938. Value *const value = NewValue(IntBoundedValueInfo::New(valueType, bounds, wasNegativeZeroPreventedByBailout, alloc));
  3939. CurrentBlockData()->InsertNewValue(value, opnd);
  3940. return value;
  3941. }
  3942. Value *
  3943. GlobOpt::NewFloatConstantValue(const FloatConstType floatValue, IR::Opnd *const opnd)
  3944. {
  3945. FloatConstantValueInfo *valueInfo = FloatConstantValueInfo::New(this->alloc, floatValue);
  3946. Value *val = NewValue(valueInfo);
  3947. CurrentBlockData()->InsertNewValue(val, opnd);
  3948. return val;
  3949. }
  3950. Value *
  3951. GlobOpt::GetVarConstantValue(IR::AddrOpnd *addrOpnd)
  3952. {
  3953. bool isVar = addrOpnd->IsVar();
  3954. bool isString = isVar && addrOpnd->m_localAddress && JITJavascriptString::Is(addrOpnd->m_localAddress);
  3955. Value *val = nullptr;
  3956. Value *cachedValue = nullptr;
  3957. if(this->addrConstantToValueMap->TryGetValue(addrOpnd->m_address, &cachedValue))
  3958. {
  3959. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  3960. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  3961. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  3962. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  3963. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  3964. // Otherwise, we have to create a new value with a new value number.
  3965. Sym *symStore = cachedValue->GetValueInfo()->GetSymStore();
  3966. if(symStore && CurrentBlockData()->IsLive(symStore))
  3967. {
  3968. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  3969. if(symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber())
  3970. {
  3971. ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo();
  3972. if(symStoreValueInfo->IsVarConstant() && symStoreValueInfo->AsVarConstant()->VarValue() == addrOpnd->m_address)
  3973. {
  3974. val = symStoreValue;
  3975. }
  3976. }
  3977. }
  3978. }
  3979. else if (isString)
  3980. {
  3981. JITJavascriptString* jsString = JITJavascriptString::FromVar(addrOpnd->m_localAddress);
  3982. Js::InternalString internalString(jsString->GetString(), jsString->GetLength());
  3983. if (this->stringConstantToValueMap->TryGetValue(internalString, &cachedValue))
  3984. {
  3985. Sym *symStore = cachedValue->GetValueInfo()->GetSymStore();
  3986. if (symStore && CurrentBlockData()->IsLive(symStore))
  3987. {
  3988. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  3989. if (symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber())
  3990. {
  3991. ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo();
  3992. if (symStoreValueInfo->IsVarConstant())
  3993. {
  3994. JITJavascriptString * cachedString = JITJavascriptString::FromVar(symStoreValue->GetValueInfo()->AsVarConstant()->VarValue(true));
  3995. Js::InternalString cachedInternalString(cachedString->GetString(), cachedString->GetLength());
  3996. if (Js::InternalStringComparer::Equals(internalString, cachedInternalString))
  3997. {
  3998. val = symStoreValue;
  3999. }
  4000. }
  4001. }
  4002. }
  4003. }
  4004. }
  4005. if(!val)
  4006. {
  4007. val = NewVarConstantValue(addrOpnd, isString);
  4008. }
  4009. addrOpnd->SetValueType(val->GetValueInfo()->Type());
  4010. return val;
  4011. }
  4012. Value *
  4013. GlobOpt::NewVarConstantValue(IR::AddrOpnd *addrOpnd, bool isString)
  4014. {
  4015. VarConstantValueInfo *valueInfo = VarConstantValueInfo::New(this->alloc, addrOpnd->m_address, addrOpnd->GetValueType(), false, addrOpnd->m_localAddress);
  4016. Value * value = NewValue(valueInfo);
  4017. this->addrConstantToValueMap->Item(addrOpnd->m_address, value);
  4018. if (isString)
  4019. {
  4020. JITJavascriptString* jsString = JITJavascriptString::FromVar(addrOpnd->m_localAddress);
  4021. Js::InternalString internalString(jsString->GetString(), jsString->GetLength());
  4022. this->stringConstantToValueMap->Item(internalString, value);
  4023. }
  4024. return value;
  4025. }
  4026. Value *
  4027. GlobOpt::HoistConstantLoadAndPropagateValueBackward(Js::Var varConst, IR::Instr * origInstr, Value * value)
  4028. {
  4029. if (this->IsLoopPrePass() ||
  4030. ((this->currentBlock == this->func->m_fg->blockList) &&
  4031. origInstr->TransfersSrcValue()))
  4032. {
  4033. return value;
  4034. }
  4035. // Only hoisting taggable int const loads for now. Could be extended to other constants (floats, strings, addr opnds) if we see some benefit.
  4036. Assert(Js::TaggedInt::Is(varConst));
  4037. // Insert a load of the constant at the top of the function
  4038. StackSym * dstSym = StackSym::New(this->func);
  4039. IR::RegOpnd * constRegOpnd = IR::RegOpnd::New(dstSym, TyVar, this->func);
  4040. IR::Instr * loadInstr = IR::Instr::NewConstantLoad(constRegOpnd, (intptr_t)varConst, ValueType::GetInt(true), this->func);
  4041. this->func->m_fg->blockList->GetFirstInstr()->InsertAfter(loadInstr);
  4042. // Type-spec the load (Support for floats needs to be added when we start hoisting float constants).
  4043. bool typeSpecedToInt = false;
  4044. if (Js::TaggedInt::Is(varConst) && !IsTypeSpecPhaseOff(this->func))
  4045. {
  4046. typeSpecedToInt = true;
  4047. loadInstr->m_opcode = Js::OpCode::Ld_I4;
  4048. ToInt32Dst(loadInstr, loadInstr->GetDst()->AsRegOpnd(), this->currentBlock);
  4049. loadInstr->GetDst()->GetStackSym()->SetIsConst();
  4050. }
  4051. else
  4052. {
  4053. CurrentBlockData()->liveVarSyms->Set(dstSym->m_id);
  4054. }
  4055. // Add the value (object) to the current block's symToValueMap and propagate the value backward to all relevant blocks so it is available on merges.
  4056. value = CurrentBlockData()->InsertNewValue(value, constRegOpnd);
  4057. BVSparse<JitArenaAllocator>* GlobOptBlockData::*bv;
  4058. bv = typeSpecedToInt ? &GlobOptBlockData::liveInt32Syms : &GlobOptBlockData::liveVarSyms; // Will need to be expanded when we start hoisting float constants.
  4059. if (this->currentBlock != this->func->m_fg->blockList)
  4060. {
  4061. for (InvariantBlockBackwardIterator it(this, this->currentBlock, this->func->m_fg->blockList, nullptr);
  4062. it.IsValid();
  4063. it.MoveNext())
  4064. {
  4065. BasicBlock * block = it.Block();
  4066. (block->globOptData.*bv)->Set(dstSym->m_id);
  4067. if (!block->globOptData.FindValue(dstSym))
  4068. {
  4069. Value *const valueCopy = CopyValue(value, value->GetValueNumber());
  4070. block->globOptData.SetValue(valueCopy, dstSym);
  4071. }
  4072. }
  4073. }
  4074. return value;
  4075. }
  4076. Value *
  4077. GlobOpt::NewFixedFunctionValue(Js::JavascriptFunction *function, IR::AddrOpnd *addrOpnd)
  4078. {
  4079. Assert(function != nullptr);
  4080. Value *val = nullptr;
  4081. Value *cachedValue = nullptr;
  4082. if(this->addrConstantToValueMap->TryGetValue(addrOpnd->m_address, &cachedValue))
  4083. {
  4084. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  4085. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  4086. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  4087. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  4088. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  4089. // Otherwise, we have to create a new value with a new value number.
  4090. Sym *symStore = cachedValue->GetValueInfo()->GetSymStore();
  4091. if(symStore && CurrentBlockData()->IsLive(symStore))
  4092. {
  4093. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  4094. if(symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber())
  4095. {
  4096. ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo();
  4097. if(symStoreValueInfo->IsVarConstant())
  4098. {
  4099. VarConstantValueInfo *const symStoreVarConstantValueInfo = symStoreValueInfo->AsVarConstant();
  4100. if(symStoreVarConstantValueInfo->VarValue() == addrOpnd->m_address &&
  4101. symStoreVarConstantValueInfo->IsFunction())
  4102. {
  4103. val = symStoreValue;
  4104. }
  4105. }
  4106. }
  4107. }
  4108. }
  4109. if(!val)
  4110. {
  4111. VarConstantValueInfo *valueInfo = VarConstantValueInfo::New(this->alloc, function, addrOpnd->GetValueType(), true, addrOpnd->m_localAddress);
  4112. val = NewValue(valueInfo);
  4113. this->addrConstantToValueMap->AddNew(addrOpnd->m_address, val);
  4114. }
  4115. CurrentBlockData()->InsertNewValue(val, addrOpnd);
  4116. return val;
  4117. }
  4118. StackSym *GlobOpt::GetTaggedIntConstantStackSym(const int32 intConstantValue) const
  4119. {
  4120. Assert(!Js::TaggedInt::IsOverflow(intConstantValue));
  4121. return intConstantToStackSymMap->Lookup(intConstantValue, nullptr);
  4122. }
  4123. StackSym *GlobOpt::GetOrCreateTaggedIntConstantStackSym(const int32 intConstantValue) const
  4124. {
  4125. StackSym *stackSym = GetTaggedIntConstantStackSym(intConstantValue);
  4126. if(stackSym)
  4127. {
  4128. return stackSym;
  4129. }
  4130. stackSym = StackSym::New(TyVar,func);
  4131. intConstantToStackSymMap->Add(intConstantValue, stackSym);
  4132. return stackSym;
  4133. }
  4134. Sym *
  4135. GlobOpt::SetSymStore(ValueInfo *valueInfo, Sym *sym)
  4136. {
  4137. if (sym->IsStackSym())
  4138. {
  4139. StackSym *stackSym = sym->AsStackSym();
  4140. if (stackSym->IsTypeSpec())
  4141. {
  4142. stackSym = stackSym->GetVarEquivSym(this->func);
  4143. sym = stackSym;
  4144. }
  4145. }
  4146. if (valueInfo->GetSymStore() == nullptr || valueInfo->GetSymStore()->IsPropertySym())
  4147. {
  4148. SetSymStoreDirect(valueInfo, sym);
  4149. }
  4150. return sym;
  4151. }
  4152. void
  4153. GlobOpt::SetSymStoreDirect(ValueInfo * valueInfo, Sym * sym)
  4154. {
  4155. Sym * prevSymStore = valueInfo->GetSymStore();
  4156. CurrentBlockData()->SetChangedSym(prevSymStore);
  4157. valueInfo->SetSymStore(sym);
  4158. }
  4159. // Figure out the Value of this dst.
  4160. Value *
  4161. GlobOpt::ValueNumberDst(IR::Instr **pInstr, Value *src1Val, Value *src2Val)
  4162. {
  4163. IR::Instr *&instr = *pInstr;
  4164. IR::Opnd *dst = instr->GetDst();
  4165. Value *dstVal = nullptr;
  4166. Sym *sym;
  4167. if (instr->CallsSetter())
  4168. {
  4169. return nullptr;
  4170. }
  4171. if (dst == nullptr)
  4172. {
  4173. return nullptr;
  4174. }
  4175. switch (dst->GetKind())
  4176. {
  4177. case IR::OpndKindSym:
  4178. sym = dst->AsSymOpnd()->m_sym;
  4179. break;
  4180. case IR::OpndKindReg:
  4181. sym = dst->AsRegOpnd()->m_sym;
  4182. if (OpCodeAttr::TempNumberProducing(instr->m_opcode))
  4183. {
  4184. CurrentBlockData()->isTempSrc->Set(sym->m_id);
  4185. }
  4186. else if (OpCodeAttr::TempNumberTransfer(instr->m_opcode))
  4187. {
  4188. IR::Opnd *src1 = instr->GetSrc1();
  4189. if (src1->IsRegOpnd() && CurrentBlockData()->isTempSrc->Test(src1->AsRegOpnd()->m_sym->m_id))
  4190. {
  4191. StackSym *src1Sym = src1->AsRegOpnd()->m_sym;
  4192. // isTempSrc is used for marking isTempLastUse, which is used to generate AddLeftDead()
  4193. // calls instead of the normal Add helpers. It tells the runtime that concats can use string
  4194. // builders.
  4195. // We need to be careful in the case where src1 points to a string builder and is getting aliased.
  4196. // Clear the bit on src and dst of the transfer instr in this case, unless we can prove src1
  4197. // isn't pointing at a string builder, like if it is single def and the def instr is not an Add,
  4198. // but TempProducing.
  4199. if (src1Sym->IsSingleDef() && src1Sym->m_instrDef->m_opcode != Js::OpCode::Add_A
  4200. && OpCodeAttr::TempNumberProducing(src1Sym->m_instrDef->m_opcode))
  4201. {
  4202. CurrentBlockData()->isTempSrc->Set(sym->m_id);
  4203. }
  4204. else
  4205. {
  4206. CurrentBlockData()->isTempSrc->Clear(src1->AsRegOpnd()->m_sym->m_id);
  4207. CurrentBlockData()->isTempSrc->Clear(sym->m_id);
  4208. }
  4209. }
  4210. else
  4211. {
  4212. CurrentBlockData()->isTempSrc->Clear(sym->m_id);
  4213. }
  4214. }
  4215. else
  4216. {
  4217. CurrentBlockData()->isTempSrc->Clear(sym->m_id);
  4218. }
  4219. break;
  4220. case IR::OpndKindIndir:
  4221. return nullptr;
  4222. default:
  4223. return nullptr;
  4224. }
  4225. int32 min1, max1, min2, max2, newMin, newMax;
  4226. ValueInfo *src1ValueInfo = (src1Val ? src1Val->GetValueInfo() : nullptr);
  4227. ValueInfo *src2ValueInfo = (src2Val ? src2Val->GetValueInfo() : nullptr);
  4228. switch (instr->m_opcode)
  4229. {
  4230. case Js::OpCode::Conv_PrimStr:
  4231. AssertMsg(instr->GetDst()->GetValueType().IsString(),
  4232. "Creator of this instruction should have set the type");
  4233. if (this->IsLoopPrePass() || src1ValueInfo == nullptr || !src1ValueInfo->IsPrimitive())
  4234. {
  4235. break;
  4236. }
  4237. instr->m_opcode = Js::OpCode::Conv_Str;
  4238. // fall-through
  4239. case Js::OpCode::Conv_Str:
  4240. // This opcode is commented out since we don't track regex information in GlobOpt now.
  4241. //case Js::OpCode::Coerce_Regex:
  4242. case Js::OpCode::Coerce_Str:
  4243. AssertMsg(instr->GetDst()->GetValueType().IsString(),
  4244. "Creator of this instruction should have set the type");
  4245. // Due to fall through and the fact that Ld_A only takes one source,
  4246. // free the other source here.
  4247. if (instr->GetSrc2() && !(this->IsLoopPrePass() || src1ValueInfo == nullptr || !src1ValueInfo->IsString()))
  4248. {
  4249. instr->FreeSrc2();
  4250. }
  4251. // fall-through
  4252. case Js::OpCode::Coerce_StrOrRegex:
  4253. // We don't set the ValueType of src1 for Coerce_StrOrRegex, hence skip the ASSERT
  4254. if (this->IsLoopPrePass() || src1ValueInfo == nullptr || !src1ValueInfo->IsString())
  4255. {
  4256. break;
  4257. }
  4258. instr->m_opcode = Js::OpCode::Ld_A;
  4259. // fall-through
  4260. case Js::OpCode::BytecodeArgOutCapture:
  4261. case Js::OpCode::LdAsmJsFunc:
  4262. case Js::OpCode::Ld_A:
  4263. case Js::OpCode::Ld_I4:
  4264. // Propagate sym attributes across the reg copy.
  4265. if (!this->IsLoopPrePass() && instr->GetSrc1()->IsRegOpnd())
  4266. {
  4267. if (dst->AsRegOpnd()->m_sym->IsSingleDef())
  4268. {
  4269. dst->AsRegOpnd()->m_sym->CopySymAttrs(instr->GetSrc1()->AsRegOpnd()->m_sym);
  4270. }
  4271. }
  4272. if (instr->IsProfiledInstr())
  4273. {
  4274. const ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType);
  4275. if(!(
  4276. profiledValueType.IsLikelyInt() &&
  4277. (
  4278. (dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber) ||
  4279. (instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber)
  4280. )
  4281. ))
  4282. {
  4283. if(!src1ValueInfo)
  4284. {
  4285. dstVal = this->NewGenericValue(profiledValueType, dst);
  4286. }
  4287. else if(src1ValueInfo->IsUninitialized())
  4288. {
  4289. if(IsLoopPrePass())
  4290. {
  4291. dstVal = this->NewGenericValue(profiledValueType, dst);
  4292. }
  4293. else
  4294. {
  4295. // Assuming the profile data gives more precise value types based on the path it took at runtime, we
  4296. // can improve the original value type.
  4297. src1ValueInfo->Type() = profiledValueType;
  4298. instr->GetSrc1()->SetValueType(profiledValueType);
  4299. }
  4300. }
  4301. }
  4302. }
  4303. if (dstVal == nullptr)
  4304. {
  4305. // Ld_A is just transferring the value
  4306. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4307. }
  4308. break;
  4309. case Js::OpCode::ExtendArg_A:
  4310. {
  4311. // SIMD_JS
  4312. // We avoid transforming EAs to Lds to keep the IR shape consistent and avoid CSEing of EAs.
  4313. // CSEOptimize only assigns a Value to the EA dst, and doesn't turn it to a Ld. If this happened, we shouldn't assign a new Value here.
  4314. if (DoCSE())
  4315. {
  4316. IR::Opnd * currDst = instr->GetDst();
  4317. Value * currDstVal = CurrentBlockData()->FindValue(currDst->GetStackSym());
  4318. if (currDstVal != nullptr)
  4319. {
  4320. return currDstVal;
  4321. }
  4322. }
  4323. break;
  4324. }
  4325. case Js::OpCode::CheckFixedFld:
  4326. AssertMsg(false, "CheckFixedFld doesn't have a dst, so we should never get here");
  4327. break;
  4328. case Js::OpCode::LdSlot:
  4329. case Js::OpCode::LdSlotArr:
  4330. case Js::OpCode::LdFld:
  4331. case Js::OpCode::LdFldForTypeOf:
  4332. case Js::OpCode::LdFldForCallApplyTarget:
  4333. // Do not transfer value type on LdRootFldForTypeOf to prevent copy-prop to LdRootFld in case the field doesn't exist since LdRootFldForTypeOf does not throw.
  4334. // Same goes for ScopedLdFldForTypeOf as we'll end up loading the property from the root object if the property is not in the scope chain.
  4335. //case Js::OpCode::LdRootFldForTypeOf:
  4336. //case Js::OpCode::ScopedLdFldForTypeOf:
  4337. case Js::OpCode::LdRootFld:
  4338. case Js::OpCode::LdMethodFld:
  4339. case Js::OpCode::LdRootMethodFld:
  4340. case Js::OpCode::ScopedLdMethodFld:
  4341. case Js::OpCode::LdMethodFromFlags:
  4342. case Js::OpCode::ScopedLdFld:
  4343. if (instr->IsProfiledInstr())
  4344. {
  4345. ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType);
  4346. if(!(profiledValueType.IsLikelyInt() && dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber))
  4347. {
  4348. if(!src1ValueInfo)
  4349. {
  4350. dstVal = this->NewGenericValue(profiledValueType, dst);
  4351. }
  4352. else if(src1ValueInfo->IsUninitialized())
  4353. {
  4354. if(IsLoopPrePass() && (!dst->IsRegOpnd() || !dst->AsRegOpnd()->m_sym->IsSingleDef()))
  4355. {
  4356. dstVal = this->NewGenericValue(profiledValueType, dst);
  4357. }
  4358. else
  4359. {
  4360. // Assuming the profile data gives more precise value types based on the path it took at runtime, we
  4361. // can improve the original value type.
  4362. src1ValueInfo->Type() = profiledValueType;
  4363. instr->GetSrc1()->SetValueType(profiledValueType);
  4364. }
  4365. }
  4366. }
  4367. }
  4368. if (dstVal == nullptr)
  4369. {
  4370. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4371. }
  4372. if(!this->IsLoopPrePass())
  4373. {
  4374. // We cannot transfer value if the field hasn't been copy prop'd because we don't generate
  4375. // an implicit call bailout between those values if we don't have "live fields" unless, we are hoisting the field.
  4376. ValueInfo *dstValueInfo = (dstVal ? dstVal->GetValueInfo() : nullptr);
  4377. // Update symStore if it isn't a stackSym
  4378. if (dstVal && (!dstValueInfo->GetSymStore() || !dstValueInfo->GetSymStore()->IsStackSym()))
  4379. {
  4380. Assert(dst->IsRegOpnd());
  4381. this->SetSymStoreDirect(dstValueInfo, dst->AsRegOpnd()->m_sym);
  4382. }
  4383. if (src1Val != dstVal)
  4384. {
  4385. CurrentBlockData()->SetValue(dstVal, instr->GetSrc1());
  4386. }
  4387. }
  4388. break;
  4389. case Js::OpCode::LdC_A_R8:
  4390. case Js::OpCode::LdC_A_I4:
  4391. case Js::OpCode::ArgIn_A:
  4392. dstVal = src1Val;
  4393. break;
  4394. case Js::OpCode::LdStr:
  4395. if (src1Val == nullptr)
  4396. {
  4397. src1Val = NewGenericValue(ValueType::String, dst);
  4398. }
  4399. dstVal = src1Val;
  4400. break;
  4401. // LdElemUndef only assign undef if the field doesn't exist.
  4402. // So we don't actually know what the value is, so we can't really copy prop it.
  4403. //case Js::OpCode::LdElemUndef:
  4404. case Js::OpCode::StSlot:
  4405. case Js::OpCode::StSlotChkUndecl:
  4406. case Js::OpCode::StFld:
  4407. case Js::OpCode::StRootFld:
  4408. case Js::OpCode::StFldStrict:
  4409. case Js::OpCode::StRootFldStrict:
  4410. case Js::OpCode::InitFld:
  4411. case Js::OpCode::InitComputedProperty:
  4412. if (DoFieldCopyProp())
  4413. {
  4414. if (src1Val == nullptr)
  4415. {
  4416. // src1 may have no value if it's not a valid var, e.g., NULL for let/const initialization.
  4417. // Consider creating generic values for such things.
  4418. return nullptr;
  4419. }
  4420. AssertMsg(!src2Val, "Bad src Values...");
  4421. Assert(sym->IsPropertySym());
  4422. SymID symId = sym->m_id;
  4423. Assert(instr->m_opcode == Js::OpCode::StSlot || instr->m_opcode == Js::OpCode::StSlotChkUndecl || !CurrentBlockData()->liveFields->Test(symId));
  4424. CurrentBlockData()->liveFields->Set(symId);
  4425. if (!this->IsLoopPrePass() && dst->GetIsDead())
  4426. {
  4427. // Take the property sym out of the live fields set (with special handling for loops).
  4428. this->EndFieldLifetime(dst->AsSymOpnd());
  4429. }
  4430. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4431. }
  4432. else
  4433. {
  4434. return nullptr;
  4435. }
  4436. break;
  4437. case Js::OpCode::Conv_Num:
  4438. if(src1ValueInfo->IsNumber())
  4439. {
  4440. dstVal = ValueNumberTransferDst(instr, src1Val);
  4441. }
  4442. else
  4443. {
  4444. return NewGenericValue(src1ValueInfo->Type().ToDefiniteAnyNumber().SetCanBeTaggedValue(true), dst);
  4445. }
  4446. break;
  4447. case Js::OpCode::Not_A:
  4448. {
  4449. if (!src1Val || !src1ValueInfo->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec()))
  4450. {
  4451. min1 = INT32_MIN;
  4452. max1 = INT32_MAX;
  4453. }
  4454. this->PropagateIntRangeForNot(min1, max1, &newMin, &newMax);
  4455. return CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val);
  4456. }
  4457. case Js::OpCode::Xor_A:
  4458. case Js::OpCode::Or_A:
  4459. case Js::OpCode::And_A:
  4460. case Js::OpCode::Shl_A:
  4461. case Js::OpCode::Shr_A:
  4462. case Js::OpCode::ShrU_A:
  4463. {
  4464. if (!src1Val || !src1ValueInfo->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec()))
  4465. {
  4466. min1 = INT32_MIN;
  4467. max1 = INT32_MAX;
  4468. }
  4469. if (!src2Val || !src2ValueInfo->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec()))
  4470. {
  4471. min2 = INT32_MIN;
  4472. max2 = INT32_MAX;
  4473. }
  4474. if (instr->m_opcode == Js::OpCode::ShrU_A &&
  4475. min1 < 0 &&
  4476. IntConstantBounds(min2, max2).And_0x1f().Contains(0))
  4477. {
  4478. // Src1 may be too large to represent as a signed int32, and src2 may be zero.
  4479. // Since the result can therefore be too large to represent as a signed int32,
  4480. // include Number in the value type.
  4481. return CreateDstUntransferredValue(
  4482. ValueType::AnyNumber.SetCanBeTaggedValue(true), instr, src1Val, src2Val);
  4483. }
  4484. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  4485. return CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val);
  4486. }
  4487. case Js::OpCode::Incr_A:
  4488. case Js::OpCode::Decr_A:
  4489. {
  4490. ValueType valueType;
  4491. if(src1Val)
  4492. {
  4493. valueType = src1Val->GetValueInfo()->Type().ToDefiniteAnyNumber();
  4494. }
  4495. else
  4496. {
  4497. valueType = ValueType::Number;
  4498. }
  4499. return CreateDstUntransferredValue(valueType.SetCanBeTaggedValue(true), instr, src1Val, src2Val);
  4500. }
  4501. case Js::OpCode::Add_A:
  4502. {
  4503. ValueType valueType;
  4504. if (src1Val && src1ValueInfo->IsLikelyNumber() && src2Val && src2ValueInfo->IsLikelyNumber())
  4505. {
  4506. if(src1ValueInfo->IsLikelyInt() && src2ValueInfo->IsLikelyInt())
  4507. {
  4508. // When doing aggressiveIntType, just assume the result is likely going to be int
  4509. // if both input is int.
  4510. const bool isLikelyTagged = src1ValueInfo->IsLikelyTaggedInt() && src2ValueInfo->IsLikelyTaggedInt();
  4511. if(src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber())
  4512. {
  4513. // If both of them are numbers then we can definitely say that the result is a number.
  4514. valueType = ValueType::GetNumberAndLikelyInt(isLikelyTagged);
  4515. }
  4516. else
  4517. {
  4518. // This is only likely going to be int but can be a string as well.
  4519. valueType = ValueType::GetInt(isLikelyTagged).ToLikely();
  4520. }
  4521. }
  4522. else
  4523. {
  4524. // We can only be certain of any thing if both of them are numbers.
  4525. // Otherwise, the result could be string.
  4526. if (src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber())
  4527. {
  4528. if (src1ValueInfo->IsFloat() || src2ValueInfo->IsFloat())
  4529. {
  4530. // If one of them is a float, the result probably is a float instead of just int
  4531. // but should always be a number.
  4532. valueType = ValueType::Float.SetCanBeTaggedValue(true);
  4533. }
  4534. else
  4535. {
  4536. // Could be int, could be number
  4537. valueType = ValueType::Number.SetCanBeTaggedValue(true);
  4538. }
  4539. }
  4540. else if (src1ValueInfo->IsLikelyFloat() || src2ValueInfo->IsLikelyFloat())
  4541. {
  4542. // Result is likely a float (but can be anything)
  4543. valueType = ValueType::Float.ToLikely();
  4544. }
  4545. else
  4546. {
  4547. // Otherwise it is a likely int or float (but can be anything)
  4548. valueType = ValueType::Number.ToLikely();
  4549. }
  4550. }
  4551. }
  4552. else if((src1Val && src1ValueInfo->IsString()) || (src2Val && src2ValueInfo->IsString()))
  4553. {
  4554. // String + anything should always result in a string
  4555. valueType = ValueType::String;
  4556. }
  4557. else if((src1Val && src1ValueInfo->IsNotString() && src1ValueInfo->IsPrimitive())
  4558. && (src2Val && src2ValueInfo->IsNotString() && src2ValueInfo->IsPrimitive()))
  4559. {
  4560. // If src1 and src2 are not strings and primitive, add should yield a number.
  4561. valueType = ValueType::Number.SetCanBeTaggedValue(true);
  4562. }
  4563. else if((src1Val && src1ValueInfo->IsLikelyString()) || (src2Val && src2ValueInfo->IsLikelyString()))
  4564. {
  4565. // likelystring + anything should always result in a likelystring
  4566. valueType = ValueType::String.ToLikely();
  4567. }
  4568. else
  4569. {
  4570. // Number or string. Could make the value a merge of Number and String, but Uninitialized is more useful at the moment.
  4571. Assert(valueType.IsUninitialized());
  4572. }
  4573. return CreateDstUntransferredValue(valueType, instr, src1Val, src2Val);
  4574. }
  4575. case Js::OpCode::Div_A:
  4576. {
  4577. ValueType divValueType = GetDivValueType(instr, src1Val, src2Val, false);
  4578. if (divValueType.IsLikelyInt() || divValueType.IsFloat())
  4579. {
  4580. return CreateDstUntransferredValue(divValueType.SetCanBeTaggedValue(true), instr, src1Val, src2Val);
  4581. }
  4582. }
  4583. // fall-through
  4584. case Js::OpCode::Sub_A:
  4585. case Js::OpCode::Mul_A:
  4586. case Js::OpCode::Rem_A:
  4587. {
  4588. ValueType valueType;
  4589. if( src1Val &&
  4590. src1ValueInfo->IsLikelyInt() &&
  4591. src2Val &&
  4592. src2ValueInfo->IsLikelyInt() &&
  4593. instr->m_opcode != Js::OpCode::Div_A)
  4594. {
  4595. const bool isLikelyTagged =
  4596. src1ValueInfo->IsLikelyTaggedInt() && (src2ValueInfo->IsLikelyTaggedInt() || instr->m_opcode == Js::OpCode::Rem_A);
  4597. if(src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber())
  4598. {
  4599. valueType = ValueType::GetNumberAndLikelyInt(isLikelyTagged);
  4600. }
  4601. else
  4602. {
  4603. valueType = ValueType::GetInt(isLikelyTagged).ToLikely();
  4604. }
  4605. }
  4606. else if ((src1Val && src1ValueInfo->IsLikelyFloat()) || (src2Val && src2ValueInfo->IsLikelyFloat()))
  4607. {
  4608. // This should ideally be NewNumberAndLikelyFloatValue since we know the result is a number but not sure if it will
  4609. // be a float value. However, that Number/LikelyFloat value type doesn't exist currently and all the necessary
  4610. // checks are done for float values (tagged int checks, etc.) so it's sufficient to just create a float value here.
  4611. valueType = ValueType::Float.SetCanBeTaggedValue(true);
  4612. }
  4613. else
  4614. {
  4615. valueType = ValueType::Number.SetCanBeTaggedValue(true);
  4616. }
  4617. return CreateDstUntransferredValue(valueType, instr, src1Val, src2Val);
  4618. }
  4619. case Js::OpCode::CallI:
  4620. Assert(dst->IsRegOpnd());
  4621. return NewGenericValue(dst->AsRegOpnd()->GetValueType(), dst);
  4622. case Js::OpCode::LdElemI_A:
  4623. {
  4624. dstVal = ValueNumberLdElemDst(pInstr, src1Val);
  4625. const ValueType baseValueType(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType());
  4626. if( (
  4627. baseValueType.IsLikelyNativeArray() ||
  4628. #ifdef _M_IX86
  4629. (
  4630. !AutoSystemInfo::Data.SSE2Available() &&
  4631. baseValueType.IsLikelyObject() &&
  4632. (
  4633. baseValueType.GetObjectType() == ObjectType::Float32Array ||
  4634. baseValueType.GetObjectType() == ObjectType::Float64Array
  4635. )
  4636. )
  4637. #else
  4638. false
  4639. #endif
  4640. ) &&
  4641. instr->GetDst()->IsVar() &&
  4642. instr->HasBailOutInfo())
  4643. {
  4644. // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast
  4645. // path. Note that the removed bailouts should not be necessary for correctness.
  4646. IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  4647. if(bailOutKind & IR::BailOutOnArrayAccessHelperCall)
  4648. {
  4649. bailOutKind -= IR::BailOutOnArrayAccessHelperCall;
  4650. }
  4651. if(bailOutKind == IR::BailOutOnImplicitCallsPreOp)
  4652. {
  4653. bailOutKind -= IR::BailOutOnImplicitCallsPreOp;
  4654. }
  4655. if(bailOutKind)
  4656. {
  4657. instr->SetBailOutKind(bailOutKind);
  4658. }
  4659. else
  4660. {
  4661. instr->ClearBailOutInfo();
  4662. }
  4663. }
  4664. return dstVal;
  4665. }
  4666. case Js::OpCode::LdMethodElem:
  4667. // Not worth profiling this, just assume it's likely object (should be likely function but ValueType does not track
  4668. // functions currently, so using ObjectType::Object instead)
  4669. dstVal = NewGenericValue(ValueType::GetObject(ObjectType::Object).ToLikely(), dst);
  4670. if(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyNativeArray() && instr->HasBailOutInfo())
  4671. {
  4672. // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast
  4673. // path. Note that the removed bailouts should not be necessary for correctness.
  4674. IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  4675. if(bailOutKind & IR::BailOutOnArrayAccessHelperCall)
  4676. {
  4677. bailOutKind -= IR::BailOutOnArrayAccessHelperCall;
  4678. }
  4679. if(bailOutKind == IR::BailOutOnImplicitCallsPreOp)
  4680. {
  4681. bailOutKind -= IR::BailOutOnImplicitCallsPreOp;
  4682. }
  4683. if(bailOutKind)
  4684. {
  4685. instr->SetBailOutKind(bailOutKind);
  4686. }
  4687. else
  4688. {
  4689. instr->ClearBailOutInfo();
  4690. }
  4691. }
  4692. return dstVal;
  4693. case Js::OpCode::StElemI_A:
  4694. case Js::OpCode::StElemI_A_Strict:
  4695. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4696. break;
  4697. case Js::OpCode::LdLen_A:
  4698. if (instr->IsProfiledInstr())
  4699. {
  4700. const ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType);
  4701. if(!(profiledValueType.IsLikelyInt() && dst->AsRegOpnd()->m_sym->m_isNotNumber))
  4702. {
  4703. return this->NewGenericValue(profiledValueType, dst);
  4704. }
  4705. }
  4706. break;
  4707. case Js::OpCode::BrOnEmpty:
  4708. case Js::OpCode::BrOnNotEmpty:
  4709. Assert(dst->IsRegOpnd());
  4710. Assert(dst->GetValueType().IsString());
  4711. return this->NewGenericValue(ValueType::String, dst);
  4712. case Js::OpCode::IsInst:
  4713. case Js::OpCode::LdTrue:
  4714. case Js::OpCode::LdFalse:
  4715. case Js::OpCode::CmEq_A:
  4716. case Js::OpCode::CmSrEq_A:
  4717. case Js::OpCode::CmNeq_A:
  4718. case Js::OpCode::CmSrNeq_A:
  4719. case Js::OpCode::CmLe_A:
  4720. case Js::OpCode::CmUnLe_A:
  4721. case Js::OpCode::CmLt_A:
  4722. case Js::OpCode::CmUnLt_A:
  4723. case Js::OpCode::CmGe_A:
  4724. case Js::OpCode::CmUnGe_A:
  4725. case Js::OpCode::CmGt_A:
  4726. case Js::OpCode::CmUnGt_A:
  4727. return this->NewGenericValue(ValueType::Boolean, dst);
  4728. case Js::OpCode::LdUndef:
  4729. return this->NewGenericValue(ValueType::Undefined, dst);
  4730. case Js::OpCode::LdC_A_Null:
  4731. return this->NewGenericValue(ValueType::Null, dst);
  4732. case Js::OpCode::LdThis:
  4733. if (!PHASE_OFF(Js::OptTagChecksPhase, this->func) &&
  4734. (src1ValueInfo == nullptr || src1ValueInfo->IsUninitialized()))
  4735. {
  4736. return this->NewGenericValue(ValueType::GetObject(ObjectType::Object).ToLikely().SetCanBeTaggedValue(false), dst);
  4737. }
  4738. break;
  4739. case Js::OpCode::Typeof:
  4740. case Js::OpCode::TypeofElem:
  4741. return this->NewGenericValue(ValueType::String, dst);
  4742. case Js::OpCode::InitLocalClosure:
  4743. Assert(instr->GetDst());
  4744. Assert(instr->GetDst()->IsRegOpnd());
  4745. IR::RegOpnd *regOpnd = instr->GetDst()->AsRegOpnd();
  4746. StackSym *opndStackSym = regOpnd->m_sym;
  4747. Assert(opndStackSym != nullptr);
  4748. ObjectSymInfo *objectSymInfo = opndStackSym->m_objectInfo;
  4749. Assert(objectSymInfo != nullptr);
  4750. for (PropertySym *localVarSlotList = objectSymInfo->m_propertySymList; localVarSlotList; localVarSlotList = localVarSlotList->m_nextInStackSymList)
  4751. {
  4752. this->slotSyms->Set(localVarSlotList->m_id);
  4753. }
  4754. break;
  4755. }
  4756. if (dstVal == nullptr)
  4757. {
  4758. return this->NewGenericValue(dst->GetValueType(), dst);
  4759. }
  4760. return CurrentBlockData()->SetValue(dstVal, dst);
  4761. }
  4762. Value *
  4763. GlobOpt::ValueNumberLdElemDst(IR::Instr **pInstr, Value *srcVal)
  4764. {
  4765. IR::Instr *&instr = *pInstr;
  4766. IR::Opnd *dst = instr->GetDst();
  4767. Value *dstVal = nullptr;
  4768. int32 newMin, newMax;
  4769. ValueInfo *srcValueInfo = (srcVal ? srcVal->GetValueInfo() : nullptr);
  4770. ValueType profiledElementType;
  4771. if (instr->IsProfiledInstr())
  4772. {
  4773. profiledElementType = instr->AsProfiledInstr()->u.ldElemInfo->GetElementType();
  4774. if(!(profiledElementType.IsLikelyInt() && dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber) &&
  4775. srcVal &&
  4776. srcValueInfo->IsUninitialized())
  4777. {
  4778. if(IsLoopPrePass())
  4779. {
  4780. dstVal = NewGenericValue(profiledElementType, dst);
  4781. }
  4782. else
  4783. {
  4784. // Assuming the profile data gives more precise value types based on the path it took at runtime, we
  4785. // can improve the original value type.
  4786. srcValueInfo->Type() = profiledElementType;
  4787. instr->GetSrc1()->SetValueType(profiledElementType);
  4788. }
  4789. }
  4790. }
  4791. IR::IndirOpnd *src = instr->GetSrc1()->AsIndirOpnd();
  4792. const ValueType baseValueType(src->GetBaseOpnd()->GetValueType());
  4793. if (instr->DoStackArgsOpt() ||
  4794. !(
  4795. baseValueType.IsLikelyOptimizedTypedArray() ||
  4796. (baseValueType.IsLikelyNativeArray() && instr->IsProfiledInstr()) // Specialized native array lowering for LdElem requires that it is profiled.
  4797. ) ||
  4798. (!this->DoTypedArrayTypeSpec() && baseValueType.IsLikelyOptimizedTypedArray()) ||
  4799. // Don't do type spec on native array with a history of accessing gaps, as this is a bailout
  4800. (!this->DoNativeArrayTypeSpec() && baseValueType.IsLikelyNativeArray()) ||
  4801. !ShouldExpectConventionalArrayIndexValue(src))
  4802. {
  4803. if(DoTypedArrayTypeSpec() && !IsLoopPrePass())
  4804. {
  4805. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access.\n"));
  4806. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  4807. {
  4808. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  4809. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  4810. baseValueType.ToString(baseValueTypeStr);
  4811. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not type specialize, because %s.\n"),
  4812. this->func->GetJITFunctionBody()->GetDisplayName(),
  4813. this->func->GetDebugNumberSet(debugStringBuffer),
  4814. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  4815. baseValueTypeStr,
  4816. instr->DoStackArgsOpt() ? _u("instruction uses the arguments object") :
  4817. baseValueType.IsLikelyOptimizedTypedArray() ? _u("index is negative or likely not int") : _u("of array type"));
  4818. Output::Flush();
  4819. }
  4820. }
  4821. if(!dstVal)
  4822. {
  4823. if(srcVal)
  4824. {
  4825. dstVal = this->ValueNumberTransferDst(instr, srcVal);
  4826. }
  4827. else
  4828. {
  4829. dstVal = NewGenericValue(profiledElementType, dst);
  4830. }
  4831. }
  4832. return dstVal;
  4833. }
  4834. Assert(instr->GetSrc1()->IsIndirOpnd());
  4835. IRType toType = TyVar;
  4836. IR::BailOutKind bailOutKind = IR::BailOutConventionalTypedArrayAccessOnly;
  4837. switch(baseValueType.GetObjectType())
  4838. {
  4839. case ObjectType::Int8Array:
  4840. case ObjectType::Int8VirtualArray:
  4841. case ObjectType::Int8MixedArray:
  4842. newMin = Int8ConstMin;
  4843. newMax = Int8ConstMax;
  4844. goto IntArrayCommon;
  4845. case ObjectType::Uint8Array:
  4846. case ObjectType::Uint8VirtualArray:
  4847. case ObjectType::Uint8MixedArray:
  4848. case ObjectType::Uint8ClampedArray:
  4849. case ObjectType::Uint8ClampedVirtualArray:
  4850. case ObjectType::Uint8ClampedMixedArray:
  4851. newMin = Uint8ConstMin;
  4852. newMax = Uint8ConstMax;
  4853. goto IntArrayCommon;
  4854. case ObjectType::Int16Array:
  4855. case ObjectType::Int16VirtualArray:
  4856. case ObjectType::Int16MixedArray:
  4857. newMin = Int16ConstMin;
  4858. newMax = Int16ConstMax;
  4859. goto IntArrayCommon;
  4860. case ObjectType::Uint16Array:
  4861. case ObjectType::Uint16VirtualArray:
  4862. case ObjectType::Uint16MixedArray:
  4863. newMin = Uint16ConstMin;
  4864. newMax = Uint16ConstMax;
  4865. goto IntArrayCommon;
  4866. case ObjectType::Int32Array:
  4867. case ObjectType::Int32VirtualArray:
  4868. case ObjectType::Int32MixedArray:
  4869. case ObjectType::Uint32Array: // int-specialized loads from uint32 arrays will bail out on values that don't fit in an int32
  4870. case ObjectType::Uint32VirtualArray:
  4871. case ObjectType::Uint32MixedArray:
  4872. Int32Array:
  4873. newMin = Int32ConstMin;
  4874. newMax = Int32ConstMax;
  4875. goto IntArrayCommon;
  4876. IntArrayCommon:
  4877. Assert(dst->IsRegOpnd());
  4878. // If int type spec is disabled, it is ok to load int values as they can help float type spec, and merging int32 with float64 => float64.
  4879. // But if float type spec is also disabled, we'll have problems because float64 merged with var => float64...
  4880. if (!this->DoAggressiveIntTypeSpec() && !this->DoFloatTypeSpec())
  4881. {
  4882. if (!dstVal)
  4883. {
  4884. if (srcVal)
  4885. {
  4886. dstVal = this->ValueNumberTransferDst(instr, srcVal);
  4887. }
  4888. else
  4889. {
  4890. dstVal = NewGenericValue(profiledElementType, dst);
  4891. }
  4892. }
  4893. return dstVal;
  4894. }
  4895. if (!this->IsLoopPrePass())
  4896. {
  4897. if (instr->HasBailOutInfo())
  4898. {
  4899. const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind();
  4900. Assert(
  4901. (
  4902. !(oldBailOutKind & ~IR::BailOutKindBits) ||
  4903. (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp
  4904. ) &&
  4905. !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject)));
  4906. if (bailOutKind == IR::BailOutConventionalTypedArrayAccessOnly)
  4907. {
  4908. // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head
  4909. // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction
  4910. // bails out for the right reason.
  4911. instr->SetBailOutKind(
  4912. bailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall)));
  4913. }
  4914. else
  4915. {
  4916. // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit
  4917. // calls to occur, so it must be merged in to eliminate generating the helper call
  4918. Assert(bailOutKind == IR::BailOutConventionalNativeArrayAccessOnly);
  4919. instr->SetBailOutKind(oldBailOutKind | bailOutKind);
  4920. }
  4921. }
  4922. else
  4923. {
  4924. GenerateBailAtOperation(&instr, bailOutKind);
  4925. }
  4926. }
  4927. TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, nullptr, nullptr, bailOutKind, newMin, newMax, &dstVal);
  4928. toType = TyInt32;
  4929. break;
  4930. case ObjectType::Float32Array:
  4931. case ObjectType::Float32VirtualArray:
  4932. case ObjectType::Float32MixedArray:
  4933. case ObjectType::Float64Array:
  4934. case ObjectType::Float64VirtualArray:
  4935. case ObjectType::Float64MixedArray:
  4936. Float64Array:
  4937. Assert(dst->IsRegOpnd());
  4938. // If float type spec is disabled, don't load float64 values
  4939. if (!this->DoFloatTypeSpec())
  4940. {
  4941. if (!dstVal)
  4942. {
  4943. if (srcVal)
  4944. {
  4945. dstVal = this->ValueNumberTransferDst(instr, srcVal);
  4946. }
  4947. else
  4948. {
  4949. dstVal = NewGenericValue(profiledElementType, dst);
  4950. }
  4951. }
  4952. return dstVal;
  4953. }
  4954. if (!this->IsLoopPrePass())
  4955. {
  4956. if (instr->HasBailOutInfo())
  4957. {
  4958. const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind();
  4959. Assert(
  4960. (
  4961. !(oldBailOutKind & ~IR::BailOutKindBits) ||
  4962. (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp
  4963. ) &&
  4964. !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject)));
  4965. if (bailOutKind == IR::BailOutConventionalTypedArrayAccessOnly)
  4966. {
  4967. // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head
  4968. // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction
  4969. // bails out for the right reason.
  4970. instr->SetBailOutKind(
  4971. bailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall)));
  4972. }
  4973. else
  4974. {
  4975. // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit
  4976. // calls to occur, so it must be merged in to eliminate generating the helper call
  4977. Assert(bailOutKind == IR::BailOutConventionalNativeArrayAccessOnly);
  4978. instr->SetBailOutKind(oldBailOutKind | bailOutKind);
  4979. }
  4980. }
  4981. else
  4982. {
  4983. GenerateBailAtOperation(&instr, bailOutKind);
  4984. }
  4985. }
  4986. TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, &dstVal);
  4987. toType = TyFloat64;
  4988. break;
  4989. default:
  4990. Assert(baseValueType.IsLikelyNativeArray());
  4991. bailOutKind = IR::BailOutConventionalNativeArrayAccessOnly;
  4992. if(baseValueType.HasIntElements())
  4993. {
  4994. goto Int32Array;
  4995. }
  4996. Assert(baseValueType.HasFloatElements());
  4997. goto Float64Array;
  4998. }
  4999. if(!dstVal)
  5000. {
  5001. dstVal = NewGenericValue(profiledElementType, dst);
  5002. }
  5003. Assert(toType != TyVar);
  5004. GOPT_TRACE_INSTR(instr, _u("Type specialized array access.\n"));
  5005. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  5006. {
  5007. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  5008. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  5009. baseValueType.ToString(baseValueTypeStr);
  5010. char dstValTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  5011. dstVal->GetValueInfo()->Type().ToString(dstValTypeStr);
  5012. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, type specialized to %s producing %S"),
  5013. this->func->GetJITFunctionBody()->GetDisplayName(),
  5014. this->func->GetDebugNumberSet(debugStringBuffer),
  5015. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  5016. baseValueTypeStr,
  5017. toType == TyInt32 ? _u("int32") : _u("float64"),
  5018. dstValTypeStr);
  5019. #if DBG_DUMP
  5020. Output::Print(_u(" ("));
  5021. dstVal->Dump();
  5022. Output::Print(_u(").\n"));
  5023. #else
  5024. Output::Print(_u(".\n"));
  5025. #endif
  5026. Output::Flush();
  5027. }
  5028. return dstVal;
  5029. }
  5030. ValueType
  5031. GlobOpt::GetPrepassValueTypeForDst(
  5032. const ValueType desiredValueType,
  5033. IR::Instr *const instr,
  5034. Value *const src1Value,
  5035. Value *const src2Value,
  5036. bool const isValueInfoPrecise,
  5037. bool const isSafeToTransferInPrepass) const
  5038. {
  5039. // Values with definite types can be created in the loop prepass only when it is guaranteed that the value type will be the
  5040. // same on any iteration of the loop. The heuristics currently used are:
  5041. // - If the source sym is not live on the back-edge, then it acquires a new value for each iteration of the loop, so
  5042. // that value type can be definite
  5043. // - Consider: A better solution for this is to track values that originate in this loop, which can have definite value
  5044. // types. That catches more cases, should look into that in the future.
  5045. // - If the source sym has a constant value that doesn't change for the duration of the function
  5046. // - The operation always results in a definite value type. For instance, signed bitwise operations always result in an
  5047. // int32, conv_num and ++ always result in a number, etc.
  5048. // - For operations that always result in an int32, the resulting int range is precise only if the source syms pass
  5049. // the above heuristics. Otherwise, the range must be expanded to the full int32 range.
  5050. Assert(IsLoopPrePass());
  5051. Assert(instr);
  5052. if(!isValueInfoPrecise)
  5053. {
  5054. if(!desiredValueType.IsDefinite())
  5055. {
  5056. return isSafeToTransferInPrepass ? desiredValueType : desiredValueType.SetCanBeTaggedValue(true);
  5057. }
  5058. // If the desired value type is not precise, the value type of the destination is derived from the value types of the
  5059. // sources. Since the value type of a source sym is not definite, the destination value type also cannot be definite.
  5060. if(desiredValueType.IsInt() && OpCodeAttr::IsInt32(instr->m_opcode))
  5061. {
  5062. // The op always produces an int32, but not always a tagged int
  5063. return ValueType::GetInt(desiredValueType.IsLikelyTaggedInt());
  5064. }
  5065. if(desiredValueType.IsNumber() && OpCodeAttr::ProducesNumber(instr->m_opcode))
  5066. {
  5067. // The op always produces a number, but not always an int
  5068. return desiredValueType.ToDefiniteAnyNumber();
  5069. }
  5070. // Note: ToLikely() also sets CanBeTaggedValue
  5071. return desiredValueType.ToLikely();
  5072. }
  5073. return desiredValueType;
  5074. }
  5075. bool
  5076. GlobOpt::IsPrepassSrcValueInfoPrecise(IR::Instr *const instr, Value *const src1Value, Value *const src2Value, bool * isSafeToTransferInPrepass) const
  5077. {
  5078. return
  5079. (!instr->GetSrc1() || IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Value, isSafeToTransferInPrepass)) &&
  5080. (!instr->GetSrc2() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Value, isSafeToTransferInPrepass));
  5081. }
  5082. bool
  5083. GlobOpt::IsPrepassSrcValueInfoPrecise(IR::Opnd *const src, Value *const srcValue, bool * isSafeToTransferInPrepass) const
  5084. {
  5085. Assert(IsLoopPrePass());
  5086. Assert(src);
  5087. if (isSafeToTransferInPrepass)
  5088. {
  5089. *isSafeToTransferInPrepass = false;
  5090. }
  5091. if (src->IsAddrOpnd() &&
  5092. srcValue->GetValueInfo()->GetSymStore() &&
  5093. srcValue->GetValueInfo()->GetSymStore()->IsStackSym() &&
  5094. srcValue->GetValueInfo()->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable())
  5095. {
  5096. if (isSafeToTransferInPrepass)
  5097. {
  5098. *isSafeToTransferInPrepass = false;
  5099. }
  5100. return true;
  5101. }
  5102. if (!src->IsRegOpnd() || !srcValue)
  5103. {
  5104. return false;
  5105. }
  5106. ValueInfo *const srcValueInfo = srcValue->GetValueInfo();
  5107. bool isValueInfoDefinite = srcValueInfo->IsDefinite();
  5108. StackSym * srcSym = src->AsRegOpnd()->m_sym;
  5109. bool isSafeToTransfer = IsSafeToTransferInPrepass(srcSym, srcValueInfo);
  5110. if (isSafeToTransferInPrepass)
  5111. {
  5112. *isSafeToTransferInPrepass = isSafeToTransfer;
  5113. }
  5114. return isValueInfoDefinite && isSafeToTransfer;
  5115. }
  5116. bool
  5117. GlobOpt::IsSafeToTransferInPrepass(StackSym * const srcSym, ValueInfo *const srcValueInfo) const
  5118. {
  5119. int32 intConstantValue;
  5120. return
  5121. srcSym->IsFromByteCodeConstantTable() ||
  5122. (
  5123. srcValueInfo->TryGetIntConstantValue(&intConstantValue) &&
  5124. !Js::TaggedInt::IsOverflow(intConstantValue) &&
  5125. GetTaggedIntConstantStackSym(intConstantValue) == srcSym
  5126. ) ||
  5127. !currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(srcSym->m_id) ||
  5128. !currentBlock->loop->IsSymAssignedToInSelfOrParents(srcSym);
  5129. }
  5130. bool
  5131. GlobOpt::SafeToCopyPropInPrepass(StackSym * const originalSym, StackSym * const copySym, Value *const value) const
  5132. {
  5133. Assert(this->currentBlock->globOptData.GetCopyPropSym(originalSym, value) == copySym);
  5134. // In the following example, to copy-prop s2 into s1, it is not enough to check if s1 and s2 are safe to transfer.
  5135. // In fact, both s1 and s2 are safe to transfer, but it is not legal to copy prop s2 into s1.
  5136. //
  5137. // s1 = s2
  5138. // $Loop:
  5139. // s3 = s1
  5140. // s2 = s4
  5141. // Br $Loop
  5142. //
  5143. // In general, requirements for copy-propping in prepass are more restricted than those for transferring values.
  5144. // For copy prop in prepass, if the original sym is live on back-edge, then the copy-prop sym should not be written to
  5145. // in the loop (or its parents)
  5146. ValueInfo* const valueInfo = value->GetValueInfo();
  5147. return IsSafeToTransferInPrepass(originalSym, valueInfo) &&
  5148. IsSafeToTransferInPrepass(copySym, valueInfo) &&
  5149. (!currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(originalSym->m_id) || !currentBlock->loop->IsSymAssignedToInSelfOrParents(copySym));
  5150. }
  5151. Value *GlobOpt::CreateDstUntransferredIntValue(
  5152. const int32 min,
  5153. const int32 max,
  5154. IR::Instr *const instr,
  5155. Value *const src1Value,
  5156. Value *const src2Value)
  5157. {
  5158. Assert(instr);
  5159. Assert(instr->GetDst());
  5160. Assert(OpCodeAttr::ProducesNumber(instr->m_opcode)
  5161. || (instr->m_opcode == Js::OpCode::Add_A && src1Value->GetValueInfo()->IsNumber()
  5162. && src2Value->GetValueInfo()->IsNumber()));
  5163. ValueType valueType(ValueType::GetInt(IntConstantBounds(min, max).IsLikelyTaggable()));
  5164. Assert(valueType.IsInt());
  5165. bool isValueInfoPrecise;
  5166. if(IsLoopPrePass())
  5167. {
  5168. isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value);
  5169. valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, isValueInfoPrecise);
  5170. }
  5171. else
  5172. {
  5173. isValueInfoPrecise = true;
  5174. }
  5175. IR::Opnd *const dst = instr->GetDst();
  5176. if(isValueInfoPrecise)
  5177. {
  5178. Assert(valueType == ValueType::GetInt(IntConstantBounds(min, max).IsLikelyTaggable()));
  5179. Assert(!(dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->IsTypeSpec()));
  5180. return NewIntRangeValue(min, max, false, dst);
  5181. }
  5182. return NewGenericValue(valueType, dst);
  5183. }
  5184. Value *
  5185. GlobOpt::CreateDstUntransferredValue(
  5186. const ValueType desiredValueType,
  5187. IR::Instr *const instr,
  5188. Value *const src1Value,
  5189. Value *const src2Value)
  5190. {
  5191. Assert(instr);
  5192. Assert(instr->GetDst());
  5193. Assert(!desiredValueType.IsInt()); // use CreateDstUntransferredIntValue instead
  5194. ValueType valueType(desiredValueType);
  5195. if(IsLoopPrePass())
  5196. {
  5197. valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value));
  5198. }
  5199. return NewGenericValue(valueType, instr->GetDst());
  5200. }
  5201. Value *
  5202. GlobOpt::ValueNumberTransferDst(IR::Instr *const instr, Value * src1Val)
  5203. {
  5204. Value *dstVal = this->IsLoopPrePass() ? this->ValueNumberTransferDstInPrepass(instr, src1Val) : src1Val;
  5205. // Don't copy-prop a temp over a user symbol. This is likely to extend the temp's lifetime, as the user symbol
  5206. // is more likely to already have later references.
  5207. // REVIEW: Enabling this does cause perf issues...
  5208. #if 0
  5209. if (dstVal != src1Val)
  5210. {
  5211. return dstVal;
  5212. }
  5213. Sym *dstSym = dst->GetStackSym();
  5214. if (dstVal && dstSym && dstSym->IsStackSym() && !dstSym->AsStackSym()->m_isBytecodeTmp)
  5215. {
  5216. Sym *dstValSym = dstVal->GetValueInfo()->GetSymStore();
  5217. if (dstValSym && dstValSym->AsStackSym()->m_isBytecodeTmp /* src->GetIsDead()*/)
  5218. {
  5219. dstVal->GetValueInfo()->SetSymStore(dstSym);
  5220. }
  5221. }
  5222. #endif
  5223. return dstVal;
  5224. }
  5225. bool
  5226. GlobOpt::IsSafeToTransferInPrePass(IR::Opnd *src, Value *srcValue)
  5227. {
  5228. if (src->IsRegOpnd())
  5229. {
  5230. StackSym *srcSym = src->AsRegOpnd()->m_sym;
  5231. if (srcSym->IsFromByteCodeConstantTable())
  5232. {
  5233. return true;
  5234. }
  5235. ValueInfo *srcValueInfo = srcValue->GetValueInfo();
  5236. int32 srcIntConstantValue;
  5237. if (srcValueInfo->TryGetIntConstantValue(&srcIntConstantValue) && !Js::TaggedInt::IsOverflow(srcIntConstantValue)
  5238. && GetTaggedIntConstantStackSym(srcIntConstantValue) == srcSym)
  5239. {
  5240. return true;
  5241. }
  5242. }
  5243. return false;
  5244. }
  5245. Value *
  5246. GlobOpt::ValueNumberTransferDstInPrepass(IR::Instr *const instr, Value *const src1Val)
  5247. {
  5248. Value *dstVal = nullptr;
  5249. if (!src1Val)
  5250. {
  5251. return nullptr;
  5252. }
  5253. bool isValueInfoPrecise;
  5254. ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  5255. // TODO: This conflicts with new values created by the type specialization code
  5256. // We should re-enable if we change that code to avoid the new values.
  5257. #if 0
  5258. if (this->IsSafeToTransferInPrePass(instr->GetSrc1(), src1Val))
  5259. {
  5260. return src1Val;
  5261. }
  5262. if (this->IsPREInstrCandidateLoad(instr->m_opcode) && instr->GetDst())
  5263. {
  5264. StackSym *dstSym = instr->GetDst()->AsRegOpnd()->m_sym;
  5265. for (Loop *curLoop = this->currentBlock->loop; curLoop; curLoop = curLoop->parent)
  5266. {
  5267. if (curLoop->fieldPRESymStore->Test(dstSym->m_id))
  5268. {
  5269. return src1Val;
  5270. }
  5271. }
  5272. }
  5273. if (instr->GetDst()->IsRegOpnd())
  5274. {
  5275. StackSym *stackSym = instr->GetDst()->AsRegOpnd()->m_sym;
  5276. if (stackSym->IsSingleDef() || this->IsLive(stackSym, this->prePassLoop->landingPad))
  5277. {
  5278. IntConstantBounds src1IntConstantBounds;
  5279. if (src1ValueInfo->TryGetIntConstantBounds(&src1IntConstantBounds) &&
  5280. !(
  5281. src1IntConstantBounds.LowerBound() == INT32_MIN &&
  5282. src1IntConstantBounds.UpperBound() == INT32_MAX
  5283. ))
  5284. {
  5285. const ValueType valueType(
  5286. GetPrepassValueTypeForDst(src1ValueInfo->Type(), instr, src1Val, nullptr, &isValueInfoPrecise));
  5287. if (isValueInfoPrecise)
  5288. {
  5289. return src1Val;
  5290. }
  5291. }
  5292. else
  5293. {
  5294. return src1Val;
  5295. }
  5296. }
  5297. }
  5298. #endif
  5299. // Src1's value could change later in the loop, so the value wouldn't be the same for each
  5300. // iteration. Since we don't iterate over loops "while (!changed)", go conservative on the
  5301. // first pass when transferring a value that is live on the back-edge.
  5302. // In prepass we are going to copy the value but with a different value number
  5303. // for aggressive int type spec.
  5304. bool isSafeToTransferInPrepass = false;
  5305. isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Val, nullptr, &isSafeToTransferInPrepass);
  5306. const ValueType valueType(GetPrepassValueTypeForDst(src1ValueInfo->Type(), instr, src1Val, nullptr, isValueInfoPrecise, isSafeToTransferInPrepass));
  5307. if(isValueInfoPrecise || isSafeToTransferInPrepass)
  5308. {
  5309. Assert(valueType == src1ValueInfo->Type());
  5310. if (!PHASE_OFF1(Js::AVTInPrePassPhase))
  5311. {
  5312. dstVal = src1Val;
  5313. }
  5314. else
  5315. {
  5316. dstVal = CopyValue(src1Val);
  5317. TrackCopiedValueForKills(dstVal);
  5318. }
  5319. }
  5320. else if (valueType == src1ValueInfo->Type() && src1ValueInfo->IsGeneric()) // this else branch is probably not needed
  5321. {
  5322. Assert(valueType == src1ValueInfo->Type());
  5323. dstVal = CopyValue(src1Val);
  5324. TrackCopiedValueForKills(dstVal);
  5325. }
  5326. else
  5327. {
  5328. dstVal = NewGenericValue(valueType);
  5329. dstVal->GetValueInfo()->SetSymStore(src1ValueInfo->GetSymStore());
  5330. }
  5331. return dstVal;
  5332. }
  5333. void
  5334. GlobOpt::PropagateIntRangeForNot(int32 minimum, int32 maximum, int32 *pNewMin, int32* pNewMax)
  5335. {
  5336. int32 tmp;
  5337. Int32Math::Not(minimum, pNewMin);
  5338. *pNewMax = *pNewMin;
  5339. Int32Math::Not(maximum, &tmp);
  5340. *pNewMin = min(*pNewMin, tmp);
  5341. *pNewMax = max(*pNewMax, tmp);
  5342. }
  5343. void
  5344. GlobOpt::PropagateIntRangeBinary(IR::Instr *instr, int32 min1, int32 max1,
  5345. int32 min2, int32 max2, int32 *pNewMin, int32* pNewMax)
  5346. {
  5347. int32 min, max, tmp, tmp2;
  5348. min = INT32_MIN;
  5349. max = INT32_MAX;
  5350. switch (instr->m_opcode)
  5351. {
  5352. case Js::OpCode::Xor_A:
  5353. case Js::OpCode::Or_A:
  5354. // Find range with highest high order bit
  5355. tmp = ::max((uint32)min1, (uint32)max1);
  5356. tmp2 = ::max((uint32)min2, (uint32)max2);
  5357. if ((uint32)tmp > (uint32)tmp2)
  5358. {
  5359. max = tmp;
  5360. }
  5361. else
  5362. {
  5363. max = tmp2;
  5364. }
  5365. if (max < 0)
  5366. {
  5367. min = INT32_MIN; // REVIEW: conservative...
  5368. max = INT32_MAX;
  5369. }
  5370. else
  5371. {
  5372. // Turn values like 0x1010 into 0x1111
  5373. max = 1 << Math::Log2(max);
  5374. max = (uint32)(max << 1) - 1;
  5375. min = 0;
  5376. }
  5377. break;
  5378. case Js::OpCode::And_A:
  5379. if (min1 == INT32_MIN && min2 == INT32_MIN)
  5380. {
  5381. // Shortcut
  5382. break;
  5383. }
  5384. // Find range with lowest higher bit
  5385. tmp = ::max((uint32)min1, (uint32)max1);
  5386. tmp2 = ::max((uint32)min2, (uint32)max2);
  5387. if ((uint32)tmp < (uint32)tmp2)
  5388. {
  5389. min = min1;
  5390. max = max1;
  5391. }
  5392. else
  5393. {
  5394. min = min2;
  5395. max = max2;
  5396. }
  5397. // To compute max, look if min has higher high bit
  5398. if ((uint32)min > (uint32)max)
  5399. {
  5400. max = min;
  5401. }
  5402. // If max is negative, max let's assume it could be -1, so result in MAX_INT
  5403. if (max < 0)
  5404. {
  5405. max = INT32_MAX;
  5406. }
  5407. // If min is positive, the resulting min is zero
  5408. if (min >= 0)
  5409. {
  5410. min = 0;
  5411. }
  5412. else
  5413. {
  5414. min = INT32_MIN;
  5415. }
  5416. break;
  5417. case Js::OpCode::Shl_A:
  5418. {
  5419. // Shift count
  5420. if (min2 != max2 && ((uint32)min2 > 0x1F || (uint32)max2 > 0x1F))
  5421. {
  5422. min2 = 0;
  5423. max2 = 0x1F;
  5424. }
  5425. else
  5426. {
  5427. min2 &= 0x1F;
  5428. max2 &= 0x1F;
  5429. }
  5430. int32 min1FreeTopBitCount = min1 ? (sizeof(int32) * 8) - (Math::Log2(min1) + 1) : (sizeof(int32) * 8);
  5431. int32 max1FreeTopBitCount = max1 ? (sizeof(int32) * 8) - (Math::Log2(max1) + 1) : (sizeof(int32) * 8);
  5432. if (min1FreeTopBitCount <= max2 || max1FreeTopBitCount <= max2)
  5433. {
  5434. // If the shift is going to touch the sign bit return the max range
  5435. min = INT32_MIN;
  5436. max = INT32_MAX;
  5437. }
  5438. else
  5439. {
  5440. // Compute max
  5441. // Turn values like 0x1010 into 0x1111
  5442. if (min1)
  5443. {
  5444. min1 = 1 << Math::Log2(min1);
  5445. min1 = (min1 << 1) - 1;
  5446. }
  5447. if (max1)
  5448. {
  5449. max1 = 1 << Math::Log2(max1);
  5450. max1 = (uint32)(max1 << 1) - 1;
  5451. }
  5452. if (max1 > 0)
  5453. {
  5454. int32 nrTopBits = (sizeof(int32) * 8) - Math::Log2(max1);
  5455. if (nrTopBits < ::min(max2, 30))
  5456. max = INT32_MAX;
  5457. else
  5458. max = ::max((max1 << ::min(max2, 30)) & ~0x80000000, (min1 << min2) & ~0x80000000);
  5459. }
  5460. else
  5461. {
  5462. max = (max1 << min2) & ~0x80000000;
  5463. }
  5464. // Compute min
  5465. if (min1 < 0)
  5466. {
  5467. min = ::min(min1 << max2, max1 << max2);
  5468. }
  5469. else
  5470. {
  5471. min = ::min(min1 << min2, max1 << max2);
  5472. }
  5473. // Turn values like 0x1110 into 0x1000
  5474. if (min)
  5475. {
  5476. min = 1 << Math::Log2(min);
  5477. }
  5478. }
  5479. }
  5480. break;
  5481. case Js::OpCode::Shr_A:
  5482. // Shift count
  5483. if (min2 != max2 && ((uint32)min2 > 0x1F || (uint32)max2 > 0x1F))
  5484. {
  5485. min2 = 0;
  5486. max2 = 0x1F;
  5487. }
  5488. else
  5489. {
  5490. min2 &= 0x1F;
  5491. max2 &= 0x1F;
  5492. }
  5493. // Compute max
  5494. if (max1 < 0)
  5495. {
  5496. max = max1 >> max2;
  5497. }
  5498. else
  5499. {
  5500. max = max1 >> min2;
  5501. }
  5502. // Compute min
  5503. if (min1 < 0)
  5504. {
  5505. min = min1 >> min2;
  5506. }
  5507. else
  5508. {
  5509. min = min1 >> max2;
  5510. }
  5511. break;
  5512. case Js::OpCode::ShrU_A:
  5513. // shift count is constant zero
  5514. if ((min2 == max2) && (max2 & 0x1f) == 0)
  5515. {
  5516. // We can't encode uint32 result, so it has to be used as int32 only or the original value is positive.
  5517. Assert(instr->ignoreIntOverflow || min1 >= 0);
  5518. // We can transfer the signed int32 range.
  5519. min = min1;
  5520. max = max1;
  5521. break;
  5522. }
  5523. const IntConstantBounds src2NewBounds = IntConstantBounds(min2, max2).And_0x1f();
  5524. // Zero is only allowed if result is always a signed int32 or always used as a signed int32
  5525. Assert(min1 >= 0 || instr->ignoreIntOverflow || !src2NewBounds.Contains(0));
  5526. min2 = src2NewBounds.LowerBound();
  5527. max2 = src2NewBounds.UpperBound();
  5528. Assert(min2 <= max2);
  5529. // zero shift count is only allowed if result is used as int32 and/or value is positive
  5530. Assert(min2 > 0 || instr->ignoreIntOverflow || min1 >= 0);
  5531. uint32 umin1 = (uint32)min1;
  5532. uint32 umax1 = (uint32)max1;
  5533. if (umin1 > umax1)
  5534. {
  5535. uint32 temp = umax1;
  5536. umax1 = umin1;
  5537. umin1 = temp;
  5538. }
  5539. Assert(min2 >= 0 && max2 < 32);
  5540. // Compute max
  5541. if (min1 < 0)
  5542. {
  5543. umax1 = UINT32_MAX;
  5544. }
  5545. max = umax1 >> min2;
  5546. // Compute min
  5547. if (min1 <= 0 && max1 >=0)
  5548. {
  5549. min = 0;
  5550. }
  5551. else
  5552. {
  5553. min = umin1 >> max2;
  5554. }
  5555. // We should be able to fit uint32 range as int32
  5556. Assert(instr->ignoreIntOverflow || (min >= 0 && max >= 0) );
  5557. if (min > max)
  5558. {
  5559. // can only happen if shift count can be zero
  5560. Assert(min2 == 0 && (instr->ignoreIntOverflow || min1 >= 0));
  5561. min = Int32ConstMin;
  5562. max = Int32ConstMax;
  5563. }
  5564. break;
  5565. }
  5566. *pNewMin = min;
  5567. *pNewMax = max;
  5568. }
  5569. IR::Instr *
  5570. GlobOpt::TypeSpecialization(
  5571. IR::Instr *instr,
  5572. Value **pSrc1Val,
  5573. Value **pSrc2Val,
  5574. Value **pDstVal,
  5575. bool *redoTypeSpecRef,
  5576. bool *const forceInvariantHoistingRef)
  5577. {
  5578. Value *&src1Val = *pSrc1Val;
  5579. Value *&src2Val = *pSrc2Val;
  5580. *redoTypeSpecRef = false;
  5581. Assert(!*forceInvariantHoistingRef);
  5582. this->ignoredIntOverflowForCurrentInstr = false;
  5583. this->ignoredNegativeZeroForCurrentInstr = false;
  5584. // - Int32 values that can't be tagged are created as float constant values instead because a JavascriptNumber var is needed
  5585. // for that value at runtime. For the purposes of type specialization, recover the int32 values so that they will be
  5586. // treated as ints.
  5587. // - If int overflow does not matter for the instruction, we can additionally treat uint32 values as int32 values because
  5588. // the value resulting from the operation will eventually be converted to int32 anyway
  5589. Value *const src1OriginalVal = src1Val;
  5590. Value *const src2OriginalVal = src2Val;
  5591. if(!instr->ShouldCheckForIntOverflow())
  5592. {
  5593. if(src1Val && src1Val->GetValueInfo()->IsFloatConstant())
  5594. {
  5595. int32 int32Value;
  5596. bool isInt32;
  5597. if(Js::JavascriptNumber::TryGetInt32OrUInt32Value(
  5598. src1Val->GetValueInfo()->AsFloatConstant()->FloatValue(),
  5599. &int32Value,
  5600. &isInt32))
  5601. {
  5602. src1Val = GetIntConstantValue(int32Value, instr);
  5603. if(!isInt32)
  5604. {
  5605. this->ignoredIntOverflowForCurrentInstr = true;
  5606. }
  5607. }
  5608. }
  5609. if(src2Val && src2Val->GetValueInfo()->IsFloatConstant())
  5610. {
  5611. int32 int32Value;
  5612. bool isInt32;
  5613. if(Js::JavascriptNumber::TryGetInt32OrUInt32Value(
  5614. src2Val->GetValueInfo()->AsFloatConstant()->FloatValue(),
  5615. &int32Value,
  5616. &isInt32))
  5617. {
  5618. src2Val = GetIntConstantValue(int32Value, instr);
  5619. if(!isInt32)
  5620. {
  5621. this->ignoredIntOverflowForCurrentInstr = true;
  5622. }
  5623. }
  5624. }
  5625. }
  5626. const AutoRestoreVal autoRestoreSrc1Val(src1OriginalVal, &src1Val);
  5627. const AutoRestoreVal autoRestoreSrc2Val(src2OriginalVal, &src2Val);
  5628. if (src1Val && instr->GetSrc2() == nullptr)
  5629. {
  5630. // Unary
  5631. // Note make sure that native array StElemI gets to TypeSpecializeStElem. Do this for typed arrays, too?
  5632. int32 intConstantValue;
  5633. if (!this->IsLoopPrePass() &&
  5634. !instr->IsBranchInstr() &&
  5635. src1Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) &&
  5636. !(
  5637. // Nothing to fold for element stores. Go into type specialization to see if they can at least be specialized.
  5638. instr->m_opcode == Js::OpCode::StElemI_A ||
  5639. instr->m_opcode == Js::OpCode::StElemI_A_Strict ||
  5640. instr->m_opcode == Js::OpCode::StElemC ||
  5641. instr->m_opcode == Js::OpCode::MultiBr ||
  5642. instr->m_opcode == Js::OpCode::InlineArrayPop
  5643. ))
  5644. {
  5645. if (OptConstFoldUnary(&instr, intConstantValue, src1Val == src1OriginalVal, pDstVal))
  5646. {
  5647. return instr;
  5648. }
  5649. }
  5650. else if (this->TypeSpecializeUnary(
  5651. &instr,
  5652. &src1Val,
  5653. pDstVal,
  5654. src1OriginalVal,
  5655. redoTypeSpecRef,
  5656. forceInvariantHoistingRef))
  5657. {
  5658. return instr;
  5659. }
  5660. else if(*redoTypeSpecRef)
  5661. {
  5662. return instr;
  5663. }
  5664. }
  5665. else if (instr->GetSrc2() && !instr->IsBranchInstr())
  5666. {
  5667. // Binary
  5668. if (!this->IsLoopPrePass())
  5669. {
  5670. if (GetIsAsmJSFunc())
  5671. {
  5672. if (CONFIG_FLAG(WasmFold))
  5673. {
  5674. bool success = instr->GetSrc1()->IsInt64() ?
  5675. this->OptConstFoldBinaryWasm<int64>(&instr, src1Val, src2Val, pDstVal) :
  5676. this->OptConstFoldBinaryWasm<int>(&instr, src1Val, src2Val, pDstVal);
  5677. if (success)
  5678. {
  5679. return instr;
  5680. }
  5681. }
  5682. }
  5683. else
  5684. {
  5685. // OptConstFoldBinary doesn't do type spec, so only deal with things we are sure are int (IntConstant and IntRange)
  5686. // and not just likely ints TypeSpecializeBinary will deal with type specializing them and fold them again
  5687. IntConstantBounds src1IntConstantBounds, src2IntConstantBounds;
  5688. if (src1Val && src1Val->GetValueInfo()->TryGetIntConstantBounds(&src1IntConstantBounds))
  5689. {
  5690. if (src2Val && src2Val->GetValueInfo()->TryGetIntConstantBounds(&src2IntConstantBounds))
  5691. {
  5692. if (this->OptConstFoldBinary(&instr, src1IntConstantBounds, src2IntConstantBounds, pDstVal))
  5693. {
  5694. return instr;
  5695. }
  5696. }
  5697. }
  5698. }
  5699. }
  5700. }
  5701. if (instr->GetSrc2() && this->TypeSpecializeBinary(&instr, pSrc1Val, pSrc2Val, pDstVal, src1OriginalVal, src2OriginalVal, redoTypeSpecRef))
  5702. {
  5703. if (!this->IsLoopPrePass() &&
  5704. instr->m_opcode != Js::OpCode::Nop &&
  5705. instr->m_opcode != Js::OpCode::Br && // We may have const fold a branch
  5706. // Cannot const-peep if the result of the operation is required for a bailout check
  5707. !(instr->HasBailOutInfo() && instr->GetBailOutKind() & IR::BailOutOnResultConditions))
  5708. {
  5709. if (src1Val && src1Val->GetValueInfo()->HasIntConstantValue())
  5710. {
  5711. if (this->OptConstPeep(instr, instr->GetSrc1(), pDstVal, src1Val->GetValueInfo()))
  5712. {
  5713. return instr;
  5714. }
  5715. }
  5716. else if (src2Val && src2Val->GetValueInfo()->HasIntConstantValue())
  5717. {
  5718. if (this->OptConstPeep(instr, instr->GetSrc2(), pDstVal, src2Val->GetValueInfo()))
  5719. {
  5720. return instr;
  5721. }
  5722. }
  5723. }
  5724. return instr;
  5725. }
  5726. else if(*redoTypeSpecRef)
  5727. {
  5728. return instr;
  5729. }
  5730. if (instr->IsBranchInstr() && !this->IsLoopPrePass())
  5731. {
  5732. if (this->OptConstFoldBranch(instr, src1Val, src2Val, pDstVal))
  5733. {
  5734. return instr;
  5735. }
  5736. }
  5737. // We didn't type specialize, make sure the srcs are unspecialized
  5738. IR::Opnd *src1 = instr->GetSrc1();
  5739. if (src1)
  5740. {
  5741. instr = this->ToVarUses(instr, src1, false, src1Val);
  5742. IR::Opnd *src2 = instr->GetSrc2();
  5743. if (src2)
  5744. {
  5745. instr = this->ToVarUses(instr, src2, false, src2Val);
  5746. }
  5747. }
  5748. IR::Opnd *dst = instr->GetDst();
  5749. if (dst)
  5750. {
  5751. instr = this->ToVarUses(instr, dst, true, nullptr);
  5752. // Handling for instructions other than built-ins that may require only dst type specialization
  5753. // should be added here.
  5754. if(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode) && !GetIsAsmJSFunc()) // don't need to do typespec for asmjs
  5755. {
  5756. this->TypeSpecializeInlineBuiltInDst(&instr, pDstVal);
  5757. return instr;
  5758. }
  5759. // Clear the int specialized bit on the dst.
  5760. if (dst->IsRegOpnd())
  5761. {
  5762. IR::RegOpnd *dstRegOpnd = dst->AsRegOpnd();
  5763. if (!dstRegOpnd->m_sym->IsTypeSpec())
  5764. {
  5765. this->ToVarRegOpnd(dstRegOpnd, this->currentBlock);
  5766. }
  5767. else if (dstRegOpnd->m_sym->IsInt32())
  5768. {
  5769. this->ToInt32Dst(instr, dstRegOpnd, this->currentBlock);
  5770. }
  5771. else if (dstRegOpnd->m_sym->IsUInt32() && GetIsAsmJSFunc())
  5772. {
  5773. this->ToUInt32Dst(instr, dstRegOpnd, this->currentBlock);
  5774. }
  5775. else if (dstRegOpnd->m_sym->IsFloat64())
  5776. {
  5777. this->ToFloat64Dst(instr, dstRegOpnd, this->currentBlock);
  5778. }
  5779. }
  5780. else if (dst->IsSymOpnd() && dst->AsSymOpnd()->m_sym->IsStackSym())
  5781. {
  5782. this->ToVarStackSym(dst->AsSymOpnd()->m_sym->AsStackSym(), this->currentBlock);
  5783. }
  5784. }
  5785. return instr;
  5786. }
  5787. bool
  5788. GlobOpt::OptConstPeep(IR::Instr *instr, IR::Opnd *constSrc, Value **pDstVal, ValueInfo *valuInfo)
  5789. {
  5790. int32 value;
  5791. IR::Opnd *src;
  5792. IR::Opnd *nonConstSrc = (constSrc == instr->GetSrc1() ? instr->GetSrc2() : instr->GetSrc1());
  5793. // Try to find the value from value info first
  5794. if (valuInfo->TryGetIntConstantValue(&value))
  5795. {
  5796. }
  5797. else if (constSrc->IsAddrOpnd())
  5798. {
  5799. IR::AddrOpnd *addrOpnd = constSrc->AsAddrOpnd();
  5800. #ifdef _M_X64
  5801. Assert(addrOpnd->IsVar() || Math::FitsInDWord((size_t)addrOpnd->m_address));
  5802. #else
  5803. Assert(sizeof(value) == sizeof(addrOpnd->m_address));
  5804. #endif
  5805. if (addrOpnd->IsVar())
  5806. {
  5807. value = Js::TaggedInt::ToInt32(addrOpnd->m_address);
  5808. }
  5809. else
  5810. {
  5811. // We asserted that the address will fit in a DWORD above
  5812. value = ::Math::PointerCastToIntegral<int32>(constSrc->AsAddrOpnd()->m_address);
  5813. }
  5814. }
  5815. else if (constSrc->IsIntConstOpnd())
  5816. {
  5817. value = constSrc->AsIntConstOpnd()->AsInt32();
  5818. }
  5819. else
  5820. {
  5821. return false;
  5822. }
  5823. switch(instr->m_opcode)
  5824. {
  5825. // Can't do all Add_A because of string concats.
  5826. // Sub_A cannot be transformed to a NEG_A because 0 - 0 != -0
  5827. case Js::OpCode::Add_A:
  5828. src = nonConstSrc;
  5829. if (!src->GetValueType().IsInt())
  5830. {
  5831. // 0 + -0 != -0
  5832. // "Foo" + 0 != "Foo
  5833. return false;
  5834. }
  5835. // fall-through
  5836. case Js::OpCode::Add_I4:
  5837. if (value != 0)
  5838. {
  5839. return false;
  5840. }
  5841. if (constSrc == instr->GetSrc1())
  5842. {
  5843. src = instr->GetSrc2();
  5844. }
  5845. else
  5846. {
  5847. src = instr->GetSrc1();
  5848. }
  5849. break;
  5850. case Js::OpCode::Mul_A:
  5851. case Js::OpCode::Mul_I4:
  5852. if (value == 0)
  5853. {
  5854. // -0 * 0 != 0
  5855. return false;
  5856. }
  5857. else if (value == 1)
  5858. {
  5859. src = nonConstSrc;
  5860. }
  5861. else
  5862. {
  5863. return false;
  5864. }
  5865. break;
  5866. case Js::OpCode::Div_A:
  5867. if (value == 1 && constSrc == instr->GetSrc2())
  5868. {
  5869. src = instr->GetSrc1();
  5870. }
  5871. else
  5872. {
  5873. return false;
  5874. }
  5875. break;
  5876. case Js::OpCode::Or_I4:
  5877. if (value == -1)
  5878. {
  5879. src = constSrc;
  5880. }
  5881. else if (value == 0)
  5882. {
  5883. src = nonConstSrc;
  5884. }
  5885. else
  5886. {
  5887. return false;
  5888. }
  5889. break;
  5890. case Js::OpCode::And_I4:
  5891. if (value == -1)
  5892. {
  5893. src = nonConstSrc;
  5894. }
  5895. else if (value == 0)
  5896. {
  5897. src = constSrc;
  5898. }
  5899. else
  5900. {
  5901. return false;
  5902. }
  5903. break;
  5904. case Js::OpCode::Shl_I4:
  5905. case Js::OpCode::ShrU_I4:
  5906. case Js::OpCode::Shr_I4:
  5907. if (value != 0 || constSrc != instr->GetSrc2())
  5908. {
  5909. return false;
  5910. }
  5911. src = instr->GetSrc1();
  5912. break;
  5913. default:
  5914. return false;
  5915. }
  5916. this->CaptureByteCodeSymUses(instr);
  5917. if (src == instr->GetSrc1())
  5918. {
  5919. instr->FreeSrc2();
  5920. }
  5921. else
  5922. {
  5923. Assert(src == instr->GetSrc2());
  5924. instr->ReplaceSrc1(instr->UnlinkSrc2());
  5925. }
  5926. instr->m_opcode = Js::OpCode::Ld_A;
  5927. InvalidateInductionVariables(instr);
  5928. return true;
  5929. }
  5930. Js::Var // TODO: michhol OOP JIT, shouldn't play with Vars
  5931. GlobOpt::GetConstantVar(IR::Opnd *opnd, Value *val)
  5932. {
  5933. ValueInfo *valueInfo = val->GetValueInfo();
  5934. if (valueInfo->IsVarConstant() && valueInfo->IsPrimitive())
  5935. {
  5936. return valueInfo->AsVarConstant()->VarValue();
  5937. }
  5938. if (opnd->IsAddrOpnd())
  5939. {
  5940. IR::AddrOpnd *addrOpnd = opnd->AsAddrOpnd();
  5941. if (addrOpnd->IsVar())
  5942. {
  5943. return addrOpnd->m_address;
  5944. }
  5945. }
  5946. else if (opnd->IsIntConstOpnd())
  5947. {
  5948. if (!Js::TaggedInt::IsOverflow(opnd->AsIntConstOpnd()->AsInt32()))
  5949. {
  5950. return Js::TaggedInt::ToVarUnchecked(opnd->AsIntConstOpnd()->AsInt32());
  5951. }
  5952. }
  5953. #if FLOATVAR
  5954. else if (opnd->IsFloatConstOpnd())
  5955. {
  5956. return Js::JavascriptNumber::ToVar(opnd->AsFloatConstOpnd()->m_value);
  5957. }
  5958. #endif
  5959. else if (opnd->IsRegOpnd() && opnd->AsRegOpnd()->m_sym->IsSingleDef())
  5960. {
  5961. if (valueInfo->IsBoolean())
  5962. {
  5963. IR::Instr * defInstr = opnd->AsRegOpnd()->m_sym->GetInstrDef();
  5964. if (defInstr->m_opcode != Js::OpCode::Ld_A || !defInstr->GetSrc1()->IsAddrOpnd())
  5965. {
  5966. return nullptr;
  5967. }
  5968. Assert(defInstr->GetSrc1()->AsAddrOpnd()->IsVar());
  5969. return defInstr->GetSrc1()->AsAddrOpnd()->m_address;
  5970. }
  5971. else if (valueInfo->IsUndefined())
  5972. {
  5973. return (Js::Var)this->func->GetScriptContextInfo()->GetUndefinedAddr();
  5974. }
  5975. else if (valueInfo->IsNull())
  5976. {
  5977. return (Js::Var)this->func->GetScriptContextInfo()->GetNullAddr();
  5978. }
  5979. #if FLOATVAR
  5980. else if (valueInfo->IsFloat())
  5981. {
  5982. IR::Instr * defInstr = opnd->AsRegOpnd()->m_sym->GetInstrDef();
  5983. if ((defInstr->m_opcode == Js::OpCode::LdC_F8_R8 || defInstr->m_opcode == Js::OpCode::LdC_A_R8) && defInstr->GetSrc1()->IsFloatConstOpnd())
  5984. {
  5985. return Js::JavascriptNumber::ToVar(defInstr->GetSrc1()->AsFloatConstOpnd()->m_value);
  5986. }
  5987. }
  5988. #endif
  5989. }
  5990. return nullptr;
  5991. }
  5992. namespace
  5993. {
  5994. bool TryCompIntAndFloat(bool * result, Js::Var left, Js::Var right)
  5995. {
  5996. if (Js::TaggedInt::Is(left))
  5997. {
  5998. // If both are tagged ints we should not get here.
  5999. Assert(!Js::TaggedInt::Is(right));
  6000. if (Js::JavascriptNumber::Is_NoTaggedIntCheck(right))
  6001. {
  6002. double value = Js::JavascriptNumber::GetValue(right);
  6003. *result = (Js::TaggedInt::ToInt32(left) == value);
  6004. return true;
  6005. }
  6006. }
  6007. return false;
  6008. }
  6009. bool Op_JitEq(bool * result, Value * src1Val, Value * src2Val, Js::Var src1Var, Js::Var src2Var, Func * func, bool isStrict)
  6010. {
  6011. Assert(src1Val != nullptr && src2Val != nullptr);
  6012. Assert(src1Var != nullptr && src2Var != nullptr);
  6013. if (src1Var == src2Var)
  6014. {
  6015. if (Js::TaggedInt::Is(src1Var))
  6016. {
  6017. *result = true;
  6018. return true;
  6019. }
  6020. if (!isStrict && src1Val->GetValueInfo()->IsNotFloat())
  6021. {
  6022. // If the vars are equal and they are not NaN, non-strict equal returns true. Not float guarantees not NaN.
  6023. *result = true;
  6024. return true;
  6025. }
  6026. #if FLOATVAR
  6027. if (Js::JavascriptNumber::Is_NoTaggedIntCheck(src1Var))
  6028. {
  6029. *result = !Js::JavascriptNumber::IsNan(Js::JavascriptNumber::GetValue(src1Var));
  6030. return true;
  6031. }
  6032. #endif
  6033. if (src1Var == reinterpret_cast<Js::Var>(func->GetScriptContextInfo()->GetTrueAddr()) ||
  6034. src1Var == reinterpret_cast<Js::Var>(func->GetScriptContextInfo()->GetFalseAddr()) ||
  6035. src1Var == reinterpret_cast<Js::Var>(func->GetScriptContextInfo()->GetNullAddr()) ||
  6036. src1Var == reinterpret_cast<Js::Var>(func->GetScriptContextInfo()->GetUndefinedAddr()))
  6037. {
  6038. *result = true;
  6039. return true;
  6040. }
  6041. // Other var comparisons require the runtime to prove.
  6042. return false;
  6043. }
  6044. #if FLOATVAR
  6045. if (TryCompIntAndFloat(result, src1Var, src2Var) || TryCompIntAndFloat(result, src2Var, src1Var))
  6046. {
  6047. return true;
  6048. }
  6049. #endif
  6050. return false;
  6051. }
  6052. bool Op_JitNeq(bool * result, Value * src1Val, Value * src2Val, Js::Var src1Var, Js::Var src2Var, Func * func, bool isStrict)
  6053. {
  6054. if (Op_JitEq(result, src1Val, src2Val, src1Var, src2Var, func, isStrict))
  6055. {
  6056. *result = !*result;
  6057. return true;
  6058. }
  6059. return false;
  6060. }
  6061. bool BoolAndIntStaticAndTypeMismatch(Value* src1Val, Value* src2Val, Js::Var src1Var, Js::Var src2Var)
  6062. {
  6063. ValueInfo *src1ValInfo = src1Val->GetValueInfo();
  6064. ValueInfo *src2ValInfo = src2Val->GetValueInfo();
  6065. return (src1ValInfo->IsNumber() && src1Var && src2ValInfo->IsBoolean() && src1Var != Js::TaggedInt::ToVarUnchecked(0) && src1Var != Js::TaggedInt::ToVarUnchecked(1)) ||
  6066. (src2ValInfo->IsNumber() && src2Var && src1ValInfo->IsBoolean() && src2Var != Js::TaggedInt::ToVarUnchecked(0) && src2Var != Js::TaggedInt::ToVarUnchecked(1));
  6067. }
  6068. }
  6069. bool
  6070. GlobOpt::CanProveConditionalBranch(IR::Instr *instr, Value *src1Val, Value *src2Val, Js::Var src1Var, Js::Var src2Var, bool *result)
  6071. {
  6072. auto AreSourcesEqual = [&](Value * val1, Value * val2, bool undefinedCmp) -> bool
  6073. {
  6074. // NaN !== NaN, and objects can have valueOf/toString
  6075. if (val1->IsEqualTo(val2))
  6076. {
  6077. if (val1->GetValueInfo()->IsUndefined())
  6078. {
  6079. return undefinedCmp;
  6080. }
  6081. ValueInfo * valInfo = val1->GetValueInfo();
  6082. return !valInfo->HasBeenUndefined() && valInfo->IsPrimitive() && valInfo->IsNotFloat();
  6083. }
  6084. return false;
  6085. };
  6086. // Make sure GetConstantVar only returns primitives.
  6087. // TODO: OOP JIT, enabled these asserts
  6088. //Assert(!src1Var || !Js::JavascriptOperators::IsObject(src1Var));
  6089. //Assert(!src2Var || !Js::JavascriptOperators::IsObject(src2Var));
  6090. int64 left64, right64;
  6091. int32 left, right;
  6092. int32 constVal;
  6093. switch (instr->m_opcode)
  6094. {
  6095. #define BRANCHSIGNED(OPCODE,CMP,TYPE,UNSIGNEDNESS,UNDEFINEDCMP) \
  6096. case Js::OpCode::##OPCODE: \
  6097. if (src1Val && src2Val) \
  6098. { \
  6099. if (src1Val->GetValueInfo()->TryGetIntConstantValue(&left, UNSIGNEDNESS) && \
  6100. src2Val->GetValueInfo()->TryGetIntConstantValue(&right, UNSIGNEDNESS)) \
  6101. { \
  6102. *result = (TYPE)left CMP(TYPE)right; \
  6103. } \
  6104. if (src1Val->GetValueInfo()->TryGetInt64ConstantValue(&left64, UNSIGNEDNESS) && \
  6105. src2Val->GetValueInfo()->TryGetInt64ConstantValue(&right64, UNSIGNEDNESS)) \
  6106. { \
  6107. *result = (TYPE)left64 CMP(TYPE)right64; \
  6108. } \
  6109. else if (AreSourcesEqual(src1Val, src2Val, UNDEFINEDCMP)) \
  6110. { \
  6111. *result = 0 CMP 0; \
  6112. } \
  6113. else \
  6114. { \
  6115. return false; \
  6116. } \
  6117. } \
  6118. else \
  6119. { \
  6120. return false; \
  6121. } \
  6122. break;
  6123. BRANCHSIGNED(BrEq_I4, == , int64, false, true)
  6124. BRANCHSIGNED(BrGe_I4, >= , int64, false, false)
  6125. BRANCHSIGNED(BrGt_I4, > , int64, false, false)
  6126. BRANCHSIGNED(BrLt_I4, < , int64, false, false)
  6127. BRANCHSIGNED(BrLe_I4, <= , int64, false, false)
  6128. BRANCHSIGNED(BrNeq_I4, != , int64, false, false)
  6129. BRANCHSIGNED(BrUnGe_I4, >= , uint64, true, false)
  6130. BRANCHSIGNED(BrUnGt_I4, > , uint64, true, false)
  6131. BRANCHSIGNED(BrUnLt_I4, < , uint64, true, false)
  6132. BRANCHSIGNED(BrUnLe_I4, <= , uint64, true, false)
  6133. #undef BRANCHSIGNED
  6134. #define BRANCH(OPCODE,CMP,VARCMPFUNC,UNDEFINEDCMP) \
  6135. case Js::OpCode::##OPCODE: \
  6136. if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) && \
  6137. src2Val->GetValueInfo()->TryGetIntConstantValue(&right)) \
  6138. { \
  6139. *result = left CMP right; \
  6140. } \
  6141. else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, UNDEFINEDCMP)) \
  6142. { \
  6143. *result = 0 CMP 0; \
  6144. } \
  6145. else if (src1Var && src2Var) \
  6146. { \
  6147. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts)) \
  6148. { \
  6149. return false; \
  6150. } \
  6151. *result = VARCMPFUNC(src1Var, src2Var, this->func->GetScriptContext()); \
  6152. } \
  6153. else \
  6154. { \
  6155. return false; \
  6156. } \
  6157. break;
  6158. BRANCH(BrGe_A, >= , Js::JavascriptOperators::GreaterEqual, /*undefinedEquality*/ false)
  6159. BRANCH(BrNotGe_A, <, !Js::JavascriptOperators::GreaterEqual, false)
  6160. BRANCH(BrLt_A, <, Js::JavascriptOperators::Less, false)
  6161. BRANCH(BrNotLt_A, >= , !Js::JavascriptOperators::Less, false)
  6162. BRANCH(BrGt_A, >, Js::JavascriptOperators::Greater, false)
  6163. BRANCH(BrNotGt_A, <= , !Js::JavascriptOperators::Greater, false)
  6164. BRANCH(BrLe_A, <= , Js::JavascriptOperators::LessEqual, false)
  6165. BRANCH(BrNotLe_A, >, !Js::JavascriptOperators::LessEqual, false)
  6166. #undef BRANCH
  6167. case Js::OpCode::BrEq_A:
  6168. case Js::OpCode::BrNotNeq_A:
  6169. if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) &&
  6170. src2Val->GetValueInfo()->TryGetIntConstantValue(&right))
  6171. {
  6172. *result = left == right;
  6173. }
  6174. else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, true))
  6175. {
  6176. *result = true;
  6177. }
  6178. else if (!src1Var || !src2Var)
  6179. {
  6180. if (BoolAndIntStaticAndTypeMismatch(src1Val, src2Val, src1Var, src2Var))
  6181. {
  6182. *result = false;
  6183. }
  6184. else
  6185. {
  6186. return false;
  6187. }
  6188. }
  6189. else
  6190. {
  6191. if (!Op_JitEq(result, src1Val, src2Val, src1Var, src2Var, this->func, false /* isStrict */))
  6192. {
  6193. return false;
  6194. }
  6195. }
  6196. break;
  6197. case Js::OpCode::BrNeq_A:
  6198. case Js::OpCode::BrNotEq_A:
  6199. if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) &&
  6200. src2Val->GetValueInfo()->TryGetIntConstantValue(&right))
  6201. {
  6202. *result = left != right;
  6203. }
  6204. else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, true))
  6205. {
  6206. *result = false;
  6207. }
  6208. else if (!src1Var || !src2Var)
  6209. {
  6210. if (BoolAndIntStaticAndTypeMismatch(src1Val, src2Val, src1Var, src2Var))
  6211. {
  6212. *result = true;
  6213. }
  6214. else
  6215. {
  6216. return false;
  6217. }
  6218. }
  6219. else
  6220. {
  6221. if (!Op_JitNeq(result, src1Val, src2Val, src1Var, src2Var, this->func, false /* isStrict */))
  6222. {
  6223. return false;
  6224. }
  6225. }
  6226. break;
  6227. case Js::OpCode::BrSrEq_A:
  6228. case Js::OpCode::BrSrNotNeq_A:
  6229. if (!src1Var || !src2Var)
  6230. {
  6231. ValueInfo *src1ValInfo = src1Val->GetValueInfo();
  6232. ValueInfo *src2ValInfo = src2Val->GetValueInfo();
  6233. if (
  6234. (src1ValInfo->IsUndefined() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenUndefined()) ||
  6235. (src1ValInfo->IsNull() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNull()) ||
  6236. (src1ValInfo->IsBoolean() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenBoolean()) ||
  6237. (src1ValInfo->IsNumber() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNumber()) ||
  6238. (src1ValInfo->IsString() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenString()) ||
  6239. (src2ValInfo->IsUndefined() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenUndefined()) ||
  6240. (src2ValInfo->IsNull() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNull()) ||
  6241. (src2ValInfo->IsBoolean() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenBoolean()) ||
  6242. (src2ValInfo->IsNumber() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNumber()) ||
  6243. (src2ValInfo->IsString() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenString())
  6244. )
  6245. {
  6246. *result = false;
  6247. }
  6248. else if (AreSourcesEqual(src1Val, src2Val, true))
  6249. {
  6250. *result = true;
  6251. }
  6252. else
  6253. {
  6254. return false;
  6255. }
  6256. }
  6257. else
  6258. {
  6259. if (!Op_JitEq(result, src1Val, src2Val, src1Var, src2Var, this->func, true /* isStrict */))
  6260. {
  6261. return false;
  6262. }
  6263. }
  6264. break;
  6265. case Js::OpCode::BrSrNeq_A:
  6266. case Js::OpCode::BrSrNotEq_A:
  6267. if (!src1Var || !src2Var)
  6268. {
  6269. ValueInfo *src1ValInfo = src1Val->GetValueInfo();
  6270. ValueInfo *src2ValInfo = src2Val->GetValueInfo();
  6271. if (
  6272. (src1ValInfo->IsUndefined() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenUndefined()) ||
  6273. (src1ValInfo->IsNull() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNull()) ||
  6274. (src1ValInfo->IsBoolean() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenBoolean()) ||
  6275. (src1ValInfo->IsNumber() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNumber()) ||
  6276. (src1ValInfo->IsString() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenString()) ||
  6277. (src2ValInfo->IsUndefined() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenUndefined()) ||
  6278. (src2ValInfo->IsNull() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNull()) ||
  6279. (src2ValInfo->IsBoolean() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenBoolean()) ||
  6280. (src2ValInfo->IsNumber() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNumber()) ||
  6281. (src2ValInfo->IsString() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenString())
  6282. )
  6283. {
  6284. *result = true;
  6285. }
  6286. else if (AreSourcesEqual(src1Val, src2Val, true))
  6287. {
  6288. *result = false;
  6289. }
  6290. else
  6291. {
  6292. return false;
  6293. }
  6294. }
  6295. else
  6296. {
  6297. if (!Op_JitNeq(result, src1Val, src2Val, src1Var, src2Var, this->func, true /* isStrict */))
  6298. {
  6299. return false;
  6300. }
  6301. }
  6302. break;
  6303. case Js::OpCode::BrFalse_A:
  6304. case Js::OpCode::BrTrue_A:
  6305. {
  6306. ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  6307. if (src1ValueInfo->IsNull() || src1ValueInfo->IsUndefined())
  6308. {
  6309. *result = instr->m_opcode == Js::OpCode::BrFalse_A;
  6310. break;
  6311. }
  6312. if (src1ValueInfo->IsObject() && src1ValueInfo->GetObjectType() > ObjectType::Object)
  6313. {
  6314. // Specific object types that are tracked are equivalent to 'true'
  6315. *result = instr->m_opcode == Js::OpCode::BrTrue_A;
  6316. break;
  6317. }
  6318. if (!src1Var)
  6319. {
  6320. return false;
  6321. }
  6322. // Set *result = (evaluates true) and negate it later for BrFalse
  6323. if (src1Var == reinterpret_cast<Js::Var>(this->func->GetScriptContextInfo()->GetTrueAddr()))
  6324. {
  6325. *result = true;
  6326. }
  6327. else if (src1Var == reinterpret_cast<Js::Var>(this->func->GetScriptContextInfo()->GetFalseAddr()))
  6328. {
  6329. *result = false;
  6330. }
  6331. else if (Js::TaggedInt::Is(src1Var))
  6332. {
  6333. *result = (src1Var != reinterpret_cast<Js::Var>(Js::AtomTag_IntPtr));
  6334. }
  6335. #if FLOATVAR
  6336. else if (Js::JavascriptNumber::Is_NoTaggedIntCheck(src1Var))
  6337. {
  6338. double value = Js::JavascriptNumber::GetValue(src1Var);
  6339. *result = (!Js::JavascriptNumber::IsNan(value)) && (!Js::JavascriptNumber::IsZero(value));
  6340. }
  6341. #endif
  6342. else
  6343. {
  6344. return false;
  6345. }
  6346. if (instr->m_opcode == Js::OpCode::BrFalse_A)
  6347. {
  6348. *result = !(*result);
  6349. }
  6350. break;
  6351. }
  6352. case Js::OpCode::BrFalse_I4:
  6353. {
  6354. constVal = 0;
  6355. if (!src1Val->GetValueInfo()->TryGetIntConstantValue(&constVal))
  6356. {
  6357. return false;
  6358. }
  6359. *result = constVal == 0;
  6360. break;
  6361. }
  6362. case Js::OpCode::BrOnObject_A:
  6363. {
  6364. ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  6365. if (!src1ValueInfo->IsDefinite())
  6366. {
  6367. return false;
  6368. }
  6369. if (src1ValueInfo->IsPrimitive())
  6370. {
  6371. *result = false;
  6372. }
  6373. else
  6374. {
  6375. if (src1ValueInfo->HasBeenPrimitive())
  6376. {
  6377. return false;
  6378. }
  6379. *result = true;
  6380. }
  6381. break;
  6382. }
  6383. default:
  6384. return false;
  6385. }
  6386. return true;
  6387. }
  6388. bool
  6389. GlobOpt::OptConstFoldBranch(IR::Instr *instr, Value *src1Val, Value*src2Val, Value **pDstVal)
  6390. {
  6391. if (!src1Val)
  6392. {
  6393. return false;
  6394. }
  6395. Js::Var src1Var = this->GetConstantVar(instr->GetSrc1(), src1Val);
  6396. Js::Var src2Var = nullptr;
  6397. if (instr->GetSrc2())
  6398. {
  6399. if (!src2Val)
  6400. {
  6401. return false;
  6402. }
  6403. src2Var = this->GetConstantVar(instr->GetSrc2(), src2Val);
  6404. }
  6405. bool result;
  6406. if (!CanProveConditionalBranch(instr, src1Val, src2Val, src1Var, src2Var, &result))
  6407. {
  6408. return false;
  6409. }
  6410. this->OptConstFoldBr(!!result, instr);
  6411. return true;
  6412. }
  6413. bool
  6414. GlobOpt::OptConstFoldUnary(
  6415. IR::Instr * *pInstr,
  6416. const int32 intConstantValue,
  6417. const bool isUsingOriginalSrc1Value,
  6418. Value **pDstVal)
  6419. {
  6420. IR::Instr * &instr = *pInstr;
  6421. int32 value = 0;
  6422. IR::Opnd *constOpnd;
  6423. bool isInt = true;
  6424. bool doSetDstVal = true;
  6425. FloatConstType fValue = 0.0;
  6426. if (!DoConstFold())
  6427. {
  6428. return false;
  6429. }
  6430. if (instr->GetDst() && !instr->GetDst()->IsRegOpnd())
  6431. {
  6432. return false;
  6433. }
  6434. switch(instr->m_opcode)
  6435. {
  6436. case Js::OpCode::Neg_A:
  6437. if (intConstantValue == 0)
  6438. {
  6439. // Could fold to -0.0
  6440. return false;
  6441. }
  6442. if (Int32Math::Neg(intConstantValue, &value))
  6443. {
  6444. return false;
  6445. }
  6446. break;
  6447. case Js::OpCode::Not_A:
  6448. Int32Math::Not(intConstantValue, &value);
  6449. break;
  6450. case Js::OpCode::Ld_A:
  6451. if (instr->HasBailOutInfo())
  6452. {
  6453. //The profile data for switch expr can be string and in GlobOpt we realize it is an int.
  6454. if(instr->GetBailOutKind() == IR::BailOutExpectingString)
  6455. {
  6456. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingString);
  6457. }
  6458. Assert(instr->GetBailOutKind() == IR::BailOutExpectingInteger);
  6459. instr->ClearBailOutInfo();
  6460. }
  6461. value = intConstantValue;
  6462. if(isUsingOriginalSrc1Value)
  6463. {
  6464. doSetDstVal = false; // Let OptDst do it by copying src1Val
  6465. }
  6466. break;
  6467. case Js::OpCode::Conv_Num:
  6468. case Js::OpCode::LdC_A_I4:
  6469. value = intConstantValue;
  6470. if(isUsingOriginalSrc1Value)
  6471. {
  6472. doSetDstVal = false; // Let OptDst do it by copying src1Val
  6473. }
  6474. break;
  6475. case Js::OpCode::Incr_A:
  6476. if (Int32Math::Inc(intConstantValue, &value))
  6477. {
  6478. return false;
  6479. }
  6480. break;
  6481. case Js::OpCode::Decr_A:
  6482. if (Int32Math::Dec(intConstantValue, &value))
  6483. {
  6484. return false;
  6485. }
  6486. break;
  6487. case Js::OpCode::InlineMathAcos:
  6488. fValue = Js::Math::Acos((double)intConstantValue);
  6489. isInt = false;
  6490. break;
  6491. case Js::OpCode::InlineMathAsin:
  6492. fValue = Js::Math::Asin((double)intConstantValue);
  6493. isInt = false;
  6494. break;
  6495. case Js::OpCode::InlineMathAtan:
  6496. fValue = Js::Math::Atan((double)intConstantValue);
  6497. isInt = false;
  6498. break;
  6499. case Js::OpCode::InlineMathCos:
  6500. fValue = Js::Math::Cos((double)intConstantValue);
  6501. isInt = false;
  6502. break;
  6503. case Js::OpCode::InlineMathExp:
  6504. fValue = Js::Math::Exp((double)intConstantValue);
  6505. isInt = false;
  6506. break;
  6507. case Js::OpCode::InlineMathLog:
  6508. fValue = Js::Math::Log((double)intConstantValue);
  6509. isInt = false;
  6510. break;
  6511. case Js::OpCode::InlineMathSin:
  6512. fValue = Js::Math::Sin((double)intConstantValue);
  6513. isInt = false;
  6514. break;
  6515. case Js::OpCode::InlineMathSqrt:
  6516. fValue = ::sqrt((double)intConstantValue);
  6517. isInt = false;
  6518. break;
  6519. case Js::OpCode::InlineMathTan:
  6520. fValue = ::tan((double)intConstantValue);
  6521. isInt = false;
  6522. break;
  6523. case Js::OpCode::InlineMathFround:
  6524. fValue = (double) (float) intConstantValue;
  6525. isInt = false;
  6526. break;
  6527. case Js::OpCode::InlineMathAbs:
  6528. if (intConstantValue == INT32_MIN)
  6529. {
  6530. if (instr->GetDst()->IsInt32())
  6531. {
  6532. // if dst is an int (e.g. in asm.js), we should coerce it, not convert to float
  6533. value = static_cast<int32>(2147483648U);
  6534. }
  6535. else
  6536. {
  6537. // Rejit with AggressiveIntTypeSpecDisabled for Math.abs(INT32_MIN) because it causes dst
  6538. // to be float type which could be different with previous type spec result in LoopPrePass
  6539. throw Js::RejitException(RejitReason::AggressiveIntTypeSpecDisabled);
  6540. }
  6541. }
  6542. else
  6543. {
  6544. value = ::abs(intConstantValue);
  6545. }
  6546. break;
  6547. case Js::OpCode::InlineMathClz:
  6548. DWORD clz;
  6549. if (_BitScanReverse(&clz, intConstantValue))
  6550. {
  6551. value = 31 - clz;
  6552. }
  6553. else
  6554. {
  6555. value = 32;
  6556. }
  6557. instr->ClearBailOutInfo();
  6558. break;
  6559. case Js::OpCode::Ctz:
  6560. Assert(func->GetJITFunctionBody()->IsWasmFunction());
  6561. Assert(!instr->HasBailOutInfo());
  6562. DWORD ctz;
  6563. if (_BitScanForward(&ctz, intConstantValue))
  6564. {
  6565. value = ctz;
  6566. }
  6567. else
  6568. {
  6569. value = 32;
  6570. }
  6571. break;
  6572. case Js::OpCode::InlineMathFloor:
  6573. value = intConstantValue;
  6574. instr->ClearBailOutInfo();
  6575. break;
  6576. case Js::OpCode::InlineMathCeil:
  6577. value = intConstantValue;
  6578. instr->ClearBailOutInfo();
  6579. break;
  6580. case Js::OpCode::InlineMathRound:
  6581. value = intConstantValue;
  6582. instr->ClearBailOutInfo();
  6583. break;
  6584. case Js::OpCode::ToVar:
  6585. if (Js::TaggedInt::IsOverflow(intConstantValue))
  6586. {
  6587. return false;
  6588. }
  6589. else
  6590. {
  6591. value = intConstantValue;
  6592. instr->ClearBailOutInfo();
  6593. break;
  6594. }
  6595. default:
  6596. return false;
  6597. }
  6598. this->CaptureByteCodeSymUses(instr);
  6599. Assert(!instr->HasBailOutInfo()); // If we are, in fact, successful in constant folding the instruction, there is no point in having the bailoutinfo around anymore.
  6600. // Make sure that it is cleared if it was initially present.
  6601. if (!isInt)
  6602. {
  6603. value = (int32)fValue;
  6604. if (fValue == (double)value)
  6605. {
  6606. isInt = true;
  6607. }
  6608. }
  6609. if (isInt)
  6610. {
  6611. constOpnd = IR::IntConstOpnd::New(value, TyInt32, instr->m_func);
  6612. GOPT_TRACE(_u("Constant folding to %d\n"), value);
  6613. }
  6614. else
  6615. {
  6616. constOpnd = IR::FloatConstOpnd::New(fValue, TyFloat64, instr->m_func);
  6617. GOPT_TRACE(_u("Constant folding to %f\n"), fValue);
  6618. }
  6619. instr->ReplaceSrc1(constOpnd);
  6620. this->OptSrc(constOpnd, &instr);
  6621. IR::Opnd *dst = instr->GetDst();
  6622. Assert(dst->IsRegOpnd());
  6623. StackSym *dstSym = dst->AsRegOpnd()->m_sym;
  6624. if (isInt)
  6625. {
  6626. if (dstSym->IsSingleDef())
  6627. {
  6628. dstSym->SetIsIntConst(value);
  6629. }
  6630. if (doSetDstVal)
  6631. {
  6632. *pDstVal = GetIntConstantValue(value, instr, dst);
  6633. }
  6634. if (IsTypeSpecPhaseOff(this->func))
  6635. {
  6636. instr->m_opcode = Js::OpCode::LdC_A_I4;
  6637. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  6638. }
  6639. else
  6640. {
  6641. instr->m_opcode = Js::OpCode::Ld_I4;
  6642. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  6643. StackSym * currDstSym = instr->GetDst()->AsRegOpnd()->m_sym;
  6644. if (currDstSym->IsSingleDef())
  6645. {
  6646. currDstSym->SetIsIntConst(value);
  6647. }
  6648. }
  6649. }
  6650. else
  6651. {
  6652. *pDstVal = NewFloatConstantValue(fValue, dst);
  6653. if (IsTypeSpecPhaseOff(this->func))
  6654. {
  6655. instr->m_opcode = Js::OpCode::LdC_A_R8;
  6656. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  6657. }
  6658. else
  6659. {
  6660. instr->m_opcode = Js::OpCode::LdC_F8_R8;
  6661. this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  6662. }
  6663. }
  6664. InvalidateInductionVariables(instr);
  6665. return true;
  6666. }
  6667. //------------------------------------------------------------------------------------------------------
  6668. // Type specialization
  6669. //------------------------------------------------------------------------------------------------------
  6670. bool
  6671. GlobOpt::IsWorthSpecializingToInt32DueToSrc(IR::Opnd *const src, Value *const val)
  6672. {
  6673. Assert(src);
  6674. Assert(val);
  6675. ValueInfo *valueInfo = val->GetValueInfo();
  6676. Assert(valueInfo->IsLikelyInt());
  6677. // If it is not known that the operand is definitely an int, the operand is not already type-specialized, and it's not live
  6678. // in the loop landing pad (if we're in a loop), it's probably not worth type-specializing this instruction. The common case
  6679. // where type-specializing this would be bad is where the operations are entirely on properties or array elements, where the
  6680. // ratio of FromVars and ToVars to the number of actual operations is high, and the conversions would dominate the time
  6681. // spent. On the other hand, if we're using a function formal parameter more than once, it would probably be worth
  6682. // type-specializing it, hence the IsDead check on the operands.
  6683. return
  6684. valueInfo->IsInt() ||
  6685. valueInfo->HasIntConstantValue(true) ||
  6686. !src->GetIsDead() ||
  6687. !src->IsRegOpnd() ||
  6688. CurrentBlockData()->IsInt32TypeSpecialized(src->AsRegOpnd()->m_sym) ||
  6689. (this->currentBlock->loop && this->currentBlock->loop->landingPad->globOptData.IsLive(src->AsRegOpnd()->m_sym));
  6690. }
  6691. bool
  6692. GlobOpt::IsWorthSpecializingToInt32DueToDst(IR::Opnd *const dst)
  6693. {
  6694. Assert(dst);
  6695. const auto sym = dst->AsRegOpnd()->m_sym;
  6696. return
  6697. CurrentBlockData()->IsInt32TypeSpecialized(sym) ||
  6698. (this->currentBlock->loop && this->currentBlock->loop->landingPad->globOptData.IsLive(sym));
  6699. }
  6700. bool
  6701. GlobOpt::IsWorthSpecializingToInt32(IR::Instr *const instr, Value *const src1Val, Value *const src2Val)
  6702. {
  6703. Assert(instr);
  6704. const auto src1 = instr->GetSrc1();
  6705. const auto src2 = instr->GetSrc2();
  6706. // In addition to checking each operand and the destination, if for any reason we only have to do a maximum of two
  6707. // conversions instead of the worst-case 3 conversions, it's probably worth specializing.
  6708. if (IsWorthSpecializingToInt32DueToSrc(src1, src1Val) ||
  6709. (src2Val && IsWorthSpecializingToInt32DueToSrc(src2, src2Val)))
  6710. {
  6711. return true;
  6712. }
  6713. IR::Opnd *dst = instr->GetDst();
  6714. if (!dst || IsWorthSpecializingToInt32DueToDst(dst))
  6715. {
  6716. return true;
  6717. }
  6718. if (dst->IsEqual(src1) || (src2Val && (dst->IsEqual(src2) || src1->IsEqual(src2))))
  6719. {
  6720. return true;
  6721. }
  6722. IR::Instr *instrNext = instr->GetNextRealInstrOrLabel();
  6723. // Skip useless Ld_A's
  6724. do
  6725. {
  6726. switch (instrNext->m_opcode)
  6727. {
  6728. case Js::OpCode::Ld_A:
  6729. if (!dst->IsEqual(instrNext->GetSrc1()))
  6730. {
  6731. goto done;
  6732. }
  6733. dst = instrNext->GetDst();
  6734. break;
  6735. case Js::OpCode::LdFld:
  6736. case Js::OpCode::LdRootFld:
  6737. case Js::OpCode::LdRootFldForTypeOf:
  6738. case Js::OpCode::LdFldForTypeOf:
  6739. case Js::OpCode::LdElemI_A:
  6740. case Js::OpCode::ByteCodeUses:
  6741. break;
  6742. default:
  6743. goto done;
  6744. }
  6745. instrNext = instrNext->GetNextRealInstrOrLabel();
  6746. } while (true);
  6747. done:
  6748. // If the next instr could also be type specialized, then it is probably worth it.
  6749. if ((instrNext->GetSrc1() && dst->IsEqual(instrNext->GetSrc1())) || (instrNext->GetSrc2() && dst->IsEqual(instrNext->GetSrc2())))
  6750. {
  6751. switch (instrNext->m_opcode)
  6752. {
  6753. case Js::OpCode::Add_A:
  6754. case Js::OpCode::Sub_A:
  6755. case Js::OpCode::Mul_A:
  6756. case Js::OpCode::Div_A:
  6757. case Js::OpCode::Rem_A:
  6758. case Js::OpCode::Xor_A:
  6759. case Js::OpCode::And_A:
  6760. case Js::OpCode::Or_A:
  6761. case Js::OpCode::Shl_A:
  6762. case Js::OpCode::Shr_A:
  6763. case Js::OpCode::Incr_A:
  6764. case Js::OpCode::Decr_A:
  6765. case Js::OpCode::Neg_A:
  6766. case Js::OpCode::Not_A:
  6767. case Js::OpCode::Conv_Num:
  6768. case Js::OpCode::BrEq_I4:
  6769. case Js::OpCode::BrTrue_I4:
  6770. case Js::OpCode::BrFalse_I4:
  6771. case Js::OpCode::BrGe_I4:
  6772. case Js::OpCode::BrGt_I4:
  6773. case Js::OpCode::BrLt_I4:
  6774. case Js::OpCode::BrLe_I4:
  6775. case Js::OpCode::BrNeq_I4:
  6776. return true;
  6777. }
  6778. }
  6779. return false;
  6780. }
  6781. bool
  6782. GlobOpt::TypeSpecializeNumberUnary(IR::Instr *instr, Value *src1Val, Value **pDstVal)
  6783. {
  6784. Assert(src1Val->GetValueInfo()->IsNumber());
  6785. if (this->IsLoopPrePass())
  6786. {
  6787. return false;
  6788. }
  6789. switch (instr->m_opcode)
  6790. {
  6791. case Js::OpCode::Conv_Num:
  6792. // Optimize Conv_Num away since we know this is a number
  6793. instr->m_opcode = Js::OpCode::Ld_A;
  6794. return false;
  6795. }
  6796. return false;
  6797. }
  6798. bool
  6799. GlobOpt::TypeSpecializeUnary(
  6800. IR::Instr **pInstr,
  6801. Value **pSrc1Val,
  6802. Value **pDstVal,
  6803. Value *const src1OriginalVal,
  6804. bool *redoTypeSpecRef,
  6805. bool *const forceInvariantHoistingRef)
  6806. {
  6807. Assert(pSrc1Val);
  6808. Value *&src1Val = *pSrc1Val;
  6809. Assert(src1Val);
  6810. // We don't need to do typespec for asmjs
  6811. if (IsTypeSpecPhaseOff(this->func) || GetIsAsmJSFunc())
  6812. {
  6813. return false;
  6814. }
  6815. IR::Instr *&instr = *pInstr;
  6816. int32 min, max;
  6817. // Inline built-ins explicitly specify how srcs/dst must be specialized.
  6818. if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  6819. {
  6820. TypeSpecializeInlineBuiltInUnary(pInstr, &src1Val, pDstVal, src1OriginalVal, redoTypeSpecRef);
  6821. return true;
  6822. }
  6823. // Consider: If type spec wasn't completely done, make sure that we don't type-spec the dst 2nd time.
  6824. if(instr->m_opcode == Js::OpCode::LdLen_A && TypeSpecializeLdLen(&instr, &src1Val, pDstVal, forceInvariantHoistingRef))
  6825. {
  6826. return true;
  6827. }
  6828. if (!src1Val->GetValueInfo()->GetIntValMinMax(&min, &max, this->DoAggressiveIntTypeSpec()))
  6829. {
  6830. src1Val = src1OriginalVal;
  6831. if (src1Val->GetValueInfo()->IsLikelyFloat())
  6832. {
  6833. // Try to type specialize to float
  6834. return this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal);
  6835. }
  6836. else if (src1Val->GetValueInfo()->IsNumber())
  6837. {
  6838. return TypeSpecializeNumberUnary(instr, src1Val, pDstVal);
  6839. }
  6840. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  6841. }
  6842. return this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, min, max, src1OriginalVal, redoTypeSpecRef);
  6843. }
  6844. // Returns true if the built-in requested type specialization, and no further action needed,
  6845. // otherwise returns false.
  6846. void
  6847. GlobOpt::TypeSpecializeInlineBuiltInUnary(IR::Instr **pInstr, Value **pSrc1Val, Value **pDstVal, Value *const src1OriginalVal, bool *redoTypeSpecRef)
  6848. {
  6849. IR::Instr *&instr = *pInstr;
  6850. Assert(pSrc1Val);
  6851. Value *&src1Val = *pSrc1Val;
  6852. Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  6853. Js::BuiltinFunction builtInId = Js::JavascriptLibrary::GetBuiltInInlineCandidateId(instr->m_opcode); // From actual instr, not profile based.
  6854. Assert(builtInId != Js::BuiltinFunction::None);
  6855. // Consider using different bailout for float/int FromVars, so that when the arg cannot be converted to number we don't disable
  6856. // type spec for other parts of the big function but rather just don't inline that built-in instr.
  6857. // E.g. could do that if the value is not likelyInt/likelyFloat.
  6858. Js::BuiltInFlags builtInFlags = Js::JavascriptLibrary::GetFlagsForBuiltIn(builtInId);
  6859. bool areAllArgsAlwaysFloat = (builtInFlags & Js::BuiltInFlags::BIF_Args) == Js::BuiltInFlags::BIF_TypeSpecUnaryToFloat;
  6860. if (areAllArgsAlwaysFloat)
  6861. {
  6862. // InlineMathAcos, InlineMathAsin, InlineMathAtan, InlineMathCos, InlineMathExp, InlineMathLog, InlineMathSin, InlineMathSqrt, InlineMathTan.
  6863. Assert(this->DoFloatTypeSpec());
  6864. // Type-spec the src.
  6865. src1Val = src1OriginalVal;
  6866. bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, /* skipDst = */ true);
  6867. AssertMsg(retVal, "For inline built-ins the args have to be type-specialized to float, but something failed during the process.");
  6868. // Type-spec the dst.
  6869. this->TypeSpecializeFloatDst(instr, nullptr, src1Val, nullptr, pDstVal);
  6870. }
  6871. else if (instr->m_opcode == Js::OpCode::InlineMathAbs)
  6872. {
  6873. // Consider the case when the value is unknown - because of bailout in abs we may disable type spec for the whole function which is too much.
  6874. // First, try int.
  6875. int minVal, maxVal;
  6876. bool shouldTypeSpecToInt = src1Val->GetValueInfo()->GetIntValMinMax(&minVal, &maxVal, /* doAggressiveIntTypeSpec = */ true);
  6877. if (shouldTypeSpecToInt)
  6878. {
  6879. Assert(this->DoAggressiveIntTypeSpec());
  6880. bool retVal = this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, minVal, maxVal, src1OriginalVal, redoTypeSpecRef, true);
  6881. AssertMsg(retVal, "For inline built-ins the args have to be type-specialized (int), but something failed during the process.");
  6882. if (!this->IsLoopPrePass())
  6883. {
  6884. // Create bailout for INT_MIN which does not have corresponding int value on the positive side.
  6885. // Check int range: if we know the range is out of overflow, we do not need the bail out at all.
  6886. if (minVal == INT32_MIN)
  6887. {
  6888. GenerateBailAtOperation(&instr, IR::BailOnIntMin);
  6889. }
  6890. }
  6891. // Account for ::abs(INT_MIN) == INT_MIN (which is less than 0).
  6892. maxVal = ::max(
  6893. ::abs(Int32Math::NearestInRangeTo(minVal, INT_MIN + 1, INT_MAX)),
  6894. ::abs(Int32Math::NearestInRangeTo(maxVal, INT_MIN + 1, INT_MAX)));
  6895. minVal = minVal >= 0 ? minVal : 0;
  6896. this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, nullptr, IR::BailOutInvalid, minVal, maxVal, pDstVal);
  6897. }
  6898. else
  6899. {
  6900. // If we couldn't do int, do float.
  6901. Assert(this->DoFloatTypeSpec());
  6902. src1Val = src1OriginalVal;
  6903. bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, true);
  6904. AssertMsg(retVal, "For inline built-ins the args have to be type-specialized (float), but something failed during the process.");
  6905. this->TypeSpecializeFloatDst(instr, nullptr, src1Val, nullptr, pDstVal);
  6906. }
  6907. }
  6908. else if (instr->m_opcode == Js::OpCode::InlineMathFloor || instr->m_opcode == Js::OpCode::InlineMathCeil || instr->m_opcode == Js::OpCode::InlineMathRound)
  6909. {
  6910. // Type specialize src to float
  6911. src1Val = src1OriginalVal;
  6912. bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, /* skipDst = */ true);
  6913. AssertMsg(retVal, "For inline Math.floor and Math.ceil the src has to be type-specialized to float, but something failed during the process.");
  6914. // Type specialize dst to int
  6915. this->TypeSpecializeIntDst(
  6916. instr,
  6917. instr->m_opcode,
  6918. nullptr,
  6919. src1Val,
  6920. nullptr,
  6921. IR::BailOutInvalid,
  6922. INT32_MIN,
  6923. INT32_MAX,
  6924. pDstVal);
  6925. }
  6926. else if(instr->m_opcode == Js::OpCode::InlineArrayPop)
  6927. {
  6928. IR::Opnd *const thisOpnd = instr->GetSrc1();
  6929. Assert(thisOpnd);
  6930. // Ensure src1 (Array) is a var
  6931. this->ToVarUses(instr, thisOpnd, false, src1Val);
  6932. if(!this->IsLoopPrePass() && thisOpnd->GetValueType().IsLikelyNativeArray())
  6933. {
  6934. // We bail out, if there is illegal access or a mismatch in the Native array type that is optimized for, during the run time.
  6935. GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly);
  6936. }
  6937. if(!instr->GetDst())
  6938. {
  6939. return;
  6940. }
  6941. // Try Type Specializing the element (return item from Pop) based on the array's profile data.
  6942. if(thisOpnd->GetValueType().IsLikelyNativeIntArray())
  6943. {
  6944. this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, nullptr, nullptr, IR::BailOutInvalid, INT32_MIN, INT32_MAX, pDstVal);
  6945. }
  6946. else if(thisOpnd->GetValueType().IsLikelyNativeFloatArray())
  6947. {
  6948. this->TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, pDstVal);
  6949. }
  6950. else
  6951. {
  6952. // We reached here so the Element is not yet type specialized. Ensure element is a var
  6953. if(instr->GetDst()->IsRegOpnd())
  6954. {
  6955. this->ToVarRegOpnd(instr->GetDst()->AsRegOpnd(), currentBlock);
  6956. }
  6957. }
  6958. }
  6959. else if (instr->m_opcode == Js::OpCode::InlineMathClz)
  6960. {
  6961. Assert(this->DoAggressiveIntTypeSpec());
  6962. Assert(this->DoLossyIntTypeSpec());
  6963. //Type specialize to int
  6964. bool retVal = this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, INT32_MIN, INT32_MAX, src1OriginalVal, redoTypeSpecRef);
  6965. AssertMsg(retVal, "For clz32, the arg has to be type-specialized to int.");
  6966. }
  6967. else
  6968. {
  6969. AssertMsg(FALSE, "Unsupported built-in!");
  6970. }
  6971. }
  6972. void
  6973. GlobOpt::TypeSpecializeInlineBuiltInBinary(IR::Instr **pInstr, Value *src1Val, Value* src2Val, Value **pDstVal, Value *const src1OriginalVal, Value *const src2OriginalVal)
  6974. {
  6975. IR::Instr *&instr = *pInstr;
  6976. Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  6977. switch(instr->m_opcode)
  6978. {
  6979. case Js::OpCode::InlineMathAtan2:
  6980. {
  6981. Js::BuiltinFunction builtInId = Js::JavascriptLibrary::GetBuiltInInlineCandidateId(instr->m_opcode); // From actual instr, not profile based.
  6982. Js::BuiltInFlags builtInFlags = Js::JavascriptLibrary::GetFlagsForBuiltIn(builtInId);
  6983. bool areAllArgsAlwaysFloat = (builtInFlags & Js::BuiltInFlags::BIF_TypeSpecAllToFloat) != 0;
  6984. Assert(areAllArgsAlwaysFloat);
  6985. Assert(this->DoFloatTypeSpec());
  6986. // Type-spec the src1, src2 and dst.
  6987. src1Val = src1OriginalVal;
  6988. src2Val = src2OriginalVal;
  6989. bool retVal = this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  6990. AssertMsg(retVal, "For pow and atnan2 the args have to be type-specialized to float, but something failed during the process.");
  6991. break;
  6992. }
  6993. case Js::OpCode::InlineMathPow:
  6994. {
  6995. #ifndef _M_ARM32_OR_ARM64
  6996. if (src2Val->GetValueInfo()->IsLikelyInt())
  6997. {
  6998. bool lossy = false;
  6999. this->ToInt32(instr, instr->GetSrc2(), this->currentBlock, src2Val, nullptr, lossy);
  7000. IR::Opnd* src1 = instr->GetSrc1();
  7001. int32 valueMin, valueMax;
  7002. if (src1Val->GetValueInfo()->IsLikelyInt() &&
  7003. this->DoPowIntIntTypeSpec() &&
  7004. src2Val->GetValueInfo()->GetIntValMinMax(&valueMin, &valueMax, this->DoAggressiveIntTypeSpec()) &&
  7005. valueMin >= 0)
  7006. {
  7007. this->ToInt32(instr, src1, this->currentBlock, src1Val, nullptr, lossy);
  7008. this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, src2Val, IR::BailOutInvalid, INT32_MIN, INT32_MAX, pDstVal);
  7009. if(!this->IsLoopPrePass())
  7010. {
  7011. GenerateBailAtOperation(&instr, IR::BailOutOnPowIntIntOverflow);
  7012. }
  7013. }
  7014. else
  7015. {
  7016. this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, IR::BailOutPrimitiveButString);
  7017. TypeSpecializeFloatDst(instr, nullptr, src1Val, src2Val, pDstVal);
  7018. }
  7019. }
  7020. else
  7021. {
  7022. #endif
  7023. this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  7024. #ifndef _M_ARM32_OR_ARM64
  7025. }
  7026. #endif
  7027. break;
  7028. }
  7029. case Js::OpCode::InlineMathImul:
  7030. {
  7031. Assert(this->DoAggressiveIntTypeSpec());
  7032. Assert(this->DoLossyIntTypeSpec());
  7033. //Type specialize to int
  7034. bool retVal = this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, INT32_MIN, INT32_MAX, false /* skipDst */);
  7035. AssertMsg(retVal, "For imul, the args have to be type-specialized to int but something failed during the process.");
  7036. break;
  7037. }
  7038. case Js::OpCode::InlineMathMin:
  7039. case Js::OpCode::InlineMathMax:
  7040. {
  7041. if(src1Val->GetValueInfo()->IsLikelyInt() && src2Val->GetValueInfo()->IsLikelyInt())
  7042. {
  7043. // Compute resulting range info
  7044. int32 min1 = INT32_MIN;
  7045. int32 max1 = INT32_MAX;
  7046. int32 min2 = INT32_MIN;
  7047. int32 max2 = INT32_MAX;
  7048. int32 newMin, newMax;
  7049. Assert(this->DoAggressiveIntTypeSpec());
  7050. src1Val->GetValueInfo()->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec());
  7051. src2Val->GetValueInfo()->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec());
  7052. if (instr->m_opcode == Js::OpCode::InlineMathMin)
  7053. {
  7054. newMin = min(min1, min2);
  7055. newMax = min(max1, max2);
  7056. }
  7057. else
  7058. {
  7059. Assert(instr->m_opcode == Js::OpCode::InlineMathMax);
  7060. newMin = max(min1, min2);
  7061. newMax = max(max1, max2);
  7062. }
  7063. // Type specialize to int
  7064. bool retVal = this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, newMin, newMax, false /* skipDst */);
  7065. AssertMsg(retVal, "For min and max, the args have to be type-specialized to int if any one of the sources is an int, but something failed during the process.");
  7066. }
  7067. // Couldn't type specialize to int, type specialize to float
  7068. else
  7069. {
  7070. Assert(this->DoFloatTypeSpec());
  7071. src1Val = src1OriginalVal;
  7072. src2Val = src2OriginalVal;
  7073. bool retVal = this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  7074. AssertMsg(retVal, "For min and max, the args have to be type-specialized to float if any one of the sources is a float, but something failed during the process.");
  7075. }
  7076. break;
  7077. }
  7078. case Js::OpCode::InlineArrayPush:
  7079. {
  7080. IR::Opnd *const thisOpnd = instr->GetSrc1();
  7081. Assert(thisOpnd);
  7082. if(instr->GetDst() && instr->GetDst()->IsRegOpnd())
  7083. {
  7084. // Set the dst as live here, as the built-ins return early from the TypeSpecialization functions - before the dst is marked as live.
  7085. // Also, we are not specializing the dst separately and we are skipping the dst to be handled when we specialize the instruction above.
  7086. this->ToVarRegOpnd(instr->GetDst()->AsRegOpnd(), currentBlock);
  7087. }
  7088. // Ensure src1 (Array) is a var
  7089. this->ToVarUses(instr, thisOpnd, false, src1Val);
  7090. if(!this->IsLoopPrePass())
  7091. {
  7092. if(thisOpnd->GetValueType().IsLikelyNativeArray())
  7093. {
  7094. // We bail out, if there is illegal access or a mismatch in the Native array type that is optimized for, during run time.
  7095. GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly);
  7096. }
  7097. else
  7098. {
  7099. GenerateBailAtOperation(&instr, IR::BailOutOnImplicitCallsPreOp);
  7100. }
  7101. }
  7102. // Try Type Specializing the element based on the array's profile data.
  7103. if(thisOpnd->GetValueType().IsLikelyNativeFloatArray())
  7104. {
  7105. src1Val = src1OriginalVal;
  7106. src2Val = src2OriginalVal;
  7107. }
  7108. if((thisOpnd->GetValueType().IsLikelyNativeIntArray() && this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, INT32_MIN, INT32_MAX, true))
  7109. || (thisOpnd->GetValueType().IsLikelyNativeFloatArray() && this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal)))
  7110. {
  7111. break;
  7112. }
  7113. // The Element is not yet type specialized. Ensure element is a var
  7114. this->ToVarUses(instr, instr->GetSrc2(), false, src2Val);
  7115. break;
  7116. }
  7117. }
  7118. }
  7119. void
  7120. GlobOpt::TypeSpecializeInlineBuiltInDst(IR::Instr **pInstr, Value **pDstVal)
  7121. {
  7122. IR::Instr *&instr = *pInstr;
  7123. Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  7124. if (instr->m_opcode == Js::OpCode::InlineMathRandom)
  7125. {
  7126. Assert(this->DoFloatTypeSpec());
  7127. // Type specialize dst to float
  7128. this->TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, pDstVal);
  7129. }
  7130. }
  7131. bool
  7132. GlobOpt::TryTypeSpecializeUnaryToFloatHelper(IR::Instr** pInstr, Value** pSrc1Val, Value* const src1OriginalVal, Value **pDstVal)
  7133. {
  7134. // It has been determined that this instruction cannot be int-specialized. We need to determine whether to attempt to
  7135. // float-specialize the instruction, or leave it unspecialized.
  7136. #if !INT32VAR
  7137. Value*& src1Val = *pSrc1Val;
  7138. if(src1Val->GetValueInfo()->IsLikelyUntaggedInt())
  7139. {
  7140. // An input range is completely outside the range of an int31. Even if the operation may overflow, it is
  7141. // unlikely to overflow on these operations, so we leave it unspecialized on 64-bit platforms. However, on
  7142. // 32-bit platforms, the value is untaggable and will be a JavascriptNumber, which is significantly slower to
  7143. // use in an unspecialized operation compared to a tagged int. So, try to float-specialize the instruction.
  7144. src1Val = src1OriginalVal;
  7145. return this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal);
  7146. }
  7147. #endif
  7148. return false;
  7149. }
  7150. bool
  7151. GlobOpt::TypeSpecializeIntBinary(IR::Instr **pInstr, Value *src1Val, Value *src2Val, Value **pDstVal, int32 min, int32 max, bool skipDst /* = false */)
  7152. {
  7153. // Consider moving the code for int type spec-ing binary functions here.
  7154. IR::Instr *&instr = *pInstr;
  7155. bool lossy = false;
  7156. if(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  7157. {
  7158. if(instr->m_opcode == Js::OpCode::InlineArrayPush)
  7159. {
  7160. int32 intConstantValue;
  7161. bool isIntConstMissingItem = src2Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue);
  7162. if(isIntConstMissingItem)
  7163. {
  7164. isIntConstMissingItem = Js::SparseArraySegment<int>::IsMissingItem(&intConstantValue);
  7165. }
  7166. // Don't specialize if the element is not likelyInt or an IntConst which is a missing item value.
  7167. if(!(src2Val->GetValueInfo()->IsLikelyInt()) || isIntConstMissingItem)
  7168. {
  7169. return false;
  7170. }
  7171. // We don't want to specialize both the source operands, though it is a binary instr.
  7172. IR::Opnd * elementOpnd = instr->GetSrc2();
  7173. this->ToInt32(instr, elementOpnd, this->currentBlock, src2Val, nullptr, lossy);
  7174. }
  7175. else
  7176. {
  7177. IR::Opnd *src1 = instr->GetSrc1();
  7178. this->ToInt32(instr, src1, this->currentBlock, src1Val, nullptr, lossy);
  7179. IR::Opnd *src2 = instr->GetSrc2();
  7180. this->ToInt32(instr, src2, this->currentBlock, src2Val, nullptr, lossy);
  7181. }
  7182. if(!skipDst)
  7183. {
  7184. IR::Opnd *dst = instr->GetDst();
  7185. if (dst)
  7186. {
  7187. TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, src2Val, IR::BailOutInvalid, min, max, pDstVal);
  7188. }
  7189. }
  7190. return true;
  7191. }
  7192. else
  7193. {
  7194. AssertMsg(false, "Yet to move code for other binary functions here");
  7195. return false;
  7196. }
  7197. }
  7198. bool
  7199. GlobOpt::TypeSpecializeIntUnary(
  7200. IR::Instr **pInstr,
  7201. Value **pSrc1Val,
  7202. Value **pDstVal,
  7203. int32 min,
  7204. int32 max,
  7205. Value *const src1OriginalVal,
  7206. bool *redoTypeSpecRef,
  7207. bool skipDst /* = false */)
  7208. {
  7209. IR::Instr *&instr = *pInstr;
  7210. Assert(pSrc1Val);
  7211. Value *&src1Val = *pSrc1Val;
  7212. bool isTransfer = false;
  7213. Js::OpCode opcode;
  7214. int32 newMin, newMax;
  7215. bool lossy = false;
  7216. IR::BailOutKind bailOutKind = IR::BailOutInvalid;
  7217. bool ignoredIntOverflow = this->ignoredIntOverflowForCurrentInstr;
  7218. bool ignoredNegativeZero = false;
  7219. bool checkTypeSpecWorth = false;
  7220. if(instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber)
  7221. {
  7222. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7223. }
  7224. AddSubConstantInfo addSubConstantInfo;
  7225. switch(instr->m_opcode)
  7226. {
  7227. case Js::OpCode::Ld_A:
  7228. if (instr->GetSrc1()->IsRegOpnd())
  7229. {
  7230. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  7231. if (CurrentBlockData()->IsInt32TypeSpecialized(sym) == false)
  7232. {
  7233. // Type specializing an Ld_A isn't worth it, unless the src
  7234. // is already type specialized.
  7235. return false;
  7236. }
  7237. }
  7238. newMin = min;
  7239. newMax = max;
  7240. opcode = Js::OpCode::Ld_I4;
  7241. isTransfer = true;
  7242. break;
  7243. case Js::OpCode::Conv_Num:
  7244. newMin = min;
  7245. newMax = max;
  7246. opcode = Js::OpCode::Ld_I4;
  7247. isTransfer = true;
  7248. break;
  7249. case Js::OpCode::LdC_A_I4:
  7250. newMin = newMax = instr->GetSrc1()->AsIntConstOpnd()->AsInt32();
  7251. opcode = Js::OpCode::Ld_I4;
  7252. break;
  7253. case Js::OpCode::Neg_A:
  7254. if (min <= 0 && max >= 0)
  7255. {
  7256. if(instr->ShouldCheckForNegativeZero())
  7257. {
  7258. // -0 matters since the sym is not a local, or is used in a way in which -0 would differ from +0
  7259. if(!DoAggressiveIntTypeSpec())
  7260. {
  7261. // May result in -0
  7262. // Consider adding a dynamic check for src1 == 0
  7263. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7264. }
  7265. if(min == 0 && max == 0)
  7266. {
  7267. // Always results in -0
  7268. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7269. }
  7270. bailOutKind |= IR::BailOutOnNegativeZero;
  7271. }
  7272. else
  7273. {
  7274. ignoredNegativeZero = true;
  7275. }
  7276. }
  7277. if (Int32Math::Neg(min, &newMax))
  7278. {
  7279. if(instr->ShouldCheckForIntOverflow())
  7280. {
  7281. if(!DoAggressiveIntTypeSpec())
  7282. {
  7283. // May overflow
  7284. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7285. }
  7286. if(min == max)
  7287. {
  7288. // Always overflows
  7289. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7290. }
  7291. bailOutKind |= IR::BailOutOnOverflow;
  7292. newMax = INT32_MAX;
  7293. }
  7294. else
  7295. {
  7296. ignoredIntOverflow = true;
  7297. }
  7298. }
  7299. if (Int32Math::Neg(max, &newMin))
  7300. {
  7301. if(instr->ShouldCheckForIntOverflow())
  7302. {
  7303. if(!DoAggressiveIntTypeSpec())
  7304. {
  7305. // May overflow
  7306. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7307. }
  7308. bailOutKind |= IR::BailOutOnOverflow;
  7309. newMin = INT32_MAX;
  7310. }
  7311. else
  7312. {
  7313. ignoredIntOverflow = true;
  7314. }
  7315. }
  7316. if(!instr->ShouldCheckForIntOverflow() && newMin > newMax)
  7317. {
  7318. // When ignoring overflow, the range needs to account for overflow. Since MIN_INT is the only int32 value that
  7319. // overflows on Neg, and the value resulting from overflow is also MIN_INT, if calculating only the new min or new
  7320. // max overflowed but not both, then the new min will be greater than the new max. In that case we need to consider
  7321. // the full range of int32s as possible resulting values.
  7322. newMin = INT32_MIN;
  7323. newMax = INT32_MAX;
  7324. }
  7325. opcode = Js::OpCode::Neg_I4;
  7326. checkTypeSpecWorth = true;
  7327. break;
  7328. case Js::OpCode::Not_A:
  7329. if(!DoLossyIntTypeSpec())
  7330. {
  7331. return false;
  7332. }
  7333. this->PropagateIntRangeForNot(min, max, &newMin, &newMax);
  7334. opcode = Js::OpCode::Not_I4;
  7335. lossy = true;
  7336. break;
  7337. case Js::OpCode::Incr_A:
  7338. do // while(false)
  7339. {
  7340. const auto CannotOverflowBasedOnRelativeBounds = [&]()
  7341. {
  7342. const ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  7343. return
  7344. (src1ValueInfo->IsInt() || DoAggressiveIntTypeSpec()) &&
  7345. src1ValueInfo->IsIntBounded() &&
  7346. src1ValueInfo->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(1);
  7347. };
  7348. if (Int32Math::Inc(min, &newMin))
  7349. {
  7350. if(CannotOverflowBasedOnRelativeBounds())
  7351. {
  7352. newMin = INT32_MAX;
  7353. }
  7354. else if(instr->ShouldCheckForIntOverflow())
  7355. {
  7356. // Always overflows
  7357. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7358. }
  7359. else
  7360. {
  7361. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow
  7362. // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints,
  7363. // we use the full range of int32s.
  7364. ignoredIntOverflow = true;
  7365. newMin = INT32_MIN;
  7366. newMax = INT32_MAX;
  7367. break;
  7368. }
  7369. }
  7370. if (Int32Math::Inc(max, &newMax))
  7371. {
  7372. if(CannotOverflowBasedOnRelativeBounds())
  7373. {
  7374. newMax = INT32_MAX;
  7375. }
  7376. else if(instr->ShouldCheckForIntOverflow())
  7377. {
  7378. if(!DoAggressiveIntTypeSpec())
  7379. {
  7380. // May overflow
  7381. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7382. }
  7383. bailOutKind |= IR::BailOutOnOverflow;
  7384. newMax = INT32_MAX;
  7385. }
  7386. else
  7387. {
  7388. // See comment about ignoring overflow above
  7389. ignoredIntOverflow = true;
  7390. newMin = INT32_MIN;
  7391. newMax = INT32_MAX;
  7392. break;
  7393. }
  7394. }
  7395. } while(false);
  7396. if(!ignoredIntOverflow && instr->GetSrc1()->IsRegOpnd())
  7397. {
  7398. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min == max, 1);
  7399. }
  7400. opcode = Js::OpCode::Add_I4;
  7401. if (!this->IsLoopPrePass())
  7402. {
  7403. instr->SetSrc2(IR::IntConstOpnd::New(1, TyInt32, instr->m_func));
  7404. }
  7405. checkTypeSpecWorth = true;
  7406. break;
  7407. case Js::OpCode::Decr_A:
  7408. do // while(false)
  7409. {
  7410. const auto CannotOverflowBasedOnRelativeBounds = [&]()
  7411. {
  7412. const ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  7413. return
  7414. (src1ValueInfo->IsInt() || DoAggressiveIntTypeSpec()) &&
  7415. src1ValueInfo->IsIntBounded() &&
  7416. src1ValueInfo->AsIntBounded()->Bounds()->SubCannotOverflowBasedOnRelativeBounds(1);
  7417. };
  7418. if (Int32Math::Dec(max, &newMax))
  7419. {
  7420. if(CannotOverflowBasedOnRelativeBounds())
  7421. {
  7422. newMax = INT32_MIN;
  7423. }
  7424. else if(instr->ShouldCheckForIntOverflow())
  7425. {
  7426. // Always overflows
  7427. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7428. }
  7429. else
  7430. {
  7431. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow
  7432. // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints, we
  7433. // use the full range of int32s.
  7434. ignoredIntOverflow = true;
  7435. newMin = INT32_MIN;
  7436. newMax = INT32_MAX;
  7437. break;
  7438. }
  7439. }
  7440. if (Int32Math::Dec(min, &newMin))
  7441. {
  7442. if(CannotOverflowBasedOnRelativeBounds())
  7443. {
  7444. newMin = INT32_MIN;
  7445. }
  7446. else if(instr->ShouldCheckForIntOverflow())
  7447. {
  7448. if(!DoAggressiveIntTypeSpec())
  7449. {
  7450. // May overflow
  7451. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7452. }
  7453. bailOutKind |= IR::BailOutOnOverflow;
  7454. newMin = INT32_MIN;
  7455. }
  7456. else
  7457. {
  7458. // See comment about ignoring overflow above
  7459. ignoredIntOverflow = true;
  7460. newMin = INT32_MIN;
  7461. newMax = INT32_MAX;
  7462. break;
  7463. }
  7464. }
  7465. } while(false);
  7466. if(!ignoredIntOverflow && instr->GetSrc1()->IsRegOpnd())
  7467. {
  7468. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min == max, -1);
  7469. }
  7470. opcode = Js::OpCode::Sub_I4;
  7471. if (!this->IsLoopPrePass())
  7472. {
  7473. instr->SetSrc2(IR::IntConstOpnd::New(1, TyInt32, instr->m_func));
  7474. }
  7475. checkTypeSpecWorth = true;
  7476. break;
  7477. case Js::OpCode::BrFalse_A:
  7478. case Js::OpCode::BrTrue_A:
  7479. {
  7480. if(DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrFalse(instr, src1Val, min, max))
  7481. {
  7482. return true;
  7483. }
  7484. bool specialize = true;
  7485. if (!src1Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc1()->IsRegOpnd())
  7486. {
  7487. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  7488. if (CurrentBlockData()->IsInt32TypeSpecialized(sym) == false)
  7489. {
  7490. // Type specializing a BrTrue_A/BrFalse_A isn't worth it, unless the src
  7491. // is already type specialized
  7492. specialize = false;
  7493. }
  7494. }
  7495. if(instr->m_opcode == Js::OpCode::BrTrue_A)
  7496. {
  7497. UpdateIntBoundsForNotEqualBranch(src1Val, nullptr, 0);
  7498. opcode = Js::OpCode::BrTrue_I4;
  7499. }
  7500. else
  7501. {
  7502. UpdateIntBoundsForEqualBranch(src1Val, nullptr, 0);
  7503. opcode = Js::OpCode::BrFalse_I4;
  7504. }
  7505. if(!specialize)
  7506. {
  7507. return false;
  7508. }
  7509. newMin = 2; newMax = 1; // We'll assert if we make a range where min > max
  7510. break;
  7511. }
  7512. case Js::OpCode::MultiBr:
  7513. newMin = min;
  7514. newMax = max;
  7515. opcode = instr->m_opcode;
  7516. break;
  7517. case Js::OpCode::StElemI_A:
  7518. case Js::OpCode::StElemI_A_Strict:
  7519. case Js::OpCode::StElemC:
  7520. if(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyAnyArrayWithNativeFloatValues())
  7521. {
  7522. src1Val = src1OriginalVal;
  7523. }
  7524. return TypeSpecializeStElem(pInstr, src1Val, pDstVal);
  7525. case Js::OpCode::NewScArray:
  7526. case Js::OpCode::NewScArrayWithMissingValues:
  7527. case Js::OpCode::InitFld:
  7528. case Js::OpCode::InitRootFld:
  7529. case Js::OpCode::StSlot:
  7530. case Js::OpCode::StSlotChkUndecl:
  7531. #if !FLOATVAR
  7532. case Js::OpCode::StSlotBoxTemp:
  7533. #endif
  7534. case Js::OpCode::StFld:
  7535. case Js::OpCode::StRootFld:
  7536. case Js::OpCode::StFldStrict:
  7537. case Js::OpCode::StRootFldStrict:
  7538. case Js::OpCode::ArgOut_A:
  7539. case Js::OpCode::ArgOut_A_Inline:
  7540. case Js::OpCode::ArgOut_A_FixupForStackArgs:
  7541. case Js::OpCode::ArgOut_A_Dynamic:
  7542. case Js::OpCode::ArgOut_A_FromStackArgs:
  7543. case Js::OpCode::ArgOut_A_SpreadArg:
  7544. // For this one we need to implement type specialization
  7545. //case Js::OpCode::ArgOut_A_InlineBuiltIn:
  7546. case Js::OpCode::Ret:
  7547. case Js::OpCode::LdElemUndef:
  7548. case Js::OpCode::LdElemUndefScoped:
  7549. return false;
  7550. default:
  7551. if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  7552. {
  7553. newMin = min;
  7554. newMax = max;
  7555. opcode = instr->m_opcode;
  7556. break; // Note: we must keep checkTypeSpecWorth = false to make sure we never return false from this function.
  7557. }
  7558. return false;
  7559. }
  7560. // If this instruction is in a range of instructions where int overflow does not matter, we will still specialize it (won't
  7561. // leave it unspecialized based on heuristics), since it is most likely worth specializing, and the dst value needs to be
  7562. // guaranteed to be an int
  7563. if(checkTypeSpecWorth &&
  7564. !ignoredIntOverflow &&
  7565. !ignoredNegativeZero &&
  7566. instr->ShouldCheckForIntOverflow() &&
  7567. !IsWorthSpecializingToInt32(instr, src1Val))
  7568. {
  7569. // Even though type specialization is being skipped since it may not be worth it, the proper value should still be
  7570. // maintained so that the result may be type specialized later. An int value is not created for the dst in any of
  7571. // the following cases.
  7572. // - A bailout check is necessary to specialize this instruction. The bailout check is what guarantees the result to be
  7573. // an int, but since we're not going to specialize this instruction, there won't be a bailout check.
  7574. // - Aggressive int type specialization is disabled and we're in a loop prepass. We're conservative on dst values in
  7575. // that case, especially if the dst sym is live on the back-edge.
  7576. if(bailOutKind == IR::BailOutInvalid &&
  7577. instr->GetDst() &&
  7578. (DoAggressiveIntTypeSpec() || !this->IsLoopPrePass()))
  7579. {
  7580. *pDstVal = CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, nullptr);
  7581. }
  7582. if(instr->GetSrc2())
  7583. {
  7584. instr->FreeSrc2();
  7585. }
  7586. return false;
  7587. }
  7588. this->ignoredIntOverflowForCurrentInstr = ignoredIntOverflow;
  7589. this->ignoredNegativeZeroForCurrentInstr = ignoredNegativeZero;
  7590. {
  7591. // Try CSE again before modifying the IR, in case some attributes are required for successful CSE
  7592. Value *src1IndirIndexVal = nullptr;
  7593. Value *src2Val = nullptr;
  7594. if(CSEOptimize(currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal, true /* intMathExprOnly */))
  7595. {
  7596. *redoTypeSpecRef = true;
  7597. return false;
  7598. }
  7599. }
  7600. const Js::OpCode originalOpCode = instr->m_opcode;
  7601. if (!this->IsLoopPrePass())
  7602. {
  7603. // No re-write on prepass
  7604. instr->m_opcode = opcode;
  7605. }
  7606. Value *src1ValueToSpecialize = src1Val;
  7607. if(lossy)
  7608. {
  7609. // Lossy conversions to int32 must be done based on the original source values. For instance, if one of the values is a
  7610. // float constant with a value that fits in a uint32 but not an int32, and the instruction can ignore int overflow, the
  7611. // source value for the purposes of int specialization would have been changed to an int constant value by ignoring
  7612. // overflow. If we were to specialize the sym using the int constant value, it would be treated as a lossless
  7613. // conversion, but since there may be subsequent uses of the same float constant value that may not ignore overflow,
  7614. // this must be treated as a lossy conversion by specializing the sym using the original float constant value.
  7615. src1ValueToSpecialize = src1OriginalVal;
  7616. }
  7617. // Make sure the srcs are specialized
  7618. IR::Opnd *src1 = instr->GetSrc1();
  7619. this->ToInt32(instr, src1, this->currentBlock, src1ValueToSpecialize, nullptr, lossy);
  7620. if(bailOutKind != IR::BailOutInvalid && !this->IsLoopPrePass())
  7621. {
  7622. GenerateBailAtOperation(&instr, bailOutKind);
  7623. }
  7624. if (!skipDst)
  7625. {
  7626. IR::Opnd *dst = instr->GetDst();
  7627. if (dst)
  7628. {
  7629. AssertMsg(!(isTransfer && !this->IsLoopPrePass()) || min == newMin && max == newMax, "If this is just a copy, old/new min/max should be the same");
  7630. TypeSpecializeIntDst(
  7631. instr,
  7632. originalOpCode,
  7633. isTransfer ? src1Val : nullptr,
  7634. src1Val,
  7635. nullptr,
  7636. bailOutKind,
  7637. newMin,
  7638. newMax,
  7639. pDstVal,
  7640. addSubConstantInfo.HasInfo() ? &addSubConstantInfo : nullptr);
  7641. }
  7642. }
  7643. if(bailOutKind == IR::BailOutInvalid)
  7644. {
  7645. GOPT_TRACE(_u("Type specialized to INT\n"));
  7646. #if ENABLE_DEBUG_CONFIG_OPTIONS
  7647. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  7648. {
  7649. Output::Print(_u("Type specialized to INT: "));
  7650. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  7651. }
  7652. #endif
  7653. }
  7654. else
  7655. {
  7656. GOPT_TRACE(_u("Type specialized to INT with bailout on:\n"));
  7657. if(bailOutKind & IR::BailOutOnOverflow)
  7658. {
  7659. GOPT_TRACE(_u(" Overflow\n"));
  7660. #if ENABLE_DEBUG_CONFIG_OPTIONS
  7661. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  7662. {
  7663. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Overflow");
  7664. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  7665. }
  7666. #endif
  7667. }
  7668. if(bailOutKind & IR::BailOutOnNegativeZero)
  7669. {
  7670. GOPT_TRACE(_u(" Zero\n"));
  7671. #if ENABLE_DEBUG_CONFIG_OPTIONS
  7672. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  7673. {
  7674. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Zero");
  7675. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  7676. }
  7677. #endif
  7678. }
  7679. }
  7680. return true;
  7681. }
  7682. void
  7683. GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, int32 newMin, int32 newMax, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo)
  7684. {
  7685. this->TypeSpecializeIntDst(instr, originalOpCode, valToTransfer, src1Value, src2Value, bailOutKind, ValueType::GetInt(IntConstantBounds(newMin, newMax).IsLikelyTaggable()), newMin, newMax, pDstVal, addSubConstantInfo);
  7686. }
  7687. void
  7688. GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, ValueType valueType, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo)
  7689. {
  7690. this->TypeSpecializeIntDst(instr, originalOpCode, valToTransfer, src1Value, src2Value, bailOutKind, valueType, 0, 0, pDstVal, addSubConstantInfo);
  7691. }
  7692. void
  7693. GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, ValueType valueType, int32 newMin, int32 newMax, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo)
  7694. {
  7695. Assert(valueType.IsInt() || (valueType.IsNumber() && valueType.IsLikelyInt() && newMin == 0 && newMax == 0));
  7696. Assert(!valToTransfer || valToTransfer == src1Value);
  7697. Assert(!addSubConstantInfo || addSubConstantInfo->HasInfo());
  7698. IR::Opnd *dst = instr->GetDst();
  7699. Assert(dst);
  7700. bool isValueInfoPrecise;
  7701. if(IsLoopPrePass())
  7702. {
  7703. isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value);
  7704. valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, isValueInfoPrecise);
  7705. }
  7706. else
  7707. {
  7708. isValueInfoPrecise = true;
  7709. }
  7710. // If dst has a circular reference in a loop, it probably won't get specialized. Don't mark the dst as type-specialized on
  7711. // the pre-pass. With aggressive int spec though, it will take care of bailing out if necessary so there's no need to assume
  7712. // that the dst will be a var even if it's live on the back-edge. Also if the op always produces an int32, then there's no
  7713. // ambiguity in the dst's value type even in the prepass.
  7714. if (!DoAggressiveIntTypeSpec() && this->IsLoopPrePass() && !valueType.IsInt())
  7715. {
  7716. if (dst->IsRegOpnd())
  7717. {
  7718. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  7719. }
  7720. return;
  7721. }
  7722. const IntBounds *dstBounds = nullptr;
  7723. if(addSubConstantInfo && !addSubConstantInfo->SrcValueIsLikelyConstant() && DoTrackRelativeIntBounds())
  7724. {
  7725. Assert(!ignoredIntOverflowForCurrentInstr);
  7726. // Track bounds for add or sub with a constant. For instance, consider (b = a + 2). The value of 'b' should track that
  7727. // it is equal to (the value of 'a') + 2. Additionally, the value of 'b' should inherit the bounds of 'a', offset by
  7728. // the constant value.
  7729. if(!valueType.IsInt() || !isValueInfoPrecise)
  7730. {
  7731. newMin = INT32_MIN;
  7732. newMax = INT32_MAX;
  7733. }
  7734. dstBounds =
  7735. IntBounds::Add(
  7736. addSubConstantInfo->SrcValue(),
  7737. addSubConstantInfo->Offset(),
  7738. isValueInfoPrecise,
  7739. IntConstantBounds(newMin, newMax),
  7740. alloc);
  7741. }
  7742. // Src1's value could change later in the loop, so the value wouldn't be the same for each
  7743. // iteration. Since we don't iterate over loops "while (!changed)", go conservative on the
  7744. // pre-pass.
  7745. if (valToTransfer)
  7746. {
  7747. // If this is just a copy, no need for creating a new value.
  7748. Assert(!addSubConstantInfo);
  7749. *pDstVal = this->ValueNumberTransferDst(instr, valToTransfer);
  7750. CurrentBlockData()->InsertNewValue(*pDstVal, dst);
  7751. }
  7752. else if (valueType.IsInt() && isValueInfoPrecise)
  7753. {
  7754. bool wasNegativeZeroPreventedByBailout = false;
  7755. if(newMin <= 0 && newMax >= 0)
  7756. {
  7757. switch(originalOpCode)
  7758. {
  7759. case Js::OpCode::Add_A:
  7760. // -0 + -0 == -0
  7761. Assert(src1Value);
  7762. Assert(src2Value);
  7763. wasNegativeZeroPreventedByBailout =
  7764. src1Value->GetValueInfo()->WasNegativeZeroPreventedByBailout() &&
  7765. src2Value->GetValueInfo()->WasNegativeZeroPreventedByBailout();
  7766. break;
  7767. case Js::OpCode::Sub_A:
  7768. // -0 - 0 == -0
  7769. Assert(src1Value);
  7770. wasNegativeZeroPreventedByBailout = src1Value->GetValueInfo()->WasNegativeZeroPreventedByBailout();
  7771. break;
  7772. case Js::OpCode::Neg_A:
  7773. case Js::OpCode::Mul_A:
  7774. case Js::OpCode::Div_A:
  7775. case Js::OpCode::Rem_A:
  7776. wasNegativeZeroPreventedByBailout = !!(bailOutKind & IR::BailOutOnNegativeZero);
  7777. break;
  7778. }
  7779. }
  7780. *pDstVal =
  7781. dstBounds
  7782. ? NewIntBoundedValue(valueType, dstBounds, wasNegativeZeroPreventedByBailout, nullptr)
  7783. : NewIntRangeValue(newMin, newMax, wasNegativeZeroPreventedByBailout, nullptr);
  7784. }
  7785. else
  7786. {
  7787. *pDstVal = dstBounds ? NewIntBoundedValue(valueType, dstBounds, false, nullptr) : NewGenericValue(valueType);
  7788. }
  7789. if(addSubConstantInfo || updateInductionVariableValueNumber)
  7790. {
  7791. TrackIntSpecializedAddSubConstant(instr, addSubConstantInfo, *pDstVal, !!dstBounds);
  7792. }
  7793. CurrentBlockData()->SetValue(*pDstVal, dst);
  7794. AssertMsg(dst->IsRegOpnd(), "What else?");
  7795. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  7796. }
  7797. bool
  7798. GlobOpt::TypeSpecializeBinary(IR::Instr **pInstr, Value **pSrc1Val, Value **pSrc2Val, Value **pDstVal, Value *const src1OriginalVal, Value *const src2OriginalVal, bool *redoTypeSpecRef)
  7799. {
  7800. IR::Instr *&instr = *pInstr;
  7801. int32 min1 = INT32_MIN, max1 = INT32_MAX, min2 = INT32_MIN, max2 = INT32_MAX, newMin, newMax, tmp;
  7802. Js::OpCode opcode;
  7803. Value *&src1Val = *pSrc1Val;
  7804. Value *&src2Val = *pSrc2Val;
  7805. // We don't need to do typespec for asmjs
  7806. if (IsTypeSpecPhaseOff(this->func) || GetIsAsmJSFunc())
  7807. {
  7808. return false;
  7809. }
  7810. if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  7811. {
  7812. this->TypeSpecializeInlineBuiltInBinary(pInstr, src1Val, src2Val, pDstVal, src1OriginalVal, src2OriginalVal);
  7813. return true;
  7814. }
  7815. if (src1Val)
  7816. {
  7817. src1Val->GetValueInfo()->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec());
  7818. }
  7819. if (src2Val)
  7820. {
  7821. src2Val->GetValueInfo()->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec());
  7822. }
  7823. // Type specialize binary operators to int32
  7824. bool src1Lossy = true;
  7825. bool src2Lossy = true;
  7826. IR::BailOutKind bailOutKind = IR::BailOutInvalid;
  7827. bool ignoredIntOverflow = this->ignoredIntOverflowForCurrentInstr;
  7828. bool ignoredNegativeZero = false;
  7829. bool skipSrc2 = false;
  7830. bool skipDst = false;
  7831. bool needsBoolConv = false;
  7832. AddSubConstantInfo addSubConstantInfo;
  7833. switch (instr->m_opcode)
  7834. {
  7835. case Js::OpCode::Or_A:
  7836. if (!DoLossyIntTypeSpec())
  7837. {
  7838. return false;
  7839. }
  7840. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7841. opcode = Js::OpCode::Or_I4;
  7842. break;
  7843. case Js::OpCode::And_A:
  7844. if (!DoLossyIntTypeSpec())
  7845. {
  7846. return false;
  7847. }
  7848. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7849. opcode = Js::OpCode::And_I4;
  7850. break;
  7851. case Js::OpCode::Xor_A:
  7852. if (!DoLossyIntTypeSpec())
  7853. {
  7854. return false;
  7855. }
  7856. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7857. opcode = Js::OpCode::Xor_I4;
  7858. break;
  7859. case Js::OpCode::Shl_A:
  7860. if (!DoLossyIntTypeSpec())
  7861. {
  7862. return false;
  7863. }
  7864. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7865. opcode = Js::OpCode::Shl_I4;
  7866. break;
  7867. case Js::OpCode::Shr_A:
  7868. if (!DoLossyIntTypeSpec())
  7869. {
  7870. return false;
  7871. }
  7872. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7873. opcode = Js::OpCode::Shr_I4;
  7874. break;
  7875. case Js::OpCode::ShrU_A:
  7876. if (!DoLossyIntTypeSpec())
  7877. {
  7878. return false;
  7879. }
  7880. if (min1 < 0 && IntConstantBounds(min2, max2).And_0x1f().Contains(0))
  7881. {
  7882. // Src1 may be too large to represent as a signed int32, and src2 may be zero. Unless the resulting value is only
  7883. // used as a signed int32 (hence allowing us to ignore the result's sign), don't specialize the instruction.
  7884. if (!instr->ignoreIntOverflow)
  7885. return false;
  7886. ignoredIntOverflow = true;
  7887. }
  7888. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7889. opcode = Js::OpCode::ShrU_I4;
  7890. break;
  7891. case Js::OpCode::BrUnLe_A:
  7892. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7893. // int already, so require that both sources are likely int for folding.
  7894. if (DoConstFold() &&
  7895. !IsLoopPrePass() &&
  7896. TryOptConstFoldBrUnsignedGreaterThan(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  7897. {
  7898. return true;
  7899. }
  7900. if (min1 >= 0 && min2 >= 0)
  7901. {
  7902. // Only handle positive values since this is unsigned...
  7903. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7904. // (INT32_MIN, INT32_MAX), so we're good.
  7905. Assert(src1Val);
  7906. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7907. Assert(src2Val);
  7908. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7909. UpdateIntBoundsForLessThanOrEqualBranch(src1Val, src2Val);
  7910. }
  7911. if (!DoLossyIntTypeSpec())
  7912. {
  7913. return false;
  7914. }
  7915. newMin = newMax = 0;
  7916. opcode = Js::OpCode::BrUnLe_I4;
  7917. break;
  7918. case Js::OpCode::BrUnLt_A:
  7919. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7920. // int already, so require that both sources are likely int for folding.
  7921. if (DoConstFold() &&
  7922. !IsLoopPrePass() &&
  7923. TryOptConstFoldBrUnsignedLessThan(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  7924. {
  7925. return true;
  7926. }
  7927. if (min1 >= 0 && min2 >= 0)
  7928. {
  7929. // Only handle positive values since this is unsigned...
  7930. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7931. // (INT32_MIN, INT32_MAX), so we're good.
  7932. Assert(src1Val);
  7933. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7934. Assert(src2Val);
  7935. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7936. UpdateIntBoundsForLessThanBranch(src1Val, src2Val);
  7937. }
  7938. if (!DoLossyIntTypeSpec())
  7939. {
  7940. return false;
  7941. }
  7942. newMin = newMax = 0;
  7943. opcode = Js::OpCode::BrUnLt_I4;
  7944. break;
  7945. case Js::OpCode::BrUnGe_A:
  7946. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7947. // int already, so require that both sources are likely int for folding.
  7948. if (DoConstFold() &&
  7949. !IsLoopPrePass() &&
  7950. TryOptConstFoldBrUnsignedLessThan(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  7951. {
  7952. return true;
  7953. }
  7954. if (min1 >= 0 && min2 >= 0)
  7955. {
  7956. // Only handle positive values since this is unsigned...
  7957. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7958. // (INT32_MIN, INT32_MAX), so we're good.
  7959. Assert(src1Val);
  7960. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7961. Assert(src2Val);
  7962. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7963. UpdateIntBoundsForGreaterThanOrEqualBranch(src1Val, src2Val);
  7964. }
  7965. if (!DoLossyIntTypeSpec())
  7966. {
  7967. return false;
  7968. }
  7969. newMin = newMax = 0;
  7970. opcode = Js::OpCode::BrUnGe_I4;
  7971. break;
  7972. case Js::OpCode::BrUnGt_A:
  7973. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7974. // int already, so require that both sources are likely int for folding.
  7975. if (DoConstFold() &&
  7976. !IsLoopPrePass() &&
  7977. TryOptConstFoldBrUnsignedGreaterThan(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  7978. {
  7979. return true;
  7980. }
  7981. if (min1 >= 0 && min2 >= 0)
  7982. {
  7983. // Only handle positive values since this is unsigned...
  7984. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7985. // (INT32_MIN, INT32_MAX), so we're good.
  7986. Assert(src1Val);
  7987. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7988. Assert(src2Val);
  7989. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7990. UpdateIntBoundsForGreaterThanBranch(src1Val, src2Val);
  7991. }
  7992. if (!DoLossyIntTypeSpec())
  7993. {
  7994. return false;
  7995. }
  7996. newMin = newMax = 0;
  7997. opcode = Js::OpCode::BrUnGt_I4;
  7998. break;
  7999. case Js::OpCode::CmUnLe_A:
  8000. if (!DoLossyIntTypeSpec())
  8001. {
  8002. return false;
  8003. }
  8004. newMin = 0;
  8005. newMax = 1;
  8006. opcode = Js::OpCode::CmUnLe_I4;
  8007. needsBoolConv = true;
  8008. break;
  8009. case Js::OpCode::CmUnLt_A:
  8010. if (!DoLossyIntTypeSpec())
  8011. {
  8012. return false;
  8013. }
  8014. newMin = 0;
  8015. newMax = 1;
  8016. opcode = Js::OpCode::CmUnLt_I4;
  8017. needsBoolConv = true;
  8018. break;
  8019. case Js::OpCode::CmUnGe_A:
  8020. if (!DoLossyIntTypeSpec())
  8021. {
  8022. return false;
  8023. }
  8024. newMin = 0;
  8025. newMax = 1;
  8026. opcode = Js::OpCode::CmUnGe_I4;
  8027. needsBoolConv = true;
  8028. break;
  8029. case Js::OpCode::CmUnGt_A:
  8030. if (!DoLossyIntTypeSpec())
  8031. {
  8032. return false;
  8033. }
  8034. newMin = 0;
  8035. newMax = 1;
  8036. opcode = Js::OpCode::CmUnGt_I4;
  8037. needsBoolConv = true;
  8038. break;
  8039. case Js::OpCode::Expo_A:
  8040. {
  8041. src1Val = src1OriginalVal;
  8042. src2Val = src2OriginalVal;
  8043. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  8044. }
  8045. case Js::OpCode::Div_A:
  8046. {
  8047. ValueType specializedValueType = GetDivValueType(instr, src1Val, src2Val, true);
  8048. if (specializedValueType.IsFloat())
  8049. {
  8050. // Either result is float or 1/x or cst1/cst2 where cst1%cst2 != 0
  8051. // Note: We should really constant fold cst1%cst2...
  8052. src1Val = src1OriginalVal;
  8053. src2Val = src2OriginalVal;
  8054. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  8055. }
  8056. #ifdef _M_ARM
  8057. if (!AutoSystemInfo::Data.ArmDivAvailable())
  8058. {
  8059. return false;
  8060. }
  8061. #endif
  8062. if (specializedValueType.IsInt())
  8063. {
  8064. if (max2 == 0x80000000 || (min2 == 0 && max2 == 00))
  8065. {
  8066. return false;
  8067. }
  8068. if (min1 == 0x80000000 && min2 <= -1 && max2 >= -1)
  8069. {
  8070. // Prevent integer overflow, as div by zero or MIN_INT / -1 will throw an exception
  8071. // Or we know we are dividing by zero (which is weird to have because the profile data
  8072. // say we got an int)
  8073. bailOutKind = IR::BailOutOnDivOfMinInt;
  8074. }
  8075. src1Lossy = false; // Detect -0 on the sources
  8076. src2Lossy = false;
  8077. opcode = Js::OpCode::Div_I4;
  8078. Assert(!instr->GetSrc1()->IsUnsigned());
  8079. bailOutKind |= IR::BailOnDivResultNotInt;
  8080. if (max2 >= 0 && min2 <= 0)
  8081. {
  8082. // Need to check for divide by zero if the denominator range includes 0
  8083. bailOutKind |= IR::BailOutOnDivByZero;
  8084. }
  8085. if (max1 >= 0 && min1 <= 0)
  8086. {
  8087. // Numerator contains 0 so the result contains 0
  8088. newMin = 0;
  8089. newMax = 0;
  8090. if (min2 < 0)
  8091. {
  8092. // Denominator may be negative, so the result could be negative 0
  8093. if (instr->ShouldCheckForNegativeZero())
  8094. {
  8095. bailOutKind |= IR::BailOutOnNegativeZero;
  8096. }
  8097. else
  8098. {
  8099. ignoredNegativeZero = true;
  8100. }
  8101. }
  8102. }
  8103. else
  8104. {
  8105. // Initialize to invalid value, one of the condition below will update it correctly
  8106. newMin = INT_MAX;
  8107. newMax = INT_MIN;
  8108. }
  8109. // Deal with the positive and negative range separately for both the numerator and the denominator,
  8110. // and integrate to the overall min and max.
  8111. // If the result is positive (positive/positive or negative/negative):
  8112. // The min should be the smallest magnitude numerator (positive_Min1 | negative_Max1)
  8113. // divided by ---------------------------------------------------------------
  8114. // largest magnitude denominator (positive_Max2 | negative_Min2)
  8115. //
  8116. // The max should be the largest magnitude numerator (positive_Max1 | negative_Max1)
  8117. // divided by ---------------------------------------------------------------
  8118. // smallest magnitude denominator (positive_Min2 | negative_Max2)
  8119. // If the result is negative (positive/negative or positive/negative):
  8120. // The min should be the largest magnitude numerator (positive_Max1 | negative_Min1)
  8121. // divided by ---------------------------------------------------------------
  8122. // smallest magnitude denominator (negative_Max2 | positive_Min2)
  8123. //
  8124. // The max should be the smallest magnitude numerator (positive_Min1 | negative_Max1)
  8125. // divided by ---------------------------------------------------------------
  8126. // largest magnitude denominator (negative_Min2 | positive_Max2)
  8127. // Consider: The range can be slightly more precise if we take care of the rounding
  8128. if (max1 > 0)
  8129. {
  8130. // Take only the positive numerator range
  8131. int32 positive_Min1 = max(1, min1);
  8132. int32 positive_Max1 = max1;
  8133. if (max2 > 0)
  8134. {
  8135. // Take only the positive denominator range
  8136. int32 positive_Min2 = max(1, min2);
  8137. int32 positive_Max2 = max2;
  8138. // Positive / Positive
  8139. int32 quadrant1_Min = positive_Min1 <= positive_Max2? 1 : positive_Min1 / positive_Max2;
  8140. int32 quadrant1_Max = positive_Max1 <= positive_Min2? 1 : positive_Max1 / positive_Min2;
  8141. Assert(1 <= quadrant1_Min && quadrant1_Min <= quadrant1_Max);
  8142. // The result should positive
  8143. newMin = min(newMin, quadrant1_Min);
  8144. newMax = max(newMax, quadrant1_Max);
  8145. }
  8146. if (min2 < 0)
  8147. {
  8148. // Take only the negative denominator range
  8149. int32 negative_Min2 = min2;
  8150. int32 negative_Max2 = min(-1, max2);
  8151. // Positive / Negative
  8152. int32 quadrant2_Min = -positive_Max1 >= negative_Max2? -1 : positive_Max1 / negative_Max2;
  8153. int32 quadrant2_Max = -positive_Min1 >= negative_Min2? -1 : positive_Min1 / negative_Min2;
  8154. // The result should negative
  8155. Assert(quadrant2_Min <= quadrant2_Max && quadrant2_Max <= -1);
  8156. newMin = min(newMin, quadrant2_Min);
  8157. newMax = max(newMax, quadrant2_Max);
  8158. }
  8159. }
  8160. if (min1 < 0)
  8161. {
  8162. // Take only the native numerator range
  8163. int32 negative_Min1 = min1;
  8164. int32 negative_Max1 = min(-1, max1);
  8165. if (max2 > 0)
  8166. {
  8167. // Take only the positive denominator range
  8168. int32 positive_Min2 = max(1, min2);
  8169. int32 positive_Max2 = max2;
  8170. // Negative / Positive
  8171. int32 quadrant4_Min = negative_Min1 >= -positive_Min2? -1 : negative_Min1 / positive_Min2;
  8172. int32 quadrant4_Max = negative_Max1 >= -positive_Max2? -1 : negative_Max1 / positive_Max2;
  8173. // The result should negative
  8174. Assert(quadrant4_Min <= quadrant4_Max && quadrant4_Max <= -1);
  8175. newMin = min(newMin, quadrant4_Min);
  8176. newMax = max(newMax, quadrant4_Max);
  8177. }
  8178. if (min2 < 0)
  8179. {
  8180. // Take only the negative denominator range
  8181. int32 negative_Min2 = min2;
  8182. int32 negative_Max2 = min(-1, max2);
  8183. int32 quadrant3_Min;
  8184. int32 quadrant3_Max;
  8185. // Negative / Negative
  8186. if (negative_Max1 == 0x80000000 && negative_Min2 == -1)
  8187. {
  8188. quadrant3_Min = negative_Max1 >= negative_Min2? 1 : (negative_Max1+1) / negative_Min2;
  8189. }
  8190. else
  8191. {
  8192. quadrant3_Min = negative_Max1 >= negative_Min2? 1 : negative_Max1 / negative_Min2;
  8193. }
  8194. if (negative_Min1 == 0x80000000 && negative_Max2 == -1)
  8195. {
  8196. quadrant3_Max = negative_Min1 >= negative_Max2? 1 : (negative_Min1+1) / negative_Max2;
  8197. }
  8198. else
  8199. {
  8200. quadrant3_Max = negative_Min1 >= negative_Max2? 1 : negative_Min1 / negative_Max2;
  8201. }
  8202. // The result should positive
  8203. Assert(1 <= quadrant3_Min && quadrant3_Min <= quadrant3_Max);
  8204. newMin = min(newMin, quadrant3_Min);
  8205. newMax = max(newMax, quadrant3_Max);
  8206. }
  8207. }
  8208. Assert(newMin <= newMax);
  8209. // Continue to int type spec
  8210. break;
  8211. }
  8212. }
  8213. // fall-through
  8214. default:
  8215. {
  8216. const bool involesLargeInt32 =
  8217. (src1Val && src1Val->GetValueInfo()->IsLikelyUntaggedInt()) ||
  8218. (src2Val && src2Val->GetValueInfo()->IsLikelyUntaggedInt());
  8219. const auto trySpecializeToFloat =
  8220. [&](const bool mayOverflow) -> bool
  8221. {
  8222. // It has been determined that this instruction cannot be int-specialized. Need to determine whether to attempt
  8223. // to float-specialize the instruction, or leave it unspecialized.
  8224. if((involesLargeInt32
  8225. #if INT32VAR
  8226. && mayOverflow
  8227. #endif
  8228. ) || (instr->m_opcode == Js::OpCode::Mul_A && !this->DoAggressiveMulIntTypeSpec())
  8229. )
  8230. {
  8231. // An input range is completely outside the range of an int31 and the operation is likely to overflow.
  8232. // Additionally, on 32-bit platforms, the value is untaggable and will be a JavascriptNumber, which is
  8233. // significantly slower to use in an unspecialized operation compared to a tagged int. So, try to
  8234. // float-specialize the instruction.
  8235. src1Val = src1OriginalVal;
  8236. src2Val = src2OriginalVal;
  8237. return TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  8238. }
  8239. return false;
  8240. };
  8241. if (instr->m_opcode != Js::OpCode::ArgOut_A_InlineBuiltIn)
  8242. {
  8243. if ((src1Val && src1Val->GetValueInfo()->IsLikelyFloat()) || (src2Val && src2Val->GetValueInfo()->IsLikelyFloat()))
  8244. {
  8245. // Try to type specialize to float
  8246. src1Val = src1OriginalVal;
  8247. src2Val = src2OriginalVal;
  8248. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  8249. }
  8250. if (src1Val == nullptr ||
  8251. src2Val == nullptr ||
  8252. !src1Val->GetValueInfo()->IsLikelyInt() ||
  8253. !src2Val->GetValueInfo()->IsLikelyInt() ||
  8254. (
  8255. !DoAggressiveIntTypeSpec() &&
  8256. (
  8257. !(src1Val->GetValueInfo()->IsInt() || CurrentBlockData()->IsSwitchInt32TypeSpecialized(instr)) ||
  8258. !src2Val->GetValueInfo()->IsInt()
  8259. )
  8260. ) ||
  8261. (instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber) ||
  8262. (instr->GetSrc2()->IsRegOpnd() && instr->GetSrc2()->AsRegOpnd()->m_sym->m_isNotNumber))
  8263. {
  8264. return trySpecializeToFloat(true);
  8265. }
  8266. }
  8267. // Try to type specialize to int32
  8268. // If one of the values is a float constant with a value that fits in a uint32 but not an int32,
  8269. // and the instruction can ignore int overflow, the source value for the purposes of int specialization
  8270. // would have been changed to an int constant value by ignoring overflow. But, the conversion is still lossy.
  8271. if (!(src1OriginalVal && src1OriginalVal->GetValueInfo()->IsFloatConstant() && src1Val && src1Val->GetValueInfo()->HasIntConstantValue()))
  8272. {
  8273. src1Lossy = false;
  8274. }
  8275. if (!(src2OriginalVal && src2OriginalVal->GetValueInfo()->IsFloatConstant() && src2Val && src2Val->GetValueInfo()->HasIntConstantValue()))
  8276. {
  8277. src2Lossy = false;
  8278. }
  8279. switch(instr->m_opcode)
  8280. {
  8281. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  8282. // If the src is already type-specialized, if we don't type-specialize ArgOut_A_InlineBuiltIn instr, we'll get additional ToVar.
  8283. // So, to avoid that, type-specialize the ArgOut_A_InlineBuiltIn instr.
  8284. // Else we don't need to type-specialize the instr, we are fine with src being Var.
  8285. if (instr->GetSrc1()->IsRegOpnd())
  8286. {
  8287. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  8288. if (CurrentBlockData()->IsInt32TypeSpecialized(sym))
  8289. {
  8290. opcode = instr->m_opcode;
  8291. skipDst = true; // We should keep dst as is, otherwise the link opnd for next ArgOut/InlineBuiltInStart would be broken.
  8292. skipSrc2 = true; // src2 is linkOpnd. We don't need to type-specialize it.
  8293. newMin = min1; newMax = max1; // Values don't matter, these are unused.
  8294. goto LOutsideSwitch; // Continue to int-type-specialize.
  8295. }
  8296. else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  8297. {
  8298. src1Val = src1OriginalVal;
  8299. src2Val = src2OriginalVal;
  8300. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  8301. }
  8302. }
  8303. return false;
  8304. case Js::OpCode::Add_A:
  8305. do // while(false)
  8306. {
  8307. const auto CannotOverflowBasedOnRelativeBounds = [&](int32 *const constantValueRef)
  8308. {
  8309. Assert(constantValueRef);
  8310. if(min2 == max2 &&
  8311. src1Val->GetValueInfo()->IsIntBounded() &&
  8312. src1Val->GetValueInfo()->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(min2))
  8313. {
  8314. *constantValueRef = min2;
  8315. return true;
  8316. }
  8317. else if(
  8318. min1 == max1 &&
  8319. src2Val->GetValueInfo()->IsIntBounded() &&
  8320. src2Val->GetValueInfo()->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(min1))
  8321. {
  8322. *constantValueRef = min1;
  8323. return true;
  8324. }
  8325. return false;
  8326. };
  8327. if (Int32Math::Add(min1, min2, &newMin))
  8328. {
  8329. int32 constantSrcValue;
  8330. if(CannotOverflowBasedOnRelativeBounds(&constantSrcValue))
  8331. {
  8332. newMin = constantSrcValue >= 0 ? INT32_MAX : INT32_MIN;
  8333. }
  8334. else if(instr->ShouldCheckForIntOverflow())
  8335. {
  8336. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8337. {
  8338. // May overflow
  8339. return trySpecializeToFloat(true);
  8340. }
  8341. bailOutKind |= IR::BailOutOnOverflow;
  8342. newMin = min1 < 0 ? INT32_MIN : INT32_MAX;
  8343. }
  8344. else
  8345. {
  8346. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since
  8347. // overflow causes the value to wrap around, and we don't have a way to specify a lower and upper
  8348. // range of ints, we use the full range of int32s.
  8349. ignoredIntOverflow = true;
  8350. newMin = INT32_MIN;
  8351. newMax = INT32_MAX;
  8352. break;
  8353. }
  8354. }
  8355. if (Int32Math::Add(max1, max2, &newMax))
  8356. {
  8357. int32 constantSrcValue;
  8358. if(CannotOverflowBasedOnRelativeBounds(&constantSrcValue))
  8359. {
  8360. newMax = constantSrcValue >= 0 ? INT32_MAX : INT32_MIN;
  8361. }
  8362. else if(instr->ShouldCheckForIntOverflow())
  8363. {
  8364. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8365. {
  8366. // May overflow
  8367. return trySpecializeToFloat(true);
  8368. }
  8369. bailOutKind |= IR::BailOutOnOverflow;
  8370. newMax = max1 < 0 ? INT32_MIN : INT32_MAX;
  8371. }
  8372. else
  8373. {
  8374. // See comment about ignoring overflow above
  8375. ignoredIntOverflow = true;
  8376. newMin = INT32_MIN;
  8377. newMax = INT32_MAX;
  8378. break;
  8379. }
  8380. }
  8381. if(bailOutKind & IR::BailOutOnOverflow)
  8382. {
  8383. Assert(bailOutKind == IR::BailOutOnOverflow);
  8384. Assert(instr->ShouldCheckForIntOverflow());
  8385. int32 temp;
  8386. if(Int32Math::Add(
  8387. Int32Math::NearestInRangeTo(0, min1, max1),
  8388. Int32Math::NearestInRangeTo(0, min2, max2),
  8389. &temp))
  8390. {
  8391. // Always overflows
  8392. return trySpecializeToFloat(true);
  8393. }
  8394. }
  8395. } while(false);
  8396. if (!this->IsLoopPrePass() && newMin == newMax && bailOutKind == IR::BailOutInvalid)
  8397. {
  8398. // Take care of Add with zero here, since we know we're dealing with 2 numbers.
  8399. this->CaptureByteCodeSymUses(instr);
  8400. IR::Opnd *src;
  8401. bool isAddZero = true;
  8402. int32 intConstantValue;
  8403. if (src1Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) && intConstantValue == 0)
  8404. {
  8405. src = instr->UnlinkSrc2();
  8406. instr->FreeSrc1();
  8407. }
  8408. else if (src2Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) && intConstantValue == 0)
  8409. {
  8410. src = instr->UnlinkSrc1();
  8411. instr->FreeSrc2();
  8412. }
  8413. else
  8414. {
  8415. // This should have been handled by const folding, unless:
  8416. // - A source's value was substituted with a different value here, which is after const folding happened
  8417. // - A value is not definitely int, but once converted to definite int, it would be zero due to a
  8418. // condition in the source code such as if(a === 0). Ideally, we would specialize the sources and
  8419. // remove the add, but doesn't seem too important for now.
  8420. Assert(
  8421. !DoConstFold() ||
  8422. src1Val != src1OriginalVal ||
  8423. src2Val != src2OriginalVal ||
  8424. !src1Val->GetValueInfo()->IsInt() ||
  8425. !src2Val->GetValueInfo()->IsInt());
  8426. isAddZero = false;
  8427. src = nullptr;
  8428. }
  8429. if (isAddZero)
  8430. {
  8431. IR::Instr *newInstr = IR::Instr::New(Js::OpCode::Ld_A, instr->UnlinkDst(), src, instr->m_func);
  8432. newInstr->SetByteCodeOffset(instr);
  8433. instr->m_opcode = Js::OpCode::Nop;
  8434. this->currentBlock->InsertInstrAfter(newInstr, instr);
  8435. return true;
  8436. }
  8437. }
  8438. if(!ignoredIntOverflow)
  8439. {
  8440. if(min2 == max2 &&
  8441. (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val)) &&
  8442. instr->GetSrc1()->IsRegOpnd())
  8443. {
  8444. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min1 == max1, min2);
  8445. }
  8446. else if(
  8447. min1 == max1 &&
  8448. (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Val)) &&
  8449. instr->GetSrc2()->IsRegOpnd())
  8450. {
  8451. addSubConstantInfo.Set(instr->GetSrc2()->AsRegOpnd()->m_sym, src2Val, min2 == max2, min1);
  8452. }
  8453. }
  8454. opcode = Js::OpCode::Add_I4;
  8455. break;
  8456. case Js::OpCode::Sub_A:
  8457. do // while(false)
  8458. {
  8459. const auto CannotOverflowBasedOnRelativeBounds = [&]()
  8460. {
  8461. return
  8462. min2 == max2 &&
  8463. src1Val->GetValueInfo()->IsIntBounded() &&
  8464. src1Val->GetValueInfo()->AsIntBounded()->Bounds()->SubCannotOverflowBasedOnRelativeBounds(min2);
  8465. };
  8466. if (Int32Math::Sub(min1, max2, &newMin))
  8467. {
  8468. if(CannotOverflowBasedOnRelativeBounds())
  8469. {
  8470. Assert(min2 == max2);
  8471. newMin = min2 >= 0 ? INT32_MIN : INT32_MAX;
  8472. }
  8473. else if(instr->ShouldCheckForIntOverflow())
  8474. {
  8475. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8476. {
  8477. // May overflow
  8478. return trySpecializeToFloat(true);
  8479. }
  8480. bailOutKind |= IR::BailOutOnOverflow;
  8481. newMin = min1 < 0 ? INT32_MIN : INT32_MAX;
  8482. }
  8483. else
  8484. {
  8485. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow
  8486. // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints,
  8487. // we use the full range of int32s.
  8488. ignoredIntOverflow = true;
  8489. newMin = INT32_MIN;
  8490. newMax = INT32_MAX;
  8491. break;
  8492. }
  8493. }
  8494. if (Int32Math::Sub(max1, min2, &newMax))
  8495. {
  8496. if(CannotOverflowBasedOnRelativeBounds())
  8497. {
  8498. Assert(min2 == max2);
  8499. newMax = min2 >= 0 ? INT32_MIN: INT32_MAX;
  8500. }
  8501. else if(instr->ShouldCheckForIntOverflow())
  8502. {
  8503. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8504. {
  8505. // May overflow
  8506. return trySpecializeToFloat(true);
  8507. }
  8508. bailOutKind |= IR::BailOutOnOverflow;
  8509. newMax = max1 < 0 ? INT32_MIN : INT32_MAX;
  8510. }
  8511. else
  8512. {
  8513. // See comment about ignoring overflow above
  8514. ignoredIntOverflow = true;
  8515. newMin = INT32_MIN;
  8516. newMax = INT32_MAX;
  8517. break;
  8518. }
  8519. }
  8520. if(bailOutKind & IR::BailOutOnOverflow)
  8521. {
  8522. Assert(bailOutKind == IR::BailOutOnOverflow);
  8523. Assert(instr->ShouldCheckForIntOverflow());
  8524. int32 temp;
  8525. if(Int32Math::Sub(
  8526. Int32Math::NearestInRangeTo(-1, min1, max1),
  8527. Int32Math::NearestInRangeTo(0, min2, max2),
  8528. &temp))
  8529. {
  8530. // Always overflows
  8531. return trySpecializeToFloat(true);
  8532. }
  8533. }
  8534. } while(false);
  8535. if(!ignoredIntOverflow &&
  8536. min2 == max2 &&
  8537. min2 != INT32_MIN &&
  8538. (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val)) &&
  8539. instr->GetSrc1()->IsRegOpnd())
  8540. {
  8541. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min1 == max1, -min2);
  8542. }
  8543. opcode = Js::OpCode::Sub_I4;
  8544. break;
  8545. case Js::OpCode::Mul_A:
  8546. {
  8547. bool isConservativeMulInt = !DoAggressiveMulIntTypeSpec() || !DoAggressiveIntTypeSpec();
  8548. // Be conservative about predicting Mul overflow in prepass.
  8549. // Operands that are live on back edge may be denied lossless-conversion to int32 and
  8550. // trigger rejit with AggressiveIntTypeSpec off.
  8551. // Besides multiplying a variable in a loop can overflow in just a few iterations even in simple cases like v *= 2
  8552. // So, make sure we definitely know the source max/min values, otherwise assume the full range.
  8553. if (isConservativeMulInt && IsLoopPrePass())
  8554. {
  8555. if (!IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Val))
  8556. {
  8557. max1 = INT32_MAX;
  8558. min1 = INT32_MIN;
  8559. }
  8560. if (!IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val))
  8561. {
  8562. max2 = INT32_MAX;
  8563. min2 = INT32_MIN;
  8564. }
  8565. }
  8566. if (Int32Math::Mul(min1, min2, &newMin))
  8567. {
  8568. if (involesLargeInt32 || isConservativeMulInt)
  8569. {
  8570. // May overflow
  8571. return trySpecializeToFloat(true);
  8572. }
  8573. bailOutKind |= IR::BailOutOnMulOverflow;
  8574. newMin = (min1 < 0) ^ (min2 < 0) ? INT32_MIN : INT32_MAX;
  8575. }
  8576. newMax = newMin;
  8577. if (Int32Math::Mul(max1, max2, &tmp))
  8578. {
  8579. if (involesLargeInt32 || isConservativeMulInt)
  8580. {
  8581. // May overflow
  8582. return trySpecializeToFloat(true);
  8583. }
  8584. bailOutKind |= IR::BailOutOnMulOverflow;
  8585. tmp = (max1 < 0) ^ (max2 < 0) ? INT32_MIN : INT32_MAX;
  8586. }
  8587. newMin = min(newMin, tmp);
  8588. newMax = max(newMax, tmp);
  8589. if (Int32Math::Mul(min1, max2, &tmp))
  8590. {
  8591. if (involesLargeInt32 || isConservativeMulInt)
  8592. {
  8593. // May overflow
  8594. return trySpecializeToFloat(true);
  8595. }
  8596. bailOutKind |= IR::BailOutOnMulOverflow;
  8597. tmp = (min1 < 0) ^ (max2 < 0) ? INT32_MIN : INT32_MAX;
  8598. }
  8599. newMin = min(newMin, tmp);
  8600. newMax = max(newMax, tmp);
  8601. if (Int32Math::Mul(max1, min2, &tmp))
  8602. {
  8603. if (involesLargeInt32 || isConservativeMulInt)
  8604. {
  8605. // May overflow
  8606. return trySpecializeToFloat(true);
  8607. }
  8608. bailOutKind |= IR::BailOutOnMulOverflow;
  8609. tmp = (max1 < 0) ^ (min2 < 0) ? INT32_MIN : INT32_MAX;
  8610. }
  8611. newMin = min(newMin, tmp);
  8612. newMax = max(newMax, tmp);
  8613. if (bailOutKind & IR::BailOutOnMulOverflow)
  8614. {
  8615. // CSE only if two MULs have the same overflow check behavior.
  8616. // Currently this is set to be ignore int32 overflow, but not 53-bit, or int32 overflow matters.
  8617. if (!instr->ShouldCheckFor32BitOverflow() && instr->ShouldCheckForNon32BitOverflow())
  8618. {
  8619. // If we allow int to overflow then there can be anything in the resulting int
  8620. newMin = IntConstMin;
  8621. newMax = IntConstMax;
  8622. ignoredIntOverflow = true;
  8623. }
  8624. int32 temp, overflowValue;
  8625. if (Int32Math::Mul(
  8626. Int32Math::NearestInRangeTo(0, min1, max1),
  8627. Int32Math::NearestInRangeTo(0, min2, max2),
  8628. &temp,
  8629. &overflowValue))
  8630. {
  8631. Assert(instr->ignoreOverflowBitCount >= 32);
  8632. int overflowMatters = 64 - instr->ignoreOverflowBitCount;
  8633. if (!ignoredIntOverflow ||
  8634. // Use shift to check high bits in case its negative
  8635. ((overflowValue << overflowMatters) >> overflowMatters) != overflowValue
  8636. )
  8637. {
  8638. // Always overflows
  8639. return trySpecializeToFloat(true);
  8640. }
  8641. }
  8642. }
  8643. if (newMin <= 0 && newMax >= 0 && // New range crosses zero
  8644. (min1 < 0 || min2 < 0) && // An operand's range contains a negative integer
  8645. !(min1 > 0 || min2 > 0) && // Neither operand's range contains only positive integers
  8646. !instr->GetSrc1()->IsEqual(instr->GetSrc2())) // The operands don't have the same value
  8647. {
  8648. if (instr->ShouldCheckForNegativeZero())
  8649. {
  8650. // -0 matters since the sym is not a local, or is used in a way in which -0 would differ from +0
  8651. if (!DoAggressiveIntTypeSpec())
  8652. {
  8653. // May result in -0
  8654. return trySpecializeToFloat(false);
  8655. }
  8656. if (((min1 == 0 && max1 == 0) || (min2 == 0 && max2 == 0)) && (max1 < 0 || max2 < 0))
  8657. {
  8658. // Always results in -0
  8659. return trySpecializeToFloat(false);
  8660. }
  8661. bailOutKind |= IR::BailOutOnNegativeZero;
  8662. }
  8663. else
  8664. {
  8665. ignoredNegativeZero = true;
  8666. }
  8667. }
  8668. opcode = Js::OpCode::Mul_I4;
  8669. break;
  8670. }
  8671. case Js::OpCode::Rem_A:
  8672. {
  8673. IR::Opnd* src2 = instr->GetSrc2();
  8674. if (!this->IsLoopPrePass() && min2 == max2 && min1 >= 0)
  8675. {
  8676. int32 value = min2;
  8677. if (value == (1 << Math::Log2(value)) && src2->IsAddrOpnd())
  8678. {
  8679. Assert(src2->AsAddrOpnd()->IsVar());
  8680. instr->m_opcode = Js::OpCode::And_A;
  8681. src2->AsAddrOpnd()->SetAddress(Js::TaggedInt::ToVarUnchecked(value - 1),
  8682. IR::AddrOpndKindConstantVar);
  8683. *pSrc2Val = GetIntConstantValue(value - 1, instr);
  8684. src2Val = *pSrc2Val;
  8685. return this->TypeSpecializeBinary(&instr, pSrc1Val, pSrc2Val, pDstVal, src1OriginalVal, src2Val, redoTypeSpecRef);
  8686. }
  8687. }
  8688. #ifdef _M_ARM
  8689. if (!AutoSystemInfo::Data.ArmDivAvailable())
  8690. {
  8691. return false;
  8692. }
  8693. #endif
  8694. if (min1 < 0)
  8695. {
  8696. // The most negative it can be is min1, unless limited by min2/max2
  8697. int32 negMaxAbs2;
  8698. if (min2 == INT32_MIN)
  8699. {
  8700. negMaxAbs2 = INT32_MIN;
  8701. }
  8702. else
  8703. {
  8704. negMaxAbs2 = -max(abs(min2), abs(max2)) + 1;
  8705. }
  8706. newMin = max(min1, negMaxAbs2);
  8707. }
  8708. else
  8709. {
  8710. newMin = 0;
  8711. }
  8712. bool isModByPowerOf2 = (instr->IsProfiledInstr() && instr->m_func->HasProfileInfo() &&
  8713. instr->m_func->GetReadOnlyProfileInfo()->IsModulusOpByPowerOf2(static_cast<Js::ProfileId>(instr->AsProfiledInstr()->u.profileId)));
  8714. if(isModByPowerOf2)
  8715. {
  8716. Assert(bailOutKind == IR::BailOutInvalid);
  8717. bailOutKind = IR::BailOnModByPowerOf2;
  8718. newMin = 0;
  8719. }
  8720. else
  8721. {
  8722. if (min2 <= 0 && max2 >= 0)
  8723. {
  8724. // Consider: We could handle the zero case with a check and bailout...
  8725. return false;
  8726. }
  8727. if (min1 == 0x80000000 && (min2 <= -1 && max2 >= -1))
  8728. {
  8729. // Prevent integer overflow, as div by zero or MIN_INT / -1 will throw an exception
  8730. return false;
  8731. }
  8732. if (min1 < 0)
  8733. {
  8734. if(instr->ShouldCheckForNegativeZero())
  8735. {
  8736. if (!DoAggressiveIntTypeSpec())
  8737. {
  8738. return false;
  8739. }
  8740. bailOutKind |= IR::BailOutOnNegativeZero;
  8741. }
  8742. else
  8743. {
  8744. ignoredNegativeZero = true;
  8745. }
  8746. }
  8747. }
  8748. {
  8749. int32 absMax2;
  8750. if (min2 == INT32_MIN)
  8751. {
  8752. // abs(INT32_MIN) == INT32_MAX because of overflow
  8753. absMax2 = INT32_MAX;
  8754. }
  8755. else
  8756. {
  8757. absMax2 = max(abs(min2), abs(max2)) - 1;
  8758. }
  8759. newMax = min(absMax2, max(max1, 0));
  8760. newMax = max(newMin, newMax);
  8761. }
  8762. opcode = Js::OpCode::Rem_I4;
  8763. Assert(!instr->GetSrc1()->IsUnsigned());
  8764. break;
  8765. }
  8766. case Js::OpCode::CmEq_A:
  8767. case Js::OpCode::CmSrEq_A:
  8768. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8769. {
  8770. return false;
  8771. }
  8772. newMin = 0;
  8773. newMax = 1;
  8774. opcode = Js::OpCode::CmEq_I4;
  8775. needsBoolConv = true;
  8776. break;
  8777. case Js::OpCode::CmNeq_A:
  8778. case Js::OpCode::CmSrNeq_A:
  8779. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8780. {
  8781. return false;
  8782. }
  8783. newMin = 0;
  8784. newMax = 1;
  8785. opcode = Js::OpCode::CmNeq_I4;
  8786. needsBoolConv = true;
  8787. break;
  8788. case Js::OpCode::CmLe_A:
  8789. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8790. {
  8791. return false;
  8792. }
  8793. newMin = 0;
  8794. newMax = 1;
  8795. opcode = Js::OpCode::CmLe_I4;
  8796. needsBoolConv = true;
  8797. break;
  8798. case Js::OpCode::CmLt_A:
  8799. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8800. {
  8801. return false;
  8802. }
  8803. newMin = 0;
  8804. newMax = 1;
  8805. opcode = Js::OpCode::CmLt_I4;
  8806. needsBoolConv = true;
  8807. break;
  8808. case Js::OpCode::CmGe_A:
  8809. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8810. {
  8811. return false;
  8812. }
  8813. newMin = 0;
  8814. newMax = 1;
  8815. opcode = Js::OpCode::CmGe_I4;
  8816. needsBoolConv = true;
  8817. break;
  8818. case Js::OpCode::CmGt_A:
  8819. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8820. {
  8821. return false;
  8822. }
  8823. newMin = 0;
  8824. newMax = 1;
  8825. opcode = Js::OpCode::CmGt_I4;
  8826. needsBoolConv = true;
  8827. break;
  8828. case Js::OpCode::BrSrEq_A:
  8829. case Js::OpCode::BrEq_A:
  8830. case Js::OpCode::BrNotNeq_A:
  8831. case Js::OpCode::BrSrNotNeq_A:
  8832. {
  8833. if(DoConstFold() &&
  8834. !IsLoopPrePass() &&
  8835. TryOptConstFoldBrEqual(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  8836. {
  8837. return true;
  8838. }
  8839. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8840. UpdateIntBoundsForEqualBranch(src1Val, src2Val);
  8841. if(!specialize)
  8842. {
  8843. return false;
  8844. }
  8845. opcode = Js::OpCode::BrEq_I4;
  8846. // We'll get a warning if we don't assign a value to these...
  8847. // We'll assert if we use them and make a range where min > max
  8848. newMin = 2; newMax = 1;
  8849. break;
  8850. }
  8851. case Js::OpCode::BrSrNeq_A:
  8852. case Js::OpCode::BrNeq_A:
  8853. case Js::OpCode::BrSrNotEq_A:
  8854. case Js::OpCode::BrNotEq_A:
  8855. {
  8856. if(DoConstFold() &&
  8857. !IsLoopPrePass() &&
  8858. TryOptConstFoldBrEqual(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  8859. {
  8860. return true;
  8861. }
  8862. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8863. UpdateIntBoundsForNotEqualBranch(src1Val, src2Val);
  8864. if(!specialize)
  8865. {
  8866. return false;
  8867. }
  8868. opcode = Js::OpCode::BrNeq_I4;
  8869. // We'll get a warning if we don't assign a value to these...
  8870. // We'll assert if we use them and make a range where min > max
  8871. newMin = 2; newMax = 1;
  8872. break;
  8873. }
  8874. case Js::OpCode::BrGt_A:
  8875. case Js::OpCode::BrNotLe_A:
  8876. {
  8877. if(DoConstFold() &&
  8878. !IsLoopPrePass() &&
  8879. TryOptConstFoldBrGreaterThan(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  8880. {
  8881. return true;
  8882. }
  8883. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8884. UpdateIntBoundsForGreaterThanBranch(src1Val, src2Val);
  8885. if(!specialize)
  8886. {
  8887. return false;
  8888. }
  8889. opcode = Js::OpCode::BrGt_I4;
  8890. // We'll get a warning if we don't assign a value to these...
  8891. // We'll assert if we use them and make a range where min > max
  8892. newMin = 2; newMax = 1;
  8893. break;
  8894. }
  8895. case Js::OpCode::BrGe_A:
  8896. case Js::OpCode::BrNotLt_A:
  8897. {
  8898. if(DoConstFold() &&
  8899. !IsLoopPrePass() &&
  8900. TryOptConstFoldBrGreaterThanOrEqual(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  8901. {
  8902. return true;
  8903. }
  8904. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8905. UpdateIntBoundsForGreaterThanOrEqualBranch(src1Val, src2Val);
  8906. if(!specialize)
  8907. {
  8908. return false;
  8909. }
  8910. opcode = Js::OpCode::BrGe_I4;
  8911. // We'll get a warning if we don't assign a value to these...
  8912. // We'll assert if we use them and make a range where min > max
  8913. newMin = 2; newMax = 1;
  8914. break;
  8915. }
  8916. case Js::OpCode::BrLt_A:
  8917. case Js::OpCode::BrNotGe_A:
  8918. {
  8919. if(DoConstFold() &&
  8920. !IsLoopPrePass() &&
  8921. TryOptConstFoldBrGreaterThanOrEqual(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  8922. {
  8923. return true;
  8924. }
  8925. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8926. UpdateIntBoundsForLessThanBranch(src1Val, src2Val);
  8927. if(!specialize)
  8928. {
  8929. return false;
  8930. }
  8931. opcode = Js::OpCode::BrLt_I4;
  8932. // We'll get a warning if we don't assign a value to these...
  8933. // We'll assert if we use them and make a range where min > max
  8934. newMin = 2; newMax = 1;
  8935. break;
  8936. }
  8937. case Js::OpCode::BrLe_A:
  8938. case Js::OpCode::BrNotGt_A:
  8939. {
  8940. if(DoConstFold() &&
  8941. !IsLoopPrePass() &&
  8942. TryOptConstFoldBrGreaterThan(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  8943. {
  8944. return true;
  8945. }
  8946. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8947. UpdateIntBoundsForLessThanOrEqualBranch(src1Val, src2Val);
  8948. if(!specialize)
  8949. {
  8950. return false;
  8951. }
  8952. opcode = Js::OpCode::BrLe_I4;
  8953. // We'll get a warning if we don't assign a value to these...
  8954. // We'll assert if we use them and make a range where min > max
  8955. newMin = 2; newMax = 1;
  8956. break;
  8957. }
  8958. default:
  8959. return false;
  8960. }
  8961. // If this instruction is in a range of instructions where int overflow does not matter, we will still specialize it
  8962. // (won't leave it unspecialized based on heuristics), since it is most likely worth specializing, and the dst value
  8963. // needs to be guaranteed to be an int
  8964. if(!ignoredIntOverflow &&
  8965. !ignoredNegativeZero &&
  8966. !needsBoolConv &&
  8967. instr->ShouldCheckForIntOverflow() &&
  8968. !IsWorthSpecializingToInt32(instr, src1Val, src2Val))
  8969. {
  8970. // Even though type specialization is being skipped since it may not be worth it, the proper value should still be
  8971. // maintained so that the result may be type specialized later. An int value is not created for the dst in any of
  8972. // the following cases.
  8973. // - A bailout check is necessary to specialize this instruction. The bailout check is what guarantees the result to
  8974. // be an int, but since we're not going to specialize this instruction, there won't be a bailout check.
  8975. // - Aggressive int type specialization is disabled and we're in a loop prepass. We're conservative on dst values in
  8976. // that case, especially if the dst sym is live on the back-edge.
  8977. if(bailOutKind == IR::BailOutInvalid &&
  8978. instr->GetDst() &&
  8979. src1Val->GetValueInfo()->IsInt() &&
  8980. src2Val->GetValueInfo()->IsInt() &&
  8981. (DoAggressiveIntTypeSpec() || !this->IsLoopPrePass()))
  8982. {
  8983. *pDstVal = CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val);
  8984. }
  8985. return false;
  8986. }
  8987. } // case default
  8988. } // switch
  8989. LOutsideSwitch:
  8990. this->ignoredIntOverflowForCurrentInstr = ignoredIntOverflow;
  8991. this->ignoredNegativeZeroForCurrentInstr = ignoredNegativeZero;
  8992. {
  8993. // Try CSE again before modifying the IR, in case some attributes are required for successful CSE
  8994. Value *src1IndirIndexVal = nullptr;
  8995. if(CSEOptimize(currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal, true /* intMathExprOnly */))
  8996. {
  8997. *redoTypeSpecRef = true;
  8998. return false;
  8999. }
  9000. }
  9001. const Js::OpCode originalOpCode = instr->m_opcode;
  9002. if (!this->IsLoopPrePass())
  9003. {
  9004. // No re-write on prepass
  9005. instr->m_opcode = opcode;
  9006. }
  9007. Value *src1ValueToSpecialize = src1Val, *src2ValueToSpecialize = src2Val;
  9008. // Lossy conversions to int32 must be done based on the original source values. For instance, if one of the values is a
  9009. // float constant with a value that fits in a uint32 but not an int32, and the instruction can ignore int overflow, the
  9010. // source value for the purposes of int specialization would have been changed to an int constant value by ignoring
  9011. // overflow. If we were to specialize the sym using the int constant value, it would be treated as a lossless
  9012. // conversion, but since there may be subsequent uses of the same float constant value that may not ignore overflow,
  9013. // this must be treated as a lossy conversion by specializing the sym using the original float constant value.
  9014. if(src1Lossy)
  9015. {
  9016. src1ValueToSpecialize = src1OriginalVal;
  9017. }
  9018. if (src2Lossy)
  9019. {
  9020. src2ValueToSpecialize = src2OriginalVal;
  9021. }
  9022. // Make sure the srcs are specialized
  9023. IR::Opnd* src1 = instr->GetSrc1();
  9024. this->ToInt32(instr, src1, this->currentBlock, src1ValueToSpecialize, nullptr, src1Lossy);
  9025. if (!skipSrc2)
  9026. {
  9027. IR::Opnd* src2 = instr->GetSrc2();
  9028. this->ToInt32(instr, src2, this->currentBlock, src2ValueToSpecialize, nullptr, src2Lossy);
  9029. }
  9030. if(bailOutKind != IR::BailOutInvalid && !this->IsLoopPrePass())
  9031. {
  9032. GenerateBailAtOperation(&instr, bailOutKind);
  9033. }
  9034. if (!skipDst && instr->GetDst())
  9035. {
  9036. if (needsBoolConv)
  9037. {
  9038. IR::RegOpnd *varDst;
  9039. if (this->IsLoopPrePass())
  9040. {
  9041. varDst = instr->GetDst()->AsRegOpnd();
  9042. this->ToVarRegOpnd(varDst, this->currentBlock);
  9043. }
  9044. else
  9045. {
  9046. // Generate:
  9047. // t1.i = CmCC t2.i, t3.i
  9048. // t1.v = Conv_bool t1.i
  9049. //
  9050. // If the only uses of t1 are ints, the conv_bool will get dead-stored
  9051. TypeSpecializeIntDst(instr, originalOpCode, nullptr, src1Val, src2Val, bailOutKind, newMin, newMax, pDstVal);
  9052. IR::RegOpnd *intDst = instr->GetDst()->AsRegOpnd();
  9053. intDst->SetIsJITOptimizedReg(true);
  9054. varDst = IR::RegOpnd::New(intDst->m_sym->GetVarEquivSym(this->func), TyVar, this->func);
  9055. IR::Instr *convBoolInstr = IR::Instr::New(Js::OpCode::Conv_Bool, varDst, intDst, this->func);
  9056. // In some cases (e.g. unsigned compare peep code), a comparison will use variables
  9057. // other than the ones initially intended for it, if we can determine that we would
  9058. // arrive at the same result. This means that we get a ByteCodeUses operation after
  9059. // the actual comparison. Since Inserting the Conv_bool just after the compare, and
  9060. // just before the ByteCodeUses, would cause issues later on with register lifetime
  9061. // calculation, we want to insert the Conv_bool after the whole compare instruction
  9062. // block.
  9063. IR::Instr *putAfter = instr;
  9064. while (putAfter->m_next && putAfter->m_next->IsByteCodeUsesInstrFor(instr))
  9065. {
  9066. putAfter = putAfter->m_next;
  9067. }
  9068. putAfter->InsertAfter(convBoolInstr);
  9069. convBoolInstr->SetByteCodeOffset(instr);
  9070. this->ToVarRegOpnd(varDst, this->currentBlock);
  9071. CurrentBlockData()->liveInt32Syms->Set(varDst->m_sym->m_id);
  9072. CurrentBlockData()->liveLossyInt32Syms->Set(varDst->m_sym->m_id);
  9073. }
  9074. *pDstVal = this->NewGenericValue(ValueType::Boolean, varDst);
  9075. }
  9076. else
  9077. {
  9078. TypeSpecializeIntDst(
  9079. instr,
  9080. originalOpCode,
  9081. nullptr,
  9082. src1Val,
  9083. src2Val,
  9084. bailOutKind,
  9085. newMin,
  9086. newMax,
  9087. pDstVal,
  9088. addSubConstantInfo.HasInfo() ? &addSubConstantInfo : nullptr);
  9089. }
  9090. }
  9091. if(bailOutKind == IR::BailOutInvalid)
  9092. {
  9093. GOPT_TRACE(_u("Type specialized to INT\n"));
  9094. #if ENABLE_DEBUG_CONFIG_OPTIONS
  9095. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  9096. {
  9097. Output::Print(_u("Type specialized to INT: "));
  9098. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  9099. }
  9100. #endif
  9101. }
  9102. else
  9103. {
  9104. GOPT_TRACE(_u("Type specialized to INT with bailout on:\n"));
  9105. if(bailOutKind & (IR::BailOutOnOverflow | IR::BailOutOnMulOverflow) )
  9106. {
  9107. GOPT_TRACE(_u(" Overflow\n"));
  9108. #if ENABLE_DEBUG_CONFIG_OPTIONS
  9109. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  9110. {
  9111. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Overflow");
  9112. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  9113. }
  9114. #endif
  9115. }
  9116. if(bailOutKind & IR::BailOutOnNegativeZero)
  9117. {
  9118. GOPT_TRACE(_u(" Zero\n"));
  9119. #if ENABLE_DEBUG_CONFIG_OPTIONS
  9120. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  9121. {
  9122. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Zero");
  9123. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  9124. }
  9125. #endif
  9126. }
  9127. }
  9128. return true;
  9129. }
  9130. bool
  9131. GlobOpt::IsWorthSpecializingToInt32Branch(IR::Instr const * instr, Value const * src1Val, Value const * src2Val) const
  9132. {
  9133. if (!src1Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc1()->IsRegOpnd())
  9134. {
  9135. StackSym const *sym1 = instr->GetSrc1()->AsRegOpnd()->m_sym;
  9136. if (CurrentBlockData()->IsInt32TypeSpecialized(sym1) == false)
  9137. {
  9138. if (!src2Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc2()->IsRegOpnd())
  9139. {
  9140. StackSym const *sym2 = instr->GetSrc2()->AsRegOpnd()->m_sym;
  9141. if (CurrentBlockData()->IsInt32TypeSpecialized(sym2) == false)
  9142. {
  9143. // Type specializing a Br itself isn't worth it, unless one src
  9144. // is already type specialized
  9145. return false;
  9146. }
  9147. }
  9148. }
  9149. }
  9150. return true;
  9151. }
  9152. bool
  9153. GlobOpt::TryOptConstFoldBrFalse(
  9154. IR::Instr *const instr,
  9155. Value *const srcValue,
  9156. const int32 min,
  9157. const int32 max)
  9158. {
  9159. Assert(instr);
  9160. Assert(instr->m_opcode == Js::OpCode::BrFalse_A || instr->m_opcode == Js::OpCode::BrTrue_A);
  9161. Assert(srcValue);
  9162. if(!(DoAggressiveIntTypeSpec() ? srcValue->GetValueInfo()->IsLikelyInt() : srcValue->GetValueInfo()->IsInt()))
  9163. {
  9164. return false;
  9165. }
  9166. if(ValueInfo::IsEqualTo(srcValue, min, max, nullptr, 0, 0))
  9167. {
  9168. OptConstFoldBr(instr->m_opcode == Js::OpCode::BrFalse_A, instr, srcValue);
  9169. return true;
  9170. }
  9171. if(ValueInfo::IsNotEqualTo(srcValue, min, max, nullptr, 0, 0))
  9172. {
  9173. OptConstFoldBr(instr->m_opcode == Js::OpCode::BrTrue_A, instr, srcValue);
  9174. return true;
  9175. }
  9176. return false;
  9177. }
  9178. bool
  9179. GlobOpt::TryOptConstFoldBrEqual(
  9180. IR::Instr *const instr,
  9181. const bool branchOnEqual,
  9182. Value *const src1Value,
  9183. const int32 min1,
  9184. const int32 max1,
  9185. Value *const src2Value,
  9186. const int32 min2,
  9187. const int32 max2)
  9188. {
  9189. Assert(instr);
  9190. Assert(src1Value);
  9191. Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt());
  9192. Assert(src2Value);
  9193. Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt());
  9194. if(ValueInfo::IsEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  9195. {
  9196. OptConstFoldBr(branchOnEqual, instr, src1Value, src2Value);
  9197. return true;
  9198. }
  9199. if(ValueInfo::IsNotEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  9200. {
  9201. OptConstFoldBr(!branchOnEqual, instr, src1Value, src2Value);
  9202. return true;
  9203. }
  9204. return false;
  9205. }
  9206. bool
  9207. GlobOpt::TryOptConstFoldBrGreaterThan(
  9208. IR::Instr *const instr,
  9209. const bool branchOnGreaterThan,
  9210. Value *const src1Value,
  9211. const int32 min1,
  9212. const int32 max1,
  9213. Value *const src2Value,
  9214. const int32 min2,
  9215. const int32 max2)
  9216. {
  9217. Assert(instr);
  9218. Assert(src1Value);
  9219. Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt());
  9220. Assert(src2Value);
  9221. Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt());
  9222. if(ValueInfo::IsGreaterThan(src1Value, min1, max1, src2Value, min2, max2))
  9223. {
  9224. OptConstFoldBr(branchOnGreaterThan, instr, src1Value, src2Value);
  9225. return true;
  9226. }
  9227. if(ValueInfo::IsLessThanOrEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  9228. {
  9229. OptConstFoldBr(!branchOnGreaterThan, instr, src1Value, src2Value);
  9230. return true;
  9231. }
  9232. return false;
  9233. }
  9234. bool
  9235. GlobOpt::TryOptConstFoldBrGreaterThanOrEqual(
  9236. IR::Instr *const instr,
  9237. const bool branchOnGreaterThanOrEqual,
  9238. Value *const src1Value,
  9239. const int32 min1,
  9240. const int32 max1,
  9241. Value *const src2Value,
  9242. const int32 min2,
  9243. const int32 max2)
  9244. {
  9245. Assert(instr);
  9246. Assert(src1Value);
  9247. Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt());
  9248. Assert(src2Value);
  9249. Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt());
  9250. if(ValueInfo::IsGreaterThanOrEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  9251. {
  9252. OptConstFoldBr(branchOnGreaterThanOrEqual, instr, src1Value, src2Value);
  9253. return true;
  9254. }
  9255. if(ValueInfo::IsLessThan(src1Value, min1, max1, src2Value, min2, max2))
  9256. {
  9257. OptConstFoldBr(!branchOnGreaterThanOrEqual, instr, src1Value, src2Value);
  9258. return true;
  9259. }
  9260. return false;
  9261. }
  9262. bool
  9263. GlobOpt::TryOptConstFoldBrUnsignedLessThan(
  9264. IR::Instr *const instr,
  9265. const bool branchOnLessThan,
  9266. Value *const src1Value,
  9267. const int32 min1,
  9268. const int32 max1,
  9269. Value *const src2Value,
  9270. const int32 min2,
  9271. const int32 max2)
  9272. {
  9273. Assert(DoConstFold());
  9274. Assert(!IsLoopPrePass());
  9275. if(!src1Value ||
  9276. !src2Value ||
  9277. !(
  9278. DoAggressiveIntTypeSpec()
  9279. ? src1Value->GetValueInfo()->IsLikelyInt() && src2Value->GetValueInfo()->IsLikelyInt()
  9280. : src1Value->GetValueInfo()->IsInt() && src2Value->GetValueInfo()->IsInt()
  9281. ))
  9282. {
  9283. return false;
  9284. }
  9285. uint uMin1 = (min1 < 0 ? (max1 < 0 ? min((uint)min1, (uint)max1) : 0) : min1);
  9286. uint uMax1 = max((uint)min1, (uint)max1);
  9287. uint uMin2 = (min2 < 0 ? (max2 < 0 ? min((uint)min2, (uint)max2) : 0) : min2);
  9288. uint uMax2 = max((uint)min2, (uint)max2);
  9289. if (uMax1 < uMin2)
  9290. {
  9291. // Range 1 is always lesser than Range 2
  9292. OptConstFoldBr(branchOnLessThan, instr, src1Value, src2Value);
  9293. return true;
  9294. }
  9295. if (uMin1 >= uMax2)
  9296. {
  9297. // Range 2 is always lesser than Range 1
  9298. OptConstFoldBr(!branchOnLessThan, instr, src1Value, src2Value);
  9299. return true;
  9300. }
  9301. return false;
  9302. }
  9303. bool
  9304. GlobOpt::TryOptConstFoldBrUnsignedGreaterThan(
  9305. IR::Instr *const instr,
  9306. const bool branchOnGreaterThan,
  9307. Value *const src1Value,
  9308. const int32 min1,
  9309. const int32 max1,
  9310. Value *const src2Value,
  9311. const int32 min2,
  9312. const int32 max2)
  9313. {
  9314. Assert(DoConstFold());
  9315. Assert(!IsLoopPrePass());
  9316. if(!src1Value ||
  9317. !src2Value ||
  9318. !(
  9319. DoAggressiveIntTypeSpec()
  9320. ? src1Value->GetValueInfo()->IsLikelyInt() && src2Value->GetValueInfo()->IsLikelyInt()
  9321. : src1Value->GetValueInfo()->IsInt() && src2Value->GetValueInfo()->IsInt()
  9322. ))
  9323. {
  9324. return false;
  9325. }
  9326. uint uMin1 = (min1 < 0 ? (max1 < 0 ? min((uint)min1, (uint)max1) : 0) : min1);
  9327. uint uMax1 = max((uint)min1, (uint)max1);
  9328. uint uMin2 = (min2 < 0 ? (max2 < 0 ? min((uint)min2, (uint)max2) : 0) : min2);
  9329. uint uMax2 = max((uint)min2, (uint)max2);
  9330. if (uMin1 > uMax2)
  9331. {
  9332. // Range 1 is always greater than Range 2
  9333. OptConstFoldBr(branchOnGreaterThan, instr, src1Value, src2Value);
  9334. return true;
  9335. }
  9336. if (uMax1 <= uMin2)
  9337. {
  9338. // Range 2 is always greater than Range 1
  9339. OptConstFoldBr(!branchOnGreaterThan, instr, src1Value, src2Value);
  9340. return true;
  9341. }
  9342. return false;
  9343. }
  9344. void
  9345. GlobOpt::SetPathDependentInfo(const bool conditionToBranch, const PathDependentInfo &info)
  9346. {
  9347. Assert(this->currentBlock->GetSuccList()->Count() == 2);
  9348. IR::Instr * fallthrough = this->currentBlock->GetNext()->GetFirstInstr();
  9349. FOREACH_SLISTBASECOUNTED_ENTRY(FlowEdge*, edge, this->currentBlock->GetSuccList())
  9350. {
  9351. if (conditionToBranch == (edge->GetSucc()->GetFirstInstr() != fallthrough))
  9352. {
  9353. edge->SetPathDependentInfo(info, alloc);
  9354. return;
  9355. }
  9356. }
  9357. NEXT_SLISTBASECOUNTED_ENTRY;
  9358. // In case flowgraph peeps is disabled, we could have conditional branch to next instr
  9359. Assert(this->func->HasTry() || PHASE_OFF(Js::FGPeepsPhase, this->func));
  9360. }
  9361. PathDependentInfoToRestore
  9362. GlobOpt::UpdatePathDependentInfo(PathDependentInfo *const info)
  9363. {
  9364. Assert(info);
  9365. if(!info->HasInfo())
  9366. {
  9367. return PathDependentInfoToRestore();
  9368. }
  9369. decltype(&GlobOpt::UpdateIntBoundsForEqual) UpdateIntBoundsForLeftValue, UpdateIntBoundsForRightValue;
  9370. switch(info->Relationship())
  9371. {
  9372. case PathDependentRelationship::Equal:
  9373. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForEqual;
  9374. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForEqual;
  9375. break;
  9376. case PathDependentRelationship::NotEqual:
  9377. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForNotEqual;
  9378. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForNotEqual;
  9379. break;
  9380. case PathDependentRelationship::GreaterThanOrEqual:
  9381. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForGreaterThanOrEqual;
  9382. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForLessThanOrEqual;
  9383. break;
  9384. case PathDependentRelationship::GreaterThan:
  9385. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForGreaterThan;
  9386. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForLessThan;
  9387. break;
  9388. case PathDependentRelationship::LessThanOrEqual:
  9389. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForLessThanOrEqual;
  9390. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForGreaterThanOrEqual;
  9391. break;
  9392. case PathDependentRelationship::LessThan:
  9393. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForLessThan;
  9394. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForGreaterThan;
  9395. break;
  9396. default:
  9397. Assert(false);
  9398. __assume(false);
  9399. }
  9400. ValueInfo *leftValueInfo = info->LeftValue()->GetValueInfo();
  9401. IntConstantBounds leftConstantBounds;
  9402. AssertVerify(leftValueInfo->TryGetIntConstantBounds(&leftConstantBounds, true));
  9403. ValueInfo *rightValueInfo;
  9404. IntConstantBounds rightConstantBounds;
  9405. if(info->RightValue())
  9406. {
  9407. rightValueInfo = info->RightValue()->GetValueInfo();
  9408. AssertVerify(rightValueInfo->TryGetIntConstantBounds(&rightConstantBounds, true));
  9409. }
  9410. else
  9411. {
  9412. rightValueInfo = nullptr;
  9413. rightConstantBounds = IntConstantBounds(info->RightConstantValue(), info->RightConstantValue());
  9414. }
  9415. ValueInfo *const newLeftValueInfo =
  9416. (this->*UpdateIntBoundsForLeftValue)(
  9417. info->LeftValue(),
  9418. leftConstantBounds,
  9419. info->RightValue(),
  9420. rightConstantBounds,
  9421. true);
  9422. if(newLeftValueInfo)
  9423. {
  9424. ChangeValueInfo(nullptr, info->LeftValue(), newLeftValueInfo);
  9425. AssertVerify(newLeftValueInfo->TryGetIntConstantBounds(&leftConstantBounds, true));
  9426. }
  9427. else
  9428. {
  9429. leftValueInfo = nullptr;
  9430. }
  9431. ValueInfo *const newRightValueInfo =
  9432. (this->*UpdateIntBoundsForRightValue)(
  9433. info->RightValue(),
  9434. rightConstantBounds,
  9435. info->LeftValue(),
  9436. leftConstantBounds,
  9437. true);
  9438. if(newRightValueInfo)
  9439. {
  9440. ChangeValueInfo(nullptr, info->RightValue(), newRightValueInfo);
  9441. }
  9442. else
  9443. {
  9444. rightValueInfo = nullptr;
  9445. }
  9446. return PathDependentInfoToRestore(leftValueInfo, rightValueInfo);
  9447. }
  9448. void
  9449. GlobOpt::RestorePathDependentInfo(PathDependentInfo *const info, const PathDependentInfoToRestore infoToRestore)
  9450. {
  9451. Assert(info);
  9452. if(infoToRestore.LeftValueInfo())
  9453. {
  9454. Assert(info->LeftValue());
  9455. ChangeValueInfo(nullptr, info->LeftValue(), infoToRestore.LeftValueInfo());
  9456. }
  9457. if(infoToRestore.RightValueInfo())
  9458. {
  9459. Assert(info->RightValue());
  9460. ChangeValueInfo(nullptr, info->RightValue(), infoToRestore.RightValueInfo());
  9461. }
  9462. }
  9463. bool
  9464. GlobOpt::TypeSpecializeFloatUnary(IR::Instr **pInstr, Value *src1Val, Value **pDstVal, bool skipDst /* = false */)
  9465. {
  9466. IR::Instr *&instr = *pInstr;
  9467. IR::Opnd *src1;
  9468. IR::Opnd *dst;
  9469. Js::OpCode opcode = instr->m_opcode;
  9470. Value *valueToTransfer = nullptr;
  9471. Assert(src1Val && src1Val->GetValueInfo()->IsLikelyNumber() || OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  9472. if (!this->DoFloatTypeSpec())
  9473. {
  9474. return false;
  9475. }
  9476. // For inline built-ins we need to do type specialization. Check upfront to avoid duplicating same case labels.
  9477. if (!OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  9478. {
  9479. switch (opcode)
  9480. {
  9481. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  9482. skipDst = true;
  9483. // fall-through
  9484. case Js::OpCode::Ld_A:
  9485. case Js::OpCode::BrTrue_A:
  9486. case Js::OpCode::BrFalse_A:
  9487. if (instr->GetSrc1()->IsRegOpnd())
  9488. {
  9489. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  9490. if (CurrentBlockData()->IsFloat64TypeSpecialized(sym) == false)
  9491. {
  9492. // Type specializing an Ld_A isn't worth it, unless the src
  9493. // is already type specialized
  9494. return false;
  9495. }
  9496. }
  9497. if (instr->m_opcode == Js::OpCode::Ld_A)
  9498. {
  9499. valueToTransfer = src1Val;
  9500. }
  9501. break;
  9502. case Js::OpCode::Neg_A:
  9503. break;
  9504. case Js::OpCode::Conv_Num:
  9505. Assert(src1Val);
  9506. opcode = Js::OpCode::Ld_A;
  9507. valueToTransfer = src1Val;
  9508. if (!src1Val->GetValueInfo()->IsNumber())
  9509. {
  9510. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  9511. valueToTransfer = NewGenericValue(ValueType::Float, instr->GetDst()->GetStackSym());
  9512. if (CurrentBlockData()->IsFloat64TypeSpecialized(sym) == false)
  9513. {
  9514. // Set the dst as a nonDeadStore. We want to keep the Ld_A to prevent the FromVar from
  9515. // being dead-stored, as it could cause implicit calls.
  9516. dst = instr->GetDst();
  9517. dst->AsRegOpnd()->m_dontDeadStore = true;
  9518. }
  9519. }
  9520. break;
  9521. case Js::OpCode::StElemI_A:
  9522. case Js::OpCode::StElemI_A_Strict:
  9523. case Js::OpCode::StElemC:
  9524. return TypeSpecializeStElem(pInstr, src1Val, pDstVal);
  9525. default:
  9526. return false;
  9527. }
  9528. }
  9529. // Make sure the srcs are specialized
  9530. src1 = instr->GetSrc1();
  9531. // Use original val when calling toFloat64 as this is what we'll use to try hoisting the fromVar if we're in a loop.
  9532. this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, IR::BailOutPrimitiveButString);
  9533. if (!skipDst)
  9534. {
  9535. dst = instr->GetDst();
  9536. if (dst)
  9537. {
  9538. this->TypeSpecializeFloatDst(instr, valueToTransfer, src1Val, nullptr, pDstVal);
  9539. if (!this->IsLoopPrePass())
  9540. {
  9541. instr->m_opcode = opcode;
  9542. }
  9543. }
  9544. }
  9545. GOPT_TRACE_INSTR(instr, _u("Type specialized to FLOAT: "));
  9546. #if ENABLE_DEBUG_CONFIG_OPTIONS
  9547. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FloatTypeSpecPhase))
  9548. {
  9549. Output::Print(_u("Type specialized to FLOAT: "));
  9550. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  9551. }
  9552. #endif
  9553. return true;
  9554. }
  9555. // Unconditionally type-spec dst to float.
  9556. void
  9557. GlobOpt::TypeSpecializeFloatDst(IR::Instr *instr, Value *valToTransfer, Value *const src1Value, Value *const src2Value, Value **pDstVal)
  9558. {
  9559. IR::Opnd* dst = instr->GetDst();
  9560. Assert(dst);
  9561. AssertMsg(dst->IsRegOpnd(), "What else?");
  9562. this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  9563. if(valToTransfer)
  9564. {
  9565. *pDstVal = this->ValueNumberTransferDst(instr, valToTransfer);
  9566. CurrentBlockData()->InsertNewValue(*pDstVal, dst);
  9567. }
  9568. else
  9569. {
  9570. *pDstVal = CreateDstUntransferredValue(ValueType::Float, instr, src1Value, src2Value);
  9571. }
  9572. }
  9573. bool
  9574. GlobOpt::TypeSpecializeLdLen(
  9575. IR::Instr * *const instrRef,
  9576. Value * *const src1ValueRef,
  9577. Value * *const dstValueRef,
  9578. bool *const forceInvariantHoistingRef)
  9579. {
  9580. Assert(instrRef);
  9581. IR::Instr *&instr = *instrRef;
  9582. Assert(instr);
  9583. Assert(instr->m_opcode == Js::OpCode::LdLen_A);
  9584. Assert(src1ValueRef);
  9585. Value *&src1Value = *src1ValueRef;
  9586. Assert(dstValueRef);
  9587. Value *&dstValue = *dstValueRef;
  9588. Assert(forceInvariantHoistingRef);
  9589. bool &forceInvariantHoisting = *forceInvariantHoistingRef;
  9590. if(!DoLdLenIntSpec(instr, instr->GetSrc1()->GetValueType()))
  9591. {
  9592. return false;
  9593. }
  9594. IR::BailOutKind bailOutKind = IR::BailOutOnIrregularLength;
  9595. if(!IsLoopPrePass())
  9596. {
  9597. IR::RegOpnd *const baseOpnd = instr->GetSrc1()->AsRegOpnd();
  9598. if(baseOpnd->IsArrayRegOpnd())
  9599. {
  9600. StackSym *const lengthSym = baseOpnd->AsArrayRegOpnd()->LengthSym();
  9601. if(lengthSym)
  9602. {
  9603. CaptureByteCodeSymUses(instr);
  9604. instr->m_opcode = Js::OpCode::Ld_I4;
  9605. instr->ReplaceSrc1(IR::RegOpnd::New(lengthSym, lengthSym->GetType(), func));
  9606. instr->ClearBailOutInfo();
  9607. // Find the hoisted length value
  9608. Value *const lengthValue = CurrentBlockData()->FindValue(lengthSym);
  9609. Assert(lengthValue);
  9610. src1Value = lengthValue;
  9611. ValueInfo *const lengthValueInfo = lengthValue->GetValueInfo();
  9612. IntConstantBounds lengthConstantBounds;
  9613. AssertVerify(lengthValueInfo->TryGetIntConstantBounds(&lengthConstantBounds));
  9614. Assert(lengthConstantBounds.LowerBound() >= 0);
  9615. if (lengthValueInfo->GetSymStore() == lengthSym)
  9616. {
  9617. // When type specializing the dst below, we will end up inserting lengthSym.u32 as symstore for a var
  9618. // Clear the symstore here, so that we dont end up with problems with copyprop later on
  9619. lengthValueInfo->SetSymStore(nullptr);
  9620. }
  9621. // Int-specialize, and transfer the value to the dst
  9622. TypeSpecializeIntDst(
  9623. instr,
  9624. Js::OpCode::LdLen_A,
  9625. src1Value,
  9626. src1Value,
  9627. nullptr,
  9628. bailOutKind,
  9629. lengthConstantBounds.LowerBound(),
  9630. lengthConstantBounds.UpperBound(),
  9631. &dstValue);
  9632. // Try to force hoisting the Ld_I4 so that the length will have an invariant sym store that can be
  9633. // copy-propped. Invariant hoisting does not automatically hoist Ld_I4.
  9634. forceInvariantHoisting = true;
  9635. return true;
  9636. }
  9637. }
  9638. if (instr->HasBailOutInfo())
  9639. {
  9640. Assert(instr->GetBailOutKind() == IR::BailOutMarkTempObject);
  9641. bailOutKind = IR::BailOutOnIrregularLength | IR::BailOutMarkTempObject;
  9642. instr->SetBailOutKind(bailOutKind);
  9643. }
  9644. else
  9645. {
  9646. Assert(bailOutKind == IR::BailOutOnIrregularLength);
  9647. GenerateBailAtOperation(&instr, bailOutKind);
  9648. }
  9649. }
  9650. TypeSpecializeIntDst(
  9651. instr,
  9652. Js::OpCode::LdLen_A,
  9653. nullptr,
  9654. nullptr,
  9655. nullptr,
  9656. bailOutKind,
  9657. 0,
  9658. INT32_MAX,
  9659. &dstValue);
  9660. return true;
  9661. }
  9662. bool
  9663. GlobOpt::TypeSpecializeFloatBinary(IR::Instr *instr, Value *src1Val, Value *src2Val, Value **pDstVal)
  9664. {
  9665. IR::Opnd *src1;
  9666. IR::Opnd *src2;
  9667. IR::Opnd *dst;
  9668. bool allowUndefinedOrNullSrc1 = true;
  9669. bool allowUndefinedOrNullSrc2 = true;
  9670. bool skipSrc1 = false;
  9671. bool skipSrc2 = false;
  9672. bool skipDst = false;
  9673. bool convertDstToBool = false;
  9674. if (!this->DoFloatTypeSpec())
  9675. {
  9676. return false;
  9677. }
  9678. // For inline built-ins we need to do type specialization. Check upfront to avoid duplicating same case labels.
  9679. if (!OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  9680. {
  9681. switch (instr->m_opcode)
  9682. {
  9683. case Js::OpCode::Sub_A:
  9684. case Js::OpCode::Mul_A:
  9685. case Js::OpCode::Div_A:
  9686. case Js::OpCode::Expo_A:
  9687. // Avoid if one source is known not to be a number.
  9688. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber())
  9689. {
  9690. return false;
  9691. }
  9692. break;
  9693. case Js::OpCode::BrSrEq_A:
  9694. case Js::OpCode::BrSrNeq_A:
  9695. case Js::OpCode::BrEq_A:
  9696. case Js::OpCode::BrNeq_A:
  9697. case Js::OpCode::BrSrNotEq_A:
  9698. case Js::OpCode::BrNotEq_A:
  9699. case Js::OpCode::BrSrNotNeq_A:
  9700. case Js::OpCode::BrNotNeq_A:
  9701. // Avoid if one source is known not to be a number.
  9702. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber())
  9703. {
  9704. return false;
  9705. }
  9706. // Undef == Undef, but +Undef != +Undef
  9707. // 0.0 != null, but 0.0 == +null
  9708. //
  9709. // So Bailout on anything but numbers for both src1 and src2
  9710. allowUndefinedOrNullSrc1 = false;
  9711. allowUndefinedOrNullSrc2 = false;
  9712. break;
  9713. case Js::OpCode::BrGt_A:
  9714. case Js::OpCode::BrGe_A:
  9715. case Js::OpCode::BrLt_A:
  9716. case Js::OpCode::BrLe_A:
  9717. case Js::OpCode::BrNotGt_A:
  9718. case Js::OpCode::BrNotGe_A:
  9719. case Js::OpCode::BrNotLt_A:
  9720. case Js::OpCode::BrNotLe_A:
  9721. // Avoid if one source is known not to be a number.
  9722. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber())
  9723. {
  9724. return false;
  9725. }
  9726. break;
  9727. case Js::OpCode::Add_A:
  9728. // For Add, we need both sources to be Numbers, otherwise it could be a string concat
  9729. if (!src1Val || !src2Val || !(src1Val->GetValueInfo()->IsLikelyNumber() && src2Val->GetValueInfo()->IsLikelyNumber()))
  9730. {
  9731. return false;
  9732. }
  9733. break;
  9734. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  9735. skipSrc2 = true;
  9736. skipDst = true;
  9737. break;
  9738. case Js::OpCode::CmEq_A:
  9739. case Js::OpCode::CmSrEq_A:
  9740. case Js::OpCode::CmNeq_A:
  9741. case Js::OpCode::CmSrNeq_A:
  9742. {
  9743. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber())
  9744. {
  9745. return false;
  9746. }
  9747. allowUndefinedOrNullSrc1 = false;
  9748. allowUndefinedOrNullSrc2 = false;
  9749. convertDstToBool = true;
  9750. break;
  9751. }
  9752. case Js::OpCode::CmLe_A:
  9753. case Js::OpCode::CmLt_A:
  9754. case Js::OpCode::CmGe_A:
  9755. case Js::OpCode::CmGt_A:
  9756. {
  9757. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber())
  9758. {
  9759. return false;
  9760. }
  9761. convertDstToBool = true;
  9762. break;
  9763. }
  9764. default:
  9765. return false;
  9766. }
  9767. }
  9768. else
  9769. {
  9770. switch (instr->m_opcode)
  9771. {
  9772. case Js::OpCode::InlineArrayPush:
  9773. bool isFloatConstMissingItem = src2Val->GetValueInfo()->IsFloatConstant();
  9774. if(isFloatConstMissingItem)
  9775. {
  9776. FloatConstType floatValue = src2Val->GetValueInfo()->AsFloatConstant()->FloatValue();
  9777. isFloatConstMissingItem = Js::SparseArraySegment<double>::IsMissingItem(&floatValue);
  9778. }
  9779. // Don't specialize if the element is not likelyNumber - we will surely bailout
  9780. if(!(src2Val->GetValueInfo()->IsLikelyNumber()) || isFloatConstMissingItem)
  9781. {
  9782. return false;
  9783. }
  9784. // Only specialize the Second source - element
  9785. skipSrc1 = true;
  9786. skipDst = true;
  9787. allowUndefinedOrNullSrc2 = false;
  9788. break;
  9789. }
  9790. }
  9791. // Make sure the srcs are specialized
  9792. if(!skipSrc1)
  9793. {
  9794. src1 = instr->GetSrc1();
  9795. this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, (allowUndefinedOrNullSrc1 ? IR::BailOutPrimitiveButString : IR::BailOutNumberOnly));
  9796. }
  9797. if (!skipSrc2)
  9798. {
  9799. src2 = instr->GetSrc2();
  9800. this->ToFloat64(instr, src2, this->currentBlock, src2Val, nullptr, (allowUndefinedOrNullSrc2 ? IR::BailOutPrimitiveButString : IR::BailOutNumberOnly));
  9801. }
  9802. if (!skipDst)
  9803. {
  9804. dst = instr->GetDst();
  9805. if (dst)
  9806. {
  9807. if (convertDstToBool)
  9808. {
  9809. *pDstVal = CreateDstUntransferredValue(ValueType::Boolean, instr, src1Val, src2Val);
  9810. ToVarRegOpnd(dst->AsRegOpnd(), currentBlock);
  9811. }
  9812. else
  9813. {
  9814. *pDstVal = CreateDstUntransferredValue(ValueType::Float, instr, src1Val, src2Val);
  9815. AssertMsg(dst->IsRegOpnd(), "What else?");
  9816. this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  9817. }
  9818. }
  9819. }
  9820. GOPT_TRACE_INSTR(instr, _u("Type specialized to FLOAT: "));
  9821. #if ENABLE_DEBUG_CONFIG_OPTIONS
  9822. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FloatTypeSpecPhase))
  9823. {
  9824. Output::Print(_u("Type specialized to FLOAT: "));
  9825. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  9826. }
  9827. #endif
  9828. return true;
  9829. }
  9830. bool
  9831. GlobOpt::TypeSpecializeStElem(IR::Instr ** pInstr, Value *src1Val, Value **pDstVal)
  9832. {
  9833. IR::Instr *&instr = *pInstr;
  9834. IR::RegOpnd *baseOpnd = instr->GetDst()->AsIndirOpnd()->GetBaseOpnd();
  9835. ValueType baseValueType(baseOpnd->GetValueType());
  9836. if (instr->DoStackArgsOpt() ||
  9837. (!this->DoTypedArrayTypeSpec() && baseValueType.IsLikelyOptimizedTypedArray()) ||
  9838. (!this->DoNativeArrayTypeSpec() && baseValueType.IsLikelyNativeArray()) ||
  9839. !(baseValueType.IsLikelyOptimizedTypedArray() || baseValueType.IsLikelyNativeArray()))
  9840. {
  9841. GOPT_TRACE_INSTR(instr, _u("Didn't type specialize array access, because typed array type specialization is disabled, or base is not an optimized typed array.\n"));
  9842. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9843. {
  9844. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9845. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9846. baseValueType.ToString(baseValueTypeStr);
  9847. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because %s.\n"),
  9848. this->func->GetJITFunctionBody()->GetDisplayName(),
  9849. this->func->GetDebugNumberSet(debugStringBuffer),
  9850. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9851. baseValueTypeStr,
  9852. instr->DoStackArgsOpt() ?
  9853. _u("instruction uses the arguments object") :
  9854. _u("typed array type specialization is disabled, or base is not an optimized typed array"));
  9855. Output::Flush();
  9856. }
  9857. return false;
  9858. }
  9859. Assert(instr->GetSrc1()->IsRegOpnd() || (src1Val && src1Val->GetValueInfo()->HasIntConstantValue()));
  9860. StackSym *sym = instr->GetSrc1()->IsRegOpnd() ? instr->GetSrc1()->AsRegOpnd()->m_sym : nullptr;
  9861. // Only type specialize the source of store element if the source symbol is already type specialized to int or float.
  9862. if (sym)
  9863. {
  9864. if (baseValueType.IsLikelyNativeArray())
  9865. {
  9866. // Gently coerce these src's into native if it seems likely to work.
  9867. // Otherwise we can't use the fast path to store.
  9868. // But don't try to put a float-specialized number into an int array this way.
  9869. if (!(
  9870. CurrentBlockData()->IsInt32TypeSpecialized(sym) ||
  9871. (
  9872. src1Val &&
  9873. (
  9874. DoAggressiveIntTypeSpec()
  9875. ? src1Val->GetValueInfo()->IsLikelyInt()
  9876. : src1Val->GetValueInfo()->IsInt()
  9877. )
  9878. )
  9879. ))
  9880. {
  9881. if (!(
  9882. CurrentBlockData()->IsFloat64TypeSpecialized(sym) ||
  9883. (src1Val && src1Val->GetValueInfo()->IsLikelyNumber())
  9884. ) ||
  9885. baseValueType.HasIntElements())
  9886. {
  9887. return false;
  9888. }
  9889. }
  9890. }
  9891. else if (!CurrentBlockData()->IsInt32TypeSpecialized(sym) && !CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  9892. {
  9893. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because src is not type specialized.\n"));
  9894. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9895. {
  9896. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9897. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9898. baseValueType.ToString(baseValueTypeStr);
  9899. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because src is not specialized.\n"),
  9900. this->func->GetJITFunctionBody()->GetDisplayName(),
  9901. this->func->GetDebugNumberSet(debugStringBuffer),
  9902. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9903. baseValueTypeStr);
  9904. Output::Flush();
  9905. }
  9906. return false;
  9907. }
  9908. }
  9909. int32 src1IntConstantValue;
  9910. if(baseValueType.IsLikelyNativeIntArray() && src1Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&src1IntConstantValue))
  9911. {
  9912. if(Js::SparseArraySegment<int32>::IsMissingItem(&src1IntConstantValue))
  9913. {
  9914. return false;
  9915. }
  9916. }
  9917. // Note: doing ToVarUses to make sure we do get the int32 version of the index before trying to access its value in
  9918. // ShouldExpectConventionalArrayIndexValue. Not sure why that never gave us a problem before.
  9919. Assert(instr->GetDst()->IsIndirOpnd());
  9920. IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd();
  9921. // Make sure we use the int32 version of the index operand symbol, if available. Otherwise, ensure the var symbol is live (by
  9922. // potentially inserting a ToVar).
  9923. this->ToVarUses(instr, dst, /* isDst = */ true, nullptr);
  9924. if (!ShouldExpectConventionalArrayIndexValue(dst))
  9925. {
  9926. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because index is negative or likely not int.\n"));
  9927. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9928. {
  9929. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9930. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9931. baseValueType.ToString(baseValueTypeStr);
  9932. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because index is negative or likely not int.\n"),
  9933. this->func->GetJITFunctionBody()->GetDisplayName(),
  9934. this->func->GetDebugNumberSet(debugStringBuffer),
  9935. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9936. baseValueTypeStr);
  9937. Output::Flush();
  9938. }
  9939. return false;
  9940. }
  9941. IRType toType = TyVar;
  9942. bool isLossyAllowed = true;
  9943. IR::BailOutKind arrayBailOutKind = IR::BailOutConventionalTypedArrayAccessOnly;
  9944. switch(baseValueType.GetObjectType())
  9945. {
  9946. case ObjectType::Int8Array:
  9947. case ObjectType::Uint8Array:
  9948. case ObjectType::Int16Array:
  9949. case ObjectType::Uint16Array:
  9950. case ObjectType::Int32Array:
  9951. case ObjectType::Int8VirtualArray:
  9952. case ObjectType::Uint8VirtualArray:
  9953. case ObjectType::Int16VirtualArray:
  9954. case ObjectType::Uint16VirtualArray:
  9955. case ObjectType::Int32VirtualArray:
  9956. case ObjectType::Int8MixedArray:
  9957. case ObjectType::Uint8MixedArray:
  9958. case ObjectType::Int16MixedArray:
  9959. case ObjectType::Uint16MixedArray:
  9960. case ObjectType::Int32MixedArray:
  9961. Int32Array:
  9962. if (this->DoAggressiveIntTypeSpec() || this->DoFloatTypeSpec())
  9963. {
  9964. toType = TyInt32;
  9965. }
  9966. break;
  9967. case ObjectType::Uint32Array:
  9968. case ObjectType::Uint32VirtualArray:
  9969. case ObjectType::Uint32MixedArray:
  9970. // Uint32Arrays may store values that overflow int32. If the value being stored comes from a symbol that's
  9971. // already losslessly type specialized to int32, we'll use it. Otherwise, if we only have a float64 specialized
  9972. // value, we don't want to force bailout if it doesn't fit in int32. Instead, we'll emit conversion in the
  9973. // lowerer, and handle overflow, if necessary.
  9974. if (!sym || CurrentBlockData()->IsInt32TypeSpecialized(sym))
  9975. {
  9976. toType = TyInt32;
  9977. }
  9978. else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  9979. {
  9980. toType = TyFloat64;
  9981. }
  9982. break;
  9983. case ObjectType::Float32Array:
  9984. case ObjectType::Float64Array:
  9985. case ObjectType::Float32VirtualArray:
  9986. case ObjectType::Float32MixedArray:
  9987. case ObjectType::Float64VirtualArray:
  9988. case ObjectType::Float64MixedArray:
  9989. Float64Array:
  9990. if (this->DoFloatTypeSpec())
  9991. {
  9992. toType = TyFloat64;
  9993. }
  9994. break;
  9995. case ObjectType::Uint8ClampedArray:
  9996. case ObjectType::Uint8ClampedVirtualArray:
  9997. case ObjectType::Uint8ClampedMixedArray:
  9998. // Uint8ClampedArray requires rounding (as opposed to truncation) of floating point values. If source symbol is
  9999. // float type specialized, type specialize this instruction to float as well, and handle rounding in the
  10000. // lowerer.
  10001. if (!sym || CurrentBlockData()->IsInt32TypeSpecialized(sym))
  10002. {
  10003. toType = TyInt32;
  10004. isLossyAllowed = false;
  10005. }
  10006. else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  10007. {
  10008. toType = TyFloat64;
  10009. }
  10010. break;
  10011. default:
  10012. Assert(baseValueType.IsLikelyNativeArray());
  10013. isLossyAllowed = false;
  10014. arrayBailOutKind = IR::BailOutConventionalNativeArrayAccessOnly;
  10015. if(baseValueType.HasIntElements())
  10016. {
  10017. goto Int32Array;
  10018. }
  10019. Assert(baseValueType.HasFloatElements());
  10020. goto Float64Array;
  10021. }
  10022. if (toType != TyVar)
  10023. {
  10024. GOPT_TRACE_INSTR(instr, _u("Type specialized array access.\n"));
  10025. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  10026. {
  10027. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  10028. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  10029. baseValueType.ToString(baseValueTypeStr);
  10030. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, type specialized to %s.\n"),
  10031. this->func->GetJITFunctionBody()->GetDisplayName(),
  10032. this->func->GetDebugNumberSet(debugStringBuffer),
  10033. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  10034. baseValueTypeStr,
  10035. toType == TyInt32 ? _u("int32") : _u("float64"));
  10036. Output::Flush();
  10037. }
  10038. IR::BailOutKind bailOutKind = ((toType == TyInt32) ? IR::BailOutIntOnly : IR::BailOutNumberOnly);
  10039. this->ToTypeSpecUse(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, toType, bailOutKind, /* lossy = */ isLossyAllowed);
  10040. if (!this->IsLoopPrePass())
  10041. {
  10042. bool bConvertToBailoutInstr = true;
  10043. // Definite StElemC doesn't need bailout, because it can't fail or cause conversion.
  10044. if (instr->m_opcode == Js::OpCode::StElemC && baseValueType.IsObject())
  10045. {
  10046. if (baseValueType.HasIntElements())
  10047. {
  10048. //Native int array requires a missing element check & bailout
  10049. int32 min = INT32_MIN;
  10050. int32 max = INT32_MAX;
  10051. if (src1Val->GetValueInfo()->GetIntValMinMax(&min, &max, false))
  10052. {
  10053. bConvertToBailoutInstr = ((min <= Js::JavascriptNativeIntArray::MissingItem) && (max >= Js::JavascriptNativeIntArray::MissingItem));
  10054. }
  10055. }
  10056. else
  10057. {
  10058. bConvertToBailoutInstr = false;
  10059. }
  10060. }
  10061. if (bConvertToBailoutInstr)
  10062. {
  10063. if(instr->HasBailOutInfo())
  10064. {
  10065. const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind();
  10066. Assert(
  10067. (
  10068. !(oldBailOutKind & ~IR::BailOutKindBits) ||
  10069. (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp
  10070. ) &&
  10071. !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject)));
  10072. if(arrayBailOutKind == IR::BailOutConventionalTypedArrayAccessOnly)
  10073. {
  10074. // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head
  10075. // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction
  10076. // bails out for the right reason.
  10077. instr->SetBailOutKind(
  10078. arrayBailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall)));
  10079. }
  10080. else
  10081. {
  10082. // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit
  10083. // calls to occur, so it must be merged in to eliminate generating the helper call.
  10084. Assert(arrayBailOutKind == IR::BailOutConventionalNativeArrayAccessOnly);
  10085. instr->SetBailOutKind(oldBailOutKind | arrayBailOutKind);
  10086. }
  10087. }
  10088. else
  10089. {
  10090. GenerateBailAtOperation(&instr, arrayBailOutKind);
  10091. }
  10092. }
  10093. }
  10094. }
  10095. else
  10096. {
  10097. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because the source was not already specialized.\n"));
  10098. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  10099. {
  10100. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  10101. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  10102. baseValueType.ToString(baseValueTypeStr);
  10103. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not type specialize, because of array type.\n"),
  10104. this->func->GetJITFunctionBody()->GetDisplayName(),
  10105. this->func->GetDebugNumberSet(debugStringBuffer),
  10106. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  10107. baseValueTypeStr);
  10108. Output::Flush();
  10109. }
  10110. }
  10111. return toType != TyVar;
  10112. }
  10113. IR::Instr *
  10114. GlobOpt::ToVarUses(IR::Instr *instr, IR::Opnd *opnd, bool isDst, Value *val)
  10115. {
  10116. Sym *sym;
  10117. switch (opnd->GetKind())
  10118. {
  10119. case IR::OpndKindReg:
  10120. if (!isDst && !CurrentBlockData()->liveVarSyms->Test(opnd->AsRegOpnd()->m_sym->m_id))
  10121. {
  10122. instr = this->ToVar(instr, opnd->AsRegOpnd(), this->currentBlock, val, true);
  10123. }
  10124. break;
  10125. case IR::OpndKindSym:
  10126. sym = opnd->AsSymOpnd()->m_sym;
  10127. if (sym->IsPropertySym() && !CurrentBlockData()->liveVarSyms->Test(sym->AsPropertySym()->m_stackSym->m_id)
  10128. && sym->AsPropertySym()->m_stackSym->IsVar())
  10129. {
  10130. StackSym *propertyBase = sym->AsPropertySym()->m_stackSym;
  10131. IR::RegOpnd *newOpnd = IR::RegOpnd::New(propertyBase, TyVar, instr->m_func);
  10132. instr = this->ToVar(instr, newOpnd, this->currentBlock, CurrentBlockData()->FindValue(propertyBase), true);
  10133. }
  10134. break;
  10135. case IR::OpndKindIndir:
  10136. IR::RegOpnd *baseOpnd = opnd->AsIndirOpnd()->GetBaseOpnd();
  10137. if (!CurrentBlockData()->liveVarSyms->Test(baseOpnd->m_sym->m_id))
  10138. {
  10139. instr = this->ToVar(instr, baseOpnd, this->currentBlock, CurrentBlockData()->FindValue(baseOpnd->m_sym), true);
  10140. }
  10141. IR::RegOpnd *indexOpnd = opnd->AsIndirOpnd()->GetIndexOpnd();
  10142. if (indexOpnd && !indexOpnd->m_sym->IsTypeSpec())
  10143. {
  10144. instr = ToTypeSpecIndex(instr, indexOpnd, opnd->AsIndirOpnd());
  10145. }
  10146. break;
  10147. }
  10148. return instr;
  10149. }
  10150. IR::Instr *
  10151. GlobOpt::ToTypeSpecIndex(IR::Instr * instr, IR::RegOpnd * indexOpnd, IR::IndirOpnd * indirOpnd)
  10152. {
  10153. Assert(indirOpnd != nullptr || indexOpnd == instr->GetSrc1());
  10154. bool isGetterOrSetter = instr->m_opcode == Js::OpCode::InitGetElemI ||
  10155. instr->m_opcode == Js::OpCode::InitSetElemI ||
  10156. instr->m_opcode == Js::OpCode::InitClassMemberGetComputedName ||
  10157. instr->m_opcode == Js::OpCode::InitClassMemberSetComputedName;
  10158. if (!isGetterOrSetter // typespec is disabled for getters, setters
  10159. && (indexOpnd->GetValueType().IsInt()
  10160. ? !IsTypeSpecPhaseOff(func)
  10161. : indexOpnd->GetValueType().IsLikelyInt() && DoAggressiveIntTypeSpec())
  10162. && !GetIsAsmJSFunc()) // typespec is disabled for asmjs
  10163. {
  10164. StackSym *const indexVarSym = indexOpnd->m_sym;
  10165. Value *const indexValue = CurrentBlockData()->FindValue(indexVarSym);
  10166. Assert(indexValue);
  10167. Assert(indexValue->GetValueInfo()->IsLikelyInt());
  10168. ToInt32(instr, indexOpnd, currentBlock, indexValue, indirOpnd, false);
  10169. Assert(indexValue->GetValueInfo()->IsInt() || IsLoopPrePass());
  10170. if (!IsLoopPrePass())
  10171. {
  10172. IR::Opnd * intOpnd = indirOpnd ? indirOpnd->GetIndexOpnd() : instr->GetSrc1();
  10173. if (intOpnd != nullptr)
  10174. {
  10175. Assert(!intOpnd->IsRegOpnd() || intOpnd->AsRegOpnd()->m_sym->IsTypeSpec());
  10176. IntConstantBounds indexConstantBounds;
  10177. AssertVerify(indexValue->GetValueInfo()->TryGetIntConstantBounds(&indexConstantBounds));
  10178. if (ValueInfo::IsGreaterThanOrEqualTo(
  10179. indexValue,
  10180. indexConstantBounds.LowerBound(),
  10181. indexConstantBounds.UpperBound(),
  10182. nullptr,
  10183. 0,
  10184. 0))
  10185. {
  10186. intOpnd->SetType(TyUint32);
  10187. }
  10188. }
  10189. }
  10190. }
  10191. else if (!CurrentBlockData()->liveVarSyms->Test(indexOpnd->m_sym->m_id))
  10192. {
  10193. instr = this->ToVar(instr, indexOpnd, this->currentBlock, CurrentBlockData()->FindValue(indexOpnd->m_sym), true);
  10194. }
  10195. return instr;
  10196. }
  10197. IR::Instr *
  10198. GlobOpt::ToVar(IR::Instr *instr, IR::RegOpnd *regOpnd, BasicBlock *block, Value *value, bool needsUpdate)
  10199. {
  10200. IR::Instr *newInstr;
  10201. StackSym *varSym = regOpnd->m_sym;
  10202. if (IsTypeSpecPhaseOff(this->func))
  10203. {
  10204. return instr;
  10205. }
  10206. if (this->IsLoopPrePass())
  10207. {
  10208. block->globOptData.liveVarSyms->Set(varSym->m_id);
  10209. return instr;
  10210. }
  10211. if (block->globOptData.liveVarSyms->Test(varSym->m_id))
  10212. {
  10213. // Already live, nothing to do
  10214. return instr;
  10215. }
  10216. if (!varSym->IsVar())
  10217. {
  10218. Assert(!varSym->IsTypeSpec());
  10219. // Leave non-vars alone.
  10220. return instr;
  10221. }
  10222. Assert(block->globOptData.IsTypeSpecialized(varSym));
  10223. if (!value)
  10224. {
  10225. value = block->globOptData.FindValue(varSym);
  10226. }
  10227. ValueInfo *valueInfo = value ? value->GetValueInfo() : nullptr;
  10228. if(valueInfo && valueInfo->IsInt())
  10229. {
  10230. // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value
  10231. // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as
  10232. // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the
  10233. // lossy state.
  10234. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10235. }
  10236. IRType fromType = TyIllegal;
  10237. StackSym *typeSpecSym = nullptr;
  10238. if (block->globOptData.liveInt32Syms->Test(varSym->m_id) && !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id))
  10239. {
  10240. fromType = TyInt32;
  10241. typeSpecSym = varSym->GetInt32EquivSym(this->func);
  10242. Assert(valueInfo);
  10243. Assert(valueInfo->IsInt());
  10244. }
  10245. else if (block->globOptData.liveFloat64Syms->Test(varSym->m_id))
  10246. {
  10247. fromType = TyFloat64;
  10248. typeSpecSym = varSym->GetFloat64EquivSym(this->func);
  10249. // Ensure that all bailout FromVars that generate a value for this type-specialized sym will bail out on any non-number
  10250. // value, even ones that have already been generated before. Float-specialized non-number values cannot be converted
  10251. // back to Var since they will not go back to the original non-number value. The dead-store pass will update the bailout
  10252. // kind on already-generated FromVars based on this bit.
  10253. typeSpecSym->m_requiresBailOnNotNumber = true;
  10254. // A previous float conversion may have used BailOutPrimitiveButString, which does not change the value type to say
  10255. // definitely float, since it can also be a non-string primitive. The convert back to Var though, will cause that
  10256. // bailout kind to be changed to BailOutNumberOnly in the dead-store phase, so from the point of the initial conversion
  10257. // to float, that the value is definitely number. Since we don't know where the FromVar is, change the value type here.
  10258. if(valueInfo)
  10259. {
  10260. if(!valueInfo->IsNumber())
  10261. {
  10262. valueInfo = valueInfo->SpecializeToFloat64(alloc);
  10263. ChangeValueInfo(block, value, valueInfo);
  10264. regOpnd->SetValueType(valueInfo->Type());
  10265. }
  10266. }
  10267. else
  10268. {
  10269. value = NewGenericValue(ValueType::Float);
  10270. valueInfo = value->GetValueInfo();
  10271. block->globOptData.SetValue(value, varSym);
  10272. regOpnd->SetValueType(valueInfo->Type());
  10273. }
  10274. }
  10275. else
  10276. {
  10277. Assert(UNREACHED);
  10278. }
  10279. AssertOrFailFast(valueInfo);
  10280. int32 intConstantValue;
  10281. if (valueInfo->TryGetIntConstantValue(&intConstantValue))
  10282. {
  10283. // Lower will tag or create a number directly
  10284. newInstr = IR::Instr::New(Js::OpCode::LdC_A_I4, regOpnd,
  10285. IR::IntConstOpnd::New(intConstantValue, TyInt32, instr->m_func), instr->m_func);
  10286. }
  10287. else
  10288. {
  10289. IR::RegOpnd * regNew = IR::RegOpnd::New(typeSpecSym, fromType, instr->m_func);
  10290. Js::OpCode opcode = Js::OpCode::ToVar;
  10291. regNew->SetIsJITOptimizedReg(true);
  10292. newInstr = IR::Instr::New(opcode, regOpnd, regNew, instr->m_func);
  10293. }
  10294. newInstr->SetByteCodeOffset(instr);
  10295. newInstr->GetDst()->AsRegOpnd()->SetIsJITOptimizedReg(true);
  10296. ValueType valueType = valueInfo->Type();
  10297. if(fromType == TyInt32)
  10298. {
  10299. #if !INT32VAR // All 32-bit ints are taggable on 64-bit architectures
  10300. IntConstantBounds constantBounds;
  10301. AssertVerify(valueInfo->TryGetIntConstantBounds(&constantBounds));
  10302. if(constantBounds.IsTaggable())
  10303. #endif
  10304. {
  10305. // The value is within the taggable range, so set the opnd value types to TaggedInt to avoid the overflow check
  10306. valueType = ValueType::GetTaggedInt();
  10307. }
  10308. }
  10309. newInstr->GetDst()->SetValueType(valueType);
  10310. newInstr->GetSrc1()->SetValueType(valueType);
  10311. IR::Instr *insertAfterInstr = instr->m_prev;
  10312. if (instr == block->GetLastInstr() &&
  10313. (instr->IsBranchInstr() || instr->m_opcode == Js::OpCode::BailTarget))
  10314. {
  10315. // Don't insert code between the branch and the preceding ByteCodeUses instrs...
  10316. while(insertAfterInstr->m_opcode == Js::OpCode::ByteCodeUses)
  10317. {
  10318. insertAfterInstr = insertAfterInstr->m_prev;
  10319. }
  10320. }
  10321. block->InsertInstrAfter(newInstr, insertAfterInstr);
  10322. block->globOptData.liveVarSyms->Set(varSym->m_id);
  10323. GOPT_TRACE_OPND(regOpnd, _u("Converting to var\n"));
  10324. if (block->loop)
  10325. {
  10326. Assert(!this->IsLoopPrePass());
  10327. this->TryHoistInvariant(newInstr, block, value, value, nullptr, false);
  10328. }
  10329. if (needsUpdate)
  10330. {
  10331. // Make sure that the kill effect of the ToVar instruction is tracked and that the kill of a property
  10332. // type is reflected in the current instruction.
  10333. this->ProcessKills(newInstr);
  10334. this->ValueNumberObjectType(newInstr->GetDst(), newInstr);
  10335. if (instr->GetSrc1() && instr->GetSrc1()->IsSymOpnd() && instr->GetSrc1()->AsSymOpnd()->IsPropertySymOpnd())
  10336. {
  10337. // Reprocess the load source. We need to reset the PropertySymOpnd fields first.
  10338. IR::PropertySymOpnd *propertySymOpnd = instr->GetSrc1()->AsPropertySymOpnd();
  10339. if (propertySymOpnd->IsTypeCheckSeqCandidate())
  10340. {
  10341. propertySymOpnd->SetTypeChecked(false);
  10342. propertySymOpnd->SetTypeAvailable(false);
  10343. propertySymOpnd->SetWriteGuardChecked(false);
  10344. }
  10345. this->FinishOptPropOp(instr, propertySymOpnd);
  10346. instr = this->SetTypeCheckBailOut(instr->GetSrc1(), instr, nullptr);
  10347. }
  10348. }
  10349. return instr;
  10350. }
  10351. IR::Instr *
  10352. GlobOpt::ToInt32(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, bool lossy)
  10353. {
  10354. return this->ToTypeSpecUse(instr, opnd, block, val, indir, TyInt32, IR::BailOutIntOnly, lossy);
  10355. }
  10356. IR::Instr *
  10357. GlobOpt::ToFloat64(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, IR::BailOutKind bailOutKind)
  10358. {
  10359. return this->ToTypeSpecUse(instr, opnd, block, val, indir, TyFloat64, bailOutKind);
  10360. }
  10361. IR::Instr *
  10362. GlobOpt::ToTypeSpecUse(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, IRType toType, IR::BailOutKind bailOutKind, bool lossy, IR::Instr *insertBeforeInstr)
  10363. {
  10364. Assert(bailOutKind != IR::BailOutInvalid);
  10365. IR::Instr *newInstr;
  10366. if (!val && opnd->IsRegOpnd())
  10367. {
  10368. val = block->globOptData.FindValue(opnd->AsRegOpnd()->m_sym);
  10369. }
  10370. ValueInfo *valueInfo = val ? val->GetValueInfo() : nullptr;
  10371. bool needReplaceSrc = false;
  10372. bool updateBlockLastInstr = false;
  10373. if (instr)
  10374. {
  10375. needReplaceSrc = true;
  10376. if (!insertBeforeInstr)
  10377. {
  10378. insertBeforeInstr = instr;
  10379. }
  10380. }
  10381. else if (!insertBeforeInstr)
  10382. {
  10383. // Insert it at the end of the block
  10384. insertBeforeInstr = block->GetLastInstr();
  10385. if (insertBeforeInstr->IsBranchInstr() || insertBeforeInstr->m_opcode == Js::OpCode::BailTarget)
  10386. {
  10387. // Don't insert code between the branch and the preceding ByteCodeUses instrs...
  10388. while(insertBeforeInstr->m_prev->m_opcode == Js::OpCode::ByteCodeUses)
  10389. {
  10390. insertBeforeInstr = insertBeforeInstr->m_prev;
  10391. }
  10392. }
  10393. else
  10394. {
  10395. insertBeforeInstr = insertBeforeInstr->m_next;
  10396. updateBlockLastInstr = true;
  10397. }
  10398. }
  10399. // Int constant values will be propagated into the instruction. For ArgOut_A_InlineBuiltIn, there's no benefit from
  10400. // const-propping, so those are excluded.
  10401. if (opnd->IsRegOpnd() &&
  10402. !(
  10403. valueInfo &&
  10404. (valueInfo->HasIntConstantValue() || valueInfo->IsFloatConstant()) &&
  10405. (!instr || instr->m_opcode != Js::OpCode::ArgOut_A_InlineBuiltIn)
  10406. ))
  10407. {
  10408. IR::RegOpnd *regSrc = opnd->AsRegOpnd();
  10409. StackSym *varSym = regSrc->m_sym;
  10410. Js::OpCode opcode = Js::OpCode::FromVar;
  10411. if (varSym->IsTypeSpec() || !block->globOptData.liveVarSyms->Test(varSym->m_id))
  10412. {
  10413. // Conversion between int32 and float64
  10414. if (varSym->IsTypeSpec())
  10415. {
  10416. varSym = varSym->GetVarEquivSym(this->func);
  10417. }
  10418. opcode = Js::OpCode::Conv_Prim;
  10419. }
  10420. Assert(block->globOptData.liveVarSyms->Test(varSym->m_id) || block->globOptData.IsTypeSpecialized(varSym));
  10421. StackSym *typeSpecSym = nullptr;
  10422. BOOL isLive = FALSE;
  10423. BVSparse<JitArenaAllocator> *livenessBv = nullptr;
  10424. if(valueInfo && valueInfo->IsInt())
  10425. {
  10426. // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value
  10427. // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as
  10428. // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the
  10429. // lossy state.
  10430. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10431. }
  10432. if (toType == TyInt32)
  10433. {
  10434. // Need to determine whether the conversion is actually lossy or lossless. If the value is an int, then it's a
  10435. // lossless conversion despite the type of conversion requested. The liveness of the converted int32 sym needs to be
  10436. // set to reflect the actual type of conversion done. Also, a lossless conversion needs the value to determine
  10437. // whether the conversion may need to bail out.
  10438. Assert(valueInfo);
  10439. if(valueInfo->IsInt())
  10440. {
  10441. lossy = false;
  10442. }
  10443. else
  10444. {
  10445. Assert(IsLoopPrePass() || !block->globOptData.IsInt32TypeSpecialized(varSym));
  10446. }
  10447. livenessBv = block->globOptData.liveInt32Syms;
  10448. isLive = livenessBv->Test(varSym->m_id) && (lossy || !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id));
  10449. if (this->IsLoopPrePass())
  10450. {
  10451. if (!isLive)
  10452. {
  10453. livenessBv->Set(varSym->m_id);
  10454. if (lossy)
  10455. {
  10456. block->globOptData.liveLossyInt32Syms->Set(varSym->m_id);
  10457. }
  10458. else
  10459. {
  10460. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10461. }
  10462. }
  10463. return instr;
  10464. }
  10465. typeSpecSym = varSym->GetInt32EquivSym(this->func);
  10466. if (!isLive)
  10467. {
  10468. if (!opnd->IsVar() ||
  10469. !block->globOptData.liveVarSyms->Test(varSym->m_id) ||
  10470. (block->globOptData.liveFloat64Syms->Test(varSym->m_id) && valueInfo && valueInfo->IsLikelyFloat()))
  10471. {
  10472. Assert(block->globOptData.liveFloat64Syms->Test(varSym->m_id));
  10473. if(!lossy && !valueInfo->IsInt())
  10474. {
  10475. // Shouldn't try to do a lossless conversion from float64 to int32 when the value is not known to be an
  10476. // int. There are cases where we need more than two passes over loops to flush out all dependencies.
  10477. // It's possible for the loop prepass to think that a sym s1 remains an int because it acquires the
  10478. // value of another sym s2 that is an int in the prepass at that time. However, s2 can become a float
  10479. // later in the loop body, in which case s1 would become a float on the second iteration of the loop. By
  10480. // that time, we would have already committed to having s1 live as a lossless int on entry into the
  10481. // loop, and we end up having to compensate by doing a lossless conversion from float to int, which will
  10482. // need a bailout and will most likely bail out.
  10483. //
  10484. // If s2 becomes a var instead of a float, then the compensation is legal although not ideal. After
  10485. // enough bailouts, rejit would be triggered with aggressive int type spec turned off. For the
  10486. // float-to-int conversion though, there's no point in emitting a bailout because we already know that
  10487. // the value is a float and has high probability of bailing out (whereas a var has a chance to be a
  10488. // tagged int), and so currently lossless conversion from float to int with bailout is not supported.
  10489. //
  10490. // So, treating this case as a compile-time bailout. The exception will trigger the jit work item to be
  10491. // restarted with aggressive int type specialization disabled.
  10492. if(bailOutKind == IR::BailOutExpectingInteger)
  10493. {
  10494. Assert(IsSwitchOptEnabledForIntTypeSpec());
  10495. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingInteger);
  10496. }
  10497. else
  10498. {
  10499. Assert(DoAggressiveIntTypeSpec());
  10500. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  10501. {
  10502. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  10503. Output::Print(
  10504. _u("BailOut (compile-time): function: %s (%s) varSym: "),
  10505. this->func->GetJITFunctionBody()->GetDisplayName(),
  10506. this->func->GetDebugNumberSet(debugStringBuffer),
  10507. varSym->m_id);
  10508. #if DBG_DUMP
  10509. varSym->Dump();
  10510. #else
  10511. Output::Print(_u("s%u"), varSym->m_id);
  10512. #endif
  10513. if(varSym->HasByteCodeRegSlot())
  10514. {
  10515. Output::Print(_u(" byteCodeReg: R%u"), varSym->GetByteCodeRegSlot());
  10516. }
  10517. Output::Print(_u(" (lossless conversion from float64 to int32)\n"));
  10518. Output::Flush();
  10519. }
  10520. if(!DoAggressiveIntTypeSpec())
  10521. {
  10522. // Aggressive int type specialization is already off for some reason. Prevent trying to rejit again
  10523. // because it won't help and the same thing will happen again. Just abort jitting this function.
  10524. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  10525. {
  10526. Output::Print(_u(" Aborting JIT because AggressiveIntTypeSpec is already off\n"));
  10527. Output::Flush();
  10528. }
  10529. throw Js::OperationAbortedException();
  10530. }
  10531. throw Js::RejitException(RejitReason::AggressiveIntTypeSpecDisabled);
  10532. }
  10533. }
  10534. if(opnd->IsVar())
  10535. {
  10536. regSrc->SetType(TyFloat64);
  10537. regSrc->m_sym = varSym->GetFloat64EquivSym(this->func);
  10538. opcode = Js::OpCode::Conv_Prim;
  10539. }
  10540. else
  10541. {
  10542. Assert(regSrc->IsFloat64());
  10543. Assert(regSrc->m_sym->IsFloat64());
  10544. Assert(opcode == Js::OpCode::Conv_Prim);
  10545. }
  10546. }
  10547. }
  10548. GOPT_TRACE_OPND(regSrc, _u("Converting to int32\n"));
  10549. }
  10550. else if (toType == TyFloat64)
  10551. {
  10552. // float64
  10553. typeSpecSym = varSym->GetFloat64EquivSym(this->func);
  10554. if(!IsLoopPrePass() && typeSpecSym->m_requiresBailOnNotNumber && block->globOptData.IsFloat64TypeSpecialized(varSym))
  10555. {
  10556. // This conversion is already protected by a BailOutNumberOnly bailout (or at least it will be after the
  10557. // dead-store phase). Since 'requiresBailOnNotNumber' is not flow-based, change the value to definitely float.
  10558. if(valueInfo)
  10559. {
  10560. if(!valueInfo->IsNumber())
  10561. {
  10562. valueInfo = valueInfo->SpecializeToFloat64(alloc);
  10563. ChangeValueInfo(block, val, valueInfo);
  10564. opnd->SetValueType(valueInfo->Type());
  10565. }
  10566. }
  10567. else
  10568. {
  10569. val = NewGenericValue(ValueType::Float);
  10570. valueInfo = val->GetValueInfo();
  10571. block->globOptData.SetValue(val, varSym);
  10572. opnd->SetValueType(valueInfo->Type());
  10573. }
  10574. }
  10575. if(bailOutKind == IR::BailOutNumberOnly)
  10576. {
  10577. if(!IsLoopPrePass())
  10578. {
  10579. // Ensure that all bailout FromVars that generate a value for this type-specialized sym will bail out on any
  10580. // non-number value, even ones that have already been generated before. The dead-store pass will update the
  10581. // bailout kind on already-generated FromVars based on this bit.
  10582. typeSpecSym->m_requiresBailOnNotNumber = true;
  10583. }
  10584. }
  10585. else if(typeSpecSym->m_requiresBailOnNotNumber)
  10586. {
  10587. Assert(bailOutKind == IR::BailOutPrimitiveButString);
  10588. bailOutKind = IR::BailOutNumberOnly;
  10589. }
  10590. livenessBv = block->globOptData.liveFloat64Syms;
  10591. isLive = livenessBv->Test(varSym->m_id);
  10592. if (this->IsLoopPrePass())
  10593. {
  10594. if(!isLive)
  10595. {
  10596. livenessBv->Set(varSym->m_id);
  10597. }
  10598. if (this->OptIsInvariant(opnd, block, this->prePassLoop, val, false, true))
  10599. {
  10600. this->prePassLoop->forceFloat64SymsOnEntry->Set(varSym->m_id);
  10601. }
  10602. else
  10603. {
  10604. Sym *symStore = (valueInfo ? valueInfo->GetSymStore() : NULL);
  10605. if (symStore && symStore != varSym
  10606. && this->OptIsInvariant(symStore, block, this->prePassLoop, block->globOptData.FindValue(symStore), false, true))
  10607. {
  10608. // If symStore is assigned to sym and we want sym to be type-specialized, for symStore to be specialized
  10609. // outside the loop.
  10610. this->prePassLoop->forceFloat64SymsOnEntry->Set(symStore->m_id);
  10611. }
  10612. }
  10613. return instr;
  10614. }
  10615. if (!isLive && regSrc->IsVar())
  10616. {
  10617. if (!block->globOptData.liveVarSyms->Test(varSym->m_id) ||
  10618. (
  10619. block->globOptData.liveInt32Syms->Test(varSym->m_id) &&
  10620. !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id) &&
  10621. valueInfo &&
  10622. valueInfo->IsLikelyInt()
  10623. ))
  10624. {
  10625. Assert(block->globOptData.liveInt32Syms->Test(varSym->m_id));
  10626. Assert(!block->globOptData.liveLossyInt32Syms->Test(varSym->m_id)); // Shouldn't try to convert a lossy int32 to anything
  10627. regSrc->SetType(TyInt32);
  10628. regSrc->m_sym = varSym->GetInt32EquivSym(this->func);
  10629. opcode = Js::OpCode::Conv_Prim;
  10630. }
  10631. }
  10632. GOPT_TRACE_OPND(regSrc, _u("Converting to float64\n"));
  10633. }
  10634. bool needLoad = false;
  10635. if (needReplaceSrc)
  10636. {
  10637. bool wasDead = regSrc->GetIsDead();
  10638. // needReplaceSrc means we are type specializing a use, and need to replace the src on the instr
  10639. if (!isLive)
  10640. {
  10641. needLoad = true;
  10642. // ReplaceSrc will delete it.
  10643. regSrc = regSrc->Copy(instr->m_func)->AsRegOpnd();
  10644. }
  10645. IR::RegOpnd * regNew = IR::RegOpnd::New(typeSpecSym, toType, instr->m_func);
  10646. if(valueInfo)
  10647. {
  10648. regNew->SetValueType(valueInfo->Type());
  10649. regNew->m_wasNegativeZeroPreventedByBailout = valueInfo->WasNegativeZeroPreventedByBailout();
  10650. }
  10651. regNew->SetIsDead(wasDead);
  10652. regNew->SetIsJITOptimizedReg(true);
  10653. this->CaptureByteCodeSymUses(instr);
  10654. if (indir == nullptr)
  10655. {
  10656. instr->ReplaceSrc(opnd, regNew);
  10657. }
  10658. else
  10659. {
  10660. indir->ReplaceIndexOpnd(regNew);
  10661. }
  10662. opnd = regNew;
  10663. if (!needLoad)
  10664. {
  10665. Assert(isLive);
  10666. return instr;
  10667. }
  10668. }
  10669. else
  10670. {
  10671. // We just need to insert a load of a type spec sym
  10672. if(isLive)
  10673. {
  10674. return instr;
  10675. }
  10676. // Insert it before the specified instruction
  10677. instr = insertBeforeInstr;
  10678. }
  10679. IR::RegOpnd *regDst = IR::RegOpnd::New(typeSpecSym, toType, instr->m_func);
  10680. bool isBailout = false;
  10681. bool isHoisted = false;
  10682. bool isInLandingPad = (block->next && !block->next->isDeleted && block->next->isLoopHeader);
  10683. if (isInLandingPad)
  10684. {
  10685. Loop *loop = block->next->loop;
  10686. Assert(loop && loop->landingPad == block);
  10687. Assert(loop->bailOutInfo);
  10688. }
  10689. if (opcode == Js::OpCode::FromVar)
  10690. {
  10691. if (toType == TyInt32)
  10692. {
  10693. Assert(valueInfo);
  10694. if (lossy)
  10695. {
  10696. if (!valueInfo->IsPrimitive() && !block->globOptData.IsTypeSpecialized(varSym))
  10697. {
  10698. // Lossy conversions to int32 on non-primitive values may have implicit calls to toString or valueOf, which
  10699. // may be overridden to have a side effect. The side effect needs to happen every time the conversion is
  10700. // supposed to happen, so the resulting lossy int32 value cannot be reused. Bail out on implicit calls.
  10701. Assert(DoLossyIntTypeSpec());
  10702. bailOutKind = IR::BailOutOnNotPrimitive;
  10703. isBailout = true;
  10704. }
  10705. }
  10706. else if (!valueInfo->IsInt())
  10707. {
  10708. // The operand is likely an int (hence the request to convert to int), so bail out if it's not an int. Only
  10709. // bail out if a lossless conversion to int is requested. Lossy conversions to int such as in (a | 0) don't
  10710. // need to bail out.
  10711. if (bailOutKind == IR::BailOutExpectingInteger)
  10712. {
  10713. Assert(IsSwitchOptEnabledForIntTypeSpec());
  10714. }
  10715. else
  10716. {
  10717. Assert(DoAggressiveIntTypeSpec());
  10718. }
  10719. isBailout = true;
  10720. }
  10721. }
  10722. else if (toType == TyFloat64 &&
  10723. (!valueInfo || !valueInfo->IsNumber()))
  10724. {
  10725. // Bailout if converting vars to float if we can't prove they are floats:
  10726. // x = str + float; -> need to bailout if str is a string
  10727. //
  10728. // x = obj * 0.1;
  10729. // y = obj * 0.2; -> if obj has valueof, we'll only call valueof once on the FromVar conversion...
  10730. Assert(bailOutKind != IR::BailOutInvalid);
  10731. isBailout = true;
  10732. }
  10733. }
  10734. if (isBailout)
  10735. {
  10736. if (isInLandingPad)
  10737. {
  10738. Loop *loop = block->next->loop;
  10739. this->EnsureBailTarget(loop);
  10740. instr = loop->bailOutInfo->bailOutInstr;
  10741. updateBlockLastInstr = false;
  10742. newInstr = IR::BailOutInstr::New(opcode, bailOutKind, loop->bailOutInfo, instr->m_func);
  10743. newInstr->SetDst(regDst);
  10744. newInstr->SetSrc1(regSrc);
  10745. }
  10746. else
  10747. {
  10748. newInstr = IR::BailOutInstr::New(opcode, regDst, regSrc, bailOutKind, instr, instr->m_func);
  10749. }
  10750. }
  10751. else
  10752. {
  10753. newInstr = IR::Instr::New(opcode, regDst, regSrc, instr->m_func);
  10754. }
  10755. newInstr->SetByteCodeOffset(instr);
  10756. instr->InsertBefore(newInstr);
  10757. if (updateBlockLastInstr)
  10758. {
  10759. block->SetLastInstr(newInstr);
  10760. }
  10761. regDst->SetIsJITOptimizedReg(true);
  10762. newInstr->GetSrc1()->AsRegOpnd()->SetIsJITOptimizedReg(true);
  10763. ValueInfo *const oldValueInfo = valueInfo;
  10764. if(valueInfo)
  10765. {
  10766. newInstr->GetSrc1()->SetValueType(valueInfo->Type());
  10767. }
  10768. if(isBailout)
  10769. {
  10770. Assert(opcode == Js::OpCode::FromVar);
  10771. if(toType == TyInt32)
  10772. {
  10773. Assert(valueInfo);
  10774. if(!lossy)
  10775. {
  10776. Assert(bailOutKind == IR::BailOutIntOnly || bailOutKind == IR::BailOutExpectingInteger);
  10777. valueInfo = valueInfo->SpecializeToInt32(alloc, isPerformingLoopBackEdgeCompensation);
  10778. ChangeValueInfo(nullptr, val, valueInfo);
  10779. int32 intConstantValue;
  10780. if(indir && needReplaceSrc && valueInfo->TryGetIntConstantValue(&intConstantValue))
  10781. {
  10782. // A likely-int value can have constant bounds due to conditional branches narrowing its range. Now that
  10783. // the sym has been proven to be an int, the likely-int value, after specialization, will be constant.
  10784. // Replace the index opnd in the indir with an offset.
  10785. Assert(opnd == indir->GetIndexOpnd());
  10786. Assert(indir->GetScale() == 0);
  10787. indir->UnlinkIndexOpnd()->Free(instr->m_func);
  10788. opnd = nullptr;
  10789. indir->SetOffset(intConstantValue);
  10790. }
  10791. }
  10792. }
  10793. else if (toType == TyFloat64)
  10794. {
  10795. if(bailOutKind == IR::BailOutNumberOnly)
  10796. {
  10797. if(valueInfo)
  10798. {
  10799. valueInfo = valueInfo->SpecializeToFloat64(alloc);
  10800. ChangeValueInfo(block, val, valueInfo);
  10801. }
  10802. else
  10803. {
  10804. val = NewGenericValue(ValueType::Float);
  10805. valueInfo = val->GetValueInfo();
  10806. block->globOptData.SetValue(val, varSym);
  10807. }
  10808. }
  10809. }
  10810. else
  10811. {
  10812. Assert(UNREACHED);
  10813. }
  10814. }
  10815. if(valueInfo)
  10816. {
  10817. newInstr->GetDst()->SetValueType(valueInfo->Type());
  10818. if(needReplaceSrc && opnd)
  10819. {
  10820. opnd->SetValueType(valueInfo->Type());
  10821. }
  10822. }
  10823. if (block->loop)
  10824. {
  10825. Assert(!this->IsLoopPrePass());
  10826. isHoisted = this->TryHoistInvariant(newInstr, block, val, val, nullptr, false, lossy, false, bailOutKind);
  10827. }
  10828. if (isBailout)
  10829. {
  10830. if (!isHoisted && !isInLandingPad)
  10831. {
  10832. if(valueInfo)
  10833. {
  10834. // Since this is a pre-op bailout, the old value info should be used for the purposes of bailout. For
  10835. // instance, the value info could be LikelyInt but with a constant range. Once specialized to int, the value
  10836. // info would be an int constant. However, the int constant is only guaranteed if the value is actually an
  10837. // int, which this conversion is verifying, so bailout cannot assume the constant value.
  10838. if(oldValueInfo)
  10839. {
  10840. val->SetValueInfo(oldValueInfo);
  10841. }
  10842. else
  10843. {
  10844. block->globOptData.ClearSymValue(varSym);
  10845. }
  10846. }
  10847. // Fill in bail out info if the FromVar is a bailout instr, and it wasn't hoisted as invariant.
  10848. // If it was hoisted, the invariant code will fill out the bailout info with the loop landing pad bailout info.
  10849. this->FillBailOutInfo(block, newInstr);
  10850. if(valueInfo)
  10851. {
  10852. // Restore the new value info after filling the bailout info
  10853. if(oldValueInfo)
  10854. {
  10855. val->SetValueInfo(valueInfo);
  10856. }
  10857. else
  10858. {
  10859. block->globOptData.SetValue(val, varSym);
  10860. }
  10861. }
  10862. }
  10863. }
  10864. // Now that we've captured the liveness in the bailout info, we can mark this as live.
  10865. // This type specialized sym isn't live if the FromVar bails out.
  10866. livenessBv->Set(varSym->m_id);
  10867. if(toType == TyInt32)
  10868. {
  10869. if(lossy)
  10870. {
  10871. block->globOptData.liveLossyInt32Syms->Set(varSym->m_id);
  10872. }
  10873. else
  10874. {
  10875. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10876. }
  10877. }
  10878. }
  10879. else
  10880. {
  10881. Assert(valueInfo);
  10882. if(opnd->IsRegOpnd() && valueInfo->IsInt())
  10883. {
  10884. // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value
  10885. // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as
  10886. // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the
  10887. // lossy state.
  10888. block->globOptData.liveLossyInt32Syms->Clear(opnd->AsRegOpnd()->m_sym->m_id);
  10889. if(toType == TyInt32)
  10890. {
  10891. lossy = false;
  10892. }
  10893. }
  10894. if (this->IsLoopPrePass())
  10895. {
  10896. if(opnd->IsRegOpnd())
  10897. {
  10898. StackSym *const sym = opnd->AsRegOpnd()->m_sym;
  10899. if(toType == TyInt32)
  10900. {
  10901. Assert(!sym->IsTypeSpec());
  10902. block->globOptData.liveInt32Syms->Set(sym->m_id);
  10903. if(lossy)
  10904. {
  10905. block->globOptData.liveLossyInt32Syms->Set(sym->m_id);
  10906. }
  10907. else
  10908. {
  10909. block->globOptData.liveLossyInt32Syms->Clear(sym->m_id);
  10910. }
  10911. }
  10912. else
  10913. {
  10914. Assert(toType == TyFloat64);
  10915. AnalysisAssert(instr);
  10916. StackSym *const varSym = sym->IsTypeSpec() ? sym->GetVarEquivSym(instr->m_func) : sym;
  10917. block->globOptData.liveFloat64Syms->Set(varSym->m_id);
  10918. }
  10919. }
  10920. return instr;
  10921. }
  10922. if (!needReplaceSrc)
  10923. {
  10924. instr = insertBeforeInstr;
  10925. }
  10926. IR::Opnd *constOpnd;
  10927. int32 intConstantValue;
  10928. if(valueInfo->TryGetIntConstantValue(&intConstantValue))
  10929. {
  10930. if(toType == TyInt32)
  10931. {
  10932. constOpnd = IR::IntConstOpnd::New(intConstantValue, TyInt32, instr->m_func);
  10933. }
  10934. else
  10935. {
  10936. Assert(toType == TyFloat64);
  10937. constOpnd = IR::FloatConstOpnd::New(static_cast<FloatConstType>(intConstantValue), TyFloat64, instr->m_func);
  10938. }
  10939. }
  10940. else if(valueInfo->IsFloatConstant())
  10941. {
  10942. const FloatConstType floatValue = valueInfo->AsFloatConstant()->FloatValue();
  10943. if(toType == TyInt32)
  10944. {
  10945. // In some loop scenarios, a sym can be specialized to int32 on loop entry
  10946. // during the prepass and then subsequentely specialized to float within
  10947. // the loop, leading to an attempted lossy conversion from float64 to int32
  10948. // on the backedge. For these cases, disable aggressive int type specialization
  10949. // and try again.
  10950. if (!lossy)
  10951. {
  10952. AssertOrFailFast(DoAggressiveIntTypeSpec());
  10953. throw Js::RejitException(RejitReason::AggressiveIntTypeSpecDisabled);
  10954. }
  10955. constOpnd =
  10956. IR::IntConstOpnd::New(
  10957. Js::JavascriptMath::ToInt32(floatValue),
  10958. TyInt32,
  10959. instr->m_func);
  10960. }
  10961. else
  10962. {
  10963. Assert(toType == TyFloat64);
  10964. constOpnd = IR::FloatConstOpnd::New(floatValue, TyFloat64, instr->m_func);
  10965. }
  10966. }
  10967. else
  10968. {
  10969. Assert(opnd->IsVar());
  10970. Assert(opnd->IsAddrOpnd());
  10971. AssertMsg(opnd->AsAddrOpnd()->IsVar(), "We only expect to see addr that are var before lower.");
  10972. // Don't need to capture uses, we are only replacing an addr opnd
  10973. if(toType == TyInt32)
  10974. {
  10975. constOpnd = IR::IntConstOpnd::New(Js::TaggedInt::ToInt32(opnd->AsAddrOpnd()->m_address), TyInt32, instr->m_func);
  10976. }
  10977. else
  10978. {
  10979. Assert(toType == TyFloat64);
  10980. constOpnd = IR::FloatConstOpnd::New(Js::TaggedInt::ToDouble(opnd->AsAddrOpnd()->m_address), TyFloat64, instr->m_func);
  10981. }
  10982. }
  10983. if (toType == TyInt32)
  10984. {
  10985. if (needReplaceSrc)
  10986. {
  10987. CaptureByteCodeSymUses(instr);
  10988. if(indir)
  10989. {
  10990. Assert(opnd == indir->GetIndexOpnd());
  10991. Assert(indir->GetScale() == 0);
  10992. indir->UnlinkIndexOpnd()->Free(instr->m_func);
  10993. indir->SetOffset(constOpnd->AsIntConstOpnd()->AsInt32());
  10994. }
  10995. else
  10996. {
  10997. instr->ReplaceSrc(opnd, constOpnd);
  10998. }
  10999. }
  11000. else
  11001. {
  11002. StackSym *varSym = opnd->AsRegOpnd()->m_sym;
  11003. if(varSym->IsTypeSpec())
  11004. {
  11005. varSym = varSym->GetVarEquivSym(nullptr);
  11006. Assert(varSym);
  11007. }
  11008. if(block->globOptData.liveInt32Syms->TestAndSet(varSym->m_id))
  11009. {
  11010. Assert(!!block->globOptData.liveLossyInt32Syms->Test(varSym->m_id) == lossy);
  11011. }
  11012. else
  11013. {
  11014. if(lossy)
  11015. {
  11016. block->globOptData.liveLossyInt32Syms->Set(varSym->m_id);
  11017. }
  11018. StackSym *int32Sym = varSym->GetInt32EquivSym(instr->m_func);
  11019. IR::RegOpnd *int32Reg = IR::RegOpnd::New(int32Sym, TyInt32, instr->m_func);
  11020. int32Reg->SetIsJITOptimizedReg(true);
  11021. newInstr = IR::Instr::New(Js::OpCode::Ld_I4, int32Reg, constOpnd, instr->m_func);
  11022. newInstr->SetByteCodeOffset(instr);
  11023. instr->InsertBefore(newInstr);
  11024. if (updateBlockLastInstr)
  11025. {
  11026. block->SetLastInstr(newInstr);
  11027. }
  11028. }
  11029. }
  11030. }
  11031. else
  11032. {
  11033. StackSym *floatSym;
  11034. bool newFloatSym = false;
  11035. StackSym* varSym;
  11036. if (opnd->IsRegOpnd())
  11037. {
  11038. varSym = opnd->AsRegOpnd()->m_sym;
  11039. if (varSym->IsTypeSpec())
  11040. {
  11041. varSym = varSym->GetVarEquivSym(nullptr);
  11042. Assert(varSym);
  11043. }
  11044. floatSym = varSym->GetFloat64EquivSym(instr->m_func);
  11045. }
  11046. else
  11047. {
  11048. varSym = block->globOptData.GetCopyPropSym(nullptr, val);
  11049. if(!varSym)
  11050. {
  11051. // Clear the symstore to ensure it's set below to this new symbol
  11052. this->SetSymStoreDirect(val->GetValueInfo(), nullptr);
  11053. varSym = StackSym::New(TyVar, instr->m_func);
  11054. newFloatSym = true;
  11055. }
  11056. floatSym = varSym->GetFloat64EquivSym(instr->m_func);
  11057. }
  11058. IR::RegOpnd *floatReg = IR::RegOpnd::New(floatSym, TyFloat64, instr->m_func);
  11059. floatReg->SetIsJITOptimizedReg(true);
  11060. // If the value is not live - let's load it.
  11061. if(!block->globOptData.liveFloat64Syms->TestAndSet(varSym->m_id))
  11062. {
  11063. newInstr = IR::Instr::New(Js::OpCode::LdC_F8_R8, floatReg, constOpnd, instr->m_func);
  11064. newInstr->SetByteCodeOffset(instr);
  11065. instr->InsertBefore(newInstr);
  11066. if (updateBlockLastInstr)
  11067. {
  11068. block->SetLastInstr(newInstr);
  11069. }
  11070. if(newFloatSym)
  11071. {
  11072. block->globOptData.SetValue(val, varSym);
  11073. }
  11074. // Src is always invariant, but check if the dst is, and then hoist.
  11075. if (block->loop &&
  11076. (
  11077. (newFloatSym && block->loop->CanHoistInvariants()) ||
  11078. this->OptIsInvariant(floatReg, block, block->loop, val, false, false)
  11079. ))
  11080. {
  11081. Assert(!this->IsLoopPrePass());
  11082. this->OptHoistInvariant(newInstr, block, block->loop, val, val, nullptr, false);
  11083. }
  11084. }
  11085. if (needReplaceSrc)
  11086. {
  11087. CaptureByteCodeSymUses(instr);
  11088. instr->ReplaceSrc(opnd, floatReg);
  11089. }
  11090. }
  11091. return instr;
  11092. }
  11093. return newInstr;
  11094. }
  11095. void
  11096. GlobOpt::ToVarRegOpnd(IR::RegOpnd *dst, BasicBlock *block)
  11097. {
  11098. ToVarStackSym(dst->m_sym, block);
  11099. }
  11100. void
  11101. GlobOpt::ToVarStackSym(StackSym *varSym, BasicBlock *block)
  11102. {
  11103. //added another check for sym , in case of asmjs there is mostly no var syms and hence added a new check to see if it is the primary sym
  11104. Assert(!varSym->IsTypeSpec());
  11105. block->globOptData.liveVarSyms->Set(varSym->m_id);
  11106. block->globOptData.liveInt32Syms->Clear(varSym->m_id);
  11107. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  11108. block->globOptData.liveFloat64Syms->Clear(varSym->m_id);
  11109. }
  11110. void
  11111. GlobOpt::ToInt32Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block)
  11112. {
  11113. StackSym *varSym = dst->m_sym;
  11114. Assert(!varSym->IsTypeSpec());
  11115. if (!this->IsLoopPrePass() && varSym->IsVar())
  11116. {
  11117. StackSym *int32Sym = varSym->GetInt32EquivSym(instr->m_func);
  11118. // Use UnlinkDst / SetDst to make sure isSingleDef is tracked properly,
  11119. // since we'll just be hammering the symbol.
  11120. dst = instr->UnlinkDst()->AsRegOpnd();
  11121. dst->m_sym = int32Sym;
  11122. dst->SetType(TyInt32);
  11123. instr->SetDst(dst);
  11124. }
  11125. block->globOptData.liveInt32Syms->Set(varSym->m_id);
  11126. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); // The store makes it lossless
  11127. block->globOptData.liveVarSyms->Clear(varSym->m_id);
  11128. block->globOptData.liveFloat64Syms->Clear(varSym->m_id);
  11129. }
  11130. void
  11131. GlobOpt::ToUInt32Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block)
  11132. {
  11133. // We should be calling only for asmjs function
  11134. Assert(GetIsAsmJSFunc());
  11135. StackSym *varSym = dst->m_sym;
  11136. Assert(!varSym->IsTypeSpec());
  11137. block->globOptData.liveInt32Syms->Set(varSym->m_id);
  11138. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); // The store makes it lossless
  11139. block->globOptData.liveVarSyms->Clear(varSym->m_id);
  11140. block->globOptData.liveFloat64Syms->Clear(varSym->m_id);
  11141. }
  11142. void
  11143. GlobOpt::ToFloat64Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block)
  11144. {
  11145. StackSym *varSym = dst->m_sym;
  11146. Assert(!varSym->IsTypeSpec());
  11147. if (!this->IsLoopPrePass() && varSym->IsVar())
  11148. {
  11149. StackSym *float64Sym = varSym->GetFloat64EquivSym(this->func);
  11150. // Use UnlinkDst / SetDst to make sure isSingleDef is tracked properly,
  11151. // since we'll just be hammering the symbol.
  11152. dst = instr->UnlinkDst()->AsRegOpnd();
  11153. dst->m_sym = float64Sym;
  11154. dst->SetType(TyFloat64);
  11155. instr->SetDst(dst);
  11156. }
  11157. block->globOptData.liveFloat64Syms->Set(varSym->m_id);
  11158. block->globOptData.liveVarSyms->Clear(varSym->m_id);
  11159. block->globOptData.liveInt32Syms->Clear(varSym->m_id);
  11160. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  11161. }
  11162. static void SetIsConstFlag(StackSym* dstSym, int64 value)
  11163. {
  11164. Assert(dstSym);
  11165. dstSym->SetIsInt64Const();
  11166. }
  11167. static void SetIsConstFlag(StackSym* dstSym, int value)
  11168. {
  11169. Assert(dstSym);
  11170. dstSym->SetIsIntConst(value);
  11171. }
  11172. static IR::Opnd* CreateIntConstOpnd(IR::Instr* instr, int64 value)
  11173. {
  11174. return (IR::Opnd*)IR::Int64ConstOpnd::New(value, instr->GetDst()->GetType(), instr->m_func);
  11175. }
  11176. static IR::Opnd* CreateIntConstOpnd(IR::Instr* instr, int value)
  11177. {
  11178. IntConstType constVal;
  11179. if (instr->GetDst()->IsUnsigned())
  11180. {
  11181. // we should zero extend in case of uint
  11182. constVal = (uint32)value;
  11183. }
  11184. else
  11185. {
  11186. constVal = value;
  11187. }
  11188. return (IR::Opnd*)IR::IntConstOpnd::New(constVal, instr->GetDst()->GetType(), instr->m_func);
  11189. }
  11190. template <typename T>
  11191. IR::Opnd* GlobOpt::ReplaceWConst(IR::Instr **pInstr, T value, Value **pDstVal)
  11192. {
  11193. IR::Instr * &instr = *pInstr;
  11194. IR::Opnd * constOpnd = CreateIntConstOpnd(instr, value);
  11195. instr->ReplaceSrc1(constOpnd);
  11196. instr->FreeSrc2();
  11197. this->OptSrc(constOpnd, &instr);
  11198. IR::Opnd *dst = instr->GetDst();
  11199. StackSym *dstSym = dst->AsRegOpnd()->m_sym;
  11200. if (dstSym->IsSingleDef())
  11201. {
  11202. SetIsConstFlag(dstSym, value);
  11203. }
  11204. GOPT_TRACE_INSTR(instr, _u("Constant folding to %d: \n"), value);
  11205. *pDstVal = GetIntConstantValue(value, instr, dst);
  11206. return dst;
  11207. }
  11208. template <typename T>
  11209. bool GlobOpt::OptConstFoldBinaryWasm(
  11210. IR::Instr** pInstr,
  11211. const Value* src1,
  11212. const Value* src2,
  11213. Value **pDstVal)
  11214. {
  11215. IR::Instr* &instr = *pInstr;
  11216. if (!DoConstFold())
  11217. {
  11218. return false;
  11219. }
  11220. T src1IntConstantValue, src2IntConstantValue;
  11221. if (!src1 || !src1->GetValueInfo()->TryGetIntConstantValue(&src1IntConstantValue, false) || //a bit sketchy: false for int32 means likelyInt = false
  11222. !src2 || !src2->GetValueInfo()->TryGetIntConstantValue(&src2IntConstantValue, false) //and unsigned = false for int64
  11223. )
  11224. {
  11225. return false;
  11226. }
  11227. int64 tmpValueOut;
  11228. if (!instr->BinaryCalculatorT<T>(src1IntConstantValue, src2IntConstantValue, &tmpValueOut, func->GetJITFunctionBody()->IsWasmFunction()))
  11229. {
  11230. return false;
  11231. }
  11232. this->CaptureByteCodeSymUses(instr);
  11233. IR::Opnd *dst = (instr->GetDst()->IsInt64()) ? //dst can be int32 for int64 comparison operators
  11234. ReplaceWConst(pInstr, tmpValueOut, pDstVal) :
  11235. ReplaceWConst(pInstr, (int)tmpValueOut, pDstVal);
  11236. instr->m_opcode = Js::OpCode::Ld_I4;
  11237. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  11238. return true;
  11239. }
  11240. bool
  11241. GlobOpt::OptConstFoldBinary(
  11242. IR::Instr * *pInstr,
  11243. const IntConstantBounds &src1IntConstantBounds,
  11244. const IntConstantBounds &src2IntConstantBounds,
  11245. Value **pDstVal)
  11246. {
  11247. IR::Instr * &instr = *pInstr;
  11248. int32 value;
  11249. IR::IntConstOpnd *constOpnd;
  11250. if (!DoConstFold())
  11251. {
  11252. return false;
  11253. }
  11254. int32 src1IntConstantValue = -1;
  11255. int32 src2IntConstantValue = -1;
  11256. int32 src1MaxIntConstantValue = -1;
  11257. int32 src2MaxIntConstantValue = -1;
  11258. int32 src1MinIntConstantValue = -1;
  11259. int32 src2MinIntConstantValue = -1;
  11260. if (instr->IsBranchInstr())
  11261. {
  11262. src1MinIntConstantValue = src1IntConstantBounds.LowerBound();
  11263. src1MaxIntConstantValue = src1IntConstantBounds.UpperBound();
  11264. src2MinIntConstantValue = src2IntConstantBounds.LowerBound();
  11265. src2MaxIntConstantValue = src2IntConstantBounds.UpperBound();
  11266. }
  11267. else if (src1IntConstantBounds.IsConstant() && src2IntConstantBounds.IsConstant())
  11268. {
  11269. src1IntConstantValue = src1IntConstantBounds.LowerBound();
  11270. src2IntConstantValue = src2IntConstantBounds.LowerBound();
  11271. }
  11272. else
  11273. {
  11274. return false;
  11275. }
  11276. IntConstType tmpValueOut;
  11277. if (!instr->BinaryCalculator(src1IntConstantValue, src2IntConstantValue, &tmpValueOut, TyInt32)
  11278. || !Math::FitsInDWord(tmpValueOut))
  11279. {
  11280. return false;
  11281. }
  11282. value = (int32)tmpValueOut;
  11283. this->CaptureByteCodeSymUses(instr);
  11284. constOpnd = IR::IntConstOpnd::New(value, TyInt32, instr->m_func);
  11285. instr->ReplaceSrc1(constOpnd);
  11286. instr->FreeSrc2();
  11287. this->OptSrc(constOpnd, &instr);
  11288. IR::Opnd *dst = instr->GetDst();
  11289. Assert(dst->IsRegOpnd());
  11290. StackSym *dstSym = dst->AsRegOpnd()->m_sym;
  11291. if (dstSym->IsSingleDef())
  11292. {
  11293. dstSym->SetIsIntConst(value);
  11294. }
  11295. GOPT_TRACE_INSTR(instr, _u("Constant folding to %d: \n"), value);
  11296. *pDstVal = GetIntConstantValue(value, instr, dst);
  11297. if (IsTypeSpecPhaseOff(this->func))
  11298. {
  11299. instr->m_opcode = Js::OpCode::LdC_A_I4;
  11300. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  11301. }
  11302. else
  11303. {
  11304. instr->m_opcode = Js::OpCode::Ld_I4;
  11305. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  11306. }
  11307. InvalidateInductionVariables(instr);
  11308. return true;
  11309. }
  11310. void
  11311. GlobOpt::OptConstFoldBr(bool test, IR::Instr *instr, Value * src1Val, Value * src2Val)
  11312. {
  11313. GOPT_TRACE_INSTR(instr, _u("Constant folding to branch: "));
  11314. BasicBlock *deadBlock;
  11315. if (src1Val)
  11316. {
  11317. this->ToInt32(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, false);
  11318. }
  11319. if (src2Val)
  11320. {
  11321. this->ToInt32(instr, instr->GetSrc2(), this->currentBlock, src2Val, nullptr, false);
  11322. }
  11323. this->CaptureByteCodeSymUses(instr);
  11324. if (test)
  11325. {
  11326. instr->m_opcode = Js::OpCode::Br;
  11327. instr->FreeSrc1();
  11328. if(instr->GetSrc2())
  11329. {
  11330. instr->FreeSrc2();
  11331. }
  11332. deadBlock = instr->m_next->AsLabelInstr()->GetBasicBlock();
  11333. }
  11334. else
  11335. {
  11336. AssertMsg(instr->m_next->IsLabelInstr(), "Next instr of branch should be a label...");
  11337. if(instr->AsBranchInstr()->IsMultiBranch())
  11338. {
  11339. return;
  11340. }
  11341. deadBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock();
  11342. instr->FreeSrc1();
  11343. if(instr->GetSrc2())
  11344. {
  11345. instr->FreeSrc2();
  11346. }
  11347. instr->m_opcode = Js::OpCode::Nop;
  11348. }
  11349. // Loop back edge: we would have already decremented data use count for the tail block when we processed the loop header.
  11350. if (!(this->currentBlock->loop && this->currentBlock->loop->GetHeadBlock() == deadBlock))
  11351. {
  11352. this->currentBlock->DecrementDataUseCount();
  11353. }
  11354. this->currentBlock->RemoveDeadSucc(deadBlock, this->func->m_fg);
  11355. if (deadBlock->GetPredList()->Count() == 0)
  11356. {
  11357. deadBlock->SetDataUseCount(0);
  11358. }
  11359. }
  11360. void
  11361. GlobOpt::ChangeValueType(
  11362. BasicBlock *const block,
  11363. Value *const value,
  11364. const ValueType newValueType,
  11365. const bool preserveSubclassInfo,
  11366. const bool allowIncompatibleType) const
  11367. {
  11368. Assert(value);
  11369. // Why are we trying to change the value type of the type sym value? Asserting here to make sure we don't deep copy the type sym's value info.
  11370. Assert(!value->GetValueInfo()->IsJsType());
  11371. ValueInfo *const valueInfo = value->GetValueInfo();
  11372. const ValueType valueType(valueInfo->Type());
  11373. if(valueType == newValueType && (preserveSubclassInfo || valueInfo->IsGeneric()))
  11374. {
  11375. return;
  11376. }
  11377. // ArrayValueInfo has information specific to the array type, so make sure that doesn't change
  11378. Assert(
  11379. !preserveSubclassInfo ||
  11380. !valueInfo->IsArrayValueInfo() ||
  11381. newValueType.IsObject() && newValueType.GetObjectType() == valueInfo->GetObjectType());
  11382. Assert(!valueInfo->GetSymStore() || !valueInfo->GetSymStore()->IsStackSym() || !valueInfo->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable());
  11383. ValueInfo *const newValueInfo =
  11384. preserveSubclassInfo
  11385. ? valueInfo->Copy(alloc)
  11386. : valueInfo->CopyWithGenericStructureKind(alloc);
  11387. newValueInfo->Type() = newValueType;
  11388. ChangeValueInfo(block, value, newValueInfo, allowIncompatibleType);
  11389. }
  11390. void
  11391. GlobOpt::ChangeValueInfo(BasicBlock *const block, Value *const value, ValueInfo *const newValueInfo, const bool allowIncompatibleType, const bool compensated) const
  11392. {
  11393. Assert(value);
  11394. Assert(newValueInfo);
  11395. // The value type must be changed to something more specific or something more generic. For instance, it would be changed to
  11396. // something more specific if the current value type is LikelyArray and checks have been done to ensure that it's an array,
  11397. // and it would be changed to something more generic if a call kills the Array value type and it must be treated as
  11398. // LikelyArray going forward.
  11399. // There are cases where we change the type because of different profile information, and because of rejit, these profile information
  11400. // may conflict. Need to allow incompatible type in those cause. However, the old type should be indefinite.
  11401. Assert((allowIncompatibleType && !value->GetValueInfo()->IsDefinite()) ||
  11402. AreValueInfosCompatible(newValueInfo, value->GetValueInfo()));
  11403. // ArrayValueInfo has information specific to the array type, so make sure that doesn't change
  11404. Assert(
  11405. !value->GetValueInfo()->IsArrayValueInfo() ||
  11406. !newValueInfo->IsArrayValueInfo() ||
  11407. newValueInfo->GetObjectType() == value->GetValueInfo()->GetObjectType());
  11408. if(block)
  11409. {
  11410. TrackValueInfoChangeForKills(block, value, newValueInfo, compensated);
  11411. }
  11412. value->SetValueInfo(newValueInfo);
  11413. }
  11414. bool
  11415. GlobOpt::AreValueInfosCompatible(const ValueInfo *const v0, const ValueInfo *const v1) const
  11416. {
  11417. Assert(v0);
  11418. Assert(v1);
  11419. if(v0->IsUninitialized() || v1->IsUninitialized())
  11420. {
  11421. return true;
  11422. }
  11423. const bool doAggressiveIntTypeSpec = DoAggressiveIntTypeSpec();
  11424. if(doAggressiveIntTypeSpec && (v0->IsInt() || v1->IsInt()))
  11425. {
  11426. // Int specialization in some uncommon loop cases involving dependencies, needs to allow specializing values of
  11427. // arbitrary types, even values that are definitely not int, to compensate for aggressive assumptions made by a loop
  11428. // prepass
  11429. return true;
  11430. }
  11431. if ((v0->Type()).IsMixedTypedArrayPair(v1->Type()) || (v1->Type()).IsMixedTypedArrayPair(v0->Type()))
  11432. {
  11433. return true;
  11434. }
  11435. const bool doFloatTypeSpec = DoFloatTypeSpec();
  11436. if(doFloatTypeSpec && (v0->IsFloat() || v1->IsFloat()))
  11437. {
  11438. // Float specialization allows specializing values of arbitrary types, even values that are definitely not float
  11439. return true;
  11440. }
  11441. const bool doArrayMissingValueCheckHoist = DoArrayMissingValueCheckHoist();
  11442. const bool doNativeArrayTypeSpec = DoNativeArrayTypeSpec();
  11443. const auto AreValueTypesCompatible = [=](const ValueType t0, const ValueType t1)
  11444. {
  11445. return
  11446. t0.IsSubsetOf(t1, doAggressiveIntTypeSpec, doFloatTypeSpec, doArrayMissingValueCheckHoist, doNativeArrayTypeSpec) ||
  11447. t1.IsSubsetOf(t0, doAggressiveIntTypeSpec, doFloatTypeSpec, doArrayMissingValueCheckHoist, doNativeArrayTypeSpec);
  11448. };
  11449. const ValueType t0(v0->Type().ToDefinite()), t1(v1->Type().ToDefinite());
  11450. if(t0.IsLikelyObject() && t1.IsLikelyObject())
  11451. {
  11452. // Check compatibility for the primitive portions and the object portions of the value types separately
  11453. if(AreValueTypesCompatible(t0.ToDefiniteObject(), t1.ToDefiniteObject()) &&
  11454. (
  11455. !t0.HasBeenPrimitive() ||
  11456. !t1.HasBeenPrimitive() ||
  11457. AreValueTypesCompatible(t0.ToDefinitePrimitiveSubset(), t1.ToDefinitePrimitiveSubset())
  11458. ))
  11459. {
  11460. return true;
  11461. }
  11462. }
  11463. else if(AreValueTypesCompatible(t0, t1))
  11464. {
  11465. return true;
  11466. }
  11467. const FloatConstantValueInfo *floatConstantValueInfo;
  11468. const ValueInfo *likelyIntValueinfo;
  11469. if(v0->IsFloatConstant() && v1->IsLikelyInt())
  11470. {
  11471. floatConstantValueInfo = v0->AsFloatConstant();
  11472. likelyIntValueinfo = v1;
  11473. }
  11474. else if(v0->IsLikelyInt() && v1->IsFloatConstant())
  11475. {
  11476. floatConstantValueInfo = v1->AsFloatConstant();
  11477. likelyIntValueinfo = v0;
  11478. }
  11479. else
  11480. {
  11481. return false;
  11482. }
  11483. // A float constant value with a value that is actually an int is a subset of a likely-int value.
  11484. // Ideally, we should create an int constant value for this up front, such that IsInt() also returns true. There
  11485. // were other issues with that, should see if that can be done.
  11486. int32 int32Value;
  11487. return
  11488. Js::JavascriptNumber::TryGetInt32Value(floatConstantValueInfo->FloatValue(), &int32Value) &&
  11489. (!likelyIntValueinfo->IsLikelyTaggedInt() || !Js::TaggedInt::IsOverflow(int32Value));
  11490. }
  11491. #if DBG
  11492. void
  11493. GlobOpt::VerifyArrayValueInfoForTracking(
  11494. const ValueInfo *const valueInfo,
  11495. const bool isJsArray,
  11496. const BasicBlock *const block,
  11497. const bool ignoreKnownImplicitCalls) const
  11498. {
  11499. Assert(valueInfo);
  11500. Assert(valueInfo->IsAnyOptimizedArray());
  11501. Assert(isJsArray == valueInfo->IsArrayOrObjectWithArray());
  11502. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11503. Assert(block);
  11504. Loop *implicitCallsLoop;
  11505. if(block->next && !block->next->isDeleted && block->next->isLoopHeader)
  11506. {
  11507. // Since a loop's landing pad does not have user code, determine whether disabling implicit calls is allowed in the
  11508. // landing pad based on the loop for which this block is the landing pad.
  11509. implicitCallsLoop = block->next->loop;
  11510. Assert(implicitCallsLoop);
  11511. Assert(implicitCallsLoop->landingPad == block);
  11512. }
  11513. else
  11514. {
  11515. implicitCallsLoop = block->loop;
  11516. }
  11517. Assert(
  11518. !isJsArray ||
  11519. DoArrayCheckHoist(valueInfo->Type(), implicitCallsLoop) ||
  11520. (
  11521. ignoreKnownImplicitCalls &&
  11522. !(implicitCallsLoop ? ImplicitCallFlagsAllowOpts(implicitCallsLoop) : ImplicitCallFlagsAllowOpts(func))
  11523. ));
  11524. Assert(!(isJsArray && valueInfo->HasNoMissingValues() && !DoArrayMissingValueCheckHoist()));
  11525. Assert(
  11526. !(
  11527. valueInfo->IsArrayValueInfo() &&
  11528. (
  11529. valueInfo->AsArrayValueInfo()->HeadSegmentSym() ||
  11530. valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()
  11531. ) &&
  11532. !DoArraySegmentHoist(valueInfo->Type())
  11533. ));
  11534. #if 0
  11535. // We can't assert here that there is only a head segment length sym if hoisting is allowed in the current block,
  11536. // because we may have propagated the sym forward out of a loop, and hoisting may be allowed inside but not
  11537. // outside the loop.
  11538. Assert(
  11539. isJsArray ||
  11540. !valueInfo->IsArrayValueInfo() ||
  11541. !valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym() ||
  11542. DoTypedArraySegmentLengthHoist(implicitCallsLoop) ||
  11543. ignoreKnownImplicitCalls ||
  11544. (implicitCallsLoop ? ImplicitCallFlagsAllowOpts(implicitCallsLoop) : ImplicitCallFlagsAllowOpts(func))
  11545. );
  11546. #endif
  11547. Assert(
  11548. !(
  11549. isJsArray &&
  11550. valueInfo->IsArrayValueInfo() &&
  11551. valueInfo->AsArrayValueInfo()->LengthSym() &&
  11552. !DoArrayLengthHoist()
  11553. ));
  11554. }
  11555. #endif
  11556. void
  11557. GlobOpt::TrackNewValueForKills(Value *const value)
  11558. {
  11559. Assert(value);
  11560. if(!value->GetValueInfo()->IsAnyOptimizedArray())
  11561. {
  11562. return;
  11563. }
  11564. DoTrackNewValueForKills(value);
  11565. }
  11566. void
  11567. GlobOpt::DoTrackNewValueForKills(Value *const value)
  11568. {
  11569. Assert(value);
  11570. ValueInfo *const valueInfo = value->GetValueInfo();
  11571. Assert(valueInfo->IsAnyOptimizedArray());
  11572. Assert(!valueInfo->IsArrayValueInfo());
  11573. // The value and value info here are new, so it's okay to modify the value info in-place
  11574. Assert(!valueInfo->GetSymStore());
  11575. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11576. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11577. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11578. Loop *implicitCallsLoop;
  11579. if(currentBlock->next && !currentBlock->next->isDeleted && currentBlock->next->isLoopHeader)
  11580. {
  11581. // Since a loop's landing pad does not have user code, determine whether disabling implicit calls is allowed in the
  11582. // landing pad based on the loop for which this block is the landing pad.
  11583. implicitCallsLoop = currentBlock->next->loop;
  11584. Assert(implicitCallsLoop);
  11585. Assert(implicitCallsLoop->landingPad == currentBlock);
  11586. }
  11587. else
  11588. {
  11589. implicitCallsLoop = currentBlock->loop;
  11590. }
  11591. if(isJsArray || isVirtualTypedArray)
  11592. {
  11593. if(!DoArrayCheckHoist(valueInfo->Type(), implicitCallsLoop))
  11594. {
  11595. // Array opts are disabled for this value type, so treat it as an indefinite value type going forward
  11596. valueInfo->Type() = valueInfo->Type().ToLikely();
  11597. return;
  11598. }
  11599. if(isJsArray && valueInfo->HasNoMissingValues() && !DoArrayMissingValueCheckHoist())
  11600. {
  11601. valueInfo->Type() = valueInfo->Type().SetHasNoMissingValues(false);
  11602. }
  11603. }
  11604. #if DBG
  11605. VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock);
  11606. #endif
  11607. if(!isJsArray && !isVirtualTypedArray)
  11608. {
  11609. return;
  11610. }
  11611. // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the
  11612. // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can
  11613. // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can
  11614. // revert the value type to a likely version.
  11615. CurrentBlockData()->valuesToKillOnCalls->Add(value);
  11616. }
  11617. void
  11618. GlobOpt::TrackCopiedValueForKills(Value *const value)
  11619. {
  11620. Assert(value);
  11621. if(!value->GetValueInfo()->IsAnyOptimizedArray())
  11622. {
  11623. return;
  11624. }
  11625. DoTrackCopiedValueForKills(value);
  11626. }
  11627. void
  11628. GlobOpt::DoTrackCopiedValueForKills(Value *const value)
  11629. {
  11630. Assert(value);
  11631. ValueInfo *const valueInfo = value->GetValueInfo();
  11632. Assert(valueInfo->IsAnyOptimizedArray());
  11633. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11634. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11635. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11636. #if DBG
  11637. VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock);
  11638. #endif
  11639. if(!isJsArray && !isVirtualTypedArray && !(valueInfo->IsArrayValueInfo() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()))
  11640. {
  11641. return;
  11642. }
  11643. // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the
  11644. // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can
  11645. // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can
  11646. // revert the value type to a likely version.
  11647. CurrentBlockData()->valuesToKillOnCalls->Add(value);
  11648. }
  11649. void
  11650. GlobOpt::TrackMergedValueForKills(
  11651. Value *const value,
  11652. GlobOptBlockData *const blockData,
  11653. BVSparse<JitArenaAllocator> *const mergedValueTypesTrackedForKills) const
  11654. {
  11655. Assert(value);
  11656. if(!value->GetValueInfo()->IsAnyOptimizedArray())
  11657. {
  11658. return;
  11659. }
  11660. DoTrackMergedValueForKills(value, blockData, mergedValueTypesTrackedForKills);
  11661. }
  11662. void
  11663. GlobOpt::DoTrackMergedValueForKills(
  11664. Value *const value,
  11665. GlobOptBlockData *const blockData,
  11666. BVSparse<JitArenaAllocator> *const mergedValueTypesTrackedForKills) const
  11667. {
  11668. Assert(value);
  11669. Assert(blockData);
  11670. ValueInfo *valueInfo = value->GetValueInfo();
  11671. Assert(valueInfo->IsAnyOptimizedArray());
  11672. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11673. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11674. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11675. #if DBG
  11676. VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock, true);
  11677. #endif
  11678. if(!isJsArray && !isVirtualTypedArray && !(valueInfo->IsArrayValueInfo() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()))
  11679. {
  11680. return;
  11681. }
  11682. // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the
  11683. // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can
  11684. // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can
  11685. // revert the value type to a likely version.
  11686. if(!mergedValueTypesTrackedForKills || !mergedValueTypesTrackedForKills->TestAndSet(value->GetValueNumber()))
  11687. {
  11688. blockData->valuesToKillOnCalls->Add(value);
  11689. }
  11690. }
  11691. void
  11692. GlobOpt::TrackValueInfoChangeForKills(BasicBlock *const block, Value *const value, ValueInfo *const newValueInfo, const bool compensated) const
  11693. {
  11694. Assert(block);
  11695. Assert(value);
  11696. Assert(newValueInfo);
  11697. ValueInfo *const oldValueInfo = value->GetValueInfo();
  11698. #if DBG
  11699. if(oldValueInfo->IsAnyOptimizedArray())
  11700. {
  11701. VerifyArrayValueInfoForTracking(oldValueInfo, oldValueInfo->IsArrayOrObjectWithArray(), block, compensated);
  11702. }
  11703. #endif
  11704. const bool trackOldValueInfo =
  11705. oldValueInfo->IsArrayOrObjectWithArray() ||
  11706. oldValueInfo->IsOptimizedVirtualTypedArray() ||
  11707. (
  11708. oldValueInfo->IsOptimizedTypedArray() &&
  11709. oldValueInfo->IsArrayValueInfo() &&
  11710. oldValueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()
  11711. );
  11712. Assert(trackOldValueInfo == block->globOptData.valuesToKillOnCalls->ContainsKey(value));
  11713. #if DBG
  11714. if(newValueInfo->IsAnyOptimizedArray())
  11715. {
  11716. VerifyArrayValueInfoForTracking(newValueInfo, newValueInfo->IsArrayOrObjectWithArray(), block, compensated);
  11717. }
  11718. #endif
  11719. const bool trackNewValueInfo =
  11720. newValueInfo->IsArrayOrObjectWithArray() ||
  11721. newValueInfo->IsOptimizedVirtualTypedArray() ||
  11722. (
  11723. newValueInfo->IsOptimizedTypedArray() &&
  11724. newValueInfo->IsArrayValueInfo() &&
  11725. newValueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()
  11726. );
  11727. if(trackOldValueInfo == trackNewValueInfo)
  11728. {
  11729. return;
  11730. }
  11731. if(trackNewValueInfo)
  11732. {
  11733. block->globOptData.valuesToKillOnCalls->Add(value);
  11734. }
  11735. else
  11736. {
  11737. block->globOptData.valuesToKillOnCalls->Remove(value);
  11738. }
  11739. }
  11740. void
  11741. GlobOpt::ProcessValueKills(IR::Instr *const instr)
  11742. {
  11743. Assert(instr);
  11744. ValueSet *const valuesToKillOnCalls = CurrentBlockData()->valuesToKillOnCalls;
  11745. if(!IsLoopPrePass() && valuesToKillOnCalls->Count() == 0)
  11746. {
  11747. return;
  11748. }
  11749. const JsArrayKills kills = CheckJsArrayKills(instr);
  11750. Assert(!kills.KillsArrayHeadSegments() || kills.KillsArrayHeadSegmentLengths());
  11751. if(IsLoopPrePass())
  11752. {
  11753. rootLoopPrePass->jsArrayKills = rootLoopPrePass->jsArrayKills.Merge(kills);
  11754. Assert(
  11755. !rootLoopPrePass->parent ||
  11756. rootLoopPrePass->jsArrayKills.AreSubsetOf(rootLoopPrePass->parent->jsArrayKills));
  11757. if(kills.KillsAllArrays())
  11758. {
  11759. rootLoopPrePass->needImplicitCallBailoutChecksForJsArrayCheckHoist = false;
  11760. }
  11761. if(valuesToKillOnCalls->Count() == 0)
  11762. {
  11763. return;
  11764. }
  11765. }
  11766. if(kills.KillsAllArrays())
  11767. {
  11768. Assert(kills.KillsTypedArrayHeadSegmentLengths());
  11769. // - Calls need to kill the value types of values in the following list. For instance, calls can transform a JS array
  11770. // into an ES5 array, so any definitely-array value types need to be killed. Also, VirtualTypeArrays do not have
  11771. // bounds checks; this can be problematic if the array is detached, so check to ensure that it is a virtual array.
  11772. // Update the value types to likley to ensure a bailout that asserts Array type is generated.
  11773. // - Calls also need to kill typed array head segment lengths. A typed array's array buffer may be transferred to a web
  11774. // worker, in which case the typed array's length is set to zero.
  11775. for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext())
  11776. {
  11777. Value *const value = it.CurrentValue();
  11778. ValueInfo *const valueInfo = value->GetValueInfo();
  11779. Assert(
  11780. valueInfo->IsArrayOrObjectWithArray() ||
  11781. valueInfo->IsOptimizedVirtualTypedArray() ||
  11782. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11783. if (valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray())
  11784. {
  11785. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11786. continue;
  11787. }
  11788. ChangeValueInfo(
  11789. nullptr,
  11790. value,
  11791. valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true));
  11792. }
  11793. valuesToKillOnCalls->Clear();
  11794. return;
  11795. }
  11796. if(kills.KillsArraysWithNoMissingValues())
  11797. {
  11798. // Some operations may kill arrays with no missing values in unlikely circumstances. Convert their value types to likely
  11799. // versions so that the checks have to be redone.
  11800. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  11801. {
  11802. Value *const value = it.CurrentValue();
  11803. ValueInfo *const valueInfo = value->GetValueInfo();
  11804. Assert(
  11805. valueInfo->IsArrayOrObjectWithArray() ||
  11806. valueInfo->IsOptimizedVirtualTypedArray() ||
  11807. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11808. if(!valueInfo->IsArrayOrObjectWithArray() || !valueInfo->HasNoMissingValues())
  11809. {
  11810. continue;
  11811. }
  11812. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11813. it.RemoveCurrent();
  11814. }
  11815. }
  11816. else if(kills.KillsObjectArraysWithNoMissingValues())
  11817. {
  11818. // Some operations may kill objects with arrays-with-no-missing-values in unlikely circumstances. Convert their value types to likely
  11819. // versions so that the checks have to be redone.
  11820. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  11821. {
  11822. Value *const value = it.CurrentValue();
  11823. ValueInfo *const valueInfo = value->GetValueInfo();
  11824. Assert(
  11825. valueInfo->IsArrayOrObjectWithArray() ||
  11826. valueInfo->IsOptimizedVirtualTypedArray() ||
  11827. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11828. if(!valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsArray() || !valueInfo->HasNoMissingValues())
  11829. {
  11830. continue;
  11831. }
  11832. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11833. it.RemoveCurrent();
  11834. }
  11835. }
  11836. if(kills.KillsNativeArrays())
  11837. {
  11838. // Some operations may kill native arrays in (what should be) unlikely circumstances. Convert their value types to
  11839. // likely versions so that the checks have to be redone.
  11840. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  11841. {
  11842. Value *const value = it.CurrentValue();
  11843. ValueInfo *const valueInfo = value->GetValueInfo();
  11844. Assert(
  11845. valueInfo->IsArrayOrObjectWithArray() ||
  11846. valueInfo->IsOptimizedVirtualTypedArray() ||
  11847. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11848. if(!valueInfo->IsArrayOrObjectWithArray() || valueInfo->HasVarElements())
  11849. {
  11850. continue;
  11851. }
  11852. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11853. it.RemoveCurrent();
  11854. }
  11855. }
  11856. const bool likelyKillsJsArraysWithNoMissingValues = IsOperationThatLikelyKillsJsArraysWithNoMissingValues(instr);
  11857. if(!kills.KillsArrayHeadSegmentLengths())
  11858. {
  11859. Assert(!kills.KillsArrayHeadSegments());
  11860. if(!likelyKillsJsArraysWithNoMissingValues && !kills.KillsArrayLengths())
  11861. {
  11862. return;
  11863. }
  11864. }
  11865. for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext())
  11866. {
  11867. Value *const value = it.CurrentValue();
  11868. ValueInfo *valueInfo = value->GetValueInfo();
  11869. Assert(
  11870. valueInfo->IsArrayOrObjectWithArray() ||
  11871. valueInfo->IsOptimizedVirtualTypedArray() ||
  11872. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11873. if(!valueInfo->IsArrayOrObjectWithArray())
  11874. {
  11875. continue;
  11876. }
  11877. if(likelyKillsJsArraysWithNoMissingValues && valueInfo->HasNoMissingValues())
  11878. {
  11879. ChangeValueType(nullptr, value, valueInfo->Type().SetHasNoMissingValues(false), true);
  11880. valueInfo = value->GetValueInfo();
  11881. }
  11882. if(!valueInfo->IsArrayValueInfo())
  11883. {
  11884. continue;
  11885. }
  11886. ArrayValueInfo *const arrayValueInfo = valueInfo->AsArrayValueInfo();
  11887. const bool removeHeadSegment = kills.KillsArrayHeadSegments() && arrayValueInfo->HeadSegmentSym();
  11888. const bool removeHeadSegmentLength = kills.KillsArrayHeadSegmentLengths() && arrayValueInfo->HeadSegmentLengthSym();
  11889. const bool removeLength = kills.KillsArrayLengths() && arrayValueInfo->LengthSym();
  11890. if(removeHeadSegment || removeHeadSegmentLength || removeLength)
  11891. {
  11892. ChangeValueInfo(
  11893. nullptr,
  11894. value,
  11895. arrayValueInfo->Copy(alloc, !removeHeadSegment, !removeHeadSegmentLength, !removeLength));
  11896. valueInfo = value->GetValueInfo();
  11897. }
  11898. }
  11899. }
  11900. void
  11901. GlobOpt::ProcessValueKills(BasicBlock *const block, GlobOptBlockData *const blockData)
  11902. {
  11903. Assert(block);
  11904. Assert(blockData);
  11905. ValueSet *const valuesToKillOnCalls = blockData->valuesToKillOnCalls;
  11906. if(!IsLoopPrePass() && valuesToKillOnCalls->Count() == 0)
  11907. {
  11908. return;
  11909. }
  11910. // If the current block or loop has implicit calls, kill all definitely-array value types, as using that info will cause
  11911. // implicit calls to be disabled, resulting in unnecessary bailouts
  11912. const bool killValuesOnImplicitCalls =
  11913. (block->loop ? !this->ImplicitCallFlagsAllowOpts(block->loop) : !this->ImplicitCallFlagsAllowOpts(func));
  11914. if (!killValuesOnImplicitCalls)
  11915. {
  11916. return;
  11917. }
  11918. if(IsLoopPrePass() && block->loop == rootLoopPrePass)
  11919. {
  11920. AnalysisAssert(rootLoopPrePass);
  11921. for (Loop * loop = rootLoopPrePass; loop != nullptr; loop = loop->parent)
  11922. {
  11923. loop->jsArrayKills.SetKillsAllArrays();
  11924. }
  11925. Assert(!rootLoopPrePass->parent || rootLoopPrePass->jsArrayKills.AreSubsetOf(rootLoopPrePass->parent->jsArrayKills));
  11926. if(valuesToKillOnCalls->Count() == 0)
  11927. {
  11928. return;
  11929. }
  11930. }
  11931. for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext())
  11932. {
  11933. Value *const value = it.CurrentValue();
  11934. ValueInfo *const valueInfo = value->GetValueInfo();
  11935. Assert(
  11936. valueInfo->IsArrayOrObjectWithArray() ||
  11937. valueInfo->IsOptimizedVirtualTypedArray() ||
  11938. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11939. if(valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray())
  11940. {
  11941. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11942. continue;
  11943. }
  11944. ChangeValueInfo(
  11945. nullptr,
  11946. value,
  11947. valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true));
  11948. }
  11949. valuesToKillOnCalls->Clear();
  11950. }
  11951. void
  11952. GlobOpt::ProcessValueKillsForLoopHeaderAfterBackEdgeMerge(BasicBlock *const block, GlobOptBlockData *const blockData)
  11953. {
  11954. Assert(block);
  11955. Assert(block->isLoopHeader);
  11956. Assert(blockData);
  11957. ValueSet *const valuesToKillOnCalls = blockData->valuesToKillOnCalls;
  11958. if(valuesToKillOnCalls->Count() == 0)
  11959. {
  11960. return;
  11961. }
  11962. const JsArrayKills loopKills(block->loop->jsArrayKills);
  11963. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  11964. {
  11965. Value *const value = it.CurrentValue();
  11966. ValueInfo *valueInfo = value->GetValueInfo();
  11967. Assert(
  11968. valueInfo->IsArrayOrObjectWithArray() ||
  11969. valueInfo->IsOptimizedVirtualTypedArray() ||
  11970. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11971. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11972. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11973. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11974. if((isJsArray || isVirtualTypedArray) ? loopKills.KillsValueType(valueInfo->Type()) : loopKills.KillsTypedArrayHeadSegmentLengths())
  11975. {
  11976. // Hoisting array checks and other related things for this type is disabled for the loop due to the kill, as
  11977. // compensation code is currently not added on back-edges. When merging values from a back-edge, the array value
  11978. // type cannot be definite, as that may require adding compensation code on the back-edge if the optimization pass
  11979. // chooses to not optimize the array.
  11980. if(isJsArray || isVirtualTypedArray)
  11981. {
  11982. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11983. }
  11984. else
  11985. {
  11986. ChangeValueInfo(
  11987. nullptr,
  11988. value,
  11989. valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true));
  11990. }
  11991. it.RemoveCurrent();
  11992. continue;
  11993. }
  11994. if(!isJsArray || !valueInfo->IsArrayValueInfo())
  11995. {
  11996. continue;
  11997. }
  11998. // Similarly, if the loop contains an operation that kills JS array segments, don't make the segment or other related
  11999. // syms available initially inside the loop
  12000. ArrayValueInfo *const arrayValueInfo = valueInfo->AsArrayValueInfo();
  12001. const bool removeHeadSegment = loopKills.KillsArrayHeadSegments() && arrayValueInfo->HeadSegmentSym();
  12002. const bool removeHeadSegmentLength = loopKills.KillsArrayHeadSegmentLengths() && arrayValueInfo->HeadSegmentLengthSym();
  12003. const bool removeLength = loopKills.KillsArrayLengths() && arrayValueInfo->LengthSym();
  12004. if(removeHeadSegment || removeHeadSegmentLength || removeLength)
  12005. {
  12006. ChangeValueInfo(
  12007. nullptr,
  12008. value,
  12009. arrayValueInfo->Copy(alloc, !removeHeadSegment, !removeHeadSegmentLength, !removeLength));
  12010. valueInfo = value->GetValueInfo();
  12011. }
  12012. }
  12013. }
  12014. bool
  12015. GlobOpt::NeedBailOnImplicitCallForLiveValues(BasicBlock const * const block, const bool isForwardPass) const
  12016. {
  12017. if(isForwardPass)
  12018. {
  12019. return block->globOptData.valuesToKillOnCalls->Count() != 0;
  12020. }
  12021. if(block->noImplicitCallUses->IsEmpty())
  12022. {
  12023. Assert(block->noImplicitCallNoMissingValuesUses->IsEmpty());
  12024. Assert(block->noImplicitCallNativeArrayUses->IsEmpty());
  12025. Assert(block->noImplicitCallJsArrayHeadSegmentSymUses->IsEmpty());
  12026. Assert(block->noImplicitCallArrayLengthSymUses->IsEmpty());
  12027. return false;
  12028. }
  12029. return true;
  12030. }
  12031. IR::Instr*
  12032. GlobOpt::CreateBoundsCheckInstr(IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset, Func* func)
  12033. {
  12034. IR::Instr* instr = IR::Instr::New(Js::OpCode::BoundCheck, func);
  12035. return AttachBoundsCheckData(instr, lowerBound, upperBound, offset);
  12036. }
  12037. IR::Instr*
  12038. GlobOpt::CreateBoundsCheckInstr(IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset, IR::BailOutKind bailoutkind, BailOutInfo* bailoutInfo, Func * func)
  12039. {
  12040. IR::Instr* instr = IR::BailOutInstr::New(Js::OpCode::BoundCheck, bailoutkind, bailoutInfo, func);
  12041. return AttachBoundsCheckData(instr, lowerBound, upperBound, offset);
  12042. }
  12043. IR::Instr*
  12044. GlobOpt::AttachBoundsCheckData(IR::Instr* instr, IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset)
  12045. {
  12046. instr->SetSrc1(lowerBound);
  12047. instr->SetSrc2(upperBound);
  12048. if (offset != 0)
  12049. {
  12050. instr->SetDst(IR::IntConstOpnd::New(offset, TyInt32, instr->m_func));
  12051. }
  12052. return instr;
  12053. }
  12054. void
  12055. GlobOpt::OptArraySrc(IR::Instr ** const instrRef, Value ** src1Val, Value ** src2Val)
  12056. {
  12057. Assert(instrRef != nullptr);
  12058. ArraySrcOpt arraySrcOpt(this, instrRef, src1Val, src2Val);
  12059. arraySrcOpt.Optimize();
  12060. }
  12061. void
  12062. GlobOpt::ProcessNoImplicitCallArrayUses(IR::RegOpnd * baseOpnd, IR::ArrayRegOpnd * baseArrayOpnd, IR::Instr * instr, bool isLikelyJsArray, bool useNoMissingValues)
  12063. {
  12064. if (isLikelyJsArray)
  12065. {
  12066. // Insert an instruction to indicate to the dead-store pass that implicit calls need to be kept disabled until this
  12067. // instruction. Operations other than LdElem, StElem and IsIn don't benefit much from arrays having no missing values,
  12068. // so no need to ensure that the array still has no missing values. For a particular array, if none of the accesses
  12069. // benefit much from the no-missing-values information, it may be beneficial to avoid checking for no missing
  12070. // values, especially in the case for a single array access, where the cost of the check could be relatively
  12071. // significant. An StElem has to do additional checks in the common path if the array may have missing values, and
  12072. // a StElem that operates on an array that has no missing values is more likely to keep the no-missing-values info
  12073. // on the array more precise, so it still benefits a little from the no-missing-values info.
  12074. this->CaptureNoImplicitCallUses(baseOpnd, isLikelyJsArray);
  12075. }
  12076. else if (baseArrayOpnd && baseArrayOpnd->HeadSegmentLengthSym())
  12077. {
  12078. // A typed array's array buffer may be transferred to a web worker as part of an implicit call, in which case the typed
  12079. // array's length is set to zero. Insert an instruction to indicate to the dead-store pass that implicit calls need to
  12080. // be disabled until this instruction.
  12081. IR::RegOpnd *const headSegmentLengthOpnd =
  12082. IR::RegOpnd::New(
  12083. baseArrayOpnd->HeadSegmentLengthSym(),
  12084. baseArrayOpnd->HeadSegmentLengthSym()->GetType(),
  12085. instr->m_func);
  12086. const IR::AutoReuseOpnd autoReuseHeadSegmentLengthOpnd(headSegmentLengthOpnd, instr->m_func);
  12087. this->CaptureNoImplicitCallUses(headSegmentLengthOpnd, false);
  12088. }
  12089. }
  12090. void
  12091. GlobOpt::OptStackArgLenAndConst(IR::Instr* instr, Value** src1Val)
  12092. {
  12093. if (!PHASE_OFF(Js::StackArgLenConstOptPhase, instr->m_func) && instr->m_func->IsStackArgsEnabled() && instr->usesStackArgumentsObject && instr->IsInlined())
  12094. {
  12095. IR::Opnd* src1 = instr->GetSrc1();
  12096. auto replaceInstr = [&](IR::Opnd* newopnd, Js::OpCode opcode)
  12097. {
  12098. if (PHASE_TESTTRACE(Js::StackArgLenConstOptPhase, instr->m_func))
  12099. {
  12100. Output::Print(_u("Inlined function %s have replaced opcode %s with opcode %s for stack arg optimization. \n"), instr->m_func->GetJITFunctionBody()->GetDisplayName(),
  12101. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode), Js::OpCodeUtil::GetOpCodeName(opcode));
  12102. Output::Flush();
  12103. }
  12104. this->CaptureByteCodeSymUses(instr);
  12105. instr->m_opcode = opcode;
  12106. instr->ReplaceSrc1(newopnd);
  12107. if (instr->HasBailOutInfo())
  12108. {
  12109. instr->ClearBailOutInfo();
  12110. }
  12111. if (instr->IsProfiledInstr())
  12112. {
  12113. Assert(opcode == Js::OpCode::Ld_A || opcode == Js::OpCode::Typeof);
  12114. instr->AsProfiledInstr()->u.FldInfo().valueType = ValueType::Uninitialized;
  12115. }
  12116. *src1Val = this->OptSrc(instr->GetSrc1(), &instr);
  12117. instr->m_func->hasArgLenAndConstOpt = true;
  12118. };
  12119. Assert(CurrentBlockData()->IsArgumentsOpnd(src1));
  12120. switch(instr->m_opcode)
  12121. {
  12122. case Js::OpCode::LdLen_A:
  12123. {
  12124. IR::AddrOpnd* newopnd = IR::AddrOpnd::New(Js::TaggedInt::ToVarUnchecked(instr->m_func->actualCount - 1), IR::AddrOpndKindConstantVar, instr->m_func);
  12125. replaceInstr(newopnd, Js::OpCode::Ld_A);
  12126. break;
  12127. }
  12128. case Js::OpCode::LdElemI_A:
  12129. case Js::OpCode::TypeofElem:
  12130. {
  12131. IR::IndirOpnd* indirOpndSrc1 = src1->AsIndirOpnd();
  12132. if (!indirOpndSrc1->GetIndexOpnd())
  12133. {
  12134. int argIndex = indirOpndSrc1->GetOffset() + 1;
  12135. IR::Instr* defInstr = nullptr;
  12136. IR::Instr* inlineeStart = instr->m_func->GetInlineeStart();
  12137. inlineeStart->IterateArgInstrs([&](IR::Instr* argInstr) {
  12138. StackSym *argSym = argInstr->GetDst()->AsSymOpnd()->m_sym->AsStackSym();
  12139. if (argSym->GetArgSlotNum() - 1 == argIndex)
  12140. {
  12141. defInstr = argInstr;
  12142. return true;
  12143. }
  12144. return false;
  12145. });
  12146. Js::OpCode replacementOpcode;
  12147. if (instr->m_opcode == Js::OpCode::TypeofElem)
  12148. {
  12149. replacementOpcode = Js::OpCode::Typeof;
  12150. }
  12151. else
  12152. {
  12153. replacementOpcode = Js::OpCode::Ld_A;
  12154. }
  12155. // If we cannot find the right instruction. I.E. When calling arguments[2] and no arguments were passed to the func
  12156. if (defInstr == nullptr)
  12157. {
  12158. IR::Opnd * undefined = IR::AddrOpnd::New(instr->m_func->GetScriptContextInfo()->GetUndefinedAddr(), IR::AddrOpndKindDynamicVar, instr->m_func, true);
  12159. undefined->SetValueType(ValueType::Undefined);
  12160. replaceInstr(undefined, replacementOpcode);
  12161. }
  12162. else
  12163. {
  12164. replaceInstr(defInstr->GetSrc1(), replacementOpcode);
  12165. }
  12166. }
  12167. else
  12168. {
  12169. instr->m_func->unoptimizableArgumentsObjReference++;
  12170. }
  12171. break;
  12172. }
  12173. }
  12174. }
  12175. }
  12176. void
  12177. GlobOpt::CaptureNoImplicitCallUses(
  12178. IR::Opnd *opnd,
  12179. const bool usesNoMissingValuesInfo,
  12180. IR::Instr *const includeCurrentInstr)
  12181. {
  12182. Assert(!IsLoopPrePass());
  12183. Assert(noImplicitCallUsesToInsert);
  12184. Assert(opnd);
  12185. // The opnd may be deleted later, so make a copy to ensure it is alive for inserting NoImplicitCallUses later
  12186. opnd = opnd->Copy(func);
  12187. if(!usesNoMissingValuesInfo)
  12188. {
  12189. const ValueType valueType(opnd->GetValueType());
  12190. if(valueType.IsArrayOrObjectWithArray() && valueType.HasNoMissingValues())
  12191. {
  12192. // Inserting NoImplicitCallUses for an opnd with a definitely-array-with-no-missing-values value type means that the
  12193. // instruction following it uses the information that the array has no missing values in some way, for instance, it
  12194. // may omit missing value checks. Based on that, the dead-store phase in turn ensures that the necessary bailouts
  12195. // are inserted to ensure that the array still has no missing values until the following instruction. Since
  12196. // 'usesNoMissingValuesInfo' is false, change the value type to indicate to the dead-store phase that the following
  12197. // instruction does not use the no-missing-values information.
  12198. opnd->SetValueType(valueType.SetHasNoMissingValues(false));
  12199. }
  12200. }
  12201. if(includeCurrentInstr)
  12202. {
  12203. IR::Instr *const noImplicitCallUses =
  12204. IR::PragmaInstr::New(Js::OpCode::NoImplicitCallUses, 0, includeCurrentInstr->m_func);
  12205. noImplicitCallUses->SetSrc1(opnd);
  12206. noImplicitCallUses->GetSrc1()->SetIsJITOptimizedReg(true);
  12207. includeCurrentInstr->InsertAfter(noImplicitCallUses);
  12208. return;
  12209. }
  12210. noImplicitCallUsesToInsert->Add(opnd);
  12211. }
  12212. void
  12213. GlobOpt::InsertNoImplicitCallUses(IR::Instr *const instr)
  12214. {
  12215. Assert(noImplicitCallUsesToInsert);
  12216. const int n = noImplicitCallUsesToInsert->Count();
  12217. if(n == 0)
  12218. {
  12219. return;
  12220. }
  12221. IR::Instr *const insertBeforeInstr = instr->GetInsertBeforeByteCodeUsesInstr();
  12222. for(int i = 0; i < n;)
  12223. {
  12224. IR::Instr *const noImplicitCallUses = IR::PragmaInstr::New(Js::OpCode::NoImplicitCallUses, 0, instr->m_func);
  12225. noImplicitCallUses->SetSrc1(noImplicitCallUsesToInsert->Item(i));
  12226. noImplicitCallUses->GetSrc1()->SetIsJITOptimizedReg(true);
  12227. ++i;
  12228. if(i < n)
  12229. {
  12230. noImplicitCallUses->SetSrc2(noImplicitCallUsesToInsert->Item(i));
  12231. noImplicitCallUses->GetSrc2()->SetIsJITOptimizedReg(true);
  12232. ++i;
  12233. }
  12234. noImplicitCallUses->SetByteCodeOffset(instr);
  12235. insertBeforeInstr->InsertBefore(noImplicitCallUses);
  12236. }
  12237. noImplicitCallUsesToInsert->Clear();
  12238. }
  12239. void
  12240. GlobOpt::PrepareLoopArrayCheckHoist()
  12241. {
  12242. if(IsLoopPrePass() || !currentBlock->loop || !currentBlock->isLoopHeader || !currentBlock->loop->parent)
  12243. {
  12244. return;
  12245. }
  12246. if(currentBlock->loop->parent->needImplicitCallBailoutChecksForJsArrayCheckHoist)
  12247. {
  12248. // If the parent loop is an array check elimination candidate, so is the current loop. Even though the current loop may
  12249. // not have array accesses, if the parent loop hoists array checks, the current loop also needs implicit call checks.
  12250. currentBlock->loop->needImplicitCallBailoutChecksForJsArrayCheckHoist = true;
  12251. }
  12252. }
  12253. JsArrayKills
  12254. GlobOpt::CheckJsArrayKills(IR::Instr *const instr)
  12255. {
  12256. Assert(instr);
  12257. JsArrayKills kills;
  12258. if(instr->UsesAllFields())
  12259. {
  12260. // Calls can (but are unlikely to) change a javascript array into an ES5 array, which may have different behavior for
  12261. // index properties.
  12262. kills.SetKillsAllArrays();
  12263. return kills;
  12264. }
  12265. const bool doArrayMissingValueCheckHoist = DoArrayMissingValueCheckHoist();
  12266. const bool doNativeArrayTypeSpec = DoNativeArrayTypeSpec();
  12267. const bool doArraySegmentHoist = DoArraySegmentHoist(ValueType::GetObject(ObjectType::Array));
  12268. Assert(doArraySegmentHoist == DoArraySegmentHoist(ValueType::GetObject(ObjectType::ObjectWithArray)));
  12269. const bool doArrayLengthHoist = DoArrayLengthHoist();
  12270. if(!doArrayMissingValueCheckHoist && !doNativeArrayTypeSpec && !doArraySegmentHoist && !doArrayLengthHoist)
  12271. {
  12272. return kills;
  12273. }
  12274. // The following operations may create missing values in an array in an unlikely circumstance. Even though they don't kill
  12275. // the fact that the 'this' parameter is an array (when implicit calls are disabled), we don't have a way to say the value
  12276. // type is definitely array but it likely has no missing values. So, these will kill the definite value type as well, making
  12277. // it likely array, such that the array checks will have to be redone.
  12278. const bool useValueTypes = !IsLoopPrePass(); // Source value types are not guaranteed to be correct in a loop prepass
  12279. switch(instr->m_opcode)
  12280. {
  12281. case Js::OpCode::StElemC:
  12282. case Js::OpCode::StElemI_A:
  12283. case Js::OpCode::StElemI_A_Strict:
  12284. {
  12285. Assert(instr->GetDst());
  12286. if(!instr->GetDst()->IsIndirOpnd())
  12287. {
  12288. break;
  12289. }
  12290. const ValueType baseValueType =
  12291. useValueTypes ? instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType() : ValueType::Uninitialized;
  12292. if(useValueTypes && baseValueType.IsNotArrayOrObjectWithArray())
  12293. {
  12294. break;
  12295. }
  12296. if(instr->IsProfiledInstr())
  12297. {
  12298. const Js::StElemInfo *const stElemInfo = instr->AsProfiledInstr()->u.stElemInfo;
  12299. if(doArraySegmentHoist && stElemInfo->LikelyStoresOutsideHeadSegmentBounds())
  12300. {
  12301. kills.SetKillsArrayHeadSegments();
  12302. kills.SetKillsArrayHeadSegmentLengths();
  12303. }
  12304. if(doArrayLengthHoist &&
  12305. !(useValueTypes && baseValueType.IsNotArray()) &&
  12306. stElemInfo->LikelyStoresOutsideArrayBounds())
  12307. {
  12308. kills.SetKillsArrayLengths();
  12309. }
  12310. }
  12311. break;
  12312. }
  12313. case Js::OpCode::DeleteElemI_A:
  12314. case Js::OpCode::DeleteElemIStrict_A:
  12315. Assert(instr->GetSrc1());
  12316. if(!instr->GetSrc1()->IsIndirOpnd() ||
  12317. (useValueTypes && instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsNotArrayOrObjectWithArray()))
  12318. {
  12319. break;
  12320. }
  12321. if(doArrayMissingValueCheckHoist)
  12322. {
  12323. kills.SetKillsArraysWithNoMissingValues();
  12324. }
  12325. if(doArraySegmentHoist)
  12326. {
  12327. kills.SetKillsArrayHeadSegmentLengths();
  12328. }
  12329. break;
  12330. case Js::OpCode::ConsoleScopedStFld:
  12331. case Js::OpCode::ConsoleScopedStFldStrict:
  12332. case Js::OpCode::ScopedStFld:
  12333. case Js::OpCode::ScopedStFldStrict:
  12334. case Js::OpCode::StFld:
  12335. case Js::OpCode::StFldStrict:
  12336. case Js::OpCode::StSuperFld:
  12337. case Js::OpCode::StSuperFldStrict:
  12338. {
  12339. Assert(instr->GetDst());
  12340. if(!doArraySegmentHoist && !doArrayLengthHoist)
  12341. {
  12342. break;
  12343. }
  12344. IR::SymOpnd *const symDst = instr->GetDst()->AsSymOpnd();
  12345. if(!symDst->IsPropertySymOpnd())
  12346. {
  12347. break;
  12348. }
  12349. IR::PropertySymOpnd *const dst = symDst->AsPropertySymOpnd();
  12350. if(dst->m_sym->AsPropertySym()->m_propertyId != Js::PropertyIds::length)
  12351. {
  12352. break;
  12353. }
  12354. if(useValueTypes && dst->GetPropertyOwnerValueType().IsNotArray())
  12355. {
  12356. // Setting the 'length' property of an object that is not an array, even if it has an internal array, does
  12357. // not kill the head segment or head segment length of any arrays.
  12358. break;
  12359. }
  12360. if(doArraySegmentHoist)
  12361. {
  12362. kills.SetKillsArrayHeadSegmentLengths();
  12363. }
  12364. if(doArrayLengthHoist)
  12365. {
  12366. kills.SetKillsArrayLengths();
  12367. }
  12368. break;
  12369. }
  12370. case Js::OpCode::InlineArrayPush:
  12371. {
  12372. Assert(instr->GetSrc2());
  12373. IR::Opnd *const arrayOpnd = instr->GetSrc1();
  12374. Assert(arrayOpnd);
  12375. const ValueType arrayValueType(arrayOpnd->GetValueType());
  12376. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  12377. {
  12378. break;
  12379. }
  12380. if(doArrayMissingValueCheckHoist)
  12381. {
  12382. kills.SetKillsArraysWithNoMissingValues();
  12383. }
  12384. if(doArraySegmentHoist)
  12385. {
  12386. kills.SetKillsArrayHeadSegments();
  12387. kills.SetKillsArrayHeadSegmentLengths();
  12388. }
  12389. if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray()))
  12390. {
  12391. kills.SetKillsArrayLengths();
  12392. }
  12393. // Don't kill NativeArray, if there is no mismatch between array's type and element's type.
  12394. if(doNativeArrayTypeSpec &&
  12395. !(useValueTypes && arrayValueType.IsNativeArray() &&
  12396. ((arrayValueType.IsLikelyNativeIntArray() && instr->GetSrc2()->IsInt32()) ||
  12397. (arrayValueType.IsLikelyNativeFloatArray() && instr->GetSrc2()->IsFloat()))
  12398. ) &&
  12399. !(useValueTypes && arrayValueType.IsNotNativeArray()))
  12400. {
  12401. kills.SetKillsNativeArrays();
  12402. }
  12403. break;
  12404. }
  12405. case Js::OpCode::InlineArrayPop:
  12406. {
  12407. IR::Opnd *const arrayOpnd = instr->GetSrc1();
  12408. Assert(arrayOpnd);
  12409. const ValueType arrayValueType(arrayOpnd->GetValueType());
  12410. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  12411. {
  12412. break;
  12413. }
  12414. if(doArraySegmentHoist)
  12415. {
  12416. kills.SetKillsArrayHeadSegmentLengths();
  12417. }
  12418. if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray()))
  12419. {
  12420. kills.SetKillsArrayLengths();
  12421. }
  12422. if(doArrayMissingValueCheckHoist && !(useValueTypes && arrayValueType.IsArray()))
  12423. {
  12424. kills.SetKillsObjectArraysWithNoMissingValues();
  12425. }
  12426. break;
  12427. }
  12428. case Js::OpCode::CallDirect:
  12429. {
  12430. Assert(instr->GetSrc1());
  12431. // Find the 'this' parameter and check if it's possible for it to be an array
  12432. IR::Opnd *const arrayOpnd = instr->FindCallArgumentOpnd(1);
  12433. Assert(arrayOpnd);
  12434. const ValueType arrayValueType(arrayOpnd->GetValueType());
  12435. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  12436. {
  12437. break;
  12438. }
  12439. const IR::JnHelperMethod helperMethod = instr->GetSrc1()->AsHelperCallOpnd()->m_fnHelper;
  12440. if(doArrayMissingValueCheckHoist)
  12441. {
  12442. switch(helperMethod)
  12443. {
  12444. case IR::HelperArray_Reverse:
  12445. case IR::HelperArray_Shift:
  12446. case IR::HelperArray_Splice:
  12447. case IR::HelperArray_Unshift:
  12448. kills.SetKillsArraysWithNoMissingValues();
  12449. break;
  12450. }
  12451. }
  12452. if(doArraySegmentHoist)
  12453. {
  12454. switch(helperMethod)
  12455. {
  12456. case IR::HelperArray_Reverse:
  12457. case IR::HelperArray_Shift:
  12458. case IR::HelperArray_Splice:
  12459. case IR::HelperArray_Unshift:
  12460. case IR::HelperArray_Concat:
  12461. kills.SetKillsArrayHeadSegments();
  12462. kills.SetKillsArrayHeadSegmentLengths();
  12463. break;
  12464. }
  12465. }
  12466. if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray()))
  12467. {
  12468. switch(helperMethod)
  12469. {
  12470. case IR::HelperArray_Shift:
  12471. case IR::HelperArray_Splice:
  12472. case IR::HelperArray_Unshift:
  12473. kills.SetKillsArrayLengths();
  12474. break;
  12475. }
  12476. }
  12477. if(doNativeArrayTypeSpec && !(useValueTypes && arrayValueType.IsNotNativeArray()))
  12478. {
  12479. switch(helperMethod)
  12480. {
  12481. case IR::HelperArray_Reverse:
  12482. case IR::HelperArray_Shift:
  12483. case IR::HelperArray_Slice:
  12484. // Currently not inlined.
  12485. //case IR::HelperArray_Sort:
  12486. case IR::HelperArray_Splice:
  12487. case IR::HelperArray_Unshift:
  12488. case IR::HelperArray_Concat:
  12489. kills.SetKillsNativeArrays();
  12490. break;
  12491. }
  12492. }
  12493. break;
  12494. }
  12495. case Js::OpCode::InitProto:
  12496. {
  12497. // Find the 'this' parameter and check if it's possible for it to be an array
  12498. IR::Opnd *const arrayOpnd = instr->GetSrc1();
  12499. Assert(arrayOpnd);
  12500. const ValueType arrayValueType(arrayOpnd->GetValueType());
  12501. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  12502. {
  12503. break;
  12504. }
  12505. if(doNativeArrayTypeSpec && !(useValueTypes && arrayValueType.IsNotNativeArray()))
  12506. {
  12507. kills.SetKillsNativeArrays();
  12508. }
  12509. break;
  12510. }
  12511. case Js::OpCode::NewClassProto:
  12512. Assert(instr->GetSrc1());
  12513. if (IR::AddrOpnd::IsEqualAddr(instr->GetSrc1(), (void*)func->GetScriptContextInfo()->GetObjectPrototypeAddr()))
  12514. {
  12515. // No extends operand, the proto parent is the Object prototype
  12516. break;
  12517. }
  12518. // Fall through
  12519. case Js::OpCode::NewScObjectNoCtor:
  12520. case Js::OpCode::NewScObjectNoCtorFull:
  12521. if(doNativeArrayTypeSpec)
  12522. {
  12523. // Class/object construction can make something a prototype
  12524. kills.SetKillsNativeArrays();
  12525. }
  12526. break;
  12527. }
  12528. return kills;
  12529. }
  12530. GlobOptBlockData const * GlobOpt::CurrentBlockData() const
  12531. {
  12532. return &this->currentBlock->globOptData;
  12533. }
  12534. GlobOptBlockData * GlobOpt::CurrentBlockData()
  12535. {
  12536. return &this->currentBlock->globOptData;
  12537. }
  12538. void GlobOpt::CommitCapturedValuesCandidate()
  12539. {
  12540. GlobOptBlockData * globOptData = CurrentBlockData();
  12541. globOptData->changedSyms->ClearAll();
  12542. if (!this->changedSymsAfterIncBailoutCandidate->IsEmpty())
  12543. {
  12544. //
  12545. // some symbols are changed after the values for current bailout have been
  12546. // captured (GlobOpt::CapturedValues), need to restore such symbols as changed
  12547. // for following incremental bailout construction, or we will miss capturing
  12548. // values for later bailout
  12549. //
  12550. // swap changedSyms and changedSymsAfterIncBailoutCandidate
  12551. // because both are from this->alloc
  12552. BVSparse<JitArenaAllocator> * tempBvSwap = globOptData->changedSyms;
  12553. globOptData->changedSyms = this->changedSymsAfterIncBailoutCandidate;
  12554. this->changedSymsAfterIncBailoutCandidate = tempBvSwap;
  12555. }
  12556. if (globOptData->capturedValues)
  12557. {
  12558. globOptData->capturedValues->DecrementRefCount();
  12559. }
  12560. globOptData->capturedValues = globOptData->capturedValuesCandidate;
  12561. // null out capturedValuesCandidate to stop tracking symbols change for it
  12562. globOptData->capturedValuesCandidate = nullptr;
  12563. }
  12564. bool
  12565. GlobOpt::IsOperationThatLikelyKillsJsArraysWithNoMissingValues(IR::Instr *const instr)
  12566. {
  12567. // StElem is profiled with information indicating whether it will likely create a missing value in the array. In that case,
  12568. // we prefer to kill the no-missing-values information in the value so that we don't bail out in a likely circumstance.
  12569. return
  12570. (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict) &&
  12571. DoArrayMissingValueCheckHoist() &&
  12572. instr->IsProfiledInstr() &&
  12573. instr->AsProfiledInstr()->u.stElemInfo->LikelyCreatesMissingValue();
  12574. }
  12575. bool
  12576. GlobOpt::NeedBailOnImplicitCallForArrayCheckHoist(BasicBlock const * const block, const bool isForwardPass) const
  12577. {
  12578. Assert(block);
  12579. return isForwardPass && block->loop && block->loop->needImplicitCallBailoutChecksForJsArrayCheckHoist;
  12580. }
  12581. bool
  12582. GlobOpt::PrepareForIgnoringIntOverflow(IR::Instr *const instr)
  12583. {
  12584. Assert(instr);
  12585. const bool isBoundary = instr->m_opcode == Js::OpCode::NoIntOverflowBoundary;
  12586. // Update the instruction's "int overflow matters" flag based on whether we are currently allowing ignoring int overflows.
  12587. // Some operations convert their srcs to int32s, those can still ignore int overflow.
  12588. if(instr->ignoreIntOverflowInRange)
  12589. {
  12590. instr->ignoreIntOverflowInRange = !intOverflowCurrentlyMattersInRange || OpCodeAttr::IsInt32(instr->m_opcode);
  12591. }
  12592. if(!intOverflowDoesNotMatterRange)
  12593. {
  12594. Assert(intOverflowCurrentlyMattersInRange);
  12595. // There are no more ranges of instructions where int overflow does not matter, in this block.
  12596. return isBoundary;
  12597. }
  12598. if(instr == intOverflowDoesNotMatterRange->LastInstr())
  12599. {
  12600. Assert(isBoundary);
  12601. // Reached the last instruction in the range
  12602. intOverflowCurrentlyMattersInRange = true;
  12603. intOverflowDoesNotMatterRange = intOverflowDoesNotMatterRange->Next();
  12604. return isBoundary;
  12605. }
  12606. if(!intOverflowCurrentlyMattersInRange)
  12607. {
  12608. return isBoundary;
  12609. }
  12610. if(instr != intOverflowDoesNotMatterRange->FirstInstr())
  12611. {
  12612. // Have not reached the next range
  12613. return isBoundary;
  12614. }
  12615. Assert(isBoundary);
  12616. // This is the first instruction in a range of instructions where int overflow does not matter. There can be many inputs to
  12617. // instructions in the range, some of which are inputs to the range itself (that is, the values are not defined in the
  12618. // range). Ignoring int overflow is only valid for int operations, so we need to ensure that all inputs to the range are
  12619. // int (not "likely int") before ignoring any overflows in the range. Ensuring that a sym with a "likely int" value is an
  12620. // int requires a bail-out. These bail-out check need to happen before any overflows are ignored, otherwise it's too late.
  12621. // The backward pass tracked all inputs into the range. Iterate over them and verify the values, and insert lossless
  12622. // conversions to int as necessary, before the first instruction in the range. If for any reason all values cannot be
  12623. // guaranteed to be ints, the optimization will be disabled for this range.
  12624. intOverflowCurrentlyMattersInRange = false;
  12625. {
  12626. BVSparse<JitArenaAllocator> tempBv1(tempAlloc);
  12627. BVSparse<JitArenaAllocator> tempBv2(tempAlloc);
  12628. {
  12629. // Just renaming the temp BVs for this section to indicate how they're used so that it makes sense
  12630. BVSparse<JitArenaAllocator> &symsToExclude = tempBv1;
  12631. BVSparse<JitArenaAllocator> &symsToInclude = tempBv2;
  12632. #if DBG_DUMP
  12633. SymID couldNotConvertSymId = 0;
  12634. #endif
  12635. FOREACH_BITSET_IN_SPARSEBV(id, intOverflowDoesNotMatterRange->SymsRequiredToBeInt())
  12636. {
  12637. Sym *const sym = func->m_symTable->Find(id);
  12638. Assert(sym);
  12639. // Some instructions with property syms are also tracked by the backward pass, and may be included in the range
  12640. // (LdSlot for instance). These property syms don't get their values until either copy-prop resolves a value for
  12641. // them, or a new value is created once the use of the property sym is reached. In either case, we're not that
  12642. // far yet, so we need to find the future value of the property sym by evaluating copy-prop in reverse.
  12643. Value *const value = sym->IsStackSym() ? CurrentBlockData()->FindValue(sym) : CurrentBlockData()->FindFuturePropertyValue(sym->AsPropertySym());
  12644. if(!value)
  12645. {
  12646. #if DBG_DUMP
  12647. couldNotConvertSymId = id;
  12648. #endif
  12649. intOverflowCurrentlyMattersInRange = true;
  12650. BREAK_BITSET_IN_SPARSEBV;
  12651. }
  12652. const bool isInt32OrUInt32Float =
  12653. value->GetValueInfo()->IsFloatConstant() &&
  12654. Js::JavascriptNumber::IsInt32OrUInt32(value->GetValueInfo()->AsFloatConstant()->FloatValue());
  12655. if(value->GetValueInfo()->IsInt() || isInt32OrUInt32Float)
  12656. {
  12657. if(!IsLoopPrePass())
  12658. {
  12659. // Input values that are already int can be excluded from int-specialization. We can treat unsigned
  12660. // int32 values as int32 values (ignoring the overflow), since the values will only be used inside the
  12661. // range where overflow does not matter.
  12662. symsToExclude.Set(sym->m_id);
  12663. }
  12664. continue;
  12665. }
  12666. if(!DoAggressiveIntTypeSpec() || !value->GetValueInfo()->IsLikelyInt())
  12667. {
  12668. // When aggressive int specialization is off, syms with "likely int" values cannot be forced to int since
  12669. // int bail-out checks are not allowed in that mode. Similarly, with aggressive int specialization on, it
  12670. // wouldn't make sense to force non-"likely int" values to int since it would almost guarantee a bail-out at
  12671. // runtime. In both cases, just disable ignoring overflow for this range.
  12672. #if DBG_DUMP
  12673. couldNotConvertSymId = id;
  12674. #endif
  12675. intOverflowCurrentlyMattersInRange = true;
  12676. BREAK_BITSET_IN_SPARSEBV;
  12677. }
  12678. if(IsLoopPrePass())
  12679. {
  12680. // The loop prepass does not modify bit-vectors. Since it doesn't add bail-out checks, it also does not need
  12681. // to specialize anything up-front. It only needs to be consistent in how it determines whether to allow
  12682. // ignoring overflow for a range, based on the values of inputs into the range.
  12683. continue;
  12684. }
  12685. // Since input syms are tracked in the backward pass, where there is no value tracking, it will not be aware of
  12686. // copy-prop. If a copy-prop sym is available, it will be used instead, so exclude the original sym and include
  12687. // the copy-prop sym for specialization.
  12688. StackSym *const copyPropSym = CurrentBlockData()->GetCopyPropSym(sym, value);
  12689. if(copyPropSym)
  12690. {
  12691. symsToExclude.Set(sym->m_id);
  12692. Assert(!symsToExclude.Test(copyPropSym->m_id));
  12693. const bool needsToBeLossless =
  12694. !intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Test(sym->m_id);
  12695. if(intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Test(copyPropSym->m_id) ||
  12696. symsToInclude.TestAndSet(copyPropSym->m_id))
  12697. {
  12698. // The copy-prop sym is already included
  12699. if(needsToBeLossless)
  12700. {
  12701. // The original sym needs to be lossless, so make the copy-prop sym lossless as well.
  12702. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Clear(copyPropSym->m_id);
  12703. }
  12704. }
  12705. else if(!needsToBeLossless)
  12706. {
  12707. // The copy-prop sym was not included before, and the original sym can be lossy, so make it lossy.
  12708. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Set(copyPropSym->m_id);
  12709. }
  12710. }
  12711. else if(!sym->IsStackSym())
  12712. {
  12713. // Only stack syms can be converted to int, and copy-prop syms are stack syms. If a copy-prop sym was not
  12714. // found for the property sym, we can't ignore overflows in this range.
  12715. #if DBG_DUMP
  12716. couldNotConvertSymId = id;
  12717. #endif
  12718. intOverflowCurrentlyMattersInRange = true;
  12719. BREAK_BITSET_IN_SPARSEBV;
  12720. }
  12721. } NEXT_BITSET_IN_SPARSEBV;
  12722. if(intOverflowCurrentlyMattersInRange)
  12723. {
  12724. #if DBG_DUMP
  12725. if(PHASE_TRACE(Js::TrackCompoundedIntOverflowPhase, func) && !IsLoopPrePass())
  12726. {
  12727. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  12728. Output::Print(
  12729. _u("TrackCompoundedIntOverflow - Top function: %s (%s), Phase: %s, Block: %u, Disabled ignoring overflows\n"),
  12730. func->GetJITFunctionBody()->GetDisplayName(),
  12731. func->GetDebugNumberSet(debugStringBuffer),
  12732. Js::PhaseNames[Js::ForwardPhase],
  12733. currentBlock->GetBlockNum());
  12734. Output::Print(_u(" Input sym could not be turned into an int: %u\n"), couldNotConvertSymId);
  12735. Output::Print(_u(" First instr: "));
  12736. instr->m_next->Dump();
  12737. Output::Flush();
  12738. }
  12739. #endif
  12740. intOverflowDoesNotMatterRange = intOverflowDoesNotMatterRange->Next();
  12741. return isBoundary;
  12742. }
  12743. if(IsLoopPrePass())
  12744. {
  12745. return isBoundary;
  12746. }
  12747. // Update the syms to specialize after enumeration
  12748. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(&symsToExclude);
  12749. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Minus(&symsToExclude);
  12750. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Or(&symsToInclude);
  12751. }
  12752. {
  12753. // Exclude syms that are already live as lossless int32, and exclude lossy conversions of syms that are already live
  12754. // as lossy int32.
  12755. // symsToExclude = liveInt32Syms - liveLossyInt32Syms // syms live as lossless int
  12756. // lossySymsToExclude = symsRequiredToBeLossyInt & liveLossyInt32Syms; // syms we want as lossy int that are already live as lossy int
  12757. // symsToExclude |= lossySymsToExclude
  12758. // symsRequiredToBeInt -= symsToExclude
  12759. // symsRequiredToBeLossyInt -= symsToExclude
  12760. BVSparse<JitArenaAllocator> &symsToExclude = tempBv1;
  12761. BVSparse<JitArenaAllocator> &lossySymsToExclude = tempBv2;
  12762. symsToExclude.Minus(CurrentBlockData()->liveInt32Syms, CurrentBlockData()->liveLossyInt32Syms);
  12763. lossySymsToExclude.And(
  12764. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt(),
  12765. CurrentBlockData()->liveLossyInt32Syms);
  12766. symsToExclude.Or(&lossySymsToExclude);
  12767. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(&symsToExclude);
  12768. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Minus(&symsToExclude);
  12769. }
  12770. #if DBG
  12771. {
  12772. // Verify that the syms to be converted are live
  12773. // liveSyms = liveInt32Syms | liveFloat64Syms | liveVarSyms
  12774. // deadSymsRequiredToBeInt = symsRequiredToBeInt - liveSyms
  12775. BVSparse<JitArenaAllocator> &liveSyms = tempBv1;
  12776. BVSparse<JitArenaAllocator> &deadSymsRequiredToBeInt = tempBv2;
  12777. liveSyms.Or(CurrentBlockData()->liveInt32Syms, CurrentBlockData()->liveFloat64Syms);
  12778. liveSyms.Or(CurrentBlockData()->liveVarSyms);
  12779. deadSymsRequiredToBeInt.Minus(intOverflowDoesNotMatterRange->SymsRequiredToBeInt(), &liveSyms);
  12780. Assert(deadSymsRequiredToBeInt.IsEmpty());
  12781. }
  12782. #endif
  12783. }
  12784. // Int-specialize the syms before the first instruction of the range (the current instruction)
  12785. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt());
  12786. #if DBG_DUMP
  12787. if(PHASE_TRACE(Js::TrackCompoundedIntOverflowPhase, func))
  12788. {
  12789. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  12790. Output::Print(
  12791. _u("TrackCompoundedIntOverflow - Top function: %s (%s), Phase: %s, Block: %u\n"),
  12792. func->GetJITFunctionBody()->GetDisplayName(),
  12793. func->GetDebugNumberSet(debugStringBuffer),
  12794. Js::PhaseNames[Js::ForwardPhase],
  12795. currentBlock->GetBlockNum());
  12796. Output::Print(_u(" Input syms to be int-specialized (lossless): "));
  12797. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Dump();
  12798. Output::Print(_u(" Input syms to be converted to int (lossy): "));
  12799. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Dump();
  12800. Output::Print(_u(" First instr: "));
  12801. instr->m_next->Dump();
  12802. Output::Flush();
  12803. }
  12804. #endif
  12805. ToInt32(intOverflowDoesNotMatterRange->SymsRequiredToBeInt(), currentBlock, false /* lossy */, instr);
  12806. ToInt32(intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt(), currentBlock, true /* lossy */, instr);
  12807. return isBoundary;
  12808. }
  12809. void
  12810. GlobOpt::VerifyIntSpecForIgnoringIntOverflow(IR::Instr *const instr)
  12811. {
  12812. if(intOverflowCurrentlyMattersInRange || IsLoopPrePass())
  12813. {
  12814. return;
  12815. }
  12816. Assert(instr->m_opcode != Js::OpCode::Mul_I4 ||
  12817. (instr->m_opcode == Js::OpCode::Mul_I4 && !instr->ShouldCheckFor32BitOverflow() && instr->ShouldCheckForNon32BitOverflow() ));
  12818. // Instructions that are marked as "overflow doesn't matter" in the range must guarantee that they operate on int values and
  12819. // result in int values, for ignoring overflow to be valid. So, int-specialization is required for such instructions in the
  12820. // range. Ld_A is an exception because it only specializes if the src sym is available as a required specialized sym, and it
  12821. // doesn't generate bailouts or cause ignoring int overflow to be invalid.
  12822. // MULs are allowed to start a region and have BailOutInfo since they will bailout on non-32 bit overflow.
  12823. if(instr->m_opcode == Js::OpCode::Ld_A ||
  12824. ((!instr->HasBailOutInfo() || instr->m_opcode == Js::OpCode::Mul_I4) &&
  12825. (!instr->GetDst() || instr->GetDst()->IsInt32()) &&
  12826. (!instr->GetSrc1() || instr->GetSrc1()->IsInt32()) &&
  12827. (!instr->GetSrc2() || instr->GetSrc2()->IsInt32())))
  12828. {
  12829. return;
  12830. }
  12831. if (!instr->HasBailOutInfo() && !instr->HasAnySideEffects())
  12832. {
  12833. return;
  12834. }
  12835. // This can happen for Neg_A if it needs to bail out on negative zero, and perhaps other cases as well. It's too late to fix
  12836. // the problem (overflows may already be ignored), so handle it by bailing out at compile-time and disabling tracking int
  12837. // overflow.
  12838. Assert(!func->IsTrackCompoundedIntOverflowDisabled());
  12839. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  12840. {
  12841. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  12842. Output::Print(
  12843. _u("BailOut (compile-time): function: %s (%s) instr: "),
  12844. func->GetJITFunctionBody()->GetDisplayName(),
  12845. func->GetDebugNumberSet(debugStringBuffer));
  12846. #if DBG_DUMP
  12847. instr->Dump();
  12848. #else
  12849. Output::Print(_u("%s "), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  12850. #endif
  12851. Output::Print(_u("(overflow does not matter but could not int-spec or needed bailout)\n"));
  12852. Output::Flush();
  12853. }
  12854. if(func->IsTrackCompoundedIntOverflowDisabled())
  12855. {
  12856. // Tracking int overflows is already off for some reason. Prevent trying to rejit again because it won't help and the
  12857. // same thing will happen again and cause an infinite loop. Just abort jitting this function.
  12858. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  12859. {
  12860. Output::Print(_u(" Aborting JIT because TrackIntOverflow is already off\n"));
  12861. Output::Flush();
  12862. }
  12863. throw Js::OperationAbortedException();
  12864. }
  12865. throw Js::RejitException(RejitReason::TrackIntOverflowDisabled);
  12866. }
  12867. // It makes lowering easier if it can assume that the first src is never a constant,
  12868. // at least for commutative operators. For non-commutative, just hoist the constant.
  12869. void
  12870. GlobOpt::PreLowerCanonicalize(IR::Instr *instr, Value **pSrc1Val, Value **pSrc2Val)
  12871. {
  12872. IR::Opnd *dst = instr->GetDst();
  12873. IR::Opnd *src1 = instr->GetSrc1();
  12874. IR::Opnd *src2 = instr->GetSrc2();
  12875. if (src1->IsImmediateOpnd())
  12876. {
  12877. // Swap for dst, src
  12878. }
  12879. else if (src2 && dst && src2->IsRegOpnd())
  12880. {
  12881. if (src2->GetIsDead() && !src1->GetIsDead() && !src1->IsEqual(dst))
  12882. {
  12883. // Swap if src2 is dead, as the reg can be reuse for the dst for opEqs like on x86 (ADD r1, r2)
  12884. }
  12885. else if (src2->IsEqual(dst))
  12886. {
  12887. // Helps lowering of opEqs
  12888. }
  12889. else
  12890. {
  12891. return;
  12892. }
  12893. // Make sure we don't swap 2 srcs with valueOf calls.
  12894. if (OpCodeAttr::OpndHasImplicitCall(instr->m_opcode))
  12895. {
  12896. if (instr->IsBranchInstr())
  12897. {
  12898. if (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive())
  12899. {
  12900. return;
  12901. }
  12902. }
  12903. else if (!src1->GetValueType().IsPrimitive() && !src2->GetValueType().IsPrimitive())
  12904. {
  12905. return;
  12906. }
  12907. }
  12908. }
  12909. else
  12910. {
  12911. return;
  12912. }
  12913. Js::OpCode opcode = instr->m_opcode;
  12914. switch (opcode)
  12915. {
  12916. case Js::OpCode::And_A:
  12917. case Js::OpCode::Mul_A:
  12918. case Js::OpCode::Or_A:
  12919. case Js::OpCode::Xor_A:
  12920. case Js::OpCode::And_I4:
  12921. case Js::OpCode::Mul_I4:
  12922. case Js::OpCode::Or_I4:
  12923. case Js::OpCode::Xor_I4:
  12924. case Js::OpCode::Add_I4:
  12925. swap_srcs:
  12926. if (!instr->GetSrc2()->IsImmediateOpnd())
  12927. {
  12928. instr->m_opcode = opcode;
  12929. instr->SwapOpnds();
  12930. Value *tempVal = *pSrc1Val;
  12931. *pSrc1Val = *pSrc2Val;
  12932. *pSrc2Val = tempVal;
  12933. return;
  12934. }
  12935. break;
  12936. case Js::OpCode::BrSrEq_A:
  12937. case Js::OpCode::BrSrNotNeq_A:
  12938. case Js::OpCode::BrEq_I4:
  12939. goto swap_srcs;
  12940. case Js::OpCode::BrSrNeq_A:
  12941. case Js::OpCode::BrNeq_A:
  12942. case Js::OpCode::BrSrNotEq_A:
  12943. case Js::OpCode::BrNotEq_A:
  12944. case Js::OpCode::BrNeq_I4:
  12945. goto swap_srcs;
  12946. case Js::OpCode::BrGe_A:
  12947. opcode = Js::OpCode::BrLe_A;
  12948. goto swap_srcs;
  12949. case Js::OpCode::BrNotGe_A:
  12950. opcode = Js::OpCode::BrNotLe_A;
  12951. goto swap_srcs;
  12952. case Js::OpCode::BrGe_I4:
  12953. opcode = Js::OpCode::BrLe_I4;
  12954. goto swap_srcs;
  12955. case Js::OpCode::BrGt_A:
  12956. opcode = Js::OpCode::BrLt_A;
  12957. goto swap_srcs;
  12958. case Js::OpCode::BrNotGt_A:
  12959. opcode = Js::OpCode::BrNotLt_A;
  12960. goto swap_srcs;
  12961. case Js::OpCode::BrGt_I4:
  12962. opcode = Js::OpCode::BrLt_I4;
  12963. goto swap_srcs;
  12964. case Js::OpCode::BrLe_A:
  12965. opcode = Js::OpCode::BrGe_A;
  12966. goto swap_srcs;
  12967. case Js::OpCode::BrNotLe_A:
  12968. opcode = Js::OpCode::BrNotGe_A;
  12969. goto swap_srcs;
  12970. case Js::OpCode::BrLe_I4:
  12971. opcode = Js::OpCode::BrGe_I4;
  12972. goto swap_srcs;
  12973. case Js::OpCode::BrLt_A:
  12974. opcode = Js::OpCode::BrGt_A;
  12975. goto swap_srcs;
  12976. case Js::OpCode::BrNotLt_A:
  12977. opcode = Js::OpCode::BrNotGt_A;
  12978. goto swap_srcs;
  12979. case Js::OpCode::BrLt_I4:
  12980. opcode = Js::OpCode::BrGt_I4;
  12981. goto swap_srcs;
  12982. case Js::OpCode::BrEq_A:
  12983. case Js::OpCode::BrNotNeq_A:
  12984. case Js::OpCode::CmEq_A:
  12985. case Js::OpCode::CmNeq_A:
  12986. // this == "" not the same as "" == this...
  12987. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12988. {
  12989. return;
  12990. }
  12991. goto swap_srcs;
  12992. case Js::OpCode::CmGe_A:
  12993. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12994. {
  12995. return;
  12996. }
  12997. opcode = Js::OpCode::CmLe_A;
  12998. goto swap_srcs;
  12999. case Js::OpCode::CmGt_A:
  13000. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  13001. {
  13002. return;
  13003. }
  13004. opcode = Js::OpCode::CmLt_A;
  13005. goto swap_srcs;
  13006. case Js::OpCode::CmLe_A:
  13007. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  13008. {
  13009. return;
  13010. }
  13011. opcode = Js::OpCode::CmGe_A;
  13012. goto swap_srcs;
  13013. case Js::OpCode::CmLt_A:
  13014. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  13015. {
  13016. return;
  13017. }
  13018. opcode = Js::OpCode::CmGt_A;
  13019. goto swap_srcs;
  13020. case Js::OpCode::CallI:
  13021. case Js::OpCode::CallIFixed:
  13022. case Js::OpCode::NewScObject:
  13023. case Js::OpCode::NewScObjectSpread:
  13024. case Js::OpCode::NewScObjArray:
  13025. case Js::OpCode::NewScObjArraySpread:
  13026. case Js::OpCode::NewScObjectNoCtor:
  13027. // Don't insert load to register if the function operand is a fixed function.
  13028. if (instr->HasFixedFunctionAddressTarget())
  13029. {
  13030. return;
  13031. }
  13032. break;
  13033. // Can't do add because <32 + "Hello"> isn't equal to <"Hello" + 32>
  13034. // Lower can do the swap. Other op-codes listed below don't need immediate source hoisting, as the fast paths handle it,
  13035. // or the lowering handles the hoisting.
  13036. case Js::OpCode::Add_A:
  13037. if (src1->IsFloat())
  13038. {
  13039. goto swap_srcs;
  13040. }
  13041. return;
  13042. case Js::OpCode::Sub_I4:
  13043. case Js::OpCode::Neg_I4:
  13044. case Js::OpCode::Not_I4:
  13045. case Js::OpCode::NewScFunc:
  13046. case Js::OpCode::NewScGenFunc:
  13047. case Js::OpCode::NewScFuncHomeObj:
  13048. case Js::OpCode::NewScGenFuncHomeObj:
  13049. case Js::OpCode::NewScArray:
  13050. case Js::OpCode::NewScIntArray:
  13051. case Js::OpCode::NewScFltArray:
  13052. case Js::OpCode::NewScArrayWithMissingValues:
  13053. case Js::OpCode::NewRegEx:
  13054. case Js::OpCode::Ld_A:
  13055. case Js::OpCode::Ld_I4:
  13056. case Js::OpCode::ThrowRuntimeError:
  13057. case Js::OpCode::TrapIfMinIntOverNegOne:
  13058. case Js::OpCode::TrapIfTruncOverflow:
  13059. case Js::OpCode::TrapIfZero:
  13060. case Js::OpCode::TrapIfUnalignedAccess:
  13061. case Js::OpCode::FromVar:
  13062. case Js::OpCode::Conv_Prim:
  13063. case Js::OpCode::Conv_Prim_Sat:
  13064. case Js::OpCode::LdC_A_I4:
  13065. case Js::OpCode::LdStr:
  13066. case Js::OpCode::InitFld:
  13067. case Js::OpCode::InitRootFld:
  13068. case Js::OpCode::StartCall:
  13069. case Js::OpCode::ArgOut_A:
  13070. case Js::OpCode::ArgOut_A_Inline:
  13071. case Js::OpCode::ArgOut_A_Dynamic:
  13072. case Js::OpCode::ArgOut_A_FromStackArgs:
  13073. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  13074. case Js::OpCode::ArgOut_A_InlineSpecialized:
  13075. case Js::OpCode::ArgOut_A_SpreadArg:
  13076. case Js::OpCode::InlineeEnd:
  13077. case Js::OpCode::EndCallForPolymorphicInlinee:
  13078. case Js::OpCode::InlineeMetaArg:
  13079. case Js::OpCode::InlineBuiltInEnd:
  13080. case Js::OpCode::InlineNonTrackingBuiltInEnd:
  13081. case Js::OpCode::CallHelper:
  13082. case Js::OpCode::LdElemUndef:
  13083. case Js::OpCode::LdElemUndefScoped:
  13084. case Js::OpCode::RuntimeTypeError:
  13085. case Js::OpCode::RuntimeReferenceError:
  13086. case Js::OpCode::Ret:
  13087. case Js::OpCode::NewScObjectSimple:
  13088. case Js::OpCode::NewScObjectLiteral:
  13089. case Js::OpCode::StFld:
  13090. case Js::OpCode::StRootFld:
  13091. case Js::OpCode::StSlot:
  13092. case Js::OpCode::StSlotChkUndecl:
  13093. case Js::OpCode::StElemC:
  13094. case Js::OpCode::StArrSegElemC:
  13095. case Js::OpCode::StElemI_A:
  13096. case Js::OpCode::StElemI_A_Strict:
  13097. case Js::OpCode::CallDirect:
  13098. case Js::OpCode::BrNotHasSideEffects:
  13099. case Js::OpCode::NewConcatStrMulti:
  13100. case Js::OpCode::NewConcatStrMultiBE:
  13101. case Js::OpCode::ExtendArg_A:
  13102. case Js::OpCode::NewScopeSlots:
  13103. case Js::OpCode::NewScopeSlotsWithoutPropIds:
  13104. case Js::OpCode::NewStackScopeSlots:
  13105. case Js::OpCode::IsInst:
  13106. case Js::OpCode::BailOnEqual:
  13107. case Js::OpCode::BailOnNotEqual:
  13108. case Js::OpCode::StArrViewElem:
  13109. return;
  13110. }
  13111. if (!src1->IsImmediateOpnd())
  13112. {
  13113. return;
  13114. }
  13115. // The fast paths or lowering of the remaining instructions may not support handling immediate opnds for the first src. The
  13116. // immediate src1 is hoisted here into a separate instruction.
  13117. if (src1->IsIntConstOpnd())
  13118. {
  13119. IR::Instr *newInstr = instr->HoistSrc1(Js::OpCode::Ld_I4);
  13120. ToInt32Dst(newInstr, newInstr->GetDst()->AsRegOpnd(), this->currentBlock);
  13121. }
  13122. else if (src1->IsInt64ConstOpnd())
  13123. {
  13124. instr->HoistSrc1(Js::OpCode::Ld_I4);
  13125. }
  13126. else
  13127. {
  13128. instr->HoistSrc1(Js::OpCode::Ld_A);
  13129. }
  13130. src1 = instr->GetSrc1();
  13131. src1->AsRegOpnd()->m_sym->SetIsConst();
  13132. }
  13133. // Clear the ValueMap pf the values invalidated by this instr.
  13134. void
  13135. GlobOpt::ProcessKills(IR::Instr *instr)
  13136. {
  13137. if (instr->m_opcode == Js::OpCode::Yield)
  13138. {
  13139. this->CurrentBlockData()->KillStateForGeneratorYield(instr);
  13140. }
  13141. this->ProcessFieldKills(instr);
  13142. this->ProcessValueKills(instr);
  13143. this->ProcessArrayValueKills(instr);
  13144. }
  13145. bool
  13146. GlobOpt::OptIsInvariant(IR::Opnd *src, BasicBlock *block, Loop *loop, Value *srcVal, bool isNotTypeSpecConv, bool allowNonPrimitives)
  13147. {
  13148. if(!loop->CanHoistInvariants())
  13149. {
  13150. return false;
  13151. }
  13152. Sym *sym;
  13153. switch(src->GetKind())
  13154. {
  13155. case IR::OpndKindAddr:
  13156. case IR::OpndKindFloatConst:
  13157. case IR::OpndKindIntConst:
  13158. return true;
  13159. case IR::OpndKindReg:
  13160. sym = src->AsRegOpnd()->m_sym;
  13161. break;
  13162. case IR::OpndKindSym:
  13163. sym = src->AsSymOpnd()->m_sym;
  13164. if (src->AsSymOpnd()->IsPropertySymOpnd())
  13165. {
  13166. if (src->AsSymOpnd()->AsPropertySymOpnd()->IsTypeChecked())
  13167. {
  13168. // We do not handle hoisting these yet. We might be hoisting this across the instr with the type check protecting this one.
  13169. // And somehow, the dead-store pass now removes the type check on that instr later on...
  13170. // For CheckFixedFld, there is no benefit hoisting these if they don't have a type check as they won't generate code.
  13171. return false;
  13172. }
  13173. }
  13174. break;
  13175. case IR::OpndKindHelperCall:
  13176. // Helper calls, like the private slot getter, can be invariant.
  13177. // Consider moving more math builtin to invariant?
  13178. return HelperMethodAttributes::IsInVariant(src->AsHelperCallOpnd()->m_fnHelper);
  13179. default:
  13180. return false;
  13181. }
  13182. return OptIsInvariant(sym, block, loop, srcVal, isNotTypeSpecConv, allowNonPrimitives);
  13183. }
  13184. bool
  13185. GlobOpt::OptIsInvariant(Sym *sym, BasicBlock *block, Loop *loop, Value *srcVal, bool isNotTypeSpecConv, bool allowNonPrimitives, Value **loopHeadValRef)
  13186. {
  13187. Value *localLoopHeadVal;
  13188. if(!loopHeadValRef)
  13189. {
  13190. loopHeadValRef = &localLoopHeadVal;
  13191. }
  13192. Value *&loopHeadVal = *loopHeadValRef;
  13193. loopHeadVal = nullptr;
  13194. if(!loop->CanHoistInvariants())
  13195. {
  13196. return false;
  13197. }
  13198. if (sym->IsStackSym())
  13199. {
  13200. if (sym->AsStackSym()->IsTypeSpec())
  13201. {
  13202. StackSym *varSym = sym->AsStackSym()->GetVarEquivSym(this->func);
  13203. // Make sure the int32/float64 version of this is available.
  13204. // Note: We could handle this by converting the src, but usually the
  13205. // conversion is hoistable if this is hoistable anyway.
  13206. // In some weird cases it may not be however, so we'll bail out.
  13207. if (sym->AsStackSym()->IsInt32())
  13208. {
  13209. Assert(block->globOptData.liveInt32Syms->Test(varSym->m_id));
  13210. if (!loop->landingPad->globOptData.liveInt32Syms->Test(varSym->m_id) ||
  13211. (loop->landingPad->globOptData.liveLossyInt32Syms->Test(varSym->m_id) &&
  13212. !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id)))
  13213. {
  13214. // Either the int32 sym is not live in the landing pad, or it's lossy in the landing pad and the
  13215. // instruction's block is using the lossless version. In either case, the instruction cannot be hoisted
  13216. // without doing a conversion of this operand.
  13217. return false;
  13218. }
  13219. }
  13220. else if (sym->AsStackSym()->IsFloat64())
  13221. {
  13222. if (!loop->landingPad->globOptData.liveFloat64Syms->Test(varSym->m_id))
  13223. {
  13224. return false;
  13225. }
  13226. }
  13227. sym = sym->AsStackSym()->GetVarEquivSym(this->func);
  13228. }
  13229. else
  13230. {
  13231. // Make sure the var version of this is available.
  13232. // Note: We could handle this by converting the src, but usually the
  13233. // conversion is hoistable if this is hoistable anyway.
  13234. // In some weird cases it may not be however, so we'll bail out.
  13235. if (!loop->landingPad->globOptData.liveVarSyms->Test(sym->m_id))
  13236. {
  13237. return false;
  13238. }
  13239. }
  13240. }
  13241. else if (sym->IsPropertySym())
  13242. {
  13243. if (!loop->landingPad->globOptData.liveVarSyms->Test(sym->AsPropertySym()->m_stackSym->m_id))
  13244. {
  13245. return false;
  13246. }
  13247. }
  13248. else
  13249. {
  13250. return false;
  13251. }
  13252. // We rely on having a value.
  13253. if (srcVal == NULL)
  13254. {
  13255. return false;
  13256. }
  13257. // A symbol is invariant if its current value is the same as it was upon entering the loop.
  13258. loopHeadVal = loop->landingPad->globOptData.FindValue(sym);
  13259. if (loopHeadVal == NULL || loopHeadVal->GetValueNumber() != srcVal->GetValueNumber())
  13260. {
  13261. return false;
  13262. }
  13263. // Can't hoist non-primitives, unless we have safeguards against valueof/tostring. Additionally, we need to consider
  13264. // the value annotations on the source *before* the loop: if we hoist this instruction outside the loop, we can't
  13265. // necessarily rely on type annotations added (and enforced) earlier in the loop's body.
  13266. //
  13267. // It might look as though !loopHeadVal->GetValueInfo()->IsPrimitive() implies
  13268. // !loop->landingPad->globOptData.IsTypeSpecialized(sym), but it turns out that this is not always the case. We
  13269. // encountered a test case in which we had previously hoisted a FromVar (to float 64) instruction, but its bailout code was
  13270. // BailoutPrimitiveButString, rather than BailoutNumberOnly, which would have allowed us to conclude that the dest was
  13271. // definitely a float64. Instead, it was only *likely* a float64, causing IsPrimitive to return false.
  13272. if (!allowNonPrimitives && !loopHeadVal->GetValueInfo()->IsPrimitive() && !loop->landingPad->globOptData.IsTypeSpecialized(sym))
  13273. {
  13274. return false;
  13275. }
  13276. if(!isNotTypeSpecConv && loop->symsDefInLoop->Test(sym->m_id))
  13277. {
  13278. // Typically, a sym is considered invariant if it has the same value in the current block and in the loop landing pad.
  13279. // The sym may have had a different value earlier in the loop or on the back-edge, but as long as it's reassigned to its
  13280. // value outside the loop, it would be considered invariant in this block. Consider that case:
  13281. // s1 = s2[invariant]
  13282. // <loop start>
  13283. // s1 = s2[invariant]
  13284. // // s1 now has the same value as in the landing pad, and is considered invariant
  13285. // s1 += s3
  13286. // // s1 is not invariant here, or on the back-edge
  13287. // ++s3 // s3 is not invariant, so the add above cannot be hoisted
  13288. // <loop end>
  13289. //
  13290. // A problem occurs at the point of (s1 += s3) when:
  13291. // - At (s1 = s2) inside the loop, s1 was made to be the sym store of that value. This by itself is legal, because
  13292. // after that transfer, s1 and s2 have the same value.
  13293. // - (s1 += s3) is type-specialized but s1 is not specialized in the loop header. This happens when s1 is not
  13294. // specialized entering the loop, and since s1 is not used before it's defined in the loop, it's not specialized
  13295. // on back-edges.
  13296. //
  13297. // With that, at (s1 += s3), the conversion of s1 to the type-specialized version would be hoisted because s1 is
  13298. // invariant just before that instruction. Since this add is specialized, the specialized version of the sym is modified
  13299. // in the loop without a reassignment at (s1 = s2) inside the loop, and (s1 += s3) would then use an incorrect value of
  13300. // s1 (it would use the value of s1 from the previous loop iteration, instead of using the value of s2).
  13301. //
  13302. // The problem here, is that we cannot hoist the conversion of s1 into its specialized version across the assignment
  13303. // (s1 = s2) inside the loop. So for the purposes of type specialization, don't consider a sym invariant if it has a def
  13304. // inside the loop.
  13305. return false;
  13306. }
  13307. // For values with an int range, require additionally that the range is the same as in the landing pad, as the range may
  13308. // have been changed on this path based on branches, and int specialization and invariant hoisting may rely on the range
  13309. // being the same. For type spec conversions, only require that if the value is an int constant in the current block, that
  13310. // it is also an int constant with the same value in the landing pad. Other range differences don't matter for type spec.
  13311. IntConstantBounds srcIntConstantBounds, loopHeadIntConstantBounds;
  13312. if(srcVal->GetValueInfo()->TryGetIntConstantBounds(&srcIntConstantBounds) &&
  13313. (isNotTypeSpecConv || srcIntConstantBounds.IsConstant()) &&
  13314. (
  13315. !loopHeadVal->GetValueInfo()->TryGetIntConstantBounds(&loopHeadIntConstantBounds) ||
  13316. loopHeadIntConstantBounds.LowerBound() != srcIntConstantBounds.LowerBound() ||
  13317. loopHeadIntConstantBounds.UpperBound() != srcIntConstantBounds.UpperBound()
  13318. ))
  13319. {
  13320. return false;
  13321. }
  13322. // Disabling this assert, because it does not hold true when we force specialize in the loop landing pad
  13323. //Assert((!loopHeadVal->GetValueInfo()->IsPrimitive()) || srcVal->GetValueInfo()->IsLikelyPrimitive());
  13324. return true;
  13325. }
  13326. bool
  13327. GlobOpt::OptIsInvariant(
  13328. IR::Instr *instr,
  13329. BasicBlock *block,
  13330. Loop *loop,
  13331. Value *src1Val,
  13332. Value *src2Val,
  13333. bool isNotTypeSpecConv,
  13334. const bool forceInvariantHoisting)
  13335. {
  13336. if (!loop->CanHoistInvariants())
  13337. {
  13338. return false;
  13339. }
  13340. if (!OpCodeAttr::CanCSE(instr->m_opcode))
  13341. {
  13342. return false;
  13343. }
  13344. bool allowNonPrimitives = !OpCodeAttr::OpndHasImplicitCall(instr->m_opcode);
  13345. switch(instr->m_opcode)
  13346. {
  13347. // Can't legally hoist these
  13348. case Js::OpCode::LdLen_A:
  13349. return false;
  13350. //Can't Hoist BailOnNotStackArgs, as it is necessary as InlineArgsOptimization relies on this opcode
  13351. //to decide whether to throw rejit exception or not.
  13352. case Js::OpCode::BailOnNotStackArgs:
  13353. return false;
  13354. // Usually not worth hoisting these
  13355. case Js::OpCode::Ld_A:
  13356. case Js::OpCode::Ld_I4:
  13357. case Js::OpCode::LdC_A_I4:
  13358. if(!forceInvariantHoisting)
  13359. {
  13360. return false;
  13361. }
  13362. break;
  13363. // Can't hoist these outside the function it's for. The LdArgumentsFromFrame for an inlinee depends on the inlinee meta arg
  13364. // that holds the arguments object, which is only initialized at the start of the inlinee. So, can't hoist this outside the
  13365. // inlinee.
  13366. case Js::OpCode::LdArgumentsFromFrame:
  13367. if(instr->m_func != loop->GetFunc())
  13368. {
  13369. return false;
  13370. }
  13371. break;
  13372. case Js::OpCode::FromVar:
  13373. if (instr->HasBailOutInfo())
  13374. {
  13375. allowNonPrimitives = true;
  13376. }
  13377. break;
  13378. case Js::OpCode::CheckObjType:
  13379. // Bug 11712101: If the operand is a field, ensure that its containing object type is invariant
  13380. // before hoisting -- that is, don't hoist a CheckObjType over a DeleteFld on that object.
  13381. // (CheckObjType only checks the operand and its immediate parent, so we don't need to go
  13382. // any farther up the object graph.)
  13383. Assert(instr->GetSrc1());
  13384. PropertySym *propertySym = instr->GetSrc1()->AsPropertySymOpnd()->GetPropertySym();
  13385. if (propertySym->HasObjectTypeSym()) {
  13386. StackSym *objectTypeSym = propertySym->GetObjectTypeSym();
  13387. if (!this->OptIsInvariant(objectTypeSym, block, loop, this->CurrentBlockData()->FindValue(objectTypeSym), true, true)) {
  13388. return false;
  13389. }
  13390. }
  13391. break;
  13392. }
  13393. IR::Opnd *dst = instr->GetDst();
  13394. if (dst && !dst->IsRegOpnd())
  13395. {
  13396. return false;
  13397. }
  13398. IR::Opnd *src1 = instr->GetSrc1();
  13399. if (src1)
  13400. {
  13401. if (!this->OptIsInvariant(src1, block, loop, src1Val, isNotTypeSpecConv, allowNonPrimitives))
  13402. {
  13403. return false;
  13404. }
  13405. IR::Opnd *src2 = instr->GetSrc2();
  13406. if (src2)
  13407. {
  13408. if (!this->OptIsInvariant(src2, block, loop, src2Val, isNotTypeSpecConv, allowNonPrimitives))
  13409. {
  13410. return false;
  13411. }
  13412. }
  13413. }
  13414. return true;
  13415. }
  13416. bool
  13417. GlobOpt::OptDstIsInvariant(IR::RegOpnd *dst)
  13418. {
  13419. StackSym *dstSym = dst->m_sym;
  13420. if (dstSym->IsTypeSpec())
  13421. {
  13422. // The type-specialized sym may be single def, but not the original...
  13423. dstSym = dstSym->GetVarEquivSym(this->func);
  13424. }
  13425. return (dstSym->m_isSingleDef);
  13426. }
  13427. void
  13428. GlobOpt::OptHoistUpdateValueType(
  13429. Loop* loop,
  13430. IR::Instr* instr,
  13431. IR::Opnd** srcOpndPtr /* All code paths that change src, should update srcOpndPtr*/,
  13432. Value* opndVal)
  13433. {
  13434. if (opndVal == nullptr || instr->m_opcode == Js::OpCode::FromVar || srcOpndPtr == nullptr || *srcOpndPtr == nullptr)
  13435. {
  13436. return;
  13437. }
  13438. IR::Opnd* srcOpnd = *srcOpndPtr;
  13439. Sym* opndSym = srcOpnd->GetSym();;
  13440. if (opndSym)
  13441. {
  13442. BasicBlock* landingPad = loop->landingPad;
  13443. Value* opndValueInLandingPad = landingPad->globOptData.FindValue(opndSym);
  13444. Assert(opndVal->GetValueNumber() == opndValueInLandingPad->GetValueNumber());
  13445. ValueType opndValueTypeInLandingPad = opndValueInLandingPad->GetValueInfo()->Type();
  13446. if (srcOpnd->GetValueType() != opndValueTypeInLandingPad)
  13447. {
  13448. srcOpnd->SetValueType(opndValueTypeInLandingPad);
  13449. if (instr->m_opcode == Js::OpCode::SetConcatStrMultiItemBE)
  13450. {
  13451. Assert(!opndSym->IsPropertySym());
  13452. Assert(!opndValueTypeInLandingPad.IsString());
  13453. Assert(instr->GetDst());
  13454. IR::RegOpnd* strOpnd = IR::RegOpnd::New(TyVar, instr->m_func);
  13455. strOpnd->SetValueType(ValueType::String);
  13456. strOpnd->SetValueTypeFixed();
  13457. IR::Instr* convPrimStrInstr =
  13458. IR::Instr::New(Js::OpCode::Conv_PrimStr, strOpnd, srcOpnd->Use(instr->m_func), instr->m_func);
  13459. instr->ReplaceSrc(srcOpnd, strOpnd);
  13460. // Replace above will free srcOpnd, so reassign it
  13461. *srcOpndPtr = srcOpnd = reinterpret_cast<IR::Opnd *>(strOpnd);
  13462. // We add ConvPrim_Str in the landingpad, and since this instruction doesn't go through the checks in OptInstr, the bailout is never added
  13463. // As we expand hoisting of instructions to new opcode, we need a better framework to handle such cases
  13464. if (IsImplicitCallBailOutCurrentlyNeeded(convPrimStrInstr, opndValueInLandingPad, nullptr, landingPad, landingPad->globOptData.liveFields->IsEmpty(), true, true))
  13465. {
  13466. EnsureBailTarget(loop);
  13467. loop->bailOutInfo->bailOutInstr->InsertBefore(convPrimStrInstr);
  13468. convPrimStrInstr = convPrimStrInstr->ConvertToBailOutInstr(convPrimStrInstr, IR::BailOutOnImplicitCallsPreOp, loop->bailOutInfo->bailOutOffset);
  13469. convPrimStrInstr->ReplaceBailOutInfo(loop->bailOutInfo);
  13470. }
  13471. else
  13472. {
  13473. if (loop->bailOutInfo->bailOutInstr)
  13474. {
  13475. loop->bailOutInfo->bailOutInstr->InsertBefore(convPrimStrInstr);
  13476. }
  13477. else
  13478. {
  13479. landingPad->InsertAfter(convPrimStrInstr);
  13480. }
  13481. }
  13482. // If we came here opndSym can't be PropertySym
  13483. return;
  13484. }
  13485. }
  13486. if (opndSym->IsPropertySym())
  13487. {
  13488. // Also fix valueInfo on objPtr
  13489. StackSym* opndObjPtrSym = opndSym->AsPropertySym()->m_stackSym;
  13490. Value* opndObjPtrSymValInLandingPad = landingPad->globOptData.FindValue(opndObjPtrSym);
  13491. ValueInfo* opndObjPtrSymValueInfoInLandingPad = opndObjPtrSymValInLandingPad->GetValueInfo();
  13492. srcOpnd->AsSymOpnd()->SetPropertyOwnerValueType(opndObjPtrSymValueInfoInLandingPad->Type());
  13493. }
  13494. }
  13495. }
  13496. void
  13497. GlobOpt::OptHoistInvariant(
  13498. IR::Instr *instr,
  13499. BasicBlock *block,
  13500. Loop *loop,
  13501. Value *dstVal,
  13502. Value *const src1Val,
  13503. Value *const src2Val,
  13504. bool isNotTypeSpecConv,
  13505. bool lossy,
  13506. IR::BailOutKind bailoutKind)
  13507. {
  13508. BasicBlock *landingPad = loop->landingPad;
  13509. IR::Opnd* src1 = instr->GetSrc1();
  13510. if (src1)
  13511. {
  13512. // We are hoisting this instruction possibly past other uses, which might invalidate the last use info. Clear it.
  13513. OptHoistUpdateValueType(loop, instr, &src1, src1Val);
  13514. if (src1->IsRegOpnd())
  13515. {
  13516. src1->AsRegOpnd()->m_isTempLastUse = false;
  13517. }
  13518. IR::Opnd* src2 = instr->GetSrc2();
  13519. if (src2)
  13520. {
  13521. OptHoistUpdateValueType(loop, instr, &src2, src2Val);
  13522. if (src2->IsRegOpnd())
  13523. {
  13524. src2->AsRegOpnd()->m_isTempLastUse = false;
  13525. }
  13526. }
  13527. }
  13528. IR::RegOpnd *dst = instr->GetDst() ? instr->GetDst()->AsRegOpnd() : nullptr;
  13529. if(dst)
  13530. {
  13531. switch (instr->m_opcode)
  13532. {
  13533. case Js::OpCode::CmEq_I4:
  13534. case Js::OpCode::CmNeq_I4:
  13535. case Js::OpCode::CmLt_I4:
  13536. case Js::OpCode::CmLe_I4:
  13537. case Js::OpCode::CmGt_I4:
  13538. case Js::OpCode::CmGe_I4:
  13539. case Js::OpCode::CmUnLt_I4:
  13540. case Js::OpCode::CmUnLe_I4:
  13541. case Js::OpCode::CmUnGt_I4:
  13542. case Js::OpCode::CmUnGe_I4:
  13543. // These operations are a special case. They generate a lossy int value, and the var sym is initialized using
  13544. // Conv_Bool. A sym cannot be live only as a lossy int sym, the var needs to be live as well since the lossy int
  13545. // sym cannot be used to convert to var. We don't know however, whether the Conv_Bool will be hoisted. The idea
  13546. // currently is that the sym is only used on the path in which it is initialized inside the loop. So, don't
  13547. // hoist any liveness info for the dst.
  13548. if (!this->GetIsAsmJSFunc())
  13549. {
  13550. lossy = true;
  13551. }
  13552. break;
  13553. case Js::OpCode::FromVar:
  13554. {
  13555. StackSym* src1StackSym = IR::RegOpnd::TryGetStackSym(instr->GetSrc1());
  13556. if (instr->HasBailOutInfo())
  13557. {
  13558. IR::BailOutKind instrBailoutKind = instr->GetBailOutKind();
  13559. Assert(instrBailoutKind == IR::BailOutIntOnly ||
  13560. instrBailoutKind == IR::BailOutExpectingInteger ||
  13561. instrBailoutKind == IR::BailOutOnNotPrimitive ||
  13562. instrBailoutKind == IR::BailOutNumberOnly ||
  13563. instrBailoutKind == IR::BailOutPrimitiveButString);
  13564. }
  13565. else if (src1StackSym && bailoutKind != IR::BailOutInvalid)
  13566. {
  13567. // We may be hoisting FromVar from a region where it didn't need a bailout (src1 had a definite value type) to a region
  13568. // where it would. In such cases, the FromVar needs a bailout based on the value type of src1 in its new position.
  13569. Assert(!src1StackSym->IsTypeSpec());
  13570. Value* landingPadSrc1val = landingPad->globOptData.FindValue(src1StackSym);
  13571. Assert(src1Val->GetValueNumber() == landingPadSrc1val->GetValueNumber());
  13572. ValueInfo *src1ValueInfo = src1Val->GetValueInfo();
  13573. ValueInfo *landingPadSrc1ValueInfo = landingPadSrc1val->GetValueInfo();
  13574. IRType dstType = dst->GetType();
  13575. const auto AddBailOutToFromVar = [&]()
  13576. {
  13577. instr->GetSrc1()->SetValueType(landingPadSrc1val->GetValueInfo()->Type());
  13578. EnsureBailTarget(loop);
  13579. if (block->IsLandingPad())
  13580. {
  13581. instr = instr->ConvertToBailOutInstr(instr, bailoutKind, loop->bailOutInfo->bailOutOffset);
  13582. }
  13583. else
  13584. {
  13585. instr = instr->ConvertToBailOutInstr(instr, bailoutKind);
  13586. }
  13587. };
  13588. // A definite type in the source position and not a definite type in the destination (landing pad)
  13589. // and no bailout on the instruction; we should put a bailout on the hoisted instruction.
  13590. if (dstType == TyInt32)
  13591. {
  13592. if (lossy)
  13593. {
  13594. if ((src1ValueInfo->IsPrimitive() || block->globOptData.IsTypeSpecialized(src1StackSym)) && // didn't need a lossy type spec bailout in the source block
  13595. (!landingPadSrc1ValueInfo->IsPrimitive() && !landingPad->globOptData.IsTypeSpecialized(src1StackSym))) // needs a lossy type spec bailout in the landing pad
  13596. {
  13597. bailoutKind = IR::BailOutOnNotPrimitive;
  13598. AddBailOutToFromVar();
  13599. }
  13600. }
  13601. else if (src1ValueInfo->IsInt() && !landingPadSrc1ValueInfo->IsInt())
  13602. {
  13603. AddBailOutToFromVar();
  13604. }
  13605. }
  13606. else if ((dstType == TyFloat64 && src1ValueInfo->IsNumber() && !landingPadSrc1ValueInfo->IsNumber()))
  13607. {
  13608. AddBailOutToFromVar();
  13609. }
  13610. }
  13611. break;
  13612. }
  13613. }
  13614. if (dstVal == NULL)
  13615. {
  13616. dstVal = this->NewGenericValue(ValueType::Uninitialized, dst);
  13617. }
  13618. // ToVar/FromVar don't need a new dst because it has to be invariant if their src is invariant.
  13619. bool dstDoesntNeedLoad = (!isNotTypeSpecConv && instr->m_opcode != Js::OpCode::LdC_A_I4);
  13620. StackSym *varSym = dst->m_sym;
  13621. if (varSym->IsTypeSpec())
  13622. {
  13623. varSym = varSym->GetVarEquivSym(this->func);
  13624. }
  13625. Value *const landingPadDstVal = loop->landingPad->globOptData.FindValue(varSym);
  13626. if(landingPadDstVal
  13627. ? dstVal->GetValueNumber() != landingPadDstVal->GetValueNumber()
  13628. : loop->symsDefInLoop->Test(varSym->m_id))
  13629. {
  13630. // We need a temp for FromVar/ToVar if dst changes in the loop.
  13631. dstDoesntNeedLoad = false;
  13632. }
  13633. if (!dstDoesntNeedLoad && this->OptDstIsInvariant(dst) == false)
  13634. {
  13635. // Keep dst in place, hoist instr using a new dst.
  13636. instr->UnlinkDst();
  13637. // Set type specialization info correctly for this new sym
  13638. StackSym *copyVarSym;
  13639. IR::RegOpnd *copyReg;
  13640. if (dst->m_sym->IsTypeSpec())
  13641. {
  13642. copyVarSym = StackSym::New(TyVar, instr->m_func);
  13643. StackSym *copySym = copyVarSym;
  13644. if (dst->m_sym->IsInt32())
  13645. {
  13646. if(lossy)
  13647. {
  13648. // The new sym would only be live as a lossy int since we're only hoisting the store to the int version
  13649. // of the sym, and cannot be converted to var. It is not legal to have a sym only live as a lossy int,
  13650. // so don't update liveness info for this sym.
  13651. }
  13652. else
  13653. {
  13654. block->globOptData.liveInt32Syms->Set(copyVarSym->m_id);
  13655. }
  13656. copySym = copySym->GetInt32EquivSym(instr->m_func);
  13657. }
  13658. else if (dst->m_sym->IsFloat64())
  13659. {
  13660. block->globOptData.liveFloat64Syms->Set(copyVarSym->m_id);
  13661. copySym = copySym->GetFloat64EquivSym(instr->m_func);
  13662. }
  13663. copyReg = IR::RegOpnd::New(copySym, copySym->GetType(), instr->m_func);
  13664. }
  13665. else
  13666. {
  13667. copyReg = IR::RegOpnd::New(dst->GetType(), instr->m_func);
  13668. copyVarSym = copyReg->m_sym;
  13669. block->globOptData.liveVarSyms->Set(copyVarSym->m_id);
  13670. }
  13671. copyReg->SetValueType(dst->GetValueType());
  13672. IR::Instr *copyInstr = IR::Instr::New(Js::OpCode::Ld_A, dst, copyReg, instr->m_func);
  13673. copyInstr->SetByteCodeOffset(instr);
  13674. instr->SetDst(copyReg);
  13675. instr->InsertBefore(copyInstr);
  13676. dst->m_sym->m_mayNotBeTempLastUse = true;
  13677. if (instr->GetSrc1() && instr->GetSrc1()->IsImmediateOpnd())
  13678. {
  13679. // Propagate IsIntConst if appropriate
  13680. switch(instr->m_opcode)
  13681. {
  13682. case Js::OpCode::Ld_A:
  13683. case Js::OpCode::Ld_I4:
  13684. case Js::OpCode::LdC_A_I4:
  13685. copyReg->m_sym->SetIsConst();
  13686. break;
  13687. }
  13688. }
  13689. ValueInfo *dstValueInfo = dstVal->GetValueInfo();
  13690. if((!dstValueInfo->GetSymStore() || dstValueInfo->GetSymStore() == varSym) && !lossy)
  13691. {
  13692. // The destination's value may have been transferred from one of the invariant sources, in which case we should
  13693. // keep the sym store intact, as that sym will likely have a better lifetime than this new copy sym. For
  13694. // instance, if we're inside a conditioned block, because we don't make the copy sym live and set its value in
  13695. // all preceding blocks, this sym would not be live after exiting this block, causing this value to not
  13696. // participate in copy-prop after this block.
  13697. this->SetSymStoreDirect(dstValueInfo, copyVarSym);
  13698. }
  13699. block->globOptData.InsertNewValue(dstVal, copyReg);
  13700. dst = copyReg;
  13701. }
  13702. }
  13703. // Move to landing pad
  13704. block->UnlinkInstr(instr);
  13705. if (loop->bailOutInfo->bailOutInstr)
  13706. {
  13707. loop->bailOutInfo->bailOutInstr->InsertBefore(instr);
  13708. }
  13709. else
  13710. {
  13711. landingPad->InsertAfter(instr);
  13712. }
  13713. GlobOpt::MarkNonByteCodeUsed(instr);
  13714. if (instr->HasBailOutInfo() || instr->HasAuxBailOut())
  13715. {
  13716. Assert(loop->bailOutInfo);
  13717. EnsureBailTarget(loop);
  13718. // Copy bailout info of loop top.
  13719. instr->ReplaceBailOutInfo(loop->bailOutInfo);
  13720. }
  13721. if(!dst)
  13722. {
  13723. return;
  13724. }
  13725. // The bailout info's liveness for the dst sym is not updated in loop landing pads because bailout instructions previously
  13726. // hoisted into the loop's landing pad may bail out before the current type of the dst sym became live (perhaps due to this
  13727. // instruction). Since the landing pad will have a shared bailout point, the bailout info cannot assume that the current
  13728. // type of the dst sym was live during every bailout hoisted into the landing pad.
  13729. StackSym *const dstSym = dst->m_sym;
  13730. StackSym *const dstVarSym = dstSym->IsTypeSpec() ? dstSym->GetVarEquivSym(nullptr) : dstSym;
  13731. Assert(dstVarSym);
  13732. if(isNotTypeSpecConv || !loop->landingPad->globOptData.IsLive(dstVarSym))
  13733. {
  13734. // A new dst is being hoisted, or the same single-def dst that would not be live before this block. So, make it live and
  13735. // update the value info with the same value info in this block.
  13736. if(lossy)
  13737. {
  13738. // This is a lossy conversion to int. The instruction was given a new dst specifically for hoisting, so this new dst
  13739. // will not be live as a var before this block. A sym cannot be live only as a lossy int sym, the var needs to be
  13740. // live as well since the lossy int sym cannot be used to convert to var. Since the var version of the sym is not
  13741. // going to be initialized, don't hoist any liveness info for the dst. The sym is only going to be used on the path
  13742. // in which it is initialized inside the loop.
  13743. Assert(dstSym->IsTypeSpec());
  13744. Assert(dstSym->IsInt32());
  13745. return;
  13746. }
  13747. // Check if the dst value was transferred from the src. If so, the value transfer needs to be replicated.
  13748. bool isTransfer = dstVal == src1Val;
  13749. StackSym *transferValueOfSym = nullptr;
  13750. if(isTransfer)
  13751. {
  13752. Assert(instr->GetSrc1());
  13753. if(instr->GetSrc1()->IsRegOpnd())
  13754. {
  13755. StackSym *src1Sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  13756. if(src1Sym->IsTypeSpec())
  13757. {
  13758. src1Sym = src1Sym->GetVarEquivSym(nullptr);
  13759. Assert(src1Sym);
  13760. }
  13761. if(dstVal == block->globOptData.FindValue(src1Sym))
  13762. {
  13763. transferValueOfSym = src1Sym;
  13764. }
  13765. }
  13766. }
  13767. // SIMD_JS
  13768. if (instr->m_opcode == Js::OpCode::ExtendArg_A)
  13769. {
  13770. // Check if we should have CSE'ed this EA
  13771. Assert(instr->GetSrc1());
  13772. // If the dstVal symstore is not the dst itself, then we copied the Value from another expression.
  13773. if (dstVal->GetValueInfo()->GetSymStore() != instr->GetDst()->GetStackSym())
  13774. {
  13775. isTransfer = true;
  13776. transferValueOfSym = dstVal->GetValueInfo()->GetSymStore()->AsStackSym();
  13777. }
  13778. }
  13779. const ValueNumber dstValueNumber = dstVal->GetValueNumber();
  13780. ValueNumber dstNewValueNumber = InvalidValueNumber;
  13781. for(InvariantBlockBackwardIterator it(this, block, loop->landingPad, nullptr); it.IsValid(); it.MoveNext())
  13782. {
  13783. BasicBlock *const hoistBlock = it.Block();
  13784. GlobOptBlockData &hoistBlockData = hoistBlock->globOptData;
  13785. Assert(!hoistBlockData.IsLive(dstVarSym));
  13786. hoistBlockData.MakeLive(dstSym, lossy);
  13787. Value *newDstValue;
  13788. do
  13789. {
  13790. if(isTransfer)
  13791. {
  13792. if(transferValueOfSym)
  13793. {
  13794. newDstValue = hoistBlockData.FindValue(transferValueOfSym);
  13795. if(newDstValue && newDstValue->GetValueNumber() == dstValueNumber)
  13796. {
  13797. break;
  13798. }
  13799. }
  13800. // It's a transfer, but we don't have a sym whose value number matches in the target block. Use a new value
  13801. // number since we don't know if there is already a value with the current number for the target block.
  13802. if(dstNewValueNumber == InvalidValueNumber)
  13803. {
  13804. dstNewValueNumber = NewValueNumber();
  13805. }
  13806. newDstValue = CopyValue(dstVal, dstNewValueNumber);
  13807. break;
  13808. }
  13809. newDstValue = CopyValue(dstVal, dstValueNumber);
  13810. } while(false);
  13811. hoistBlockData.SetValue(newDstValue, dstVarSym);
  13812. }
  13813. return;
  13814. }
  13815. #if DBG
  13816. if(instr->GetSrc1()->IsRegOpnd()) // Type spec conversion may load a constant into a dst sym
  13817. {
  13818. StackSym *const srcSym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  13819. Assert(srcSym != dstSym); // Type spec conversion must be changing the type, so the syms must be different
  13820. StackSym *const srcVarSym = srcSym->IsTypeSpec() ? srcSym->GetVarEquivSym(nullptr) : srcSym;
  13821. Assert(srcVarSym == dstVarSym); // Type spec conversion must be between variants of the same var sym
  13822. }
  13823. #endif
  13824. bool changeValueType = false, changeValueTypeToInt = false;
  13825. if(dstSym->IsTypeSpec())
  13826. {
  13827. if(dst->IsInt32())
  13828. {
  13829. if(!lossy)
  13830. {
  13831. Assert(
  13832. !instr->HasBailOutInfo() ||
  13833. instr->GetBailOutKind() == IR::BailOutIntOnly ||
  13834. instr->GetBailOutKind() == IR::BailOutExpectingInteger);
  13835. changeValueType = changeValueTypeToInt = true;
  13836. }
  13837. }
  13838. else if (dst->IsFloat64())
  13839. {
  13840. if(instr->HasBailOutInfo() && instr->GetBailOutKind() == IR::BailOutNumberOnly)
  13841. {
  13842. changeValueType = true;
  13843. }
  13844. }
  13845. }
  13846. ValueInfo *previousValueInfoBeforeUpdate = nullptr, *previousValueInfoAfterUpdate = nullptr;
  13847. for(InvariantBlockBackwardIterator it(
  13848. this,
  13849. block,
  13850. loop->landingPad,
  13851. dstVarSym,
  13852. dstVal->GetValueNumber());
  13853. it.IsValid();
  13854. it.MoveNext())
  13855. {
  13856. BasicBlock *const hoistBlock = it.Block();
  13857. GlobOptBlockData &hoistBlockData = hoistBlock->globOptData;
  13858. #if DBG
  13859. // TODO: There are some odd cases with field hoisting where the sym is invariant in only part of the loop and the info
  13860. // does not flow through all blocks. Un-comment the verification below after PRE replaces field hoisting.
  13861. //// Verify that the src sym is live as the required type, and that the conversion is valid
  13862. //Assert(IsLive(dstVarSym, &hoistBlockData));
  13863. //if(instr->GetSrc1()->IsRegOpnd())
  13864. //{
  13865. // IR::RegOpnd *const src = instr->GetSrc1()->AsRegOpnd();
  13866. // StackSym *const srcSym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  13867. // if(srcSym->IsTypeSpec())
  13868. // {
  13869. // if(src->IsInt32())
  13870. // {
  13871. // Assert(hoistBlockData.liveInt32Syms->Test(dstVarSym->m_id));
  13872. // Assert(!hoistBlockData.liveLossyInt32Syms->Test(dstVarSym->m_id)); // shouldn't try to convert a lossy int32 to anything
  13873. // }
  13874. // else
  13875. // {
  13876. // Assert(src->IsFloat64());
  13877. // Assert(hoistBlockData.liveFloat64Syms->Test(dstVarSym->m_id));
  13878. // if(dstSym->IsTypeSpec() && dst->IsInt32())
  13879. // {
  13880. // Assert(lossy); // shouldn't try to do a lossless conversion from float64 to int32
  13881. // }
  13882. // }
  13883. // }
  13884. // else
  13885. // {
  13886. // Assert(hoistBlockData.liveVarSyms->Test(dstVarSym->m_id));
  13887. // }
  13888. //}
  13889. //if(dstSym->IsTypeSpec() && dst->IsInt32())
  13890. //{
  13891. // // If the sym is already specialized as required in the block to which we are attempting to hoist the conversion,
  13892. // // that info should have flowed into this block
  13893. // if(lossy)
  13894. // {
  13895. // Assert(!hoistBlockData.liveInt32Syms->Test(dstVarSym->m_id));
  13896. // }
  13897. // else
  13898. // {
  13899. // Assert(!IsInt32TypeSpecialized(dstVarSym, hoistBlock));
  13900. // }
  13901. //}
  13902. #endif
  13903. hoistBlockData.MakeLive(dstSym, lossy);
  13904. if(!changeValueType)
  13905. {
  13906. continue;
  13907. }
  13908. Value *const hoistBlockValue = it.InvariantSymValue();
  13909. ValueInfo *const hoistBlockValueInfo = hoistBlockValue->GetValueInfo();
  13910. if(hoistBlockValueInfo == previousValueInfoBeforeUpdate)
  13911. {
  13912. if(hoistBlockValueInfo != previousValueInfoAfterUpdate)
  13913. {
  13914. HoistInvariantValueInfo(previousValueInfoAfterUpdate, hoistBlockValue, hoistBlock);
  13915. }
  13916. }
  13917. else
  13918. {
  13919. previousValueInfoBeforeUpdate = hoistBlockValueInfo;
  13920. ValueInfo *const newValueInfo =
  13921. changeValueTypeToInt
  13922. ? hoistBlockValueInfo->SpecializeToInt32(alloc)
  13923. : hoistBlockValueInfo->SpecializeToFloat64(alloc);
  13924. previousValueInfoAfterUpdate = newValueInfo;
  13925. ChangeValueInfo(changeValueTypeToInt ? nullptr : hoistBlock, hoistBlockValue, newValueInfo);
  13926. }
  13927. }
  13928. }
  13929. bool
  13930. GlobOpt::TryHoistInvariant(
  13931. IR::Instr *instr,
  13932. BasicBlock *block,
  13933. Value *dstVal,
  13934. Value *src1Val,
  13935. Value *src2Val,
  13936. bool isNotTypeSpecConv,
  13937. const bool lossy,
  13938. const bool forceInvariantHoisting,
  13939. IR::BailOutKind bailoutKind)
  13940. {
  13941. Assert(!this->IsLoopPrePass());
  13942. if (OptIsInvariant(instr, block, block->loop, src1Val, src2Val, isNotTypeSpecConv, forceInvariantHoisting))
  13943. {
  13944. #if DBG
  13945. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::InvariantsPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()))
  13946. {
  13947. Output::Print(_u(" **** INVARIANT *** "));
  13948. instr->Dump();
  13949. }
  13950. #endif
  13951. #if ENABLE_DEBUG_CONFIG_OPTIONS
  13952. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::InvariantsPhase))
  13953. {
  13954. Output::Print(_u(" **** INVARIANT *** "));
  13955. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  13956. }
  13957. #endif
  13958. Loop *loop = block->loop;
  13959. // Try hoisting from to outer most loop
  13960. while (loop->parent && OptIsInvariant(instr, block, loop->parent, src1Val, src2Val, isNotTypeSpecConv, forceInvariantHoisting))
  13961. {
  13962. loop = loop->parent;
  13963. }
  13964. // Record the byte code use here since we are going to move this instruction up
  13965. if (isNotTypeSpecConv)
  13966. {
  13967. InsertNoImplicitCallUses(instr);
  13968. this->CaptureByteCodeSymUses(instr);
  13969. this->InsertByteCodeUses(instr, true);
  13970. }
  13971. #if DBG
  13972. else
  13973. {
  13974. PropertySym *propertySymUse = NULL;
  13975. NoRecoverMemoryJitArenaAllocator tempAllocator(_u("BE-GlobOpt-Temp"), this->alloc->GetPageAllocator(), Js::Throw::OutOfMemory);
  13976. BVSparse<JitArenaAllocator> * tempByteCodeUse = JitAnew(&tempAllocator, BVSparse<JitArenaAllocator>, &tempAllocator);
  13977. GlobOpt::TrackByteCodeSymUsed(instr, tempByteCodeUse, &propertySymUse);
  13978. Assert(tempByteCodeUse->Count() == 0 && propertySymUse == NULL);
  13979. }
  13980. #endif
  13981. OptHoistInvariant(instr, block, loop, dstVal, src1Val, src2Val, isNotTypeSpecConv, lossy, bailoutKind);
  13982. return true;
  13983. }
  13984. return false;
  13985. }
  13986. InvariantBlockBackwardIterator::InvariantBlockBackwardIterator(
  13987. GlobOpt *const globOpt,
  13988. BasicBlock *const exclusiveBeginBlock,
  13989. BasicBlock *const inclusiveEndBlock,
  13990. StackSym *const invariantSym,
  13991. const ValueNumber invariantSymValueNumber,
  13992. bool followFlow)
  13993. : globOpt(globOpt),
  13994. exclusiveEndBlock(inclusiveEndBlock->prev),
  13995. invariantSym(invariantSym),
  13996. invariantSymValueNumber(invariantSymValueNumber),
  13997. block(exclusiveBeginBlock),
  13998. blockBV(globOpt->tempAlloc),
  13999. followFlow(followFlow)
  14000. #if DBG
  14001. ,
  14002. inclusiveEndBlock(inclusiveEndBlock)
  14003. #endif
  14004. {
  14005. Assert(exclusiveBeginBlock);
  14006. Assert(inclusiveEndBlock);
  14007. Assert(!inclusiveEndBlock->isDeleted);
  14008. Assert(exclusiveBeginBlock != inclusiveEndBlock);
  14009. Assert(!invariantSym == (invariantSymValueNumber == InvalidValueNumber));
  14010. MoveNext();
  14011. }
  14012. bool
  14013. InvariantBlockBackwardIterator::IsValid() const
  14014. {
  14015. return block != exclusiveEndBlock;
  14016. }
  14017. void
  14018. InvariantBlockBackwardIterator::MoveNext()
  14019. {
  14020. Assert(IsValid());
  14021. while(true)
  14022. {
  14023. #if DBG
  14024. BasicBlock *const previouslyIteratedBlock = block;
  14025. #endif
  14026. block = block->prev;
  14027. if(!IsValid())
  14028. {
  14029. Assert(previouslyIteratedBlock == inclusiveEndBlock);
  14030. break;
  14031. }
  14032. if (!this->UpdatePredBlockBV())
  14033. {
  14034. continue;
  14035. }
  14036. if (!this->UpdatePredBlockBV())
  14037. {
  14038. continue;
  14039. }
  14040. if(block->isDeleted)
  14041. {
  14042. continue;
  14043. }
  14044. if(!block->globOptData.HasData())
  14045. {
  14046. // This block's info has already been merged with all of its successors
  14047. continue;
  14048. }
  14049. if(!invariantSym)
  14050. {
  14051. break;
  14052. }
  14053. invariantSymValue = block->globOptData.FindValue(invariantSym);
  14054. if(!invariantSymValue || invariantSymValue->GetValueNumber() != invariantSymValueNumber)
  14055. {
  14056. // BailOnNoProfile and throw blocks are not moved outside loops. A sym table cleanup on these paths may delete the
  14057. // values. Field hoisting also has some odd cases where the hoisted stack sym is invariant in only part of the loop.
  14058. continue;
  14059. }
  14060. break;
  14061. }
  14062. }
  14063. bool
  14064. InvariantBlockBackwardIterator::UpdatePredBlockBV()
  14065. {
  14066. if (!this->followFlow)
  14067. {
  14068. return true;
  14069. }
  14070. // Track blocks we've visited to ensure that we only iterate over predecessor blocks
  14071. if (!this->blockBV.IsEmpty() && !this->blockBV.Test(this->block->GetBlockNum()))
  14072. {
  14073. return false;
  14074. }
  14075. FOREACH_SLISTBASECOUNTED_ENTRY(FlowEdge*, edge, this->block->GetPredList())
  14076. {
  14077. this->blockBV.Set(edge->GetPred()->GetBlockNum());
  14078. } NEXT_SLISTBASECOUNTED_ENTRY;
  14079. return true;
  14080. }
  14081. BasicBlock *
  14082. InvariantBlockBackwardIterator::Block() const
  14083. {
  14084. Assert(IsValid());
  14085. return block;
  14086. }
  14087. Value *
  14088. InvariantBlockBackwardIterator::InvariantSymValue() const
  14089. {
  14090. Assert(IsValid());
  14091. Assert(invariantSym);
  14092. return invariantSymValue;
  14093. }
  14094. void
  14095. GlobOpt::HoistInvariantValueInfo(
  14096. ValueInfo *const invariantValueInfoToHoist,
  14097. Value *const valueToUpdate,
  14098. BasicBlock *const targetBlock)
  14099. {
  14100. Assert(invariantValueInfoToHoist);
  14101. Assert(valueToUpdate);
  14102. Assert(targetBlock);
  14103. // Why are we trying to change the value type of the type sym value? Asserting here to make sure we don't deep copy the type sym's value info.
  14104. Assert(!invariantValueInfoToHoist->IsJsType());
  14105. Sym *const symStore = valueToUpdate->GetValueInfo()->GetSymStore();
  14106. ValueInfo *newValueInfo;
  14107. if(invariantValueInfoToHoist->GetSymStore() == symStore)
  14108. {
  14109. newValueInfo = invariantValueInfoToHoist;
  14110. }
  14111. else
  14112. {
  14113. newValueInfo = invariantValueInfoToHoist->Copy(alloc);
  14114. this->SetSymStoreDirect(newValueInfo, symStore);
  14115. }
  14116. ChangeValueInfo(targetBlock, valueToUpdate, newValueInfo, true);
  14117. }
  14118. // static
  14119. bool
  14120. GlobOpt::DoInlineArgsOpt(Func const * func)
  14121. {
  14122. Func const * topFunc = func->GetTopFunc();
  14123. Assert(topFunc != func);
  14124. bool doInlineArgsOpt =
  14125. !PHASE_OFF(Js::InlineArgsOptPhase, topFunc) &&
  14126. !func->GetHasCalls() &&
  14127. !func->GetHasUnoptimizedArgumentsAccess() &&
  14128. func->m_canDoInlineArgsOpt;
  14129. return doInlineArgsOpt;
  14130. }
  14131. bool
  14132. GlobOpt::IsSwitchOptEnabled(Func const * func)
  14133. {
  14134. Assert(func->IsTopFunc());
  14135. return !PHASE_OFF(Js::SwitchOptPhase, func) && !func->IsSwitchOptDisabled() && func->DoGlobOpt();
  14136. }
  14137. bool
  14138. GlobOpt::IsSwitchOptEnabledForIntTypeSpec(Func const * func)
  14139. {
  14140. return IsSwitchOptEnabled(func) && !IsTypeSpecPhaseOff(func) && DoAggressiveIntTypeSpec(func);
  14141. }
  14142. bool
  14143. GlobOpt::DoConstFold() const
  14144. {
  14145. return !PHASE_OFF(Js::ConstFoldPhase, func);
  14146. }
  14147. bool
  14148. GlobOpt::IsTypeSpecPhaseOff(Func const *func)
  14149. {
  14150. return PHASE_OFF(Js::TypeSpecPhase, func) || func->IsJitInDebugMode();
  14151. }
  14152. bool
  14153. GlobOpt::DoTypeSpec() const
  14154. {
  14155. return doTypeSpec;
  14156. }
  14157. bool
  14158. GlobOpt::DoAggressiveIntTypeSpec(Func const * func)
  14159. {
  14160. return
  14161. !PHASE_OFF(Js::AggressiveIntTypeSpecPhase, func) &&
  14162. !IsTypeSpecPhaseOff(func) &&
  14163. !func->IsAggressiveIntTypeSpecDisabled();
  14164. }
  14165. bool
  14166. GlobOpt::DoAggressiveIntTypeSpec() const
  14167. {
  14168. return doAggressiveIntTypeSpec;
  14169. }
  14170. bool
  14171. GlobOpt::DoAggressiveMulIntTypeSpec() const
  14172. {
  14173. return doAggressiveMulIntTypeSpec;
  14174. }
  14175. bool
  14176. GlobOpt::DoDivIntTypeSpec() const
  14177. {
  14178. return doDivIntTypeSpec;
  14179. }
  14180. // static
  14181. bool
  14182. GlobOpt::DoLossyIntTypeSpec(Func const * func)
  14183. {
  14184. return
  14185. !PHASE_OFF(Js::LossyIntTypeSpecPhase, func) &&
  14186. !IsTypeSpecPhaseOff(func) &&
  14187. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsLossyIntTypeSpecDisabled());
  14188. }
  14189. bool
  14190. GlobOpt::DoLossyIntTypeSpec() const
  14191. {
  14192. return doLossyIntTypeSpec;
  14193. }
  14194. // static
  14195. bool
  14196. GlobOpt::DoFloatTypeSpec(Func const * func)
  14197. {
  14198. return
  14199. !PHASE_OFF(Js::FloatTypeSpecPhase, func) &&
  14200. !IsTypeSpecPhaseOff(func) &&
  14201. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsFloatTypeSpecDisabled()) &&
  14202. AutoSystemInfo::Data.SSE2Available();
  14203. }
  14204. bool
  14205. GlobOpt::DoFloatTypeSpec() const
  14206. {
  14207. return doFloatTypeSpec;
  14208. }
  14209. bool
  14210. GlobOpt::DoStringTypeSpec(Func const * func)
  14211. {
  14212. return !PHASE_OFF(Js::StringTypeSpecPhase, func) && !IsTypeSpecPhaseOff(func);
  14213. }
  14214. // static
  14215. bool
  14216. GlobOpt::DoTypedArrayTypeSpec(Func const * func)
  14217. {
  14218. return !PHASE_OFF(Js::TypedArrayTypeSpecPhase, func) &&
  14219. !IsTypeSpecPhaseOff(func) &&
  14220. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsTypedArrayTypeSpecDisabled(func->IsLoopBody()))
  14221. #if defined(_M_IX86)
  14222. && AutoSystemInfo::Data.SSE2Available()
  14223. #endif
  14224. ;
  14225. }
  14226. // static
  14227. bool
  14228. GlobOpt::DoNativeArrayTypeSpec(Func const * func)
  14229. {
  14230. return !PHASE_OFF(Js::NativeArrayPhase, func) &&
  14231. !IsTypeSpecPhaseOff(func)
  14232. #if defined(_M_IX86)
  14233. && AutoSystemInfo::Data.SSE2Available()
  14234. #endif
  14235. ;
  14236. }
  14237. bool
  14238. GlobOpt::DoArrayCheckHoist(Func const * const func)
  14239. {
  14240. Assert(func->IsTopFunc());
  14241. return
  14242. !PHASE_OFF(Js::ArrayCheckHoistPhase, func) &&
  14243. !func->IsArrayCheckHoistDisabled() &&
  14244. !func->IsJitInDebugMode(); // StElemI fast path is not allowed when in debug mode, so it cannot have bailout
  14245. }
  14246. bool
  14247. GlobOpt::DoArrayCheckHoist() const
  14248. {
  14249. return doArrayCheckHoist;
  14250. }
  14251. bool
  14252. GlobOpt::DoArrayCheckHoist(const ValueType baseValueType, Loop* loop, IR::Instr const * const instr) const
  14253. {
  14254. if(!DoArrayCheckHoist() || (instr && !IsLoopPrePass() && instr->DoStackArgsOpt()))
  14255. {
  14256. return false;
  14257. }
  14258. // This includes typed arrays, but not virtual typed arrays, whose vtable can change if the buffer goes away.
  14259. // Note that in the virtual case the vtable check is the only way to catch this, since there's no bound check.
  14260. if(!(baseValueType.IsLikelyArrayOrObjectWithArray() || baseValueType.IsLikelyOptimizedVirtualTypedArray()) ||
  14261. (loop ? ImplicitCallFlagsAllowOpts(loop) : ImplicitCallFlagsAllowOpts(func)))
  14262. {
  14263. return true;
  14264. }
  14265. // The function or loop does not allow disabling implicit calls, which is required to eliminate redundant JS array checks
  14266. #if DBG_DUMP
  14267. if((((loop ? loop->GetImplicitCallFlags() : func->m_fg->implicitCallFlags) & ~Js::ImplicitCall_External) == 0) &&
  14268. Js::Configuration::Global.flags.Trace.IsEnabled(Js::HostOptPhase))
  14269. {
  14270. Output::Print(_u("DoArrayCheckHoist disabled for JS arrays because of external: "));
  14271. func->DumpFullFunctionName();
  14272. Output::Print(_u("\n"));
  14273. Output::Flush();
  14274. }
  14275. #endif
  14276. return false;
  14277. }
  14278. bool
  14279. GlobOpt::DoArrayMissingValueCheckHoist(Func const * const func)
  14280. {
  14281. return
  14282. DoArrayCheckHoist(func) &&
  14283. !PHASE_OFF(Js::ArrayMissingValueCheckHoistPhase, func) &&
  14284. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsArrayMissingValueCheckHoistDisabled(func->IsLoopBody()));
  14285. }
  14286. bool
  14287. GlobOpt::DoArrayMissingValueCheckHoist() const
  14288. {
  14289. return doArrayMissingValueCheckHoist;
  14290. }
  14291. bool
  14292. GlobOpt::DoArraySegmentHoist(const ValueType baseValueType, Func const * const func)
  14293. {
  14294. Assert(baseValueType.IsLikelyAnyOptimizedArray());
  14295. if(!DoArrayCheckHoist(func) || PHASE_OFF(Js::ArraySegmentHoistPhase, func))
  14296. {
  14297. return false;
  14298. }
  14299. if(!baseValueType.IsLikelyArrayOrObjectWithArray())
  14300. {
  14301. return true;
  14302. }
  14303. return
  14304. !PHASE_OFF(Js::JsArraySegmentHoistPhase, func) &&
  14305. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsJsArraySegmentHoistDisabled(func->IsLoopBody()));
  14306. }
  14307. bool
  14308. GlobOpt::DoArraySegmentHoist(const ValueType baseValueType) const
  14309. {
  14310. Assert(baseValueType.IsLikelyAnyOptimizedArray());
  14311. return baseValueType.IsLikelyArrayOrObjectWithArray() ? doJsArraySegmentHoist : doArraySegmentHoist;
  14312. }
  14313. bool
  14314. GlobOpt::DoTypedArraySegmentLengthHoist(Loop *const loop) const
  14315. {
  14316. if(!DoArraySegmentHoist(ValueType::GetObject(ObjectType::Int32Array)))
  14317. {
  14318. return false;
  14319. }
  14320. if(loop ? ImplicitCallFlagsAllowOpts(loop) : ImplicitCallFlagsAllowOpts(func))
  14321. {
  14322. return true;
  14323. }
  14324. // The function or loop does not allow disabling implicit calls, which is required to eliminate redundant typed array
  14325. // segment length loads.
  14326. #if DBG_DUMP
  14327. if((((loop ? loop->GetImplicitCallFlags() : func->m_fg->implicitCallFlags) & ~Js::ImplicitCall_External) == 0) &&
  14328. Js::Configuration::Global.flags.Trace.IsEnabled(Js::HostOptPhase))
  14329. {
  14330. Output::Print(_u("DoArraySegmentLengthHoist disabled for typed arrays because of external: "));
  14331. func->DumpFullFunctionName();
  14332. Output::Print(_u("\n"));
  14333. Output::Flush();
  14334. }
  14335. #endif
  14336. return false;
  14337. }
  14338. bool
  14339. GlobOpt::DoArrayLengthHoist(Func const * const func)
  14340. {
  14341. return
  14342. DoArrayCheckHoist(func) &&
  14343. !PHASE_OFF(Js::Phase::ArrayLengthHoistPhase, func) &&
  14344. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsArrayLengthHoistDisabled(func->IsLoopBody()));
  14345. }
  14346. bool
  14347. GlobOpt::DoArrayLengthHoist() const
  14348. {
  14349. return doArrayLengthHoist;
  14350. }
  14351. bool
  14352. GlobOpt::DoEliminateArrayAccessHelperCall(Func *const func)
  14353. {
  14354. return DoArrayCheckHoist(func);
  14355. }
  14356. bool
  14357. GlobOpt::DoEliminateArrayAccessHelperCall() const
  14358. {
  14359. return doEliminateArrayAccessHelperCall;
  14360. }
  14361. bool
  14362. GlobOpt::DoLdLenIntSpec(IR::Instr * const instr, const ValueType baseValueType)
  14363. {
  14364. Assert(!instr || instr->m_opcode == Js::OpCode::LdLen_A);
  14365. Assert(!instr || instr->GetDst());
  14366. Assert(!instr || instr->GetSrc1());
  14367. if(PHASE_OFF(Js::LdLenIntSpecPhase, func) ||
  14368. IsTypeSpecPhaseOff(func) ||
  14369. (func->HasProfileInfo() && func->GetReadOnlyProfileInfo()->IsLdLenIntSpecDisabled()) ||
  14370. (instr && !IsLoopPrePass() && instr->DoStackArgsOpt()))
  14371. {
  14372. return false;
  14373. }
  14374. if(instr &&
  14375. instr->IsProfiledInstr() &&
  14376. (
  14377. !instr->AsProfiledInstr()->u.FldInfo().valueType.IsLikelyInt() ||
  14378. instr->GetDst()->AsRegOpnd()->m_sym->m_isNotNumber
  14379. ))
  14380. {
  14381. return false;
  14382. }
  14383. Assert(!instr || baseValueType == instr->GetSrc1()->GetValueType());
  14384. return
  14385. baseValueType.HasBeenString() ||
  14386. (baseValueType.IsLikelyAnyOptimizedArray() && baseValueType.GetObjectType() != ObjectType::ObjectWithArray);
  14387. }
  14388. bool
  14389. GlobOpt::DoPathDependentValues() const
  14390. {
  14391. return !PHASE_OFF(Js::Phase::PathDependentValuesPhase, func);
  14392. }
  14393. bool
  14394. GlobOpt::DoTrackRelativeIntBounds() const
  14395. {
  14396. return doTrackRelativeIntBounds;
  14397. }
  14398. bool
  14399. GlobOpt::DoBoundCheckElimination() const
  14400. {
  14401. return doBoundCheckElimination;
  14402. }
  14403. bool
  14404. GlobOpt::DoBoundCheckHoist() const
  14405. {
  14406. return doBoundCheckHoist;
  14407. }
  14408. bool
  14409. GlobOpt::DoLoopCountBasedBoundCheckHoist() const
  14410. {
  14411. return doLoopCountBasedBoundCheckHoist;
  14412. }
  14413. bool
  14414. GlobOpt::DoPowIntIntTypeSpec() const
  14415. {
  14416. return doPowIntIntTypeSpec;
  14417. }
  14418. bool
  14419. GlobOpt::DoTagChecks() const
  14420. {
  14421. return doTagChecks;
  14422. }
  14423. bool
  14424. GlobOpt::TrackArgumentsObject()
  14425. {
  14426. if (PHASE_OFF(Js::StackArgOptPhase, this->func))
  14427. {
  14428. this->CannotAllocateArgumentsObjectOnStack(nullptr);
  14429. return false;
  14430. }
  14431. return func->GetHasStackArgs();
  14432. }
  14433. void
  14434. GlobOpt::CannotAllocateArgumentsObjectOnStack(Func * curFunc)
  14435. {
  14436. if (curFunc != nullptr && curFunc->hasArgLenAndConstOpt)
  14437. {
  14438. Assert(!curFunc->GetJITOutput()->GetOutputData()->disableStackArgOpt);
  14439. curFunc->GetJITOutput()->GetOutputData()->disableStackArgOpt = true;
  14440. throw Js::RejitException(RejitReason::DisableStackArgLenAndConstOpt);
  14441. }
  14442. func->SetHasStackArgs(false);
  14443. #ifdef ENABLE_DEBUG_CONFIG_OPTIONS
  14444. if (PHASE_TESTTRACE(Js::StackArgOptPhase, this->func))
  14445. {
  14446. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  14447. Output::Print(_u("Stack args disabled for function %s(%s)\n"), func->GetJITFunctionBody()->GetDisplayName(), func->GetDebugNumberSet(debugStringBuffer));
  14448. Output::Flush();
  14449. }
  14450. #endif
  14451. }
  14452. IR::Instr *
  14453. GlobOpt::PreOptPeep(IR::Instr *instr)
  14454. {
  14455. if (OpCodeAttr::HasDeadFallThrough(instr->m_opcode))
  14456. {
  14457. switch (instr->m_opcode)
  14458. {
  14459. case Js::OpCode::BailOnNoProfile:
  14460. {
  14461. // Handle BailOnNoProfile
  14462. if (instr->HasBailOutInfo())
  14463. {
  14464. if (!this->prePassLoop)
  14465. {
  14466. FillBailOutInfo(this->currentBlock, instr);
  14467. }
  14468. // Already processed.
  14469. return instr;
  14470. }
  14471. // Convert to bailout instr
  14472. IR::Instr *nextBytecodeOffsetInstr = instr->GetNextRealInstrOrLabel();
  14473. while(nextBytecodeOffsetInstr->GetByteCodeOffset() == Js::Constants::NoByteCodeOffset)
  14474. {
  14475. nextBytecodeOffsetInstr = nextBytecodeOffsetInstr->GetNextRealInstrOrLabel();
  14476. Assert(!nextBytecodeOffsetInstr->IsLabelInstr());
  14477. }
  14478. instr = instr->ConvertToBailOutInstr(nextBytecodeOffsetInstr, IR::BailOutOnNoProfile);
  14479. instr->ClearByteCodeOffset();
  14480. instr->SetByteCodeOffset(nextBytecodeOffsetInstr);
  14481. if (!this->currentBlock->loop)
  14482. {
  14483. FillBailOutInfo(this->currentBlock, instr);
  14484. }
  14485. else
  14486. {
  14487. Assert(this->prePassLoop);
  14488. }
  14489. break;
  14490. }
  14491. case Js::OpCode::BailOnException:
  14492. {
  14493. Assert(
  14494. (
  14495. this->func->HasTry() && this->func->DoOptimizeTry() &&
  14496. instr->m_prev->m_opcode == Js::OpCode::Catch &&
  14497. instr->m_prev->m_prev->IsLabelInstr() &&
  14498. instr->m_prev->m_prev->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeCatch
  14499. )
  14500. ||
  14501. (
  14502. this->func->HasFinally() && this->func->DoOptimizeTry() &&
  14503. instr->m_prev->AsLabelInstr() &&
  14504. instr->m_prev->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeFinally
  14505. )
  14506. );
  14507. break;
  14508. }
  14509. case Js::OpCode::BailOnEarlyExit:
  14510. {
  14511. Assert(this->func->HasFinally() && this->func->DoOptimizeTry());
  14512. break;
  14513. }
  14514. default:
  14515. {
  14516. if(this->currentBlock->loop && !this->IsLoopPrePass())
  14517. {
  14518. return instr;
  14519. }
  14520. break;
  14521. }
  14522. }
  14523. RemoveCodeAfterNoFallthroughInstr(instr);
  14524. }
  14525. return instr;
  14526. }
  14527. void
  14528. GlobOpt::RemoveCodeAfterNoFallthroughInstr(IR::Instr *instr)
  14529. {
  14530. if (instr != this->currentBlock->GetLastInstr())
  14531. {
  14532. // Remove dead code after bailout
  14533. IR::Instr *instrDead = instr->m_next;
  14534. IR::Instr *instrNext;
  14535. for (; instrDead != this->currentBlock->GetLastInstr(); instrDead = instrNext)
  14536. {
  14537. instrNext = instrDead->m_next;
  14538. if (instrNext->m_opcode == Js::OpCode::FunctionExit)
  14539. {
  14540. break;
  14541. }
  14542. this->func->m_fg->RemoveInstr(instrDead, this);
  14543. }
  14544. IR::Instr *instrNextBlock = instrDead->m_next;
  14545. this->func->m_fg->RemoveInstr(instrDead, this);
  14546. this->currentBlock->SetLastInstr(instrNextBlock->m_prev);
  14547. }
  14548. // Cleanup dead successors
  14549. FOREACH_SUCCESSOR_BLOCK_EDITING(deadBlock, this->currentBlock, iter)
  14550. {
  14551. this->currentBlock->RemoveDeadSucc(deadBlock, this->func->m_fg);
  14552. if (this->currentBlock->GetDataUseCount() > 0)
  14553. {
  14554. this->currentBlock->DecrementDataUseCount();
  14555. }
  14556. } NEXT_SUCCESSOR_BLOCK_EDITING;
  14557. }
  14558. void
  14559. GlobOpt::ProcessTryHandler(IR::Instr* instr)
  14560. {
  14561. Assert(instr->m_next->IsLabelInstr() && instr->m_next->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeTry);
  14562. Region* tryRegion = instr->m_next->AsLabelInstr()->GetRegion();
  14563. BVSparse<JitArenaAllocator> * writeThroughSymbolsSet = tryRegion->writeThroughSymbolsSet;
  14564. ToVar(writeThroughSymbolsSet, this->currentBlock);
  14565. }
  14566. bool
  14567. GlobOpt::ProcessExceptionHandlingEdges(IR::Instr* instr)
  14568. {
  14569. Assert(instr->m_opcode == Js::OpCode::BrOnException || instr->m_opcode == Js::OpCode::BrOnNoException);
  14570. if (instr->m_opcode == Js::OpCode::BrOnException)
  14571. {
  14572. if (instr->AsBranchInstr()->GetTarget()->GetRegion()->GetType() == RegionType::RegionTypeCatch)
  14573. {
  14574. // BrOnException was added to model flow from try region to the catch region to assist
  14575. // the backward pass in propagating bytecode upward exposed info from the catch block
  14576. // to the try, and to handle break blocks. Removing it here as it has served its purpose
  14577. // and keeping it around might also have unintended effects while merging block data for
  14578. // the catch block's predecessors.
  14579. // Note that the Deadstore pass will still be able to propagate bytecode upward exposed info
  14580. // because it doesn't skip dead blocks for that.
  14581. this->RemoveFlowEdgeToCatchBlock(instr);
  14582. this->currentBlock->RemoveInstr(instr);
  14583. return true;
  14584. }
  14585. else
  14586. {
  14587. // We add BrOnException from a finally region to early exit, remove that since it has served its purpose
  14588. return this->RemoveFlowEdgeToFinallyOnExceptionBlock(instr);
  14589. }
  14590. }
  14591. else if (instr->m_opcode == Js::OpCode::BrOnNoException)
  14592. {
  14593. if (instr->AsBranchInstr()->GetTarget()->GetRegion()->GetType() == RegionType::RegionTypeCatch)
  14594. {
  14595. this->RemoveFlowEdgeToCatchBlock(instr);
  14596. }
  14597. else
  14598. {
  14599. this->RemoveFlowEdgeToFinallyOnExceptionBlock(instr);
  14600. }
  14601. }
  14602. return false;
  14603. }
  14604. void
  14605. GlobOpt::InsertToVarAtDefInTryRegion(IR::Instr * instr, IR::Opnd * dstOpnd)
  14606. {
  14607. if ((this->currentRegion->GetType() == RegionTypeTry || this->currentRegion->GetType() == RegionTypeFinally) &&
  14608. dstOpnd->IsRegOpnd() && dstOpnd->AsRegOpnd()->m_sym->HasByteCodeRegSlot())
  14609. {
  14610. StackSym * sym = dstOpnd->AsRegOpnd()->m_sym;
  14611. if (sym->IsVar())
  14612. {
  14613. return;
  14614. }
  14615. StackSym * varSym = sym->GetVarEquivSym(nullptr);
  14616. if ((this->currentRegion->GetType() == RegionTypeTry && this->currentRegion->writeThroughSymbolsSet->Test(varSym->m_id)) ||
  14617. ((this->currentRegion->GetType() == RegionTypeFinally && this->currentRegion->GetMatchingTryRegion()->writeThroughSymbolsSet->Test(varSym->m_id))))
  14618. {
  14619. IR::RegOpnd * regOpnd = IR::RegOpnd::New(varSym, IRType::TyVar, instr->m_func);
  14620. this->ToVar(instr->m_next, regOpnd, this->currentBlock, NULL, false);
  14621. }
  14622. }
  14623. }
  14624. void
  14625. GlobOpt::RemoveFlowEdgeToCatchBlock(IR::Instr * instr)
  14626. {
  14627. Assert(instr->IsBranchInstr());
  14628. BasicBlock * catchBlock = nullptr;
  14629. BasicBlock * predBlock = nullptr;
  14630. if (instr->m_opcode == Js::OpCode::BrOnException)
  14631. {
  14632. catchBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock();
  14633. predBlock = this->currentBlock;
  14634. }
  14635. else
  14636. {
  14637. Assert(instr->m_opcode == Js::OpCode::BrOnNoException);
  14638. IR::Instr * nextInstr = instr->GetNextRealInstrOrLabel();
  14639. Assert(nextInstr->IsLabelInstr());
  14640. IR::LabelInstr * nextLabel = nextInstr->AsLabelInstr();
  14641. if (nextLabel->GetRegion() && nextLabel->GetRegion()->GetType() == RegionTypeCatch)
  14642. {
  14643. catchBlock = nextLabel->GetBasicBlock();
  14644. predBlock = this->currentBlock;
  14645. }
  14646. else
  14647. {
  14648. Assert(nextLabel->m_next->IsBranchInstr() && nextLabel->m_next->AsBranchInstr()->IsUnconditional());
  14649. BasicBlock * nextBlock = nextLabel->GetBasicBlock();
  14650. IR::BranchInstr * branchToCatchBlock = nextLabel->m_next->AsBranchInstr();
  14651. IR::LabelInstr * catchBlockLabel = branchToCatchBlock->GetTarget();
  14652. Assert(catchBlockLabel->GetRegion()->GetType() == RegionTypeCatch);
  14653. catchBlock = catchBlockLabel->GetBasicBlock();
  14654. predBlock = nextBlock;
  14655. }
  14656. }
  14657. Assert(catchBlock);
  14658. Assert(predBlock);
  14659. if (this->func->m_fg->FindEdge(predBlock, catchBlock))
  14660. {
  14661. predBlock->RemoveDeadSucc(catchBlock, this->func->m_fg);
  14662. if (predBlock == this->currentBlock)
  14663. {
  14664. predBlock->DecrementDataUseCount();
  14665. }
  14666. }
  14667. }
  14668. bool
  14669. GlobOpt::RemoveFlowEdgeToFinallyOnExceptionBlock(IR::Instr * instr)
  14670. {
  14671. Assert(instr->IsBranchInstr());
  14672. if (instr->m_opcode == Js::OpCode::BrOnNoException && instr->AsBranchInstr()->m_brFinallyToEarlyExit)
  14673. {
  14674. // We add edge from finally to early exit block
  14675. // We should not remove this edge
  14676. // If a loop has continue, and we add edge in finally to continue
  14677. // Break block removal can move all continues inside the loop to branch to the continue added within finally
  14678. // If we get rid of this edge, then loop may loose all backedges
  14679. // Ideally, doing tail duplication before globopt would enable us to remove these edges, but since we do it after globopt, keep it this way for now
  14680. // See test1() in core/test/tryfinallytests.js
  14681. return false;
  14682. }
  14683. BasicBlock * finallyBlock = nullptr;
  14684. BasicBlock * predBlock = nullptr;
  14685. if (instr->m_opcode == Js::OpCode::BrOnException)
  14686. {
  14687. finallyBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock();
  14688. predBlock = this->currentBlock;
  14689. }
  14690. else
  14691. {
  14692. Assert(instr->m_opcode == Js::OpCode::BrOnNoException);
  14693. IR::Instr * nextInstr = instr->GetNextRealInstrOrLabel();
  14694. Assert(nextInstr->IsLabelInstr());
  14695. IR::LabelInstr * nextLabel = nextInstr->AsLabelInstr();
  14696. if (nextLabel->GetRegion() && nextLabel->GetRegion()->GetType() == RegionTypeFinally)
  14697. {
  14698. finallyBlock = nextLabel->GetBasicBlock();
  14699. predBlock = this->currentBlock;
  14700. }
  14701. else
  14702. {
  14703. if (!(nextLabel->m_next->IsBranchInstr() && nextLabel->m_next->AsBranchInstr()->IsUnconditional()))
  14704. {
  14705. return false;
  14706. }
  14707. BasicBlock * nextBlock = nextLabel->GetBasicBlock();
  14708. IR::BranchInstr * branchTofinallyBlockOrEarlyExit = nextLabel->m_next->AsBranchInstr();
  14709. IR::LabelInstr * finallyBlockLabelOrEarlyExitLabel = branchTofinallyBlockOrEarlyExit->GetTarget();
  14710. finallyBlock = finallyBlockLabelOrEarlyExitLabel->GetBasicBlock();
  14711. predBlock = nextBlock;
  14712. }
  14713. }
  14714. Assert(finallyBlock && predBlock);
  14715. if (this->func->m_fg->FindEdge(predBlock, finallyBlock))
  14716. {
  14717. predBlock->RemoveDeadSucc(finallyBlock, this->func->m_fg);
  14718. if (instr->m_opcode == Js::OpCode::BrOnException)
  14719. {
  14720. this->currentBlock->RemoveInstr(instr);
  14721. }
  14722. if (finallyBlock->GetFirstInstr()->AsLabelInstr()->IsUnreferenced())
  14723. {
  14724. // Traverse predBlocks of finallyBlock, if any of the preds have a different region, set m_hasNonBranchRef to true
  14725. // If not, this label can get eliminated and an incorrect region from the predecessor can get propagated in lowered code
  14726. // See test3() in tryfinallytests.js
  14727. Region * finallyRegion = finallyBlock->GetFirstInstr()->AsLabelInstr()->GetRegion();
  14728. FOREACH_PREDECESSOR_BLOCK(pred, finallyBlock)
  14729. {
  14730. Region * predRegion = pred->GetFirstInstr()->AsLabelInstr()->GetRegion();
  14731. if (predRegion != finallyRegion)
  14732. {
  14733. finallyBlock->GetFirstInstr()->AsLabelInstr()->m_hasNonBranchRef = true;
  14734. }
  14735. } NEXT_PREDECESSOR_BLOCK;
  14736. }
  14737. if (predBlock == this->currentBlock)
  14738. {
  14739. predBlock->DecrementDataUseCount();
  14740. }
  14741. }
  14742. return true;
  14743. }
  14744. IR::Instr *
  14745. GlobOpt::OptPeep(IR::Instr *instr, Value *src1Val, Value *src2Val)
  14746. {
  14747. IR::Opnd *dst, *src1, *src2;
  14748. if (this->IsLoopPrePass())
  14749. {
  14750. return instr;
  14751. }
  14752. switch (instr->m_opcode)
  14753. {
  14754. case Js::OpCode::DeadBrEqual:
  14755. case Js::OpCode::DeadBrRelational:
  14756. case Js::OpCode::DeadBrSrEqual:
  14757. src1 = instr->GetSrc1();
  14758. src2 = instr->GetSrc2();
  14759. // These branches were turned into dead branches because they were unnecessary (branch to next, ...).
  14760. // The DeadBr are necessary in case the evaluation of the sources have side-effects.
  14761. // If we know for sure the srcs are primitive or have been type specialized, we don't need these instructions
  14762. if (((src1Val && src1Val->GetValueInfo()->IsPrimitive()) || (src1->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src1->AsRegOpnd()->m_sym))) &&
  14763. ((src2Val && src2Val->GetValueInfo()->IsPrimitive()) || (src2->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src2->AsRegOpnd()->m_sym))))
  14764. {
  14765. this->CaptureByteCodeSymUses(instr);
  14766. instr->m_opcode = Js::OpCode::Nop;
  14767. }
  14768. break;
  14769. case Js::OpCode::DeadBrOnHasProperty:
  14770. src1 = instr->GetSrc1();
  14771. if (((src1Val && src1Val->GetValueInfo()->IsPrimitive()) || (src1->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src1->AsRegOpnd()->m_sym))))
  14772. {
  14773. this->CaptureByteCodeSymUses(instr);
  14774. instr->m_opcode = Js::OpCode::Nop;
  14775. }
  14776. break;
  14777. case Js::OpCode::Ld_A:
  14778. case Js::OpCode::Ld_I4:
  14779. src1 = instr->GetSrc1();
  14780. dst = instr->GetDst();
  14781. if (dst->IsRegOpnd() && dst->IsEqual(src1))
  14782. {
  14783. dst = instr->UnlinkDst();
  14784. if (!dst->GetIsJITOptimizedReg())
  14785. {
  14786. IR::ByteCodeUsesInstr *bytecodeUse = IR::ByteCodeUsesInstr::New(instr);
  14787. bytecodeUse->SetDst(dst);
  14788. instr->InsertAfter(bytecodeUse);
  14789. }
  14790. instr->FreeSrc1();
  14791. instr->m_opcode = Js::OpCode::Nop;
  14792. }
  14793. break;
  14794. }
  14795. return instr;
  14796. }
  14797. void
  14798. GlobOpt::OptimizeIndirUses(IR::IndirOpnd *indirOpnd, IR::Instr * *pInstr, Value **indirIndexValRef)
  14799. {
  14800. IR::Instr * &instr = *pInstr;
  14801. Assert(!indirIndexValRef || !*indirIndexValRef);
  14802. // Update value types and copy-prop the base
  14803. OptSrc(indirOpnd->GetBaseOpnd(), &instr, nullptr, indirOpnd);
  14804. IR::RegOpnd *indexOpnd = indirOpnd->GetIndexOpnd();
  14805. if (!indexOpnd)
  14806. {
  14807. return;
  14808. }
  14809. // Update value types and copy-prop the index
  14810. Value *indexVal = OptSrc(indexOpnd, &instr, nullptr, indirOpnd);
  14811. if(indirIndexValRef)
  14812. {
  14813. *indirIndexValRef = indexVal;
  14814. }
  14815. }
  14816. bool
  14817. GlobOpt::IsPREInstrCandidateLoad(Js::OpCode opcode)
  14818. {
  14819. switch (opcode)
  14820. {
  14821. case Js::OpCode::LdFld:
  14822. case Js::OpCode::LdFldForTypeOf:
  14823. case Js::OpCode::LdRootFld:
  14824. case Js::OpCode::LdRootFldForTypeOf:
  14825. case Js::OpCode::LdMethodFld:
  14826. case Js::OpCode::LdRootMethodFld:
  14827. case Js::OpCode::LdSlot:
  14828. case Js::OpCode::LdSlotArr:
  14829. return true;
  14830. }
  14831. return false;
  14832. }
  14833. bool
  14834. GlobOpt::IsPREInstrSequenceCandidateLoad(Js::OpCode opcode)
  14835. {
  14836. switch (opcode)
  14837. {
  14838. default:
  14839. return IsPREInstrCandidateLoad(opcode);
  14840. case Js::OpCode::Ld_A:
  14841. case Js::OpCode::BytecodeArgOutCapture:
  14842. return true;
  14843. }
  14844. }
  14845. bool
  14846. GlobOpt::IsPREInstrCandidateStore(Js::OpCode opcode)
  14847. {
  14848. switch (opcode)
  14849. {
  14850. case Js::OpCode::StFld:
  14851. case Js::OpCode::StRootFld:
  14852. case Js::OpCode::StSlot:
  14853. return true;
  14854. }
  14855. return false;
  14856. }
  14857. bool
  14858. GlobOpt::ImplicitCallFlagsAllowOpts(Loop *loop)
  14859. {
  14860. return loop->GetImplicitCallFlags() != Js::ImplicitCall_HasNoInfo &&
  14861. (((loop->GetImplicitCallFlags() & ~Js::ImplicitCall_Accessor) | Js::ImplicitCall_None) == Js::ImplicitCall_None);
  14862. }
  14863. bool
  14864. GlobOpt::ImplicitCallFlagsAllowOpts(Func const *func)
  14865. {
  14866. return func->m_fg->implicitCallFlags != Js::ImplicitCall_HasNoInfo &&
  14867. (((func->m_fg->implicitCallFlags & ~Js::ImplicitCall_Accessor) | Js::ImplicitCall_None) == Js::ImplicitCall_None);
  14868. }
  14869. #if DBG_DUMP
  14870. void
  14871. GlobOpt::Dump() const
  14872. {
  14873. this->DumpSymToValueMap();
  14874. }
  14875. void
  14876. GlobOpt::DumpSymToValueMap(BasicBlock const * block) const
  14877. {
  14878. Output::Print(_u("\n*** SymToValueMap ***\n"));
  14879. block->globOptData.DumpSymToValueMap();
  14880. }
  14881. void
  14882. GlobOpt::DumpSymToValueMap() const
  14883. {
  14884. DumpSymToValueMap(this->currentBlock);
  14885. }
  14886. void
  14887. GlobOpt::DumpSymVal(int index)
  14888. {
  14889. SymID id = index;
  14890. extern Func *CurrentFunc;
  14891. Sym *sym = this->func->m_symTable->Find(id);
  14892. AssertMsg(sym, "Sym not found!!!");
  14893. Output::Print(_u("Sym: "));
  14894. sym->Dump();
  14895. Output::Print(_u("\t\tValueNumber: "));
  14896. Value * pValue = CurrentBlockData()->FindValueFromMapDirect(sym->m_id);
  14897. pValue->Dump();
  14898. Output::Print(_u("\n"));
  14899. }
  14900. void
  14901. GlobOpt::Trace(BasicBlock * block, bool before) const
  14902. {
  14903. bool globOptTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::GlobOptPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14904. bool typeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::TypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14905. bool floatTypeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FloatTypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14906. bool fieldCopyPropTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14907. bool objTypeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::ObjTypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14908. bool valueTableTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::ValueTablePhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14909. bool fieldPRETrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldPREPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14910. bool anyTrace = globOptTrace || typeSpecTrace || floatTypeSpecTrace || fieldCopyPropTrace || objTypeSpecTrace || valueTableTrace || fieldPRETrace;
  14911. if (!anyTrace)
  14912. {
  14913. return;
  14914. }
  14915. if (fieldPRETrace && this->IsLoopPrePass())
  14916. {
  14917. if (block->isLoopHeader && before)
  14918. {
  14919. Output::Print(_u("==== Loop Prepass block header #%-3d, Visiting Loop block head #%-3d\n"),
  14920. this->prePassLoop->GetHeadBlock()->GetBlockNum(), block->GetBlockNum());
  14921. }
  14922. }
  14923. if (!typeSpecTrace && !floatTypeSpecTrace && !valueTableTrace && !Js::Configuration::Global.flags.Verbose)
  14924. {
  14925. return;
  14926. }
  14927. if (before)
  14928. {
  14929. Output::Print(_u("========================================================================\n"));
  14930. Output::Print(_u("Begin OptBlock: Block #%-3d"), block->GetBlockNum());
  14931. if (block->loop)
  14932. {
  14933. Output::Print(_u(" Loop block header:%-3d currentLoop block head:%-3d %s"),
  14934. block->loop->GetHeadBlock()->GetBlockNum(),
  14935. this->prePassLoop ? this->prePassLoop->GetHeadBlock()->GetBlockNum() : 0,
  14936. this->IsLoopPrePass() ? _u("PrePass") : _u(""));
  14937. }
  14938. Output::Print(_u("\n"));
  14939. }
  14940. else
  14941. {
  14942. Output::Print(_u("-----------------------------------------------------------------------\n"));
  14943. Output::Print(_u("After OptBlock: Block #%-3d\n"), block->GetBlockNum());
  14944. }
  14945. if ((typeSpecTrace || floatTypeSpecTrace) && !block->globOptData.liveVarSyms->IsEmpty())
  14946. {
  14947. Output::Print(_u(" Live var syms: "));
  14948. block->globOptData.liveVarSyms->Dump();
  14949. }
  14950. if (typeSpecTrace && !block->globOptData.liveInt32Syms->IsEmpty())
  14951. {
  14952. Assert(this->tempBv->IsEmpty());
  14953. this->tempBv->Minus(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms);
  14954. if(!this->tempBv->IsEmpty())
  14955. {
  14956. Output::Print(_u(" Int32 type specialized (lossless) syms: "));
  14957. this->tempBv->Dump();
  14958. }
  14959. this->tempBv->ClearAll();
  14960. if(!block->globOptData.liveLossyInt32Syms->IsEmpty())
  14961. {
  14962. Output::Print(_u(" Int32 converted (lossy) syms: "));
  14963. block->globOptData.liveLossyInt32Syms->Dump();
  14964. }
  14965. }
  14966. if (floatTypeSpecTrace && !block->globOptData.liveFloat64Syms->IsEmpty())
  14967. {
  14968. Output::Print(_u(" Float64 type specialized syms: "));
  14969. block->globOptData.liveFloat64Syms->Dump();
  14970. }
  14971. if ((fieldCopyPropTrace || objTypeSpecTrace) && this->DoFieldCopyProp(block->loop) && !block->globOptData.liveFields->IsEmpty())
  14972. {
  14973. Output::Print(_u(" Live field syms: "));
  14974. block->globOptData.liveFields->Dump();
  14975. }
  14976. if (objTypeSpecTrace || valueTableTrace)
  14977. {
  14978. Output::Print(_u(" Value table:\n"));
  14979. block->globOptData.DumpSymToValueMap();
  14980. }
  14981. if (before)
  14982. {
  14983. Output::Print(_u("-----------------------------------------------------------------------\n")); \
  14984. }
  14985. Output::Flush();
  14986. }
  14987. void
  14988. GlobOpt::TraceSettings() const
  14989. {
  14990. Output::Print(_u("GlobOpt Settings:\r\n"));
  14991. Output::Print(_u(" FloatTypeSpec: %s\r\n"), this->DoFloatTypeSpec() ? _u("enabled") : _u("disabled"));
  14992. Output::Print(_u(" AggressiveIntTypeSpec: %s\r\n"), this->DoAggressiveIntTypeSpec() ? _u("enabled") : _u("disabled"));
  14993. Output::Print(_u(" LossyIntTypeSpec: %s\r\n"), this->DoLossyIntTypeSpec() ? _u("enabled") : _u("disabled"));
  14994. Output::Print(_u(" ArrayCheckHoist: %s\r\n"), this->func->IsArrayCheckHoistDisabled() ? _u("disabled") : _u("enabled"));
  14995. Output::Print(_u(" ImplicitCallFlags: %s\r\n"), Js::DynamicProfileInfo::GetImplicitCallFlagsString(this->func->m_fg->implicitCallFlags));
  14996. for (Loop * loop = this->func->m_fg->loopList; loop != NULL; loop = loop->next)
  14997. {
  14998. Output::Print(_u(" loop: %d, ImplicitCallFlags: %s\r\n"), loop->GetLoopNumber(),
  14999. Js::DynamicProfileInfo::GetImplicitCallFlagsString(loop->GetImplicitCallFlags()));
  15000. }
  15001. Output::Flush();
  15002. }
  15003. #endif // DBG_DUMP
  15004. IR::Instr *
  15005. GlobOpt::TrackMarkTempObject(IR::Instr * instrStart, IR::Instr * instrLast)
  15006. {
  15007. if (!this->func->GetHasMarkTempObjects())
  15008. {
  15009. return instrLast;
  15010. }
  15011. IR::Instr * instr = instrStart;
  15012. IR::Instr * instrEnd = instrLast->m_next;
  15013. IR::Instr * lastInstr = nullptr;
  15014. GlobOptBlockData& globOptData = *CurrentBlockData();
  15015. do
  15016. {
  15017. bool mayNeedBailOnImplicitCallsPreOp = !this->IsLoopPrePass()
  15018. && instr->HasAnyImplicitCalls()
  15019. && globOptData.maybeTempObjectSyms != nullptr;
  15020. if (mayNeedBailOnImplicitCallsPreOp)
  15021. {
  15022. IR::Opnd * src1 = instr->GetSrc1();
  15023. if (src1)
  15024. {
  15025. instr = GenerateBailOutMarkTempObjectIfNeeded(instr, src1, false);
  15026. IR::Opnd * src2 = instr->GetSrc2();
  15027. if (src2)
  15028. {
  15029. instr = GenerateBailOutMarkTempObjectIfNeeded(instr, src2, false);
  15030. }
  15031. }
  15032. }
  15033. IR::Opnd *dst = instr->GetDst();
  15034. if (dst)
  15035. {
  15036. if (dst->IsRegOpnd())
  15037. {
  15038. TrackTempObjectSyms(instr, dst->AsRegOpnd());
  15039. }
  15040. else if (mayNeedBailOnImplicitCallsPreOp)
  15041. {
  15042. instr = GenerateBailOutMarkTempObjectIfNeeded(instr, dst, true);
  15043. }
  15044. }
  15045. lastInstr = instr;
  15046. instr = instr->m_next;
  15047. }
  15048. while (instr != instrEnd);
  15049. return lastInstr;
  15050. }
  15051. void
  15052. GlobOpt::TrackTempObjectSyms(IR::Instr * instr, IR::RegOpnd * opnd)
  15053. {
  15054. // If it is marked as dstIsTempObject, we should have mark temped it, or type specialized it to Ld_I4.
  15055. Assert(!instr->dstIsTempObject || ObjectTempVerify::CanMarkTemp(instr, nullptr));
  15056. GlobOptBlockData& globOptData = *CurrentBlockData();
  15057. bool canStoreTemp = false;
  15058. bool maybeTemp = false;
  15059. if (OpCodeAttr::TempObjectProducing(instr->m_opcode))
  15060. {
  15061. maybeTemp = instr->dstIsTempObject;
  15062. // We have to make sure that lower will always generate code to do stack allocation
  15063. // before we can store any other stack instance onto it. Otherwise, we would not
  15064. // walk object to box the stack property.
  15065. canStoreTemp = instr->dstIsTempObject && ObjectTemp::CanStoreTemp(instr);
  15066. }
  15067. else if (OpCodeAttr::TempObjectTransfer(instr->m_opcode))
  15068. {
  15069. // Need to check both sources, GetNewScObject has two srcs for transfer.
  15070. // No need to get var equiv sym here as transfer of type spec value does not transfer a mark temp object.
  15071. maybeTemp = globOptData.maybeTempObjectSyms && (
  15072. (instr->GetSrc1()->IsRegOpnd() && globOptData.maybeTempObjectSyms->Test(instr->GetSrc1()->AsRegOpnd()->m_sym->m_id))
  15073. || (instr->GetSrc2() && instr->GetSrc2()->IsRegOpnd() && globOptData.maybeTempObjectSyms->Test(instr->GetSrc2()->AsRegOpnd()->m_sym->m_id)));
  15074. canStoreTemp = globOptData.canStoreTempObjectSyms && (
  15075. (instr->GetSrc1()->IsRegOpnd() && globOptData.canStoreTempObjectSyms->Test(instr->GetSrc1()->AsRegOpnd()->m_sym->m_id))
  15076. && (!instr->GetSrc2() || (instr->GetSrc2()->IsRegOpnd() && globOptData.canStoreTempObjectSyms->Test(instr->GetSrc2()->AsRegOpnd()->m_sym->m_id))));
  15077. AssertOrFailFast(!canStoreTemp || instr->dstIsTempObject);
  15078. AssertOrFailFast(!maybeTemp || instr->dstIsTempObject);
  15079. }
  15080. // Need to get the var equiv sym as assignment of type specialized sym kill the var sym value anyway.
  15081. StackSym * sym = opnd->m_sym;
  15082. if (!sym->IsVar())
  15083. {
  15084. sym = sym->GetVarEquivSym(nullptr);
  15085. if (sym == nullptr)
  15086. {
  15087. return;
  15088. }
  15089. }
  15090. SymID symId = sym->m_id;
  15091. if (maybeTemp)
  15092. {
  15093. // Only var sym should be temp objects
  15094. Assert(opnd->m_sym == sym);
  15095. if (globOptData.maybeTempObjectSyms == nullptr)
  15096. {
  15097. globOptData.maybeTempObjectSyms = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  15098. }
  15099. globOptData.maybeTempObjectSyms->Set(symId);
  15100. if (canStoreTemp)
  15101. {
  15102. if (instr->m_opcode == Js::OpCode::NewScObjectLiteral && !this->IsLoopPrePass())
  15103. {
  15104. // For object literal, we install the final type up front.
  15105. // If there are bailout before we finish initializing all the fields, we need to
  15106. // zero out the rest if we stack allocate the literal, so that the boxing would not
  15107. // try to box trash pointer in the properties.
  15108. // Although object Literal initialization can be done lexically, BailOnNoProfile may cause some path
  15109. // to disappear. Do it is flow base make it easier to stop propagate those entries.
  15110. IR::IntConstOpnd * propertyArrayIdOpnd = instr->GetSrc1()->AsIntConstOpnd();
  15111. const Js::PropertyIdArray * propIds = instr->m_func->GetJITFunctionBody()->ReadPropertyIdArrayFromAuxData(propertyArrayIdOpnd->AsUint32());
  15112. // Duplicates are removed by parser
  15113. Assert(!propIds->hadDuplicates);
  15114. if (globOptData.stackLiteralInitFldDataMap == nullptr)
  15115. {
  15116. globOptData.stackLiteralInitFldDataMap = JitAnew(alloc, StackLiteralInitFldDataMap, alloc);
  15117. }
  15118. else
  15119. {
  15120. Assert(!globOptData.stackLiteralInitFldDataMap->ContainsKey(sym));
  15121. }
  15122. StackLiteralInitFldData data = { propIds, 0};
  15123. globOptData.stackLiteralInitFldDataMap->AddNew(sym, data);
  15124. }
  15125. if (globOptData.canStoreTempObjectSyms == nullptr)
  15126. {
  15127. globOptData.canStoreTempObjectSyms = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  15128. }
  15129. globOptData.canStoreTempObjectSyms->Set(symId);
  15130. }
  15131. else if (globOptData.canStoreTempObjectSyms)
  15132. {
  15133. globOptData.canStoreTempObjectSyms->Clear(symId);
  15134. }
  15135. }
  15136. else
  15137. {
  15138. Assert(!canStoreTemp);
  15139. if (globOptData.maybeTempObjectSyms)
  15140. {
  15141. if (globOptData.canStoreTempObjectSyms)
  15142. {
  15143. globOptData.canStoreTempObjectSyms->Clear(symId);
  15144. }
  15145. globOptData.maybeTempObjectSyms->Clear(symId);
  15146. }
  15147. else
  15148. {
  15149. Assert(!globOptData.canStoreTempObjectSyms);
  15150. }
  15151. // The symbol is being assigned to, the sym shouldn't still be in the stackLiteralInitFldDataMap
  15152. Assert(this->IsLoopPrePass() ||
  15153. globOptData.stackLiteralInitFldDataMap == nullptr
  15154. || globOptData.stackLiteralInitFldDataMap->Count() == 0
  15155. || !globOptData.stackLiteralInitFldDataMap->ContainsKey(sym));
  15156. }
  15157. }
  15158. IR::Instr *
  15159. GlobOpt::GenerateBailOutMarkTempObjectIfNeeded(IR::Instr * instr, IR::Opnd * opnd, bool isDst)
  15160. {
  15161. Assert(opnd);
  15162. Assert(isDst == (opnd == instr->GetDst()));
  15163. Assert(opnd != instr->GetDst() || !opnd->IsRegOpnd());
  15164. Assert(!this->IsLoopPrePass());
  15165. Assert(instr->HasAnyImplicitCalls());
  15166. // Only dst reg opnd opcode or ArgOut_A should have dstIsTempObject marked
  15167. Assert(!isDst || !instr->dstIsTempObject || instr->m_opcode == Js::OpCode::ArgOut_A);
  15168. // Post-op implicit call shouldn't have installed yet
  15169. Assert(!instr->HasBailOutInfo() || (instr->GetBailOutKind() & IR::BailOutKindBits) != IR::BailOutOnImplicitCalls);
  15170. GlobOptBlockData& globOptData = *CurrentBlockData();
  15171. Assert(globOptData.maybeTempObjectSyms != nullptr);
  15172. IR::PropertySymOpnd * propertySymOpnd = nullptr;
  15173. StackSym * stackSym = ObjectTemp::GetStackSym(opnd, &propertySymOpnd);
  15174. // It is okay to not get the var equiv sym here, as use of a type specialized sym is not use of the temp object
  15175. // so no need to add mark temp bailout.
  15176. // TempObjectSysm doesn't contain any type spec sym, so we will get false here for all type spec sym.
  15177. if (stackSym && globOptData.maybeTempObjectSyms->Test(stackSym->m_id))
  15178. {
  15179. if (instr->HasBailOutInfo())
  15180. {
  15181. instr->SetBailOutKind(instr->GetBailOutKind() | IR::BailOutMarkTempObject);
  15182. instr->GetBailOutInfo()->canDeadStore = false;
  15183. }
  15184. else
  15185. {
  15186. // On insert the pre op bailout if it is not Direct field access do nothing, don't check the dst yet.
  15187. // SetTypeCheckBailout will clear this out if it is direct field access.
  15188. if (isDst
  15189. || (instr->m_opcode == Js::OpCode::FromVar && !opnd->GetValueType().IsPrimitive())
  15190. || propertySymOpnd == nullptr
  15191. || !propertySymOpnd->IsTypeCheckProtected())
  15192. {
  15193. this->GenerateBailAtOperation(&instr, IR::BailOutMarkTempObject);
  15194. instr->GetBailOutInfo()->canDeadStore = false;
  15195. }
  15196. else if (propertySymOpnd->MayHaveImplicitCall())
  15197. {
  15198. this->GenerateBailAtOperation(&instr, IR::BailOutMarkTempObject);
  15199. }
  15200. }
  15201. if (!opnd->IsRegOpnd() && (!isDst || (globOptData.canStoreTempObjectSyms && globOptData.canStoreTempObjectSyms->Test(stackSym->m_id))))
  15202. {
  15203. // If this opnd is a dst, that means that the object pointer is a stack object,
  15204. // and we can store temp object/number on it.
  15205. // If the opnd is a src, that means that the object pointer may be a stack object
  15206. // so the load may be a temp object/number and we need to track its use.
  15207. // Don't mark start of indir as can store temp, because we don't actually know
  15208. // what it is assigning to.
  15209. if (!isDst || !opnd->IsIndirOpnd())
  15210. {
  15211. opnd->SetCanStoreTemp();
  15212. }
  15213. if (propertySymOpnd)
  15214. {
  15215. // Track initfld of stack literals
  15216. if (isDst && instr->m_opcode == Js::OpCode::InitFld)
  15217. {
  15218. const Js::PropertyId propertyId = propertySymOpnd->m_sym->AsPropertySym()->m_propertyId;
  15219. // We don't need to track numeric properties init
  15220. if (!this->func->GetThreadContextInfo()->IsNumericProperty(propertyId))
  15221. {
  15222. DebugOnly(bool found = false);
  15223. globOptData.stackLiteralInitFldDataMap->RemoveIf(stackSym,
  15224. [&](StackSym * key, StackLiteralInitFldData & data)
  15225. {
  15226. DebugOnly(found = true);
  15227. Assert(key == stackSym);
  15228. Assert(data.currentInitFldCount < data.propIds->count);
  15229. if (data.propIds->elements[data.currentInitFldCount] != propertyId)
  15230. {
  15231. #if DBG
  15232. bool duplicate = false;
  15233. for (uint i = 0; i < data.currentInitFldCount; i++)
  15234. {
  15235. if (data.propIds->elements[i] == propertyId)
  15236. {
  15237. duplicate = true;
  15238. break;
  15239. }
  15240. }
  15241. Assert(duplicate);
  15242. #endif
  15243. // duplicate initialization
  15244. return false;
  15245. }
  15246. bool finished = (++data.currentInitFldCount == data.propIds->count);
  15247. #if DBG
  15248. if (finished)
  15249. {
  15250. // We can still track the finished stack literal InitFld lexically.
  15251. this->finishedStackLiteralInitFld->Set(stackSym->m_id);
  15252. }
  15253. #endif
  15254. return finished;
  15255. });
  15256. // We might still see InitFld even we have finished with all the property Id because
  15257. // of duplicate entries at the end
  15258. Assert(found || finishedStackLiteralInitFld->Test(stackSym->m_id));
  15259. }
  15260. }
  15261. }
  15262. }
  15263. }
  15264. return instr;
  15265. }
  15266. LoopCount *
  15267. GlobOpt::GetOrGenerateLoopCountForMemOp(Loop *loop)
  15268. {
  15269. LoopCount *loopCount = loop->loopCount;
  15270. if (loopCount && !loopCount->HasGeneratedLoopCountSym())
  15271. {
  15272. Assert(loop->bailOutInfo);
  15273. EnsureBailTarget(loop);
  15274. GenerateLoopCountPlusOne(loop, loopCount);
  15275. }
  15276. return loopCount;
  15277. }
  15278. IR::Opnd *
  15279. GlobOpt::GenerateInductionVariableChangeForMemOp(Loop *loop, byte unroll, IR::Instr *insertBeforeInstr)
  15280. {
  15281. AssertOrFailFast(unroll != Js::Constants::InvalidLoopUnrollFactor);
  15282. LoopCount *loopCount = loop->loopCount;
  15283. IR::Opnd *sizeOpnd = nullptr;
  15284. Assert(loopCount);
  15285. Assert(loop->memOpInfo->inductionVariableOpndPerUnrollMap);
  15286. if (loop->memOpInfo->inductionVariableOpndPerUnrollMap->TryGetValue(unroll, &sizeOpnd))
  15287. {
  15288. return sizeOpnd;
  15289. }
  15290. Func *localFunc = loop->GetFunc();
  15291. const auto InsertInstr = [&](IR::Instr *instr)
  15292. {
  15293. if (insertBeforeInstr == nullptr)
  15294. {
  15295. loop->landingPad->InsertAfter(instr);
  15296. }
  15297. else
  15298. {
  15299. insertBeforeInstr->InsertBefore(instr);
  15300. }
  15301. };
  15302. if (loopCount->LoopCountMinusOneSym())
  15303. {
  15304. IRType type = loopCount->LoopCountSym()->GetType();
  15305. // Loop count is off by one, so add one
  15306. IR::RegOpnd *loopCountOpnd = IR::RegOpnd::New(loopCount->LoopCountSym(), type, localFunc);
  15307. sizeOpnd = loopCountOpnd;
  15308. if (unroll != 1)
  15309. {
  15310. sizeOpnd = IR::RegOpnd::New(TyUint32, this->func);
  15311. IR::Opnd *unrollOpnd = IR::IntConstOpnd::New(unroll, type, localFunc);
  15312. IR::Instr *inductionChangeMultiplier = IR::Instr::New(
  15313. Js::OpCode::Mul_I4, sizeOpnd, loopCountOpnd, unrollOpnd, localFunc);
  15314. InsertInstr(inductionChangeMultiplier);
  15315. inductionChangeMultiplier->ConvertToBailOutInstr(loop->bailOutInfo, IR::BailOutOnOverflow);
  15316. }
  15317. }
  15318. else
  15319. {
  15320. int32 loopCountMinusOnePlusOne;
  15321. int32 size;
  15322. if (Int32Math::Add(loopCount->LoopCountMinusOneConstantValue(), 1, &loopCountMinusOnePlusOne) ||
  15323. Int32Math::Mul(loopCountMinusOnePlusOne, unroll, &size))
  15324. {
  15325. throw Js::RejitException(RejitReason::MemOpDisabled);
  15326. }
  15327. Assert(size > 0);
  15328. sizeOpnd = IR::IntConstOpnd::New(size, IRType::TyUint32, localFunc);
  15329. }
  15330. loop->memOpInfo->inductionVariableOpndPerUnrollMap->Add(unroll, sizeOpnd);
  15331. return sizeOpnd;
  15332. }
  15333. IR::RegOpnd*
  15334. GlobOpt::GenerateStartIndexOpndForMemop(Loop *loop, IR::Opnd *indexOpnd, IR::Opnd *sizeOpnd, bool isInductionVariableChangeIncremental, bool bIndexAlreadyChanged, IR::Instr *insertBeforeInstr)
  15335. {
  15336. IR::RegOpnd *startIndexOpnd = nullptr;
  15337. Func *localFunc = loop->GetFunc();
  15338. IRType type = indexOpnd->GetType();
  15339. const int cacheIndex = ((int)isInductionVariableChangeIncremental << 1) | (int)bIndexAlreadyChanged;
  15340. if (loop->memOpInfo->startIndexOpndCache[cacheIndex])
  15341. {
  15342. return loop->memOpInfo->startIndexOpndCache[cacheIndex];
  15343. }
  15344. const auto InsertInstr = [&](IR::Instr *instr)
  15345. {
  15346. if (insertBeforeInstr == nullptr)
  15347. {
  15348. loop->landingPad->InsertAfter(instr);
  15349. }
  15350. else
  15351. {
  15352. insertBeforeInstr->InsertBefore(instr);
  15353. }
  15354. };
  15355. startIndexOpnd = IR::RegOpnd::New(type, localFunc);
  15356. // If the 2 are different we can simply use indexOpnd
  15357. if (isInductionVariableChangeIncremental != bIndexAlreadyChanged)
  15358. {
  15359. InsertInstr(IR::Instr::New(Js::OpCode::Ld_A,
  15360. startIndexOpnd,
  15361. indexOpnd,
  15362. localFunc));
  15363. }
  15364. else
  15365. {
  15366. // Otherwise add 1 to it
  15367. InsertInstr(IR::Instr::New(Js::OpCode::Add_I4,
  15368. startIndexOpnd,
  15369. indexOpnd,
  15370. IR::IntConstOpnd::New(1, type, localFunc, true),
  15371. localFunc));
  15372. }
  15373. if (!isInductionVariableChangeIncremental)
  15374. {
  15375. InsertInstr(IR::Instr::New(Js::OpCode::Sub_I4,
  15376. startIndexOpnd,
  15377. startIndexOpnd,
  15378. sizeOpnd,
  15379. localFunc));
  15380. }
  15381. loop->memOpInfo->startIndexOpndCache[cacheIndex] = startIndexOpnd;
  15382. return startIndexOpnd;
  15383. }
  15384. IR::Instr*
  15385. GlobOpt::FindUpperBoundsCheckInstr(IR::Instr* fromInstr)
  15386. {
  15387. IR::Instr *upperBoundCheck = fromInstr;
  15388. do
  15389. {
  15390. upperBoundCheck = upperBoundCheck->m_prev;
  15391. Assert(upperBoundCheck);
  15392. Assert(!upperBoundCheck->IsLabelInstr());
  15393. } while (upperBoundCheck->m_opcode != Js::OpCode::BoundCheck);
  15394. return upperBoundCheck;
  15395. }
  15396. IR::Instr*
  15397. GlobOpt::FindArraySegmentLoadInstr(IR::Instr* fromInstr)
  15398. {
  15399. IR::Instr *headSegmentLengthLoad = fromInstr;
  15400. do
  15401. {
  15402. headSegmentLengthLoad = headSegmentLengthLoad->m_prev;
  15403. Assert(headSegmentLengthLoad);
  15404. Assert(!headSegmentLengthLoad->IsLabelInstr());
  15405. } while (headSegmentLengthLoad->m_opcode != Js::OpCode::LdIndir);
  15406. return headSegmentLengthLoad;
  15407. }
  15408. void
  15409. GlobOpt::RemoveMemOpSrcInstr(IR::Instr* memopInstr, IR::Instr* srcInstr, BasicBlock* block)
  15410. {
  15411. Assert(srcInstr && (srcInstr->m_opcode == Js::OpCode::LdElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A_Strict));
  15412. Assert(memopInstr && (memopInstr->m_opcode == Js::OpCode::Memcopy || memopInstr->m_opcode == Js::OpCode::Memset));
  15413. Assert(block);
  15414. const bool isDst = srcInstr->m_opcode == Js::OpCode::StElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A_Strict;
  15415. IR::RegOpnd* opnd = (isDst ? memopInstr->GetDst() : memopInstr->GetSrc1())->AsIndirOpnd()->GetBaseOpnd();
  15416. IR::ArrayRegOpnd* arrayOpnd = opnd->IsArrayRegOpnd() ? opnd->AsArrayRegOpnd() : nullptr;
  15417. IR::Instr* topInstr = srcInstr;
  15418. if (srcInstr->extractedUpperBoundCheckWithoutHoisting)
  15419. {
  15420. IR::Instr *upperBoundCheck = FindUpperBoundsCheckInstr(srcInstr);
  15421. Assert(upperBoundCheck && upperBoundCheck != srcInstr);
  15422. topInstr = upperBoundCheck;
  15423. }
  15424. if (srcInstr->loadedArrayHeadSegmentLength && arrayOpnd && arrayOpnd->HeadSegmentLengthSym())
  15425. {
  15426. IR::Instr *arrayLoadSegmentHeadLength = FindArraySegmentLoadInstr(topInstr);
  15427. Assert(arrayLoadSegmentHeadLength);
  15428. topInstr = arrayLoadSegmentHeadLength;
  15429. arrayOpnd->RemoveHeadSegmentLengthSym();
  15430. }
  15431. if (srcInstr->loadedArrayHeadSegment && arrayOpnd && arrayOpnd->HeadSegmentSym())
  15432. {
  15433. IR::Instr *arrayLoadSegmentHead = FindArraySegmentLoadInstr(topInstr);
  15434. Assert(arrayLoadSegmentHead);
  15435. topInstr = arrayLoadSegmentHead;
  15436. arrayOpnd->RemoveHeadSegmentSym();
  15437. }
  15438. // If no bounds check are present, simply look up for instruction added for instrumentation
  15439. if(topInstr == srcInstr)
  15440. {
  15441. bool checkPrev = true;
  15442. while (checkPrev)
  15443. {
  15444. switch (topInstr->m_prev->m_opcode)
  15445. {
  15446. case Js::OpCode::BailOnNotArray:
  15447. case Js::OpCode::NoImplicitCallUses:
  15448. case Js::OpCode::ByteCodeUses:
  15449. topInstr = topInstr->m_prev;
  15450. checkPrev = !!topInstr->m_prev;
  15451. break;
  15452. default:
  15453. checkPrev = false;
  15454. break;
  15455. }
  15456. }
  15457. }
  15458. while (topInstr != srcInstr)
  15459. {
  15460. IR::Instr* removeInstr = topInstr;
  15461. topInstr = topInstr->m_next;
  15462. Assert(
  15463. removeInstr->m_opcode == Js::OpCode::BailOnNotArray ||
  15464. removeInstr->m_opcode == Js::OpCode::NoImplicitCallUses ||
  15465. removeInstr->m_opcode == Js::OpCode::ByteCodeUses ||
  15466. removeInstr->m_opcode == Js::OpCode::LdIndir ||
  15467. removeInstr->m_opcode == Js::OpCode::BoundCheck
  15468. );
  15469. if (removeInstr->m_opcode != Js::OpCode::ByteCodeUses)
  15470. {
  15471. block->RemoveInstr(removeInstr);
  15472. }
  15473. }
  15474. this->ConvertToByteCodeUses(srcInstr);
  15475. }
  15476. void
  15477. GlobOpt::GetMemOpSrcInfo(Loop* loop, IR::Instr* instr, IR::RegOpnd*& base, IR::RegOpnd*& index, IRType& arrayType)
  15478. {
  15479. Assert(instr && (instr->m_opcode == Js::OpCode::LdElemI_A || instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict));
  15480. IR::Opnd* arrayOpnd = instr->m_opcode == Js::OpCode::LdElemI_A ? instr->GetSrc1() : instr->GetDst();
  15481. Assert(arrayOpnd->IsIndirOpnd());
  15482. IR::IndirOpnd* indirArrayOpnd = arrayOpnd->AsIndirOpnd();
  15483. IR::RegOpnd* baseOpnd = (IR::RegOpnd*)indirArrayOpnd->GetBaseOpnd();
  15484. IR::RegOpnd* indexOpnd = (IR::RegOpnd*)indirArrayOpnd->GetIndexOpnd();
  15485. Assert(baseOpnd);
  15486. Assert(indexOpnd);
  15487. // Process Out Params
  15488. base = baseOpnd;
  15489. index = indexOpnd;
  15490. arrayType = indirArrayOpnd->GetType();
  15491. }
  15492. void
  15493. GlobOpt::EmitMemop(Loop * loop, LoopCount *loopCount, const MemOpEmitData* emitData)
  15494. {
  15495. Assert(emitData);
  15496. Assert(emitData->candidate);
  15497. Assert(emitData->stElemInstr);
  15498. Assert(emitData->stElemInstr->m_opcode == Js::OpCode::StElemI_A || emitData->stElemInstr->m_opcode == Js::OpCode::StElemI_A_Strict);
  15499. IR::BailOutKind bailOutKind = emitData->bailOutKind;
  15500. const byte unroll = emitData->inductionVar.unroll;
  15501. Assert(unroll == 1);
  15502. const bool isInductionVariableChangeIncremental = emitData->inductionVar.isIncremental;
  15503. const bool bIndexAlreadyChanged = emitData->candidate->bIndexAlreadyChanged;
  15504. IR::RegOpnd *baseOpnd = nullptr;
  15505. IR::RegOpnd *indexOpnd = nullptr;
  15506. IRType dstType;
  15507. GetMemOpSrcInfo(loop, emitData->stElemInstr, baseOpnd, indexOpnd, dstType);
  15508. Func *localFunc = loop->GetFunc();
  15509. // Handle bailout info
  15510. EnsureBailTarget(loop);
  15511. Assert(bailOutKind != IR::BailOutInvalid);
  15512. // Keep only Array bits bailOuts. Consider handling these bailouts instead of simply ignoring them
  15513. bailOutKind &= IR::BailOutForArrayBits;
  15514. // Add our custom bailout to handle Op_MemCopy return value.
  15515. bailOutKind |= IR::BailOutOnMemOpError;
  15516. BailOutInfo *const bailOutInfo = loop->bailOutInfo;
  15517. Assert(bailOutInfo);
  15518. IR::Instr *insertBeforeInstr = bailOutInfo->bailOutInstr;
  15519. Assert(insertBeforeInstr);
  15520. IR::Opnd *sizeOpnd = GenerateInductionVariableChangeForMemOp(loop, unroll, insertBeforeInstr);
  15521. IR::RegOpnd *startIndexOpnd = GenerateStartIndexOpndForMemop(loop, indexOpnd, sizeOpnd, isInductionVariableChangeIncremental, bIndexAlreadyChanged, insertBeforeInstr);
  15522. IR::IndirOpnd* dstOpnd = IR::IndirOpnd::New(baseOpnd, startIndexOpnd, dstType, localFunc);
  15523. IR::Opnd *src1;
  15524. const bool isMemset = emitData->candidate->IsMemSet();
  15525. // Get the source according to the memop type
  15526. if (isMemset)
  15527. {
  15528. MemSetEmitData* data = (MemSetEmitData*)emitData;
  15529. const Loop::MemSetCandidate* candidate = data->candidate->AsMemSet();
  15530. if (candidate->srcSym)
  15531. {
  15532. IR::RegOpnd* regSrc = IR::RegOpnd::New(candidate->srcSym, candidate->srcSym->GetType(), func);
  15533. regSrc->SetIsJITOptimizedReg(true);
  15534. src1 = regSrc;
  15535. }
  15536. else
  15537. {
  15538. src1 = IR::AddrOpnd::New(candidate->constant.ToVar(localFunc), IR::AddrOpndKindConstantAddress, localFunc);
  15539. }
  15540. }
  15541. else
  15542. {
  15543. Assert(emitData->candidate->IsMemCopy());
  15544. MemCopyEmitData* data = (MemCopyEmitData*)emitData;
  15545. Assert(data->ldElemInstr);
  15546. Assert(data->ldElemInstr->m_opcode == Js::OpCode::LdElemI_A);
  15547. IR::RegOpnd *srcBaseOpnd = nullptr;
  15548. IR::RegOpnd *srcIndexOpnd = nullptr;
  15549. IRType srcType;
  15550. GetMemOpSrcInfo(loop, data->ldElemInstr, srcBaseOpnd, srcIndexOpnd, srcType);
  15551. Assert(GetVarSymID(srcIndexOpnd->GetStackSym()) == GetVarSymID(indexOpnd->GetStackSym()));
  15552. src1 = IR::IndirOpnd::New(srcBaseOpnd, startIndexOpnd, srcType, localFunc);
  15553. }
  15554. // Generate memcopy
  15555. IR::Instr* memopInstr = IR::BailOutInstr::New(isMemset ? Js::OpCode::Memset : Js::OpCode::Memcopy, bailOutKind, bailOutInfo, localFunc);
  15556. memopInstr->SetDst(dstOpnd);
  15557. memopInstr->SetSrc1(src1);
  15558. memopInstr->SetSrc2(sizeOpnd);
  15559. insertBeforeInstr->InsertBefore(memopInstr);
  15560. loop->memOpInfo->instr = memopInstr;
  15561. #if DBG_DUMP
  15562. if (DO_MEMOP_TRACE())
  15563. {
  15564. char valueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  15565. baseOpnd->GetValueType().ToString(valueTypeStr);
  15566. const int loopCountBufSize = 16;
  15567. char16 loopCountBuf[loopCountBufSize];
  15568. if (loopCount->LoopCountMinusOneSym())
  15569. {
  15570. swprintf_s(loopCountBuf, _u("s%u"), loopCount->LoopCountMinusOneSym()->m_id);
  15571. }
  15572. else
  15573. {
  15574. swprintf_s(loopCountBuf, _u("%u"), loopCount->LoopCountMinusOneConstantValue() + 1);
  15575. }
  15576. if (isMemset)
  15577. {
  15578. const Loop::MemSetCandidate* candidate = emitData->candidate->AsMemSet();
  15579. const int constBufSize = 32;
  15580. char16 constBuf[constBufSize];
  15581. if (candidate->srcSym)
  15582. {
  15583. swprintf_s(constBuf, _u("s%u"), candidate->srcSym->m_id);
  15584. }
  15585. else
  15586. {
  15587. switch (candidate->constant.type)
  15588. {
  15589. case TyInt8:
  15590. case TyInt16:
  15591. case TyInt32:
  15592. case TyInt64:
  15593. swprintf_s(constBuf, sizeof(IntConstType) == 8 ? _u("%lld") : _u("%d"), candidate->constant.u.intConst.value);
  15594. break;
  15595. case TyFloat32:
  15596. case TyFloat64:
  15597. swprintf_s(constBuf, _u("%.4f"), candidate->constant.u.floatConst.value);
  15598. break;
  15599. case TyVar:
  15600. swprintf_s(constBuf, sizeof(Js::Var) == 8 ? _u("0x%.16llX") : _u("0x%.8X"), candidate->constant.u.varConst.value);
  15601. break;
  15602. default:
  15603. AssertMsg(false, "Unsupported constant type");
  15604. swprintf_s(constBuf, _u("Unknown"));
  15605. break;
  15606. }
  15607. }
  15608. TRACE_MEMOP_PHASE(MemSet, loop, emitData->stElemInstr,
  15609. _u("ValueType: %S, Base: s%u, Index: s%u, Constant: %s, LoopCount: %s, IsIndexChangedBeforeUse: %d"),
  15610. valueTypeStr,
  15611. candidate->base,
  15612. candidate->index,
  15613. constBuf,
  15614. loopCountBuf,
  15615. bIndexAlreadyChanged);
  15616. }
  15617. else
  15618. {
  15619. const Loop::MemCopyCandidate* candidate = emitData->candidate->AsMemCopy();
  15620. TRACE_MEMOP_PHASE(MemCopy, loop, emitData->stElemInstr,
  15621. _u("ValueType: %S, StBase: s%u, Index: s%u, LdBase: s%u, LoopCount: %s, IsIndexChangedBeforeUse: %d"),
  15622. valueTypeStr,
  15623. candidate->base,
  15624. candidate->index,
  15625. candidate->ldBase,
  15626. loopCountBuf,
  15627. bIndexAlreadyChanged);
  15628. }
  15629. }
  15630. #endif
  15631. Assert(noImplicitCallUsesToInsert->Count() == 0);
  15632. bool isLikelyJsArray;
  15633. if (emitData->stElemInstr->GetDst()->IsIndirOpnd())
  15634. {
  15635. baseOpnd = emitData->stElemInstr->GetDst()->AsIndirOpnd()->GetBaseOpnd();
  15636. isLikelyJsArray = baseOpnd->GetValueType().IsLikelyArrayOrObjectWithArray();
  15637. ProcessNoImplicitCallArrayUses(baseOpnd, baseOpnd->IsArrayRegOpnd() ? baseOpnd->AsArrayRegOpnd() : nullptr, emitData->stElemInstr, isLikelyJsArray, true);
  15638. }
  15639. RemoveMemOpSrcInstr(memopInstr, emitData->stElemInstr, emitData->block);
  15640. if (!isMemset)
  15641. {
  15642. IR::Instr* ldElemInstr = ((MemCopyEmitData*)emitData)->ldElemInstr;
  15643. if (ldElemInstr->GetSrc1()->IsIndirOpnd())
  15644. {
  15645. baseOpnd = ldElemInstr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd();
  15646. isLikelyJsArray = baseOpnd->GetValueType().IsLikelyArrayOrObjectWithArray();
  15647. ProcessNoImplicitCallArrayUses(baseOpnd, baseOpnd->IsArrayRegOpnd() ? baseOpnd->AsArrayRegOpnd() : nullptr, ldElemInstr, isLikelyJsArray, true);
  15648. }
  15649. RemoveMemOpSrcInstr(memopInstr, ldElemInstr, emitData->block);
  15650. }
  15651. InsertNoImplicitCallUses(memopInstr);
  15652. noImplicitCallUsesToInsert->Clear();
  15653. }
  15654. bool
  15655. GlobOpt::InspectInstrForMemSetCandidate(Loop* loop, IR::Instr* instr, MemSetEmitData* emitData, bool& errorInInstr)
  15656. {
  15657. Assert(emitData && emitData->candidate && emitData->candidate->IsMemSet());
  15658. Loop::MemSetCandidate* candidate = (Loop::MemSetCandidate*)emitData->candidate;
  15659. if (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict)
  15660. {
  15661. if (instr->GetDst()->IsIndirOpnd()
  15662. && (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->base)
  15663. && (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index)
  15664. )
  15665. {
  15666. Assert(instr->IsProfiledInstr());
  15667. emitData->stElemInstr = instr;
  15668. emitData->bailOutKind = instr->GetBailOutKind();
  15669. return true;
  15670. }
  15671. TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Orphan StElemI_A detected"));
  15672. errorInInstr = true;
  15673. }
  15674. else if (instr->m_opcode == Js::OpCode::LdElemI_A)
  15675. {
  15676. TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Orphan LdElemI_A detected"));
  15677. errorInInstr = true;
  15678. }
  15679. return false;
  15680. }
  15681. bool
  15682. GlobOpt::InspectInstrForMemCopyCandidate(Loop* loop, IR::Instr* instr, MemCopyEmitData* emitData, bool& errorInInstr)
  15683. {
  15684. Assert(emitData && emitData->candidate && emitData->candidate->IsMemCopy());
  15685. Loop::MemCopyCandidate* candidate = (Loop::MemCopyCandidate*)emitData->candidate;
  15686. if (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict)
  15687. {
  15688. if (
  15689. instr->GetDst()->IsIndirOpnd() &&
  15690. (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->base) &&
  15691. (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index)
  15692. )
  15693. {
  15694. Assert(instr->IsProfiledInstr());
  15695. emitData->stElemInstr = instr;
  15696. emitData->bailOutKind = instr->GetBailOutKind();
  15697. // Still need to find the LdElem
  15698. return false;
  15699. }
  15700. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Orphan StElemI_A detected"));
  15701. errorInInstr = true;
  15702. }
  15703. else if (instr->m_opcode == Js::OpCode::LdElemI_A)
  15704. {
  15705. if (
  15706. emitData->stElemInstr &&
  15707. instr->GetSrc1()->IsIndirOpnd() &&
  15708. (GetVarSymID(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->ldBase) &&
  15709. (GetVarSymID(instr->GetSrc1()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index)
  15710. )
  15711. {
  15712. Assert(instr->IsProfiledInstr());
  15713. emitData->ldElemInstr = instr;
  15714. ValueType stValueType = emitData->stElemInstr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType();
  15715. ValueType ldValueType = emitData->ldElemInstr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType();
  15716. if (stValueType != ldValueType)
  15717. {
  15718. #if DBG_DUMP
  15719. char16 stValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  15720. stValueType.ToString(stValueTypeStr);
  15721. char16 ldValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  15722. ldValueType.ToString(ldValueTypeStr);
  15723. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("for mismatch in Load(%s) and Store(%s) value type"), ldValueTypeStr, stValueTypeStr);
  15724. #endif
  15725. errorInInstr = true;
  15726. return false;
  15727. }
  15728. // We found both instruction for this candidate
  15729. return true;
  15730. }
  15731. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Orphan LdElemI_A detected"));
  15732. errorInInstr = true;
  15733. }
  15734. return false;
  15735. }
  15736. // The caller is responsible to free the memory allocated between inOrderEmitData[iEmitData -> end]
  15737. bool
  15738. GlobOpt::ValidateMemOpCandidates(Loop * loop, _Out_writes_(iEmitData) MemOpEmitData** inOrderEmitData, int& iEmitData)
  15739. {
  15740. AnalysisAssert(iEmitData == (int)loop->memOpInfo->candidates->Count());
  15741. // We iterate over the second block of the loop only. MemOp Works only if the loop has exactly 2 blocks
  15742. Assert(loop->blockList.HasTwo());
  15743. Loop::MemOpList::Iterator iter(loop->memOpInfo->candidates);
  15744. BasicBlock* bblock = loop->blockList.Head()->next;
  15745. Loop::MemOpCandidate* candidate = nullptr;
  15746. MemOpEmitData* emitData = nullptr;
  15747. // Iterate backward because the list of candidate is reversed
  15748. FOREACH_INSTR_BACKWARD_IN_BLOCK(instr, bblock)
  15749. {
  15750. if (!candidate)
  15751. {
  15752. // Time to check next candidate
  15753. if (!iter.Next())
  15754. {
  15755. // We have been through the whole list of candidates, finish
  15756. break;
  15757. }
  15758. candidate = iter.Data();
  15759. if (!candidate)
  15760. {
  15761. continue;
  15762. }
  15763. // Common check for memset and memcopy
  15764. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 };
  15765. // Get the inductionVariable changeInfo
  15766. if (!loop->memOpInfo->inductionVariableChangeInfoMap->TryGetValue(candidate->index, &inductionVariableChangeInfo))
  15767. {
  15768. TRACE_MEMOP_VERBOSE(loop, nullptr, _u("MemOp skipped (s%d): no induction variable"), candidate->base);
  15769. return false;
  15770. }
  15771. if (inductionVariableChangeInfo.unroll != candidate->count)
  15772. {
  15773. TRACE_MEMOP_VERBOSE(loop, nullptr, _u("MemOp skipped (s%d): not matching unroll count"), candidate->base);
  15774. return false;
  15775. }
  15776. if (candidate->IsMemSet())
  15777. {
  15778. Assert(!PHASE_OFF(Js::MemSetPhase, this->func));
  15779. emitData = JitAnew(this->alloc, MemSetEmitData);
  15780. }
  15781. else
  15782. {
  15783. Assert(!PHASE_OFF(Js::MemCopyPhase, this->func));
  15784. // Specific check for memcopy
  15785. Assert(candidate->IsMemCopy());
  15786. Loop::MemCopyCandidate* memcopyCandidate = candidate->AsMemCopy();
  15787. if (memcopyCandidate->base == Js::Constants::InvalidSymID
  15788. || memcopyCandidate->ldBase == Js::Constants::InvalidSymID
  15789. || (memcopyCandidate->ldCount != memcopyCandidate->count))
  15790. {
  15791. TRACE_MEMOP_PHASE(MemCopy, loop, nullptr, _u("(s%d): not matching ldElem and stElem"), candidate->base);
  15792. return false;
  15793. }
  15794. emitData = JitAnew(this->alloc, MemCopyEmitData);
  15795. }
  15796. Assert(emitData);
  15797. emitData->block = bblock;
  15798. emitData->inductionVar = inductionVariableChangeInfo;
  15799. emitData->candidate = candidate;
  15800. }
  15801. bool errorInInstr = false;
  15802. bool candidateFound = candidate->IsMemSet() ?
  15803. InspectInstrForMemSetCandidate(loop, instr, (MemSetEmitData*)emitData, errorInInstr)
  15804. : InspectInstrForMemCopyCandidate(loop, instr, (MemCopyEmitData*)emitData, errorInInstr);
  15805. if (errorInInstr)
  15806. {
  15807. JitAdelete(this->alloc, emitData);
  15808. return false;
  15809. }
  15810. if (candidateFound)
  15811. {
  15812. AnalysisAssert(iEmitData > 0);
  15813. if (iEmitData == 0)
  15814. {
  15815. // Explicit for OACR
  15816. break;
  15817. }
  15818. inOrderEmitData[--iEmitData] = emitData;
  15819. candidate = nullptr;
  15820. emitData = nullptr;
  15821. }
  15822. } NEXT_INSTR_BACKWARD_IN_BLOCK;
  15823. if (iter.IsValid())
  15824. {
  15825. TRACE_MEMOP(loop, nullptr, _u("Candidates not found in loop while validating"));
  15826. return false;
  15827. }
  15828. return true;
  15829. }
  15830. void
  15831. GlobOpt::ProcessMemOp()
  15832. {
  15833. FOREACH_LOOP_IN_FUNC_EDITING(loop, this->func)
  15834. {
  15835. if (HasMemOp(loop))
  15836. {
  15837. const int candidateCount = loop->memOpInfo->candidates->Count();
  15838. Assert(candidateCount > 0);
  15839. LoopCount * loopCount = GetOrGenerateLoopCountForMemOp(loop);
  15840. // If loopCount is not available we can not continue with memop
  15841. if (!loopCount || !(loopCount->LoopCountMinusOneSym() || loopCount->LoopCountMinusOneConstantValue()))
  15842. {
  15843. TRACE_MEMOP(loop, nullptr, _u("MemOp skipped for no loop count"));
  15844. loop->doMemOp = false;
  15845. loop->memOpInfo->candidates->Clear();
  15846. continue;
  15847. }
  15848. // The list is reversed, check them and place them in order in the following array
  15849. MemOpEmitData** inOrderCandidates = JitAnewArray(this->alloc, MemOpEmitData*, candidateCount);
  15850. int i = candidateCount;
  15851. if (ValidateMemOpCandidates(loop, inOrderCandidates, i))
  15852. {
  15853. Assert(i == 0);
  15854. // Process the valid MemOp candidate in order.
  15855. for (; i < candidateCount; ++i)
  15856. {
  15857. // Emit
  15858. EmitMemop(loop, loopCount, inOrderCandidates[i]);
  15859. JitAdelete(this->alloc, inOrderCandidates[i]);
  15860. }
  15861. }
  15862. else
  15863. {
  15864. Assert(i != 0);
  15865. for (; i < candidateCount; ++i)
  15866. {
  15867. JitAdelete(this->alloc, inOrderCandidates[i]);
  15868. }
  15869. // One of the memop candidates did not validate. Do not emit for this loop.
  15870. loop->doMemOp = false;
  15871. loop->memOpInfo->candidates->Clear();
  15872. }
  15873. // Free memory
  15874. JitAdeleteArray(this->alloc, candidateCount, inOrderCandidates);
  15875. }
  15876. } NEXT_LOOP_EDITING;
  15877. }
  15878. void GlobOpt::PRE::FieldPRE(Loop *loop)
  15879. {
  15880. JitArenaAllocator *alloc = this->globOpt->tempAlloc;
  15881. this->FindPossiblePRECandidates(loop, alloc);
  15882. this->PreloadPRECandidates(loop);
  15883. this->RemoveOverlyOptimisticInitialValues(loop);
  15884. }
  15885. bool
  15886. GlobOpt::PRE::InsertSymDefinitionInLandingPad(StackSym * sym, Loop * loop, Sym ** objPtrCopyPropSym)
  15887. {
  15888. Assert(sym->IsSingleDef());
  15889. IR::Instr * symDefInstr = sym->GetInstrDef();
  15890. if (!GlobOpt::IsPREInstrSequenceCandidateLoad(symDefInstr->m_opcode))
  15891. {
  15892. return false;
  15893. }
  15894. IR::Opnd * symDefInstrSrc1 = symDefInstr->GetSrc1();
  15895. if (symDefInstrSrc1->IsSymOpnd())
  15896. {
  15897. Assert(symDefInstrSrc1->AsSymOpnd()->m_sym->IsPropertySym());
  15898. // $L1
  15899. // T1 = o.x (v1|T3)
  15900. // T2 = T1.y (v2|T4) <-- T1 is not live in the loop landing pad
  15901. // jmp $L1
  15902. // Trying to make T1 live in the landing pad
  15903. // o.x
  15904. PropertySym* propSym = symDefInstrSrc1->AsSymOpnd()->m_sym->AsPropertySym();
  15905. if (candidates->candidatesBv->Test(propSym->m_id))
  15906. {
  15907. // If propsym is a PRE candidate, then it must have had the same value on all back edges.
  15908. // So, just look up the value on one of the back edges.
  15909. BasicBlock* loopTail = loop->GetAnyTailBlock();
  15910. Value * valueOnBackEdge = loopTail->globOptData.FindValue(propSym);
  15911. // If o.x is not invariant in the loop, we can't use the preloaded value of o.x.y in the landing pad
  15912. Value * valueInLandingPad = loop->landingPad->globOptData.FindValue(propSym);
  15913. if (valueOnBackEdge->GetValueNumber() != valueInLandingPad->GetValueNumber())
  15914. {
  15915. return false;
  15916. }
  15917. *objPtrCopyPropSym = valueOnBackEdge->GetValueInfo()->GetSymStore();
  15918. if (candidates->candidatesToProcess->Test(propSym->m_id))
  15919. {
  15920. GlobHashBucket bucket;
  15921. bucket.element = valueOnBackEdge;
  15922. bucket.value = propSym;
  15923. if (!PreloadPRECandidate(loop, &bucket))
  15924. {
  15925. return false;
  15926. }
  15927. Assert(!candidates->candidatesToProcess->Test(propSym->m_id));
  15928. Assert(loop->landingPad->globOptData.IsLive(valueOnBackEdge->GetValueInfo()->GetSymStore()));
  15929. // Inserted T3 = o.x
  15930. // Now, we want to
  15931. // 1. Insert T1 = o.x
  15932. // 2. Insert T4 = T1.y
  15933. // 3. Indentify T3 as the objptr copy prop sym for T1, and make T3.y live on the back-edges
  15934. // #1 is done next. #2 and #3 are done as part of preloading T1.y
  15935. // Insert T1 = o.x
  15936. if (!InsertPropertySymPreloadInLandingPad(symDefInstr->Copy(), loop, propSym))
  15937. {
  15938. return false;
  15939. }
  15940. return true;
  15941. }
  15942. else
  15943. {
  15944. // o.x was already processed as a PRE candidate. If we were successful in preloading o.x,
  15945. // we can now insert T1 = o.x
  15946. if (loop->landingPad->globOptData.IsLive(*objPtrCopyPropSym))
  15947. {
  15948. // insert T1 = o.x
  15949. if (!InsertPropertySymPreloadInLandingPad(symDefInstr->Copy(), loop, propSym))
  15950. {
  15951. return false;
  15952. }
  15953. return true;
  15954. }
  15955. else
  15956. {
  15957. return false;
  15958. }
  15959. }
  15960. }
  15961. else
  15962. {
  15963. return false;
  15964. }
  15965. }
  15966. else if (symDefInstrSrc1->IsRegOpnd())
  15967. {
  15968. // T2 = T1
  15969. // T3 = T2.y
  15970. // trying to insert def of T2
  15971. // T1
  15972. StackSym * symDefInstrSrc1Sym = symDefInstrSrc1->AsRegOpnd()->GetStackSym();
  15973. if (!loop->landingPad->globOptData.IsLive(symDefInstrSrc1Sym))
  15974. {
  15975. if (symDefInstrSrc1Sym->IsSingleDef())
  15976. {
  15977. if (!InsertSymDefinitionInLandingPad(symDefInstrSrc1Sym, loop, objPtrCopyPropSym))
  15978. {
  15979. return false;
  15980. }
  15981. }
  15982. }
  15983. else
  15984. {
  15985. *objPtrCopyPropSym = symDefInstrSrc1Sym;
  15986. }
  15987. if (!(OpCodeAttr::TempNumberTransfer(symDefInstr->m_opcode) && OpCodeAttr::TempObjectTransfer(symDefInstr->m_opcode)))
  15988. {
  15989. *objPtrCopyPropSym = sym;
  15990. }
  15991. IR::Instr * instr = symDefInstr->Copy();
  15992. if (instr->m_opcode == Js::OpCode::BytecodeArgOutCapture)
  15993. {
  15994. instr->m_opcode = Js::OpCode::Ld_A;
  15995. }
  15996. InsertInstrInLandingPad(instr, loop);
  15997. return true;
  15998. }
  15999. else
  16000. {
  16001. return false;
  16002. }
  16003. }
  16004. void
  16005. GlobOpt::PRE::InsertInstrInLandingPad(IR::Instr * instr, Loop * loop)
  16006. {
  16007. instr->GetSrc1()->SetIsJITOptimizedReg(true);
  16008. if (instr->GetDst())
  16009. {
  16010. instr->GetDst()->SetIsJITOptimizedReg(true);
  16011. loop->landingPad->globOptData.liveVarSyms->Set(instr->GetDst()->GetStackSym()->m_id);
  16012. }
  16013. if (instr->HasAnyImplicitCalls())
  16014. {
  16015. IR::Instr * bailInstr = globOpt->EnsureDisableImplicitCallRegion(loop);
  16016. bailInstr->InsertBefore(instr);
  16017. }
  16018. else if (loop->endDisableImplicitCall)
  16019. {
  16020. loop->endDisableImplicitCall->InsertBefore(instr);
  16021. }
  16022. else
  16023. {
  16024. loop->landingPad->InsertAfter(instr);
  16025. }
  16026. instr->ClearByteCodeOffset();
  16027. instr->SetByteCodeOffset(loop->landingPad->GetFirstInstr());
  16028. }
  16029. IR::Instr *
  16030. GlobOpt::PRE::InsertPropertySymPreloadInLandingPad(IR::Instr * ldInstr, Loop * loop, PropertySym * propertySym)
  16031. {
  16032. IR::SymOpnd *ldSrc = ldInstr->GetSrc1()->AsSymOpnd();
  16033. if (ldSrc->m_sym != propertySym)
  16034. {
  16035. // It's possible that the property syms are different but have equivalent objPtrs. Verify their values.
  16036. Value *val1 = globOpt->CurrentBlockData()->FindValue(ldSrc->m_sym->AsPropertySym()->m_stackSym);
  16037. Value *val2 = globOpt->CurrentBlockData()->FindValue(propertySym->m_stackSym);
  16038. if (!val1 || !val2 || val1->GetValueNumber() != val2->GetValueNumber())
  16039. {
  16040. return nullptr;
  16041. }
  16042. }
  16043. // Consider: Shouldn't be necessary once we have copy-prop in prepass...
  16044. ldInstr->GetSrc1()->AsSymOpnd()->m_sym = propertySym;
  16045. ldSrc = ldInstr->GetSrc1()->AsSymOpnd();
  16046. if (ldSrc->IsPropertySymOpnd())
  16047. {
  16048. IR::PropertySymOpnd *propSymOpnd = ldSrc->AsPropertySymOpnd();
  16049. IR::PropertySymOpnd *newPropSymOpnd;
  16050. newPropSymOpnd = propSymOpnd->AsPropertySymOpnd()->CopyWithoutFlowSensitiveInfo(this->globOpt->func);
  16051. ldInstr->ReplaceSrc1(newPropSymOpnd);
  16052. }
  16053. if (ldInstr->GetDst())
  16054. {
  16055. loop->landingPad->globOptData.liveVarSyms->Set(ldInstr->GetDst()->GetStackSym()->m_id);
  16056. }
  16057. InsertInstrInLandingPad(ldInstr, loop);
  16058. return ldInstr;
  16059. }
  16060. void
  16061. GlobOpt::PRE::MakePropertySymLiveOnBackEdges(PropertySym * propertySym, Loop * loop, Value * valueToAdd)
  16062. {
  16063. BasicBlock * loopHeader = loop->GetHeadBlock();
  16064. FOREACH_PREDECESSOR_BLOCK(blockPred, loopHeader)
  16065. {
  16066. if (!loop->IsDescendentOrSelf(blockPred->loop))
  16067. {
  16068. // Not a loop back-edge
  16069. continue;
  16070. }
  16071. // Insert it in the value table
  16072. blockPred->globOptData.SetValue(valueToAdd, propertySym);
  16073. // Make it a live field
  16074. blockPred->globOptData.liveFields->Set(propertySym->m_id);
  16075. } NEXT_PREDECESSOR_BLOCK;
  16076. }
  16077. void GlobOpt::PRE::RemoveOverlyOptimisticInitialValues(Loop * loop)
  16078. {
  16079. BasicBlock * landingPad = loop->landingPad;
  16080. // For a property sym whose obj ptr sym wasn't live in the landing pad, we can optimistically (if the obj ptr sym was
  16081. // single def) insert an initial value in the landing pad, with the hope that PRE could make the obj ptr sym live.
  16082. // But, if PRE couldn't make the obj ptr sym live, we need to clear the value for the property sym from the landing pad
  16083. for (auto it = loop->initialValueFieldMap.GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  16084. {
  16085. PropertySym * propertySym = it.CurrentKey();
  16086. StackSym * objPtrSym = propertySym->m_stackSym;
  16087. if (!landingPad->globOptData.IsLive(objPtrSym))
  16088. {
  16089. Value * landingPadPropSymValue = landingPad->globOptData.FindValue(propertySym);
  16090. Assert(landingPadPropSymValue);
  16091. Assert(landingPadPropSymValue->GetValueNumber() == it.CurrentValue()->GetValueNumber());
  16092. Assert(landingPadPropSymValue->GetValueInfo()->GetSymStore() == propertySym);
  16093. landingPad->globOptData.ClearSymValue(propertySym);
  16094. it.RemoveCurrent();
  16095. }
  16096. }
  16097. }
  16098. #if DBG_DUMP
  16099. void GlobOpt::PRE::TraceFailedPreloadInLandingPad(const Loop *const loop, PropertySym * propertySym, const char16* reason) const
  16100. {
  16101. if (PHASE_TRACE(Js::FieldPREPhase, this->globOpt->func))
  16102. {
  16103. int32 propertyId = propertySym->m_propertyId;
  16104. SymID objectSymId = propertySym->m_stackSym->m_id;
  16105. char16 propSymStr[32];
  16106. switch (propertySym->m_fieldKind)
  16107. {
  16108. case PropertyKindData:
  16109. if (JITManager::GetJITManager()->IsOOPJITEnabled())
  16110. {
  16111. swprintf_s(propSymStr, _u("s%d->#%d"), objectSymId, propertyId);
  16112. }
  16113. else
  16114. {
  16115. Js::PropertyRecord const* fieldName = propertySym->m_func->GetInProcThreadContext()->GetPropertyRecord(propertyId);
  16116. swprintf_s(propSymStr, _u("s%d->%s"), objectSymId, fieldName->GetBuffer());
  16117. }
  16118. break;
  16119. case PropertyKindSlots:
  16120. case PropertyKindSlotArray:
  16121. swprintf_s(propSymStr, _u("s%d[%d]"), objectSymId, propertyId);
  16122. break;
  16123. case PropertyKindLocalSlots:
  16124. swprintf_s(propSymStr, _u("s%dl[%d]"), objectSymId, propertyId);
  16125. break;
  16126. default:
  16127. AssertMsg(0, "Unknown field kind");
  16128. break;
  16129. }
  16130. Output::Print(_u("** TRACE: Field PRE: "));
  16131. this->globOpt->func->DumpFullFunctionName();
  16132. Output::Print(_u(": Failed to pre-load (%s) in landing pad of loop #%d. Reason: %s "), propSymStr, loop->GetLoopNumber(), reason);
  16133. Output::Print(_u("\n"));
  16134. }
  16135. }
  16136. #endif