GlobOpt.cpp 644 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761107621076310764107651076610767107681076910770107711077210773107741077510776107771077810779107801078110782107831078410785107861078710788107891079010791107921079310794107951079610797107981079910800108011080210803108041080510806108071080810809108101081110812108131081410815108161081710818108191082010821108221082310824108251082610827108281082910830108311083210833108341083510836108371083810839108401084110842108431084410845108461084710848108491085010851108521085310854108551085610857108581085910860108611086210863108641086510866108671086810869108701087110872108731087410875108761087710878108791088010881108821088310884108851088610887108881088910890108911089210893108941089510896108971089810899109001090110902109031090410905109061090710908109091091010911109121091310914109151091610917109181091910920109211092210923109241092510926109271092810929109301093110932109331093410935109361093710938109391094010941109421094310944109451094610947109481094910950109511095210953109541095510956109571095810959109601096110962109631096410965109661096710968109691097010971109721097310974109751097610977109781097910980109811098210983109841098510986109871098810989109901099110992109931099410995109961099710998109991100011001110021100311004110051100611007110081100911010110111101211013110141101511016110171101811019110201102111022110231102411025110261102711028110291103011031110321103311034110351103611037110381103911040110411104211043110441104511046110471104811049110501105111052110531105411055110561105711058110591106011061110621106311064110651106611067110681106911070110711107211073110741107511076110771107811079110801108111082110831108411085110861108711088110891109011091110921109311094110951109611097110981109911100111011110211103111041110511106111071110811109111101111111112111131111411115111161111711118111191112011121111221112311124111251112611127111281112911130111311113211133111341113511136111371113811139111401114111142111431114411145111461114711148111491115011151111521115311154111551115611157111581115911160111611116211163111641116511166111671116811169111701117111172111731117411175111761117711178111791118011181111821118311184111851118611187111881118911190111911119211193111941119511196111971119811199112001120111202112031120411205112061120711208112091121011211112121121311214112151121611217112181121911220112211122211223112241122511226112271122811229112301123111232112331123411235112361123711238112391124011241112421124311244112451124611247112481124911250112511125211253112541125511256112571125811259112601126111262112631126411265112661126711268112691127011271112721127311274112751127611277112781127911280112811128211283112841128511286112871128811289112901129111292112931129411295112961129711298112991130011301113021130311304113051130611307113081130911310113111131211313113141131511316113171131811319113201132111322113231132411325113261132711328113291133011331113321133311334113351133611337113381133911340113411134211343113441134511346113471134811349113501135111352113531135411355113561135711358113591136011361113621136311364113651136611367113681136911370113711137211373113741137511376113771137811379113801138111382113831138411385113861138711388113891139011391113921139311394113951139611397113981139911400114011140211403114041140511406114071140811409114101141111412114131141411415114161141711418114191142011421114221142311424114251142611427114281142911430114311143211433114341143511436114371143811439114401144111442114431144411445114461144711448114491145011451114521145311454114551145611457114581145911460114611146211463114641146511466114671146811469114701147111472114731147411475114761147711478114791148011481114821148311484114851148611487114881148911490114911149211493114941149511496114971149811499115001150111502115031150411505115061150711508115091151011511115121151311514115151151611517115181151911520115211152211523115241152511526115271152811529115301153111532115331153411535115361153711538115391154011541115421154311544115451154611547115481154911550115511155211553115541155511556115571155811559115601156111562115631156411565115661156711568115691157011571115721157311574115751157611577115781157911580115811158211583115841158511586115871158811589115901159111592115931159411595115961159711598115991160011601116021160311604116051160611607116081160911610116111161211613116141161511616116171161811619116201162111622116231162411625116261162711628116291163011631116321163311634116351163611637116381163911640116411164211643116441164511646116471164811649116501165111652116531165411655116561165711658116591166011661116621166311664116651166611667116681166911670116711167211673116741167511676116771167811679116801168111682116831168411685116861168711688116891169011691116921169311694116951169611697116981169911700117011170211703117041170511706117071170811709117101171111712117131171411715117161171711718117191172011721117221172311724117251172611727117281172911730117311173211733117341173511736117371173811739117401174111742117431174411745117461174711748117491175011751117521175311754117551175611757117581175911760117611176211763117641176511766117671176811769117701177111772117731177411775117761177711778117791178011781117821178311784117851178611787117881178911790117911179211793117941179511796117971179811799118001180111802118031180411805118061180711808118091181011811118121181311814118151181611817118181181911820118211182211823118241182511826118271182811829118301183111832118331183411835118361183711838118391184011841118421184311844118451184611847118481184911850118511185211853118541185511856118571185811859118601186111862118631186411865118661186711868118691187011871118721187311874118751187611877118781187911880118811188211883118841188511886118871188811889118901189111892118931189411895118961189711898118991190011901119021190311904119051190611907119081190911910119111191211913119141191511916119171191811919119201192111922119231192411925119261192711928119291193011931119321193311934119351193611937119381193911940119411194211943119441194511946119471194811949119501195111952119531195411955119561195711958119591196011961119621196311964119651196611967119681196911970119711197211973119741197511976119771197811979119801198111982119831198411985119861198711988119891199011991119921199311994119951199611997119981199912000120011200212003120041200512006120071200812009120101201112012120131201412015120161201712018120191202012021120221202312024120251202612027120281202912030120311203212033120341203512036120371203812039120401204112042120431204412045120461204712048120491205012051120521205312054120551205612057120581205912060120611206212063120641206512066120671206812069120701207112072120731207412075120761207712078120791208012081120821208312084120851208612087120881208912090120911209212093120941209512096120971209812099121001210112102121031210412105121061210712108121091211012111121121211312114121151211612117121181211912120121211212212123121241212512126121271212812129121301213112132121331213412135121361213712138121391214012141121421214312144121451214612147121481214912150121511215212153121541215512156121571215812159121601216112162121631216412165121661216712168121691217012171121721217312174121751217612177121781217912180121811218212183121841218512186121871218812189121901219112192121931219412195121961219712198121991220012201122021220312204122051220612207122081220912210122111221212213122141221512216122171221812219122201222112222122231222412225122261222712228122291223012231122321223312234122351223612237122381223912240122411224212243122441224512246122471224812249122501225112252122531225412255122561225712258122591226012261122621226312264122651226612267122681226912270122711227212273122741227512276122771227812279122801228112282122831228412285122861228712288122891229012291122921229312294122951229612297122981229912300123011230212303123041230512306123071230812309123101231112312123131231412315123161231712318123191232012321123221232312324123251232612327123281232912330123311233212333123341233512336123371233812339123401234112342123431234412345123461234712348123491235012351123521235312354123551235612357123581235912360123611236212363123641236512366123671236812369123701237112372123731237412375123761237712378123791238012381123821238312384123851238612387123881238912390123911239212393123941239512396123971239812399124001240112402124031240412405124061240712408124091241012411124121241312414124151241612417124181241912420124211242212423124241242512426124271242812429124301243112432124331243412435124361243712438124391244012441124421244312444124451244612447124481244912450124511245212453124541245512456124571245812459124601246112462124631246412465124661246712468124691247012471124721247312474124751247612477124781247912480124811248212483124841248512486124871248812489124901249112492124931249412495124961249712498124991250012501125021250312504125051250612507125081250912510125111251212513125141251512516125171251812519125201252112522125231252412525125261252712528125291253012531125321253312534125351253612537125381253912540125411254212543125441254512546125471254812549125501255112552125531255412555125561255712558125591256012561125621256312564125651256612567125681256912570125711257212573125741257512576125771257812579125801258112582125831258412585125861258712588125891259012591125921259312594125951259612597125981259912600126011260212603126041260512606126071260812609126101261112612126131261412615126161261712618126191262012621126221262312624126251262612627126281262912630126311263212633126341263512636126371263812639126401264112642126431264412645126461264712648126491265012651126521265312654126551265612657126581265912660126611266212663126641266512666126671266812669126701267112672126731267412675126761267712678126791268012681126821268312684126851268612687126881268912690126911269212693126941269512696126971269812699127001270112702127031270412705127061270712708127091271012711127121271312714127151271612717127181271912720127211272212723127241272512726127271272812729127301273112732127331273412735127361273712738127391274012741127421274312744127451274612747127481274912750127511275212753127541275512756127571275812759127601276112762127631276412765127661276712768127691277012771127721277312774127751277612777127781277912780127811278212783127841278512786127871278812789127901279112792127931279412795127961279712798127991280012801128021280312804128051280612807128081280912810128111281212813128141281512816128171281812819128201282112822128231282412825128261282712828128291283012831128321283312834128351283612837128381283912840128411284212843128441284512846128471284812849128501285112852128531285412855128561285712858128591286012861128621286312864128651286612867128681286912870128711287212873128741287512876128771287812879128801288112882128831288412885128861288712888128891289012891128921289312894128951289612897128981289912900129011290212903129041290512906129071290812909129101291112912129131291412915129161291712918129191292012921129221292312924129251292612927129281292912930129311293212933129341293512936129371293812939129401294112942129431294412945129461294712948129491295012951129521295312954129551295612957129581295912960129611296212963129641296512966129671296812969129701297112972129731297412975129761297712978129791298012981129821298312984129851298612987129881298912990129911299212993129941299512996129971299812999130001300113002130031300413005130061300713008130091301013011130121301313014130151301613017130181301913020130211302213023130241302513026130271302813029130301303113032130331303413035130361303713038130391304013041130421304313044130451304613047130481304913050130511305213053130541305513056130571305813059130601306113062130631306413065130661306713068130691307013071130721307313074130751307613077130781307913080130811308213083130841308513086130871308813089130901309113092130931309413095130961309713098130991310013101131021310313104131051310613107131081310913110131111311213113131141311513116131171311813119131201312113122131231312413125131261312713128131291313013131131321313313134131351313613137131381313913140131411314213143131441314513146131471314813149131501315113152131531315413155131561315713158131591316013161131621316313164131651316613167131681316913170131711317213173131741317513176131771317813179131801318113182131831318413185131861318713188131891319013191131921319313194131951319613197131981319913200132011320213203132041320513206132071320813209132101321113212132131321413215132161321713218132191322013221132221322313224132251322613227132281322913230132311323213233132341323513236132371323813239132401324113242132431324413245132461324713248132491325013251132521325313254132551325613257132581325913260132611326213263132641326513266132671326813269132701327113272132731327413275132761327713278132791328013281132821328313284132851328613287132881328913290132911329213293132941329513296132971329813299133001330113302133031330413305133061330713308133091331013311133121331313314133151331613317133181331913320133211332213323133241332513326133271332813329133301333113332133331333413335133361333713338133391334013341133421334313344133451334613347133481334913350133511335213353133541335513356133571335813359133601336113362133631336413365133661336713368133691337013371133721337313374133751337613377133781337913380133811338213383133841338513386133871338813389133901339113392133931339413395133961339713398133991340013401134021340313404134051340613407134081340913410134111341213413134141341513416134171341813419134201342113422134231342413425134261342713428134291343013431134321343313434134351343613437134381343913440134411344213443134441344513446134471344813449134501345113452134531345413455134561345713458134591346013461134621346313464134651346613467134681346913470134711347213473134741347513476134771347813479134801348113482134831348413485134861348713488134891349013491134921349313494134951349613497134981349913500135011350213503135041350513506135071350813509135101351113512135131351413515135161351713518135191352013521135221352313524135251352613527135281352913530135311353213533135341353513536135371353813539135401354113542135431354413545135461354713548135491355013551135521355313554135551355613557135581355913560135611356213563135641356513566135671356813569135701357113572135731357413575135761357713578135791358013581135821358313584135851358613587135881358913590135911359213593135941359513596135971359813599136001360113602136031360413605136061360713608136091361013611136121361313614136151361613617136181361913620136211362213623136241362513626136271362813629136301363113632136331363413635136361363713638136391364013641136421364313644136451364613647136481364913650136511365213653136541365513656136571365813659136601366113662136631366413665136661366713668136691367013671136721367313674136751367613677136781367913680136811368213683136841368513686136871368813689136901369113692136931369413695136961369713698136991370013701137021370313704137051370613707137081370913710137111371213713137141371513716137171371813719137201372113722137231372413725137261372713728137291373013731137321373313734137351373613737137381373913740137411374213743137441374513746137471374813749137501375113752137531375413755137561375713758137591376013761137621376313764137651376613767137681376913770137711377213773137741377513776137771377813779137801378113782137831378413785137861378713788137891379013791137921379313794137951379613797137981379913800138011380213803138041380513806138071380813809138101381113812138131381413815138161381713818138191382013821138221382313824138251382613827138281382913830138311383213833138341383513836138371383813839138401384113842138431384413845138461384713848138491385013851138521385313854138551385613857138581385913860138611386213863138641386513866138671386813869138701387113872138731387413875138761387713878138791388013881138821388313884138851388613887138881388913890138911389213893138941389513896138971389813899139001390113902139031390413905139061390713908139091391013911139121391313914139151391613917139181391913920139211392213923139241392513926139271392813929139301393113932139331393413935139361393713938139391394013941139421394313944139451394613947139481394913950139511395213953139541395513956139571395813959139601396113962139631396413965139661396713968139691397013971139721397313974139751397613977139781397913980139811398213983139841398513986139871398813989139901399113992139931399413995139961399713998139991400014001140021400314004140051400614007140081400914010140111401214013140141401514016140171401814019140201402114022140231402414025140261402714028140291403014031140321403314034140351403614037140381403914040140411404214043140441404514046140471404814049140501405114052140531405414055140561405714058140591406014061140621406314064140651406614067140681406914070140711407214073140741407514076140771407814079140801408114082140831408414085140861408714088140891409014091140921409314094140951409614097140981409914100141011410214103141041410514106141071410814109141101411114112141131411414115141161411714118141191412014121141221412314124141251412614127141281412914130141311413214133141341413514136141371413814139141401414114142141431414414145141461414714148141491415014151141521415314154141551415614157141581415914160141611416214163141641416514166141671416814169141701417114172141731417414175141761417714178141791418014181141821418314184141851418614187141881418914190141911419214193141941419514196141971419814199142001420114202142031420414205142061420714208142091421014211142121421314214142151421614217142181421914220142211422214223142241422514226142271422814229142301423114232142331423414235142361423714238142391424014241142421424314244142451424614247142481424914250142511425214253142541425514256142571425814259142601426114262142631426414265142661426714268142691427014271142721427314274142751427614277142781427914280142811428214283142841428514286142871428814289142901429114292142931429414295142961429714298142991430014301143021430314304143051430614307143081430914310143111431214313143141431514316143171431814319143201432114322143231432414325143261432714328143291433014331143321433314334143351433614337143381433914340143411434214343143441434514346143471434814349143501435114352143531435414355143561435714358143591436014361143621436314364143651436614367143681436914370143711437214373143741437514376143771437814379143801438114382143831438414385143861438714388143891439014391143921439314394143951439614397143981439914400144011440214403144041440514406144071440814409144101441114412144131441414415144161441714418144191442014421144221442314424144251442614427144281442914430144311443214433144341443514436144371443814439144401444114442144431444414445144461444714448144491445014451144521445314454144551445614457144581445914460144611446214463144641446514466144671446814469144701447114472144731447414475144761447714478144791448014481144821448314484144851448614487144881448914490144911449214493144941449514496144971449814499145001450114502145031450414505145061450714508145091451014511145121451314514145151451614517145181451914520145211452214523145241452514526145271452814529145301453114532145331453414535145361453714538145391454014541145421454314544145451454614547145481454914550145511455214553145541455514556145571455814559145601456114562145631456414565145661456714568145691457014571145721457314574145751457614577145781457914580145811458214583145841458514586145871458814589145901459114592145931459414595145961459714598145991460014601146021460314604146051460614607146081460914610146111461214613146141461514616146171461814619146201462114622146231462414625146261462714628146291463014631146321463314634146351463614637146381463914640146411464214643146441464514646146471464814649146501465114652146531465414655146561465714658146591466014661146621466314664146651466614667146681466914670146711467214673146741467514676146771467814679146801468114682146831468414685146861468714688146891469014691146921469314694146951469614697146981469914700147011470214703147041470514706147071470814709147101471114712147131471414715147161471714718147191472014721147221472314724147251472614727147281472914730147311473214733147341473514736147371473814739147401474114742147431474414745147461474714748147491475014751147521475314754147551475614757147581475914760147611476214763147641476514766147671476814769147701477114772147731477414775147761477714778147791478014781147821478314784147851478614787147881478914790147911479214793147941479514796147971479814799148001480114802148031480414805148061480714808148091481014811148121481314814148151481614817148181481914820148211482214823148241482514826148271482814829148301483114832148331483414835148361483714838148391484014841148421484314844148451484614847148481484914850148511485214853148541485514856148571485814859148601486114862148631486414865148661486714868148691487014871148721487314874148751487614877148781487914880148811488214883148841488514886148871488814889148901489114892148931489414895148961489714898148991490014901149021490314904149051490614907149081490914910149111491214913149141491514916149171491814919149201492114922149231492414925149261492714928149291493014931149321493314934149351493614937149381493914940149411494214943149441494514946149471494814949149501495114952149531495414955149561495714958149591496014961149621496314964149651496614967149681496914970149711497214973149741497514976149771497814979149801498114982149831498414985149861498714988149891499014991149921499314994149951499614997149981499915000150011500215003150041500515006150071500815009150101501115012150131501415015150161501715018150191502015021150221502315024150251502615027150281502915030150311503215033150341503515036150371503815039150401504115042150431504415045150461504715048150491505015051150521505315054150551505615057150581505915060150611506215063150641506515066150671506815069150701507115072150731507415075150761507715078150791508015081150821508315084150851508615087150881508915090150911509215093150941509515096150971509815099151001510115102151031510415105151061510715108151091511015111151121511315114151151511615117151181511915120151211512215123151241512515126151271512815129151301513115132151331513415135151361513715138151391514015141151421514315144151451514615147151481514915150151511515215153151541515515156151571515815159151601516115162151631516415165151661516715168151691517015171151721517315174151751517615177151781517915180151811518215183151841518515186151871518815189151901519115192151931519415195151961519715198151991520015201152021520315204152051520615207152081520915210152111521215213152141521515216152171521815219152201522115222152231522415225152261522715228152291523015231152321523315234152351523615237152381523915240152411524215243152441524515246152471524815249152501525115252152531525415255152561525715258152591526015261152621526315264152651526615267152681526915270152711527215273152741527515276152771527815279152801528115282152831528415285152861528715288152891529015291152921529315294152951529615297152981529915300153011530215303153041530515306153071530815309153101531115312153131531415315153161531715318153191532015321153221532315324153251532615327153281532915330153311533215333153341533515336153371533815339153401534115342153431534415345153461534715348153491535015351153521535315354153551535615357153581535915360153611536215363153641536515366153671536815369153701537115372153731537415375153761537715378153791538015381153821538315384153851538615387153881538915390153911539215393153941539515396153971539815399154001540115402154031540415405154061540715408154091541015411154121541315414154151541615417154181541915420154211542215423154241542515426154271542815429154301543115432154331543415435154361543715438154391544015441154421544315444154451544615447154481544915450154511545215453154541545515456154571545815459154601546115462154631546415465154661546715468154691547015471154721547315474154751547615477154781547915480154811548215483154841548515486154871548815489154901549115492154931549415495154961549715498154991550015501155021550315504155051550615507155081550915510155111551215513155141551515516155171551815519155201552115522155231552415525155261552715528155291553015531155321553315534155351553615537155381553915540155411554215543155441554515546155471554815549155501555115552155531555415555155561555715558155591556015561155621556315564155651556615567155681556915570155711557215573155741557515576155771557815579155801558115582155831558415585155861558715588155891559015591155921559315594155951559615597155981559915600156011560215603156041560515606156071560815609156101561115612156131561415615156161561715618156191562015621156221562315624156251562615627156281562915630156311563215633156341563515636156371563815639156401564115642156431564415645156461564715648156491565015651156521565315654156551565615657156581565915660156611566215663156641566515666156671566815669156701567115672156731567415675156761567715678156791568015681156821568315684156851568615687156881568915690156911569215693156941569515696156971569815699157001570115702157031570415705157061570715708157091571015711157121571315714157151571615717157181571915720157211572215723157241572515726157271572815729157301573115732157331573415735157361573715738157391574015741157421574315744157451574615747157481574915750157511575215753157541575515756157571575815759157601576115762157631576415765157661576715768157691577015771157721577315774157751577615777157781577915780157811578215783157841578515786157871578815789157901579115792157931579415795157961579715798157991580015801158021580315804158051580615807158081580915810158111581215813158141581515816158171581815819158201582115822158231582415825158261582715828158291583015831158321583315834158351583615837158381583915840158411584215843158441584515846158471584815849158501585115852158531585415855158561585715858158591586015861158621586315864158651586615867158681586915870158711587215873158741587515876158771587815879158801588115882158831588415885158861588715888158891589015891158921589315894158951589615897158981589915900159011590215903159041590515906159071590815909159101591115912159131591415915159161591715918159191592015921159221592315924159251592615927159281592915930159311593215933159341593515936159371593815939159401594115942159431594415945159461594715948159491595015951159521595315954159551595615957159581595915960159611596215963159641596515966159671596815969159701597115972159731597415975159761597715978159791598015981159821598315984159851598615987159881598915990159911599215993159941599515996159971599815999160001600116002160031600416005160061600716008160091601016011160121601316014160151601616017160181601916020160211602216023160241602516026160271602816029160301603116032160331603416035160361603716038160391604016041160421604316044160451604616047160481604916050160511605216053160541605516056160571605816059160601606116062160631606416065160661606716068160691607016071160721607316074160751607616077160781607916080160811608216083160841608516086160871608816089160901609116092160931609416095160961609716098160991610016101161021610316104161051610616107161081610916110161111611216113161141611516116161171611816119161201612116122161231612416125161261612716128161291613016131161321613316134161351613616137161381613916140161411614216143161441614516146161471614816149161501615116152161531615416155161561615716158161591616016161161621616316164161651616616167161681616916170161711617216173161741617516176161771617816179161801618116182161831618416185161861618716188161891619016191161921619316194161951619616197161981619916200162011620216203162041620516206162071620816209162101621116212162131621416215162161621716218162191622016221162221622316224162251622616227162281622916230162311623216233162341623516236162371623816239162401624116242162431624416245162461624716248162491625016251162521625316254162551625616257162581625916260162611626216263162641626516266162671626816269162701627116272162731627416275162761627716278162791628016281162821628316284162851628616287162881628916290162911629216293162941629516296162971629816299163001630116302163031630416305163061630716308163091631016311163121631316314163151631616317163181631916320163211632216323163241632516326163271632816329163301633116332163331633416335163361633716338163391634016341163421634316344163451634616347163481634916350163511635216353163541635516356163571635816359163601636116362163631636416365163661636716368163691637016371163721637316374163751637616377163781637916380163811638216383163841638516386163871638816389163901639116392163931639416395163961639716398163991640016401164021640316404164051640616407164081640916410164111641216413164141641516416164171641816419164201642116422164231642416425164261642716428164291643016431164321643316434164351643616437164381643916440164411644216443164441644516446164471644816449164501645116452164531645416455164561645716458164591646016461164621646316464164651646616467164681646916470164711647216473164741647516476164771647816479164801648116482164831648416485164861648716488164891649016491164921649316494164951649616497164981649916500165011650216503165041650516506165071650816509165101651116512165131651416515165161651716518165191652016521165221652316524165251652616527165281652916530165311653216533165341653516536165371653816539165401654116542165431654416545165461654716548165491655016551165521655316554165551655616557165581655916560165611656216563165641656516566165671656816569165701657116572165731657416575165761657716578165791658016581165821658316584165851658616587165881658916590165911659216593165941659516596165971659816599166001660116602166031660416605166061660716608166091661016611166121661316614166151661616617166181661916620166211662216623166241662516626166271662816629166301663116632166331663416635166361663716638166391664016641166421664316644166451664616647166481664916650166511665216653166541665516656166571665816659166601666116662166631666416665166661666716668166691667016671166721667316674166751667616677166781667916680166811668216683166841668516686166871668816689166901669116692166931669416695166961669716698166991670016701167021670316704167051670616707167081670916710167111671216713167141671516716167171671816719167201672116722167231672416725167261672716728167291673016731167321673316734167351673616737167381673916740167411674216743167441674516746167471674816749167501675116752167531675416755167561675716758167591676016761167621676316764167651676616767167681676916770167711677216773167741677516776167771677816779167801678116782167831678416785167861678716788167891679016791167921679316794167951679616797167981679916800168011680216803168041680516806168071680816809168101681116812168131681416815168161681716818168191682016821168221682316824168251682616827168281682916830168311683216833168341683516836168371683816839168401684116842168431684416845168461684716848168491685016851168521685316854168551685616857168581685916860168611686216863168641686516866168671686816869168701687116872168731687416875168761687716878168791688016881168821688316884168851688616887168881688916890168911689216893168941689516896168971689816899169001690116902169031690416905169061690716908169091691016911169121691316914169151691616917169181691916920169211692216923169241692516926169271692816929169301693116932169331693416935169361693716938169391694016941169421694316944169451694616947169481694916950169511695216953169541695516956169571695816959169601696116962169631696416965169661696716968169691697016971169721697316974169751697616977169781697916980169811698216983169841698516986169871698816989169901699116992169931699416995169961699716998169991700017001170021700317004170051700617007170081700917010170111701217013170141701517016170171701817019170201702117022170231702417025170261702717028170291703017031170321703317034170351703617037170381703917040170411704217043170441704517046170471704817049170501705117052170531705417055170561705717058170591706017061170621706317064170651706617067170681706917070170711707217073170741707517076170771707817079170801708117082170831708417085170861708717088170891709017091170921709317094170951709617097170981709917100171011710217103171041710517106171071710817109171101711117112171131711417115171161711717118171191712017121171221712317124171251712617127171281712917130171311713217133171341713517136171371713817139171401714117142171431714417145171461714717148171491715017151171521715317154171551715617157171581715917160171611716217163171641716517166171671716817169171701717117172171731717417175171761717717178171791718017181171821718317184171851718617187171881718917190171911719217193171941719517196171971719817199172001720117202172031720417205172061720717208172091721017211172121721317214172151721617217172181721917220172211722217223172241722517226172271722817229172301723117232172331723417235172361723717238172391724017241172421724317244172451724617247172481724917250172511725217253172541725517256172571725817259172601726117262172631726417265172661726717268172691727017271172721727317274172751727617277172781727917280172811728217283172841728517286172871728817289172901729117292172931729417295172961729717298172991730017301173021730317304173051730617307173081730917310173111731217313173141731517316173171731817319173201732117322173231732417325173261732717328173291733017331173321733317334173351733617337173381733917340173411734217343173441734517346173471734817349173501735117352173531735417355173561735717358173591736017361173621736317364173651736617367173681736917370173711737217373173741737517376173771737817379173801738117382173831738417385173861738717388173891739017391173921739317394173951739617397173981739917400174011740217403174041740517406174071740817409174101741117412174131741417415174161741717418174191742017421174221742317424174251742617427174281742917430174311743217433174341743517436174371743817439174401744117442174431744417445174461744717448174491745017451174521745317454174551745617457174581745917460174611746217463174641746517466174671746817469174701747117472174731747417475174761747717478174791748017481174821748317484174851748617487174881748917490174911749217493174941749517496174971749817499175001750117502175031750417505175061750717508175091751017511175121751317514175151751617517175181751917520175211752217523175241752517526175271752817529175301753117532175331753417535175361753717538175391754017541175421754317544175451754617547175481754917550175511755217553175541755517556175571755817559175601756117562175631756417565
  1. //-------------------------------------------------------------------------------------------------------
  2. // Copyright (C) Microsoft Corporation and contributors. All rights reserved.
  3. // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
  4. //-------------------------------------------------------------------------------------------------------
  5. #include "Backend.h"
  6. #if DBG_DUMP
  7. #define DO_MEMOP_TRACE() (PHASE_TRACE(Js::MemOpPhase, this->func) ||\
  8. PHASE_TRACE(Js::MemSetPhase, this->func) ||\
  9. PHASE_TRACE(Js::MemCopyPhase, this->func))
  10. #define DO_MEMOP_TRACE_PHASE(phase) (PHASE_TRACE(Js::MemOpPhase, this->func) || PHASE_TRACE(Js::phase ## Phase, this->func))
  11. #define OUTPUT_MEMOP_TRACE(loop, instr, ...) {\
  12. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];\
  13. Output::Print(15, _u("Function: %s%s, Loop: %u: "), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), loop->GetLoopNumber());\
  14. Output::Print(__VA_ARGS__);\
  15. IR::Instr* __instr__ = instr;\
  16. if(__instr__) __instr__->DumpByteCodeOffset();\
  17. if(__instr__) Output::Print(_u(" (%s)"), Js::OpCodeUtil::GetOpCodeName(__instr__->m_opcode));\
  18. Output::Print(_u("\n"));\
  19. Output::Flush(); \
  20. }
  21. #define TRACE_MEMOP(loop, instr, ...) \
  22. if (DO_MEMOP_TRACE()) {\
  23. Output::Print(_u("TRACE MemOp:"));\
  24. OUTPUT_MEMOP_TRACE(loop, instr, __VA_ARGS__)\
  25. }
  26. #define TRACE_MEMOP_VERBOSE(loop, instr, ...) if(CONFIG_FLAG(Verbose)) {TRACE_MEMOP(loop, instr, __VA_ARGS__)}
  27. #define TRACE_MEMOP_PHASE(phase, loop, instr, ...) \
  28. if (DO_MEMOP_TRACE_PHASE(phase))\
  29. {\
  30. Output::Print(_u("TRACE ") _u(#phase) _u(":"));\
  31. OUTPUT_MEMOP_TRACE(loop, instr, __VA_ARGS__)\
  32. }
  33. #define TRACE_MEMOP_PHASE_VERBOSE(phase, loop, instr, ...) if(CONFIG_FLAG(Verbose)) {TRACE_MEMOP_PHASE(phase, loop, instr, __VA_ARGS__)}
  34. #else
  35. #define DO_MEMOP_TRACE()
  36. #define DO_MEMOP_TRACE_PHASE(phase)
  37. #define OUTPUT_MEMOP_TRACE(loop, instr, ...)
  38. #define TRACE_MEMOP(loop, instr, ...)
  39. #define TRACE_MEMOP_VERBOSE(loop, instr, ...)
  40. #define TRACE_MEMOP_PHASE(phase, loop, instr, ...)
  41. #define TRACE_MEMOP_PHASE_VERBOSE(phase, loop, instr, ...)
  42. #endif
  43. class AutoRestoreVal
  44. {
  45. private:
  46. Value *const originalValue;
  47. Value *const tempValue;
  48. Value * *const valueRef;
  49. public:
  50. AutoRestoreVal(Value *const originalValue, Value * *const tempValueRef)
  51. : originalValue(originalValue), tempValue(*tempValueRef), valueRef(tempValueRef)
  52. {
  53. }
  54. ~AutoRestoreVal()
  55. {
  56. if(*valueRef == tempValue)
  57. {
  58. *valueRef = originalValue;
  59. }
  60. }
  61. PREVENT_COPY(AutoRestoreVal);
  62. };
  63. GlobOpt::GlobOpt(Func * func)
  64. : func(func),
  65. intConstantToStackSymMap(nullptr),
  66. intConstantToValueMap(nullptr),
  67. currentValue(FirstNewValueNumber),
  68. prePassLoop(nullptr),
  69. alloc(nullptr),
  70. isCallHelper(false),
  71. inInlinedBuiltIn(false),
  72. rootLoopPrePass(nullptr),
  73. noImplicitCallUsesToInsert(nullptr),
  74. valuesCreatedForClone(nullptr),
  75. valuesCreatedForMerge(nullptr),
  76. instrCountSinceLastCleanUp(0),
  77. isRecursiveCallOnLandingPad(false),
  78. updateInductionVariableValueNumber(false),
  79. isPerformingLoopBackEdgeCompensation(false),
  80. currentRegion(nullptr),
  81. changedSymsAfterIncBailoutCandidate(nullptr),
  82. doTypeSpec(
  83. !IsTypeSpecPhaseOff(func)),
  84. doAggressiveIntTypeSpec(
  85. doTypeSpec &&
  86. DoAggressiveIntTypeSpec(func)),
  87. doAggressiveMulIntTypeSpec(
  88. doTypeSpec &&
  89. !PHASE_OFF(Js::AggressiveMulIntTypeSpecPhase, func) &&
  90. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsAggressiveMulIntTypeSpecDisabled(func->IsLoopBody()))),
  91. doDivIntTypeSpec(
  92. doAggressiveIntTypeSpec &&
  93. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsDivIntTypeSpecDisabled(func->IsLoopBody()))),
  94. doLossyIntTypeSpec(
  95. doTypeSpec &&
  96. DoLossyIntTypeSpec(func)),
  97. doFloatTypeSpec(
  98. doTypeSpec &&
  99. DoFloatTypeSpec(func)),
  100. doArrayCheckHoist(
  101. DoArrayCheckHoist(func)),
  102. doArrayMissingValueCheckHoist(
  103. doArrayCheckHoist &&
  104. DoArrayMissingValueCheckHoist(func)),
  105. doArraySegmentHoist(
  106. doArrayCheckHoist &&
  107. DoArraySegmentHoist(ValueType::GetObject(ObjectType::Int32Array), func)),
  108. doJsArraySegmentHoist(
  109. doArraySegmentHoist &&
  110. DoArraySegmentHoist(ValueType::GetObject(ObjectType::Array), func)),
  111. doArrayLengthHoist(
  112. doArrayCheckHoist &&
  113. DoArrayLengthHoist(func)),
  114. doEliminateArrayAccessHelperCall(
  115. doArrayCheckHoist &&
  116. !PHASE_OFF(Js::EliminateArrayAccessHelperCallPhase, func)),
  117. doTrackRelativeIntBounds(
  118. doAggressiveIntTypeSpec &&
  119. DoPathDependentValues() &&
  120. !PHASE_OFF(Js::Phase::TrackRelativeIntBoundsPhase, func)),
  121. doBoundCheckElimination(
  122. doTrackRelativeIntBounds &&
  123. !PHASE_OFF(Js::Phase::BoundCheckEliminationPhase, func)),
  124. doBoundCheckHoist(
  125. doEliminateArrayAccessHelperCall &&
  126. doBoundCheckElimination &&
  127. DoConstFold() &&
  128. !PHASE_OFF(Js::Phase::BoundCheckHoistPhase, func) &&
  129. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsBoundCheckHoistDisabled(func->IsLoopBody()))),
  130. doLoopCountBasedBoundCheckHoist(
  131. doBoundCheckHoist &&
  132. !PHASE_OFF(Js::Phase::LoopCountBasedBoundCheckHoistPhase, func) &&
  133. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsLoopCountBasedBoundCheckHoistDisabled(func->IsLoopBody()))),
  134. doPowIntIntTypeSpec(
  135. doAggressiveIntTypeSpec &&
  136. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsPowIntIntTypeSpecDisabled())),
  137. doTagChecks(
  138. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsTagCheckDisabled())),
  139. isAsmJSFunc(func->GetJITFunctionBody()->IsAsmJsMode())
  140. {
  141. }
  142. void
  143. GlobOpt::BackwardPass(Js::Phase tag)
  144. {
  145. BEGIN_CODEGEN_PHASE(this->func, tag);
  146. ::BackwardPass backwardPass(this->func, this, tag);
  147. backwardPass.Optimize();
  148. END_CODEGEN_PHASE(this->func, tag);
  149. }
  150. void
  151. GlobOpt::Optimize()
  152. {
  153. this->objectTypeSyms = nullptr;
  154. this->func->argInsCount = this->func->GetInParamsCount() - 1; //Don't include "this" pointer in the count.
  155. if (!func->DoGlobOpt())
  156. {
  157. this->lengthEquivBv = nullptr;
  158. this->argumentsEquivBv = nullptr;
  159. this->callerEquivBv = nullptr;
  160. // Still need to run the dead store phase to calculate the live reg on back edge
  161. this->BackwardPass(Js::DeadStorePhase);
  162. CannotAllocateArgumentsObjectOnStack();
  163. return;
  164. }
  165. {
  166. this->lengthEquivBv = this->func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::length, nullptr); // Used to kill live "length" properties
  167. this->argumentsEquivBv = func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::arguments, nullptr); // Used to kill live "arguments" properties
  168. this->callerEquivBv = func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::caller, nullptr); // Used to kill live "caller" properties
  169. // The backward phase needs the glob opt's allocator to allocate the propertyTypeValueMap
  170. // in GlobOpt::EnsurePropertyTypeValue and ranges of instructions where int overflow may be ignored.
  171. // (see BackwardPass::TrackIntUsage)
  172. PageAllocator * pageAllocator = this->func->m_alloc->GetPageAllocator();
  173. NoRecoverMemoryJitArenaAllocator localAlloc(_u("BE-GlobOpt"), pageAllocator, Js::Throw::OutOfMemory);
  174. this->alloc = &localAlloc;
  175. NoRecoverMemoryJitArenaAllocator localTempAlloc(_u("BE-GlobOpt temp"), pageAllocator, Js::Throw::OutOfMemory);
  176. this->tempAlloc = &localTempAlloc;
  177. // The forward passes use info (upwardExposedUses) from the backward pass. This info
  178. // isn't available for some of the symbols created during the backward pass, or the forward pass.
  179. // Keep track of the last symbol for which we're guaranteed to have data.
  180. this->maxInitialSymID = this->func->m_symTable->GetMaxSymID();
  181. #if DBG
  182. this->BackwardPass(Js::CaptureByteCodeRegUsePhase);
  183. #endif
  184. this->BackwardPass(Js::BackwardPhase);
  185. this->ForwardPass();
  186. this->BackwardPass(Js::DeadStorePhase);
  187. }
  188. this->TailDupPass();
  189. }
  190. bool GlobOpt::ShouldExpectConventionalArrayIndexValue(IR::IndirOpnd *const indirOpnd)
  191. {
  192. Assert(indirOpnd);
  193. if(!indirOpnd->GetIndexOpnd())
  194. {
  195. return indirOpnd->GetOffset() >= 0;
  196. }
  197. IR::RegOpnd *const indexOpnd = indirOpnd->GetIndexOpnd();
  198. if(indexOpnd->m_sym->m_isNotNumber)
  199. {
  200. // Typically, single-def or any sym-specific information for type-specialized syms should not be used because all of
  201. // their defs will not have been accounted for until after the forward pass. But m_isNotNumber is only ever changed from
  202. // false to true, so it's okay in this case.
  203. return false;
  204. }
  205. StackSym *indexVarSym = indexOpnd->m_sym;
  206. if(indexVarSym->IsTypeSpec())
  207. {
  208. indexVarSym = indexVarSym->GetVarEquivSym(nullptr);
  209. Assert(indexVarSym);
  210. }
  211. else if(!IsLoopPrePass())
  212. {
  213. // Don't use single-def info or const flags for type-specialized syms, as all of their defs will not have been accounted
  214. // for until after the forward pass. Also, don't use the const flags in a loop prepass because the const flags may not
  215. // be up-to-date.
  216. if (indexOpnd->IsNotInt())
  217. {
  218. return false;
  219. }
  220. StackSym *const indexSym = indexOpnd->m_sym;
  221. if(indexSym->IsIntConst())
  222. {
  223. return indexSym->GetIntConstValue() >= 0;
  224. }
  225. }
  226. Value *const indexValue = CurrentBlockData()->FindValue(indexVarSym);
  227. if(!indexValue)
  228. {
  229. // Treat it as Uninitialized, assume it's going to be valid
  230. return true;
  231. }
  232. ValueInfo *const indexValueInfo = indexValue->GetValueInfo();
  233. int32 indexConstantValue;
  234. if(indexValueInfo->TryGetIntConstantValue(&indexConstantValue))
  235. {
  236. return indexConstantValue >= 0;
  237. }
  238. if(indexValueInfo->IsUninitialized())
  239. {
  240. // Assume it's going to be valid
  241. return true;
  242. }
  243. return indexValueInfo->HasBeenNumber() && !indexValueInfo->HasBeenFloat();
  244. }
  245. //
  246. // Either result is float or 1/x or cst1/cst2 where cst1%cst2 != 0
  247. //
  248. ValueType GlobOpt::GetDivValueType(IR::Instr* instr, Value* src1Val, Value* src2Val, bool specialize)
  249. {
  250. ValueInfo *src1ValueInfo = (src1Val ? src1Val->GetValueInfo() : nullptr);
  251. ValueInfo *src2ValueInfo = (src2Val ? src2Val->GetValueInfo() : nullptr);
  252. if (instr->IsProfiledInstr() && instr->m_func->HasProfileInfo())
  253. {
  254. ValueType resultType = instr->m_func->GetReadOnlyProfileInfo()->GetDivProfileInfo(static_cast<Js::ProfileId>(instr->AsProfiledInstr()->u.profileId));
  255. if (resultType.IsLikelyInt())
  256. {
  257. if (specialize && src1ValueInfo && src2ValueInfo
  258. && ((src1ValueInfo->IsInt() && src2ValueInfo->IsInt()) ||
  259. (this->DoDivIntTypeSpec() && src1ValueInfo->IsLikelyInt() && src2ValueInfo->IsLikelyInt())))
  260. {
  261. return ValueType::GetInt(true);
  262. }
  263. return resultType;
  264. }
  265. // Consider: Checking that the sources are numbers.
  266. if (resultType.IsLikelyFloat())
  267. {
  268. return ValueType::Float;
  269. }
  270. return resultType;
  271. }
  272. int32 src1IntConstantValue;
  273. if(!src1ValueInfo || !src1ValueInfo->TryGetIntConstantValue(&src1IntConstantValue))
  274. {
  275. return ValueType::Number;
  276. }
  277. if (src1IntConstantValue == 1)
  278. {
  279. return ValueType::Float;
  280. }
  281. int32 src2IntConstantValue;
  282. if(!src2Val || !src2ValueInfo->TryGetIntConstantValue(&src2IntConstantValue))
  283. {
  284. return ValueType::Number;
  285. }
  286. if (src2IntConstantValue // Avoid divide by zero
  287. && !(src1IntConstantValue == 0x80000000 && src2IntConstantValue == -1) // Avoid integer overflow
  288. && (src1IntConstantValue % src2IntConstantValue) != 0)
  289. {
  290. return ValueType::Float;
  291. }
  292. return ValueType::Number;
  293. }
  294. void
  295. GlobOpt::ForwardPass()
  296. {
  297. BEGIN_CODEGEN_PHASE(this->func, Js::ForwardPhase);
  298. #if DBG_DUMP
  299. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::GlobOptPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()))
  300. {
  301. this->func->DumpHeader();
  302. }
  303. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::GlobOptPhase))
  304. {
  305. this->TraceSettings();
  306. }
  307. #endif
  308. // GetConstantCount() gives us the right size to pick for the SparseArray, but we may need more if we've inlined
  309. // functions with constants. There will be a gap in the symbol numbering between the main constants and
  310. // the inlined ones, so we'll most likely need a new array chunk. Make the min size of the array chunks be 64
  311. // in case we have a main function with very few constants and a bunch of constants from inlined functions.
  312. this->byteCodeConstantValueArray = SparseArray<Value>::New(this->alloc, max(this->func->GetJITFunctionBody()->GetConstCount(), 64U));
  313. this->byteCodeConstantValueNumbersBv = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  314. this->tempBv = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  315. this->prePassCopyPropSym = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  316. this->slotSyms = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  317. this->byteCodeUses = nullptr;
  318. this->propertySymUse = nullptr;
  319. // changedSymsAfterIncBailoutCandidate helps track building incremental bailout in ForwardPass
  320. this->changedSymsAfterIncBailoutCandidate = JitAnew(alloc, BVSparse<JitArenaAllocator>, alloc);
  321. #if DBG
  322. this->byteCodeUsesBeforeOpt = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  323. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase) && this->DoFunctionFieldCopyProp())
  324. {
  325. Output::Print(_u("TRACE: CanDoFieldCopyProp Func: "));
  326. this->func->DumpFullFunctionName();
  327. Output::Print(_u("\n"));
  328. }
  329. #endif
  330. OpndList localNoImplicitCallUsesToInsert(alloc);
  331. this->noImplicitCallUsesToInsert = &localNoImplicitCallUsesToInsert;
  332. IntConstantToStackSymMap localIntConstantToStackSymMap(alloc);
  333. this->intConstantToStackSymMap = &localIntConstantToStackSymMap;
  334. IntConstantToValueMap localIntConstantToValueMap(alloc);
  335. this->intConstantToValueMap = &localIntConstantToValueMap;
  336. Int64ConstantToValueMap localInt64ConstantToValueMap(alloc);
  337. this->int64ConstantToValueMap = &localInt64ConstantToValueMap;
  338. AddrConstantToValueMap localAddrConstantToValueMap(alloc);
  339. this->addrConstantToValueMap = &localAddrConstantToValueMap;
  340. StringConstantToValueMap localStringConstantToValueMap(alloc);
  341. this->stringConstantToValueMap = &localStringConstantToValueMap;
  342. SymIdToInstrMap localPrePassInstrMap(alloc);
  343. this->prePassInstrMap = &localPrePassInstrMap;
  344. ValueSetByValueNumber localValuesCreatedForClone(alloc, 64);
  345. this->valuesCreatedForClone = &localValuesCreatedForClone;
  346. ValueNumberPairToValueMap localValuesCreatedForMerge(alloc, 64);
  347. this->valuesCreatedForMerge = &localValuesCreatedForMerge;
  348. #if DBG
  349. BVSparse<JitArenaAllocator> localFinishedStackLiteralInitFld(alloc);
  350. this->finishedStackLiteralInitFld = &localFinishedStackLiteralInitFld;
  351. #endif
  352. FOREACH_BLOCK_IN_FUNC_EDITING(block, this->func)
  353. {
  354. this->OptBlock(block);
  355. } NEXT_BLOCK_IN_FUNC_EDITING;
  356. if (!PHASE_OFF(Js::MemOpPhase, this->func))
  357. {
  358. ProcessMemOp();
  359. }
  360. this->noImplicitCallUsesToInsert = nullptr;
  361. this->intConstantToStackSymMap = nullptr;
  362. this->intConstantToValueMap = nullptr;
  363. this->int64ConstantToValueMap = nullptr;
  364. this->addrConstantToValueMap = nullptr;
  365. this->stringConstantToValueMap = nullptr;
  366. #if DBG
  367. this->finishedStackLiteralInitFld = nullptr;
  368. uint freedCount = 0;
  369. uint spilledCount = 0;
  370. #endif
  371. FOREACH_BLOCK_IN_FUNC(block, this->func)
  372. {
  373. #if DBG
  374. if (block->GetDataUseCount() == 0)
  375. {
  376. freedCount++;
  377. }
  378. else
  379. {
  380. spilledCount++;
  381. }
  382. #endif
  383. block->SetDataUseCount(0);
  384. if (block->cloneStrCandidates)
  385. {
  386. JitAdelete(this->alloc, block->cloneStrCandidates);
  387. block->cloneStrCandidates = nullptr;
  388. }
  389. } NEXT_BLOCK_IN_FUNC;
  390. // Make sure we free most of them.
  391. Assert(freedCount >= spilledCount);
  392. // this->alloc will be freed right after return, no need to free it here
  393. this->changedSymsAfterIncBailoutCandidate = nullptr;
  394. END_CODEGEN_PHASE(this->func, Js::ForwardPhase);
  395. }
  396. void
  397. GlobOpt::OptBlock(BasicBlock *block)
  398. {
  399. if (this->func->m_fg->RemoveUnreachableBlock(block, this))
  400. {
  401. GOPT_TRACE(_u("Removing unreachable block #%d\n"), block->GetBlockNum());
  402. return;
  403. }
  404. Loop * loop = block->loop;
  405. if (loop && block->isLoopHeader)
  406. {
  407. if (loop != this->prePassLoop)
  408. {
  409. OptLoops(loop);
  410. if (!IsLoopPrePass() && loop->parent)
  411. {
  412. loop->fieldPRESymStores->Or(loop->parent->fieldPRESymStores);
  413. }
  414. if (!this->IsLoopPrePass() && DoFieldPRE(loop))
  415. {
  416. // Note: !IsLoopPrePass means this was a root loop pre-pass. FieldPre() is called once per loop.
  417. this->FieldPRE(loop);
  418. // Re-optimize the landing pad
  419. BasicBlock *landingPad = loop->landingPad;
  420. this->isRecursiveCallOnLandingPad = true;
  421. this->OptBlock(landingPad);
  422. this->isRecursiveCallOnLandingPad = false;
  423. }
  424. }
  425. }
  426. this->currentBlock = block;
  427. PrepareLoopArrayCheckHoist();
  428. block->MergePredBlocksValueMaps(this);
  429. this->intOverflowCurrentlyMattersInRange = true;
  430. this->intOverflowDoesNotMatterRange = this->currentBlock->intOverflowDoesNotMatterRange;
  431. if (!DoFieldCopyProp() && !DoFieldRefOpts())
  432. {
  433. this->KillAllFields(CurrentBlockData()->liveFields);
  434. }
  435. this->tempAlloc->Reset();
  436. if(loop && block->isLoopHeader)
  437. {
  438. loop->firstValueNumberInLoop = this->currentValue;
  439. }
  440. GOPT_TRACE_BLOCK(block, true);
  441. FOREACH_INSTR_IN_BLOCK_EDITING(instr, instrNext, block)
  442. {
  443. GOPT_TRACE_INSTRTRACE(instr);
  444. BailOutInfo* oldBailOutInfo = nullptr;
  445. bool isCheckAuxBailoutNeeded = this->func->IsJitInDebugMode() && !this->IsLoopPrePass();
  446. if (isCheckAuxBailoutNeeded && instr->HasAuxBailOut() && !instr->HasBailOutInfo())
  447. {
  448. oldBailOutInfo = instr->GetBailOutInfo();
  449. Assert(oldBailOutInfo);
  450. }
  451. bool isInstrRemoved = false;
  452. instrNext = this->OptInstr(instr, &isInstrRemoved);
  453. // If we still have instrs with only aux bail out, convert aux bail out back to regular bail out and fill it.
  454. // During OptInstr some instr can be moved out to a different block, in this case bailout info is going to be replaced
  455. // with e.g. loop bailout info which is filled as part of processing that block, thus we don't need to fill it here.
  456. if (isCheckAuxBailoutNeeded && !isInstrRemoved && instr->HasAuxBailOut() && !instr->HasBailOutInfo())
  457. {
  458. if (instr->GetBailOutInfo() == oldBailOutInfo)
  459. {
  460. instr->PromoteAuxBailOut();
  461. FillBailOutInfo(block, instr);
  462. }
  463. else
  464. {
  465. AssertMsg(instr->GetBailOutInfo(), "With aux bailout, the bailout info should not be removed by OptInstr.");
  466. }
  467. }
  468. } NEXT_INSTR_IN_BLOCK_EDITING;
  469. GOPT_TRACE_BLOCK(block, false);
  470. if (block->loop)
  471. {
  472. if (IsLoopPrePass())
  473. {
  474. if (DoBoundCheckHoist())
  475. {
  476. DetectUnknownChangesToInductionVariables(&block->globOptData);
  477. }
  478. }
  479. else
  480. {
  481. isPerformingLoopBackEdgeCompensation = true;
  482. Assert(this->tempBv->IsEmpty());
  483. BVSparse<JitArenaAllocator> tempBv2(this->tempAlloc);
  484. // On loop back-edges, we need to restore the state of the type specialized
  485. // symbols to that of the loop header.
  486. FOREACH_SUCCESSOR_BLOCK(succ, block)
  487. {
  488. if (succ->isLoopHeader && succ->loop->IsDescendentOrSelf(block->loop))
  489. {
  490. BVSparse<JitArenaAllocator> *liveOnBackEdge = block->loop->regAlloc.liveOnBackEdgeSyms;
  491. liveOnBackEdge->Or(block->loop->fieldPRESymStores);
  492. this->tempBv->Minus(block->loop->varSymsOnEntry, block->globOptData.liveVarSyms);
  493. this->tempBv->And(liveOnBackEdge);
  494. this->ToVar(this->tempBv, block);
  495. // Lossy int in the loop header, and no int on the back-edge - need a lossy conversion to int
  496. this->tempBv->Minus(block->loop->lossyInt32SymsOnEntry, block->globOptData.liveInt32Syms);
  497. this->tempBv->And(liveOnBackEdge);
  498. this->ToInt32(this->tempBv, block, true /* lossy */);
  499. // Lossless int in the loop header, and no lossless int on the back-edge - need a lossless conversion to int
  500. this->tempBv->Minus(block->loop->int32SymsOnEntry, block->loop->lossyInt32SymsOnEntry);
  501. tempBv2.Minus(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms);
  502. this->tempBv->Minus(&tempBv2);
  503. this->tempBv->And(liveOnBackEdge);
  504. this->ToInt32(this->tempBv, block, false /* lossy */);
  505. this->tempBv->Minus(block->loop->float64SymsOnEntry, block->globOptData.liveFloat64Syms);
  506. this->tempBv->And(liveOnBackEdge);
  507. this->ToFloat64(this->tempBv, block);
  508. // For ints and floats, go aggressive and type specialize in the landing pad any symbol which was specialized on
  509. // entry to the loop body (in the loop header), and is still specialized on this tail, but wasn't specialized in
  510. // the landing pad.
  511. // Lossy int in the loop header and no int in the landing pad - need a lossy conversion to int
  512. // (entry.lossyInt32 - landingPad.int32)
  513. this->tempBv->Minus(block->loop->lossyInt32SymsOnEntry, block->loop->landingPad->globOptData.liveInt32Syms);
  514. this->tempBv->And(liveOnBackEdge);
  515. this->ToInt32(this->tempBv, block->loop->landingPad, true /* lossy */);
  516. // Lossless int in the loop header, and no lossless int in the landing pad - need a lossless conversion to int
  517. // ((entry.int32 - entry.lossyInt32) - (landingPad.int32 - landingPad.lossyInt32))
  518. this->tempBv->Minus(block->loop->int32SymsOnEntry, block->loop->lossyInt32SymsOnEntry);
  519. tempBv2.Minus(
  520. block->loop->landingPad->globOptData.liveInt32Syms,
  521. block->loop->landingPad->globOptData.liveLossyInt32Syms);
  522. this->tempBv->Minus(&tempBv2);
  523. this->tempBv->And(liveOnBackEdge);
  524. this->ToInt32(this->tempBv, block->loop->landingPad, false /* lossy */);
  525. // ((entry.float64 - landingPad.float64) & block.float64)
  526. this->tempBv->Minus(block->loop->float64SymsOnEntry, block->loop->landingPad->globOptData.liveFloat64Syms);
  527. this->tempBv->And(block->globOptData.liveFloat64Syms);
  528. this->tempBv->And(liveOnBackEdge);
  529. this->ToFloat64(this->tempBv, block->loop->landingPad);
  530. if (block->loop->symsRequiringCompensationToMergedValueInfoMap)
  531. {
  532. InsertValueCompensation(block, succ, block->loop->symsRequiringCompensationToMergedValueInfoMap);
  533. }
  534. // Now that we're done with the liveFields within this loop, trim the set to those syms
  535. // that the backward pass told us were live out of the loop.
  536. // This assumes we have no further need of the liveFields within the loop.
  537. if (block->loop->liveOutFields)
  538. {
  539. block->globOptData.liveFields->And(block->loop->liveOutFields);
  540. }
  541. }
  542. } NEXT_SUCCESSOR_BLOCK;
  543. this->tempBv->ClearAll();
  544. isPerformingLoopBackEdgeCompensation = false;
  545. }
  546. }
  547. block->PathDepBranchFolding(this);
  548. #if DBG
  549. // The set of live lossy int32 syms should be a subset of all live int32 syms
  550. this->tempBv->And(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms);
  551. Assert(this->tempBv->Count() == block->globOptData.liveLossyInt32Syms->Count());
  552. // The set of live lossy int32 syms should be a subset of live var or float syms (var or float sym containing the lossless
  553. // value of the sym should be live)
  554. this->tempBv->Or(block->globOptData.liveVarSyms, block->globOptData.liveFloat64Syms);
  555. this->tempBv->And(block->globOptData.liveLossyInt32Syms);
  556. Assert(this->tempBv->Count() == block->globOptData.liveLossyInt32Syms->Count());
  557. this->tempBv->ClearAll();
  558. Assert(this->currentBlock == block);
  559. #endif
  560. }
  561. void
  562. GlobOpt::OptLoops(Loop *loop)
  563. {
  564. Assert(loop != nullptr);
  565. #if DBG
  566. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase) &&
  567. !DoFunctionFieldCopyProp() && DoFieldCopyProp(loop))
  568. {
  569. Output::Print(_u("TRACE: CanDoFieldCopyProp Loop: "));
  570. this->func->DumpFullFunctionName();
  571. uint loopNumber = loop->GetLoopNumber();
  572. Assert(loopNumber != Js::LoopHeader::NoLoop);
  573. Output::Print(_u(" Loop: %d\n"), loopNumber);
  574. }
  575. #endif
  576. Loop *previousLoop = this->prePassLoop;
  577. this->prePassLoop = loop;
  578. if (previousLoop == nullptr)
  579. {
  580. Assert(this->rootLoopPrePass == nullptr);
  581. this->rootLoopPrePass = loop;
  582. this->prePassInstrMap->Clear();
  583. if (loop->parent == nullptr)
  584. {
  585. // Outer most loop...
  586. this->prePassCopyPropSym->ClearAll();
  587. }
  588. }
  589. Assert(loop->symsAssignedToInLoop != nullptr);
  590. if (loop->symsUsedBeforeDefined == nullptr)
  591. {
  592. loop->symsUsedBeforeDefined = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  593. loop->likelyIntSymsUsedBeforeDefined = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  594. loop->likelyNumberSymsUsedBeforeDefined = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  595. loop->forceFloat64SymsOnEntry = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  596. loop->symsDefInLoop = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  597. loop->fieldKilled = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  598. loop->fieldPRESymStores = JitAnew(alloc, BVSparse<JitArenaAllocator>, this->alloc);
  599. loop->allFieldsKilled = false;
  600. }
  601. else
  602. {
  603. loop->symsUsedBeforeDefined->ClearAll();
  604. loop->likelyIntSymsUsedBeforeDefined->ClearAll();
  605. loop->likelyNumberSymsUsedBeforeDefined->ClearAll();
  606. loop->forceFloat64SymsOnEntry->ClearAll();
  607. loop->symsDefInLoop->ClearAll();
  608. loop->fieldKilled->ClearAll();
  609. loop->allFieldsKilled = false;
  610. loop->initialValueFieldMap.Reset();
  611. }
  612. FOREACH_BLOCK_IN_LOOP(block, loop)
  613. {
  614. block->SetDataUseCount(block->GetSuccList()->Count());
  615. OptBlock(block);
  616. } NEXT_BLOCK_IN_LOOP;
  617. if (previousLoop == nullptr)
  618. {
  619. Assert(this->rootLoopPrePass == loop);
  620. this->rootLoopPrePass = nullptr;
  621. }
  622. this->prePassLoop = previousLoop;
  623. }
  624. void
  625. GlobOpt::TailDupPass()
  626. {
  627. FOREACH_LOOP_IN_FUNC_EDITING(loop, this->func)
  628. {
  629. BasicBlock* header = loop->GetHeadBlock();
  630. BasicBlock* loopTail = nullptr;
  631. FOREACH_PREDECESSOR_BLOCK(pred, header)
  632. {
  633. if (loop->IsDescendentOrSelf(pred->loop))
  634. {
  635. loopTail = pred;
  636. break;
  637. }
  638. } NEXT_PREDECESSOR_BLOCK;
  639. if (loopTail)
  640. {
  641. AssertMsg(loopTail->GetLastInstr()->IsBranchInstr(), "LastInstr of loop should always be a branch no?");
  642. if (!loopTail->GetPredList()->HasOne())
  643. {
  644. TryTailDup(loopTail->GetLastInstr()->AsBranchInstr());
  645. }
  646. }
  647. } NEXT_LOOP_IN_FUNC_EDITING;
  648. }
  649. bool
  650. GlobOpt::TryTailDup(IR::BranchInstr *tailBranch)
  651. {
  652. if (PHASE_OFF(Js::TailDupPhase, tailBranch->m_func->GetTopFunc()))
  653. {
  654. return false;
  655. }
  656. if (tailBranch->IsConditional())
  657. {
  658. return false;
  659. }
  660. IR::Instr *instr;
  661. uint instrCount = 0;
  662. for (instr = tailBranch->GetPrevRealInstrOrLabel(); !instr->IsLabelInstr(); instr = instr->GetPrevRealInstrOrLabel())
  663. {
  664. if (instr->HasBailOutInfo())
  665. {
  666. break;
  667. }
  668. if (!OpCodeAttr::CanCSE(instr->m_opcode))
  669. {
  670. // Consider: We could be more aggressive here
  671. break;
  672. }
  673. instrCount++;
  674. if (instrCount > 1)
  675. {
  676. // Consider: If copy handled single-def tmps renaming, we could do more instrs
  677. break;
  678. }
  679. }
  680. if (!instr->IsLabelInstr())
  681. {
  682. return false;
  683. }
  684. IR::LabelInstr *mergeLabel = instr->AsLabelInstr();
  685. IR::Instr *mergeLabelPrev = mergeLabel->m_prev;
  686. // Skip unreferenced labels
  687. while (mergeLabelPrev->IsLabelInstr() && mergeLabelPrev->AsLabelInstr()->labelRefs.Empty())
  688. {
  689. mergeLabelPrev = mergeLabelPrev->m_prev;
  690. }
  691. BasicBlock* labelBlock = mergeLabel->GetBasicBlock();
  692. uint origPredCount = labelBlock->GetPredList()->Count();
  693. uint dupCount = 0;
  694. // We are good to go. Let's do the tail duplication.
  695. FOREACH_SLISTCOUNTED_ENTRY_EDITING(IR::BranchInstr*, branchEntry, &mergeLabel->labelRefs, iter)
  696. {
  697. if (branchEntry->IsUnconditional() && !branchEntry->IsMultiBranch() && branchEntry != mergeLabelPrev && branchEntry != tailBranch)
  698. {
  699. for (instr = mergeLabel->m_next; instr != tailBranch; instr = instr->m_next)
  700. {
  701. branchEntry->InsertBefore(instr->Copy());
  702. }
  703. instr = branchEntry;
  704. branchEntry->ReplaceTarget(mergeLabel, tailBranch->GetTarget());
  705. while(!instr->IsLabelInstr())
  706. {
  707. instr = instr->m_prev;
  708. }
  709. BasicBlock* branchBlock = instr->AsLabelInstr()->GetBasicBlock();
  710. labelBlock->RemovePred(branchBlock, func->m_fg);
  711. func->m_fg->AddEdge(branchBlock, tailBranch->GetTarget()->GetBasicBlock());
  712. dupCount++;
  713. }
  714. } NEXT_SLISTCOUNTED_ENTRY_EDITING;
  715. // If we've duplicated everywhere, tail block is dead and should be removed.
  716. if (dupCount == origPredCount)
  717. {
  718. AssertMsg(mergeLabel->labelRefs.Empty(), "Should not remove block with referenced label.");
  719. func->m_fg->RemoveBlock(labelBlock, nullptr, true);
  720. }
  721. return true;
  722. }
  723. void
  724. GlobOpt::ToVar(BVSparse<JitArenaAllocator> *bv, BasicBlock *block)
  725. {
  726. FOREACH_BITSET_IN_SPARSEBV(id, bv)
  727. {
  728. StackSym *stackSym = this->func->m_symTable->FindStackSym(id);
  729. IR::RegOpnd *newOpnd = IR::RegOpnd::New(stackSym, TyVar, this->func);
  730. IR::Instr *lastInstr = block->GetLastInstr();
  731. if (lastInstr->IsBranchInstr() || lastInstr->m_opcode == Js::OpCode::BailTarget)
  732. {
  733. // If branch is using this symbol, hoist the operand as the ToVar load will get
  734. // inserted right before the branch.
  735. IR::Opnd *src1 = lastInstr->GetSrc1();
  736. if (src1)
  737. {
  738. if (src1->IsRegOpnd() && src1->AsRegOpnd()->m_sym == stackSym)
  739. {
  740. lastInstr->HoistSrc1(Js::OpCode::Ld_A);
  741. }
  742. IR::Opnd *src2 = lastInstr->GetSrc2();
  743. if (src2)
  744. {
  745. if (src2->IsRegOpnd() && src2->AsRegOpnd()->m_sym == stackSym)
  746. {
  747. lastInstr->HoistSrc2(Js::OpCode::Ld_A);
  748. }
  749. }
  750. }
  751. this->ToVar(lastInstr, newOpnd, block, nullptr, false);
  752. }
  753. else
  754. {
  755. IR::Instr *lastNextInstr = lastInstr->m_next;
  756. this->ToVar(lastNextInstr, newOpnd, block, nullptr, false);
  757. }
  758. } NEXT_BITSET_IN_SPARSEBV;
  759. }
  760. void
  761. GlobOpt::ToInt32(BVSparse<JitArenaAllocator> *bv, BasicBlock *block, bool lossy, IR::Instr *insertBeforeInstr)
  762. {
  763. return this->ToTypeSpec(bv, block, TyInt32, IR::BailOutIntOnly, lossy, insertBeforeInstr);
  764. }
  765. void
  766. GlobOpt::ToFloat64(BVSparse<JitArenaAllocator> *bv, BasicBlock *block)
  767. {
  768. return this->ToTypeSpec(bv, block, TyFloat64, IR::BailOutNumberOnly);
  769. }
  770. void
  771. GlobOpt::ToTypeSpec(BVSparse<JitArenaAllocator> *bv, BasicBlock *block, IRType toType, IR::BailOutKind bailOutKind, bool lossy, IR::Instr *insertBeforeInstr)
  772. {
  773. FOREACH_BITSET_IN_SPARSEBV(id, bv)
  774. {
  775. StackSym *stackSym = this->func->m_symTable->FindStackSym(id);
  776. IRType fromType = TyIllegal;
  777. // Win8 bug: 757126. If we are trying to type specialize the arguments object,
  778. // let's make sure stack args optimization is not enabled. This is a problem, particularly,
  779. // if the instruction comes from an unreachable block. In other cases, the pass on the
  780. // instruction itself should disable arguments object optimization.
  781. if(block->globOptData.argObjSyms && block->globOptData.IsArgumentsSymID(id))
  782. {
  783. CannotAllocateArgumentsObjectOnStack();
  784. }
  785. if (block->globOptData.liveVarSyms->Test(id))
  786. {
  787. fromType = TyVar;
  788. }
  789. else if (block->globOptData.liveInt32Syms->Test(id) && !block->globOptData.liveLossyInt32Syms->Test(id))
  790. {
  791. fromType = TyInt32;
  792. stackSym = stackSym->GetInt32EquivSym(this->func);
  793. }
  794. else if (block->globOptData.liveFloat64Syms->Test(id))
  795. {
  796. fromType = TyFloat64;
  797. stackSym = stackSym->GetFloat64EquivSym(this->func);
  798. }
  799. else
  800. {
  801. Assert(UNREACHED);
  802. }
  803. IR::RegOpnd *newOpnd = IR::RegOpnd::New(stackSym, fromType, this->func);
  804. this->ToTypeSpecUse(nullptr, newOpnd, block, nullptr, nullptr, toType, bailOutKind, lossy, insertBeforeInstr);
  805. } NEXT_BITSET_IN_SPARSEBV;
  806. }
  807. void GlobOpt::PRE::FindPossiblePRECandidates(Loop *loop, JitArenaAllocator *alloc)
  808. {
  809. // Find the set of PRE candidates
  810. BasicBlock *loopHeader = loop->GetHeadBlock();
  811. PRECandidates *candidates = nullptr;
  812. bool firstBackEdge = true;
  813. FOREACH_PREDECESSOR_BLOCK(blockPred, loopHeader)
  814. {
  815. if (!loop->IsDescendentOrSelf(blockPred->loop))
  816. {
  817. // Not a loop back-edge
  818. continue;
  819. }
  820. if (firstBackEdge)
  821. {
  822. candidates = this->globOpt->FindBackEdgePRECandidates(blockPred, alloc);
  823. }
  824. else
  825. {
  826. blockPred->globOptData.RemoveUnavailableCandidates(candidates);
  827. }
  828. } NEXT_PREDECESSOR_BLOCK;
  829. this->candidates = candidates;
  830. }
  831. BOOL GlobOpt::PRE::PreloadPRECandidate(Loop *loop, GlobHashBucket* candidate)
  832. {
  833. // Insert a load for each field PRE candidate.
  834. PropertySym *propertySym = candidate->value->AsPropertySym();
  835. if (!candidates->candidatesToProcess->TestAndClear(propertySym->m_id))
  836. {
  837. return false;
  838. }
  839. Value * propSymValueOnBackEdge = candidate->element;
  840. StackSym *objPtrSym = propertySym->m_stackSym;
  841. Sym * objPtrCopyPropSym = nullptr;
  842. if (!loop->landingPad->globOptData.IsLive(objPtrSym))
  843. {
  844. if (PHASE_OFF(Js::MakeObjSymLiveInLandingPadPhase, this->globOpt->func))
  845. {
  846. return false;
  847. }
  848. if (objPtrSym->IsSingleDef())
  849. {
  850. // We can still try to do PRE if the object sym is single def, even if its not live in the landing pad.
  851. // We'll have to add a def instruction for the object sym in the landing pad, and then we can continue
  852. // pre-loading the current PRE candidate.
  853. // Case in point:
  854. // $L1
  855. // value|symStore
  856. // t1 = o.x (v1|t3)
  857. // t2 = t1.y (v2|t4) <-- t1 is not live in the loop landing pad
  858. // jmp $L1
  859. if (!InsertSymDefinitionInLandingPad(objPtrSym, loop, &objPtrCopyPropSym))
  860. {
  861. #if DBG_DUMP
  862. TraceFailedPreloadInLandingPad(loop, propertySym, _u("Failed to insert load of object sym in landing pad"));
  863. #endif
  864. return false;
  865. }
  866. }
  867. else
  868. {
  869. #if DBG_DUMP
  870. TraceFailedPreloadInLandingPad(loop, propertySym, _u("Object sym not live in landing pad and not single-def"));
  871. #endif
  872. return false;
  873. }
  874. }
  875. Assert(loop->landingPad->globOptData.IsLive(objPtrSym));
  876. BasicBlock *landingPad = loop->landingPad;
  877. Sym *symStore = propSymValueOnBackEdge->GetValueInfo()->GetSymStore();
  878. // The symStore can't be live into the loop
  879. // The symStore needs to still have the same value
  880. Assert(symStore && symStore->IsStackSym());
  881. if (loop->landingPad->globOptData.IsLive(symStore))
  882. {
  883. // May have already been hoisted:
  884. // o.x = t1;
  885. // o.y = t1;
  886. return false;
  887. }
  888. Value *landingPadValue = landingPad->globOptData.FindValue(propertySym);
  889. // Value should be added as initial value or already be there.
  890. Assert(landingPadValue);
  891. IR::Instr * ldInstrInLoop = this->globOpt->prePassInstrMap->Lookup(propertySym->m_id, nullptr);
  892. Assert(ldInstrInLoop);
  893. Assert(ldInstrInLoop->GetDst() == nullptr);
  894. // Create instr to put in landing pad for compensation
  895. Assert(IsPREInstrCandidateLoad(ldInstrInLoop->m_opcode));
  896. IR::Instr * ldInstr = InsertPropertySymPreloadInLandingPad(ldInstrInLoop, loop, propertySym);
  897. if (!ldInstr)
  898. {
  899. return false;
  900. }
  901. Assert(ldInstr->GetDst() == nullptr);
  902. ldInstr->SetDst(IR::RegOpnd::New(symStore->AsStackSym(), TyVar, this->globOpt->func));
  903. loop->fieldPRESymStores->Set(symStore->m_id);
  904. landingPad->globOptData.liveVarSyms->Set(symStore->m_id);
  905. Value * objPtrValue = landingPad->globOptData.FindValue(objPtrSym);
  906. objPtrCopyPropSym = objPtrCopyPropSym ? objPtrCopyPropSym : objPtrValue ? landingPad->globOptData.GetCopyPropSym(objPtrSym, objPtrValue) : nullptr;
  907. if (objPtrCopyPropSym)
  908. {
  909. // If we inserted T4 = T1.y, and T3 is the copy prop sym for T1 in the landing pad, we need T3.y
  910. // to be live on back edges to have the merge produce a value for T3.y. Having a value for T1.y
  911. // produced from the merge is not enough as the T1.y in the loop will get obj-ptr-copy-propped to
  912. // T3.y
  913. // T3.y
  914. PropertySym *newPropSym = PropertySym::FindOrCreate(
  915. objPtrCopyPropSym->m_id, propertySym->m_propertyId, propertySym->GetPropertyIdIndex(), propertySym->GetInlineCacheIndex(), propertySym->m_fieldKind, this->globOpt->func);
  916. if (!landingPad->globOptData.FindValue(newPropSym))
  917. {
  918. landingPad->globOptData.SetValue(landingPadValue, newPropSym);
  919. landingPad->globOptData.liveFields->Set(newPropSym->m_id);
  920. MakePropertySymLiveOnBackEdges(newPropSym, loop, propSymValueOnBackEdge);
  921. }
  922. }
  923. ValueType valueType(ValueType::Uninitialized);
  924. Value *initialValue = nullptr;
  925. if (loop->initialValueFieldMap.TryGetValue(propertySym, &initialValue))
  926. {
  927. if (ldInstr->IsProfiledInstr())
  928. {
  929. if (initialValue->GetValueNumber() == propSymValueOnBackEdge->GetValueNumber())
  930. {
  931. if (propSymValueOnBackEdge->GetValueInfo()->IsUninitialized())
  932. {
  933. valueType = ldInstr->AsProfiledInstr()->u.FldInfo().valueType;
  934. }
  935. else
  936. {
  937. valueType = propSymValueOnBackEdge->GetValueInfo()->Type();
  938. }
  939. }
  940. else
  941. {
  942. valueType = ValueType::Uninitialized;
  943. }
  944. ldInstr->AsProfiledInstr()->u.FldInfo().valueType = valueType;
  945. }
  946. }
  947. else
  948. {
  949. valueType = landingPadValue->GetValueInfo()->Type();
  950. }
  951. loop->symsUsedBeforeDefined->Set(symStore->m_id);
  952. if (valueType.IsLikelyNumber())
  953. {
  954. loop->likelyNumberSymsUsedBeforeDefined->Set(symStore->m_id);
  955. if (globOpt->DoAggressiveIntTypeSpec() ? valueType.IsLikelyInt() : valueType.IsInt())
  956. {
  957. // Can only force int conversions in the landing pad based on likely-int values if aggressive int type
  958. // specialization is enabled
  959. loop->likelyIntSymsUsedBeforeDefined->Set(symStore->m_id);
  960. }
  961. }
  962. #if DBG_DUMP
  963. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldPREPhase, this->globOpt->func->GetSourceContextId(), this->globOpt->func->GetLocalFunctionId()))
  964. {
  965. Output::Print(_u("** TRACE: Field PRE: field pre-loaded in landing pad of loop head #%-3d: "), loop->GetHeadBlock()->GetBlockNum());
  966. ldInstr->Dump();
  967. Output::Print(_u("\n"));
  968. Output::Flush();
  969. }
  970. #endif
  971. return true;
  972. }
  973. void GlobOpt::PRE::PreloadPRECandidates(Loop *loop)
  974. {
  975. // Insert loads in landing pad for field PRE candidates. Iterate while(changed)
  976. // for the o.x.y cases.
  977. BOOL changed = true;
  978. if (!candidates || !candidates->candidatesList)
  979. {
  980. return;
  981. }
  982. Assert(loop->landingPad->GetFirstInstr() == loop->landingPad->GetLastInstr());
  983. while (changed)
  984. {
  985. changed = false;
  986. FOREACH_SLIST_ENTRY_EDITING(GlobHashBucket*, candidate, (SList<GlobHashBucket*>*)candidates->candidatesList, iter)
  987. {
  988. if (this->PreloadPRECandidate(loop, candidate))
  989. {
  990. changed = true;
  991. iter.RemoveCurrent();
  992. }
  993. if (PHASE_TRACE(Js::FieldPREPhase, this->globOpt->func))
  994. {
  995. Output::Print(_u("============================\n"));
  996. Output::Flush();
  997. }
  998. } NEXT_SLIST_ENTRY_EDITING;
  999. }
  1000. }
  1001. void GlobOpt::FieldPRE(Loop *loop)
  1002. {
  1003. if (!DoFieldPRE(loop))
  1004. {
  1005. return;
  1006. }
  1007. GlobOpt::PRE pre(this);
  1008. pre.FieldPRE(loop);
  1009. }
  1010. void GlobOpt::InsertValueCompensation(
  1011. BasicBlock *const predecessor,
  1012. BasicBlock *const successor,
  1013. const SymToValueInfoMap *symsRequiringCompensationToMergedValueInfoMap)
  1014. {
  1015. Assert(predecessor);
  1016. Assert(successor);
  1017. AssertOrFailFast(predecessor != successor);
  1018. Assert(symsRequiringCompensationToMergedValueInfoMap->Count() != 0);
  1019. IR::Instr *insertBeforeInstr = predecessor->GetLastInstr();
  1020. Func *const func = insertBeforeInstr->m_func;
  1021. bool setLastInstrInPredecessor;
  1022. if(insertBeforeInstr->IsBranchInstr() || insertBeforeInstr->m_opcode == Js::OpCode::BailTarget)
  1023. {
  1024. // Don't insert code between the branch and the corresponding ByteCodeUses instructions
  1025. while(insertBeforeInstr->m_prev->m_opcode == Js::OpCode::ByteCodeUses)
  1026. {
  1027. insertBeforeInstr = insertBeforeInstr->m_prev;
  1028. }
  1029. setLastInstrInPredecessor = false;
  1030. }
  1031. else
  1032. {
  1033. // Insert at the end of the block and set the last instruction
  1034. Assert(insertBeforeInstr->m_next);
  1035. insertBeforeInstr = insertBeforeInstr->m_next; // Instruction after the last instruction in the predecessor
  1036. setLastInstrInPredecessor = true;
  1037. }
  1038. GlobOptBlockData &predecessorBlockData = predecessor->globOptData;
  1039. GlobOptBlockData &successorBlockData = successor->globOptData;
  1040. struct DelayChangeValueInfo
  1041. {
  1042. Value* predecessorValue;
  1043. ArrayValueInfo* valueInfo;
  1044. void ChangeValueInfo(BasicBlock* predecessor, GlobOpt* g)
  1045. {
  1046. g->ChangeValueInfo(
  1047. predecessor,
  1048. predecessorValue,
  1049. valueInfo,
  1050. false /*allowIncompatibleType*/,
  1051. true /*compensated*/);
  1052. }
  1053. };
  1054. JsUtil::List<DelayChangeValueInfo, ArenaAllocator> delayChangeValueInfo(alloc);
  1055. for(auto it = symsRequiringCompensationToMergedValueInfoMap->GetIterator(); it.IsValid(); it.MoveNext())
  1056. {
  1057. const auto &entry = it.Current();
  1058. Sym *const sym = entry.Key();
  1059. Value *const predecessorValue = predecessorBlockData.FindValue(sym);
  1060. Assert(predecessorValue);
  1061. ValueInfo *const predecessorValueInfo = predecessorValue->GetValueInfo();
  1062. // Currently, array value infos are the only ones that require compensation based on values
  1063. Assert(predecessorValueInfo->IsAnyOptimizedArray());
  1064. const ArrayValueInfo *const predecessorArrayValueInfo = predecessorValueInfo->AsArrayValueInfo();
  1065. StackSym *const predecessorHeadSegmentSym = predecessorArrayValueInfo->HeadSegmentSym();
  1066. StackSym *const predecessorHeadSegmentLengthSym = predecessorArrayValueInfo->HeadSegmentLengthSym();
  1067. StackSym *const predecessorLengthSym = predecessorArrayValueInfo->LengthSym();
  1068. ValueInfo *const mergedValueInfo = entry.Value();
  1069. const ArrayValueInfo *const mergedArrayValueInfo = mergedValueInfo->AsArrayValueInfo();
  1070. StackSym *const mergedHeadSegmentSym = mergedArrayValueInfo->HeadSegmentSym();
  1071. StackSym *const mergedHeadSegmentLengthSym = mergedArrayValueInfo->HeadSegmentLengthSym();
  1072. StackSym *const mergedLengthSym = mergedArrayValueInfo->LengthSym();
  1073. Assert(!mergedHeadSegmentSym || predecessorHeadSegmentSym);
  1074. Assert(!mergedHeadSegmentLengthSym || predecessorHeadSegmentLengthSym);
  1075. Assert(!mergedLengthSym || predecessorLengthSym);
  1076. bool compensated = false;
  1077. if(mergedHeadSegmentSym && predecessorHeadSegmentSym != mergedHeadSegmentSym)
  1078. {
  1079. IR::Instr *const newInstr =
  1080. IR::Instr::New(
  1081. Js::OpCode::Ld_A,
  1082. IR::RegOpnd::New(mergedHeadSegmentSym, mergedHeadSegmentSym->GetType(), func),
  1083. IR::RegOpnd::New(predecessorHeadSegmentSym, predecessorHeadSegmentSym->GetType(), func),
  1084. func);
  1085. newInstr->GetDst()->SetIsJITOptimizedReg(true);
  1086. newInstr->GetSrc1()->SetIsJITOptimizedReg(true);
  1087. newInstr->SetByteCodeOffset(insertBeforeInstr);
  1088. insertBeforeInstr->InsertBefore(newInstr);
  1089. compensated = true;
  1090. }
  1091. if(mergedHeadSegmentLengthSym && predecessorHeadSegmentLengthSym != mergedHeadSegmentLengthSym)
  1092. {
  1093. IR::Instr *const newInstr =
  1094. IR::Instr::New(
  1095. Js::OpCode::Ld_A,
  1096. IR::RegOpnd::New(mergedHeadSegmentLengthSym, mergedHeadSegmentLengthSym->GetType(), func),
  1097. IR::RegOpnd::New(predecessorHeadSegmentLengthSym, predecessorHeadSegmentLengthSym->GetType(), func),
  1098. func);
  1099. newInstr->GetDst()->SetIsJITOptimizedReg(true);
  1100. newInstr->GetSrc1()->SetIsJITOptimizedReg(true);
  1101. newInstr->SetByteCodeOffset(insertBeforeInstr);
  1102. insertBeforeInstr->InsertBefore(newInstr);
  1103. compensated = true;
  1104. // Merge the head segment length value
  1105. Assert(predecessorBlockData.liveVarSyms->Test(predecessorHeadSegmentLengthSym->m_id));
  1106. predecessorBlockData.liveVarSyms->Set(mergedHeadSegmentLengthSym->m_id);
  1107. successorBlockData.liveVarSyms->Set(mergedHeadSegmentLengthSym->m_id);
  1108. Value *const predecessorHeadSegmentLengthValue =
  1109. predecessorBlockData.FindValue(predecessorHeadSegmentLengthSym);
  1110. Assert(predecessorHeadSegmentLengthValue);
  1111. predecessorBlockData.SetValue(predecessorHeadSegmentLengthValue, mergedHeadSegmentLengthSym);
  1112. Value *const mergedHeadSegmentLengthValue = successorBlockData.FindValue(mergedHeadSegmentLengthSym);
  1113. if(mergedHeadSegmentLengthValue)
  1114. {
  1115. Assert(mergedHeadSegmentLengthValue->GetValueNumber() != predecessorHeadSegmentLengthValue->GetValueNumber());
  1116. if(predecessorHeadSegmentLengthValue->GetValueInfo() != mergedHeadSegmentLengthValue->GetValueInfo())
  1117. {
  1118. mergedHeadSegmentLengthValue->SetValueInfo(
  1119. ValueInfo::MergeLikelyIntValueInfo(
  1120. this->alloc,
  1121. mergedHeadSegmentLengthValue,
  1122. predecessorHeadSegmentLengthValue,
  1123. mergedHeadSegmentLengthValue->GetValueInfo()->Type()
  1124. .Merge(predecessorHeadSegmentLengthValue->GetValueInfo()->Type())));
  1125. }
  1126. }
  1127. else
  1128. {
  1129. successorBlockData.SetValue(CopyValue(predecessorHeadSegmentLengthValue), mergedHeadSegmentLengthSym);
  1130. }
  1131. }
  1132. if(mergedLengthSym && predecessorLengthSym != mergedLengthSym)
  1133. {
  1134. IR::Instr *const newInstr =
  1135. IR::Instr::New(
  1136. Js::OpCode::Ld_I4,
  1137. IR::RegOpnd::New(mergedLengthSym, mergedLengthSym->GetType(), func),
  1138. IR::RegOpnd::New(predecessorLengthSym, predecessorLengthSym->GetType(), func),
  1139. func);
  1140. newInstr->GetDst()->SetIsJITOptimizedReg(true);
  1141. newInstr->GetSrc1()->SetIsJITOptimizedReg(true);
  1142. newInstr->SetByteCodeOffset(insertBeforeInstr);
  1143. insertBeforeInstr->InsertBefore(newInstr);
  1144. compensated = true;
  1145. // Merge the length value
  1146. Assert(predecessorBlockData.liveVarSyms->Test(predecessorLengthSym->m_id));
  1147. predecessorBlockData.liveVarSyms->Set(mergedLengthSym->m_id);
  1148. successorBlockData.liveVarSyms->Set(mergedLengthSym->m_id);
  1149. Value *const predecessorLengthValue = predecessorBlockData.FindValue(predecessorLengthSym);
  1150. Assert(predecessorLengthValue);
  1151. predecessorBlockData.SetValue(predecessorLengthValue, mergedLengthSym);
  1152. Value *const mergedLengthValue = successorBlockData.FindValue(mergedLengthSym);
  1153. if(mergedLengthValue)
  1154. {
  1155. Assert(mergedLengthValue->GetValueNumber() != predecessorLengthValue->GetValueNumber());
  1156. if(predecessorLengthValue->GetValueInfo() != mergedLengthValue->GetValueInfo())
  1157. {
  1158. mergedLengthValue->SetValueInfo(
  1159. ValueInfo::MergeLikelyIntValueInfo(
  1160. this->alloc,
  1161. mergedLengthValue,
  1162. predecessorLengthValue,
  1163. mergedLengthValue->GetValueInfo()->Type().Merge(predecessorLengthValue->GetValueInfo()->Type())));
  1164. }
  1165. }
  1166. else
  1167. {
  1168. successorBlockData.SetValue(CopyValue(predecessorLengthValue), mergedLengthSym);
  1169. }
  1170. }
  1171. if(compensated)
  1172. {
  1173. // Save the new ValueInfo for later.
  1174. // We don't want other symbols needing compensation to see this new one
  1175. delayChangeValueInfo.Add({
  1176. predecessorValue,
  1177. ArrayValueInfo::New(
  1178. alloc,
  1179. predecessorValueInfo->Type(),
  1180. mergedHeadSegmentSym ? mergedHeadSegmentSym : predecessorHeadSegmentSym,
  1181. mergedHeadSegmentLengthSym ? mergedHeadSegmentLengthSym : predecessorHeadSegmentLengthSym,
  1182. mergedLengthSym ? mergedLengthSym : predecessorLengthSym,
  1183. predecessorValueInfo->GetSymStore())
  1184. });
  1185. }
  1186. }
  1187. // Once we've compensated all the symbols, update the new ValueInfo.
  1188. delayChangeValueInfo.Map([predecessor, this](int, DelayChangeValueInfo d) { d.ChangeValueInfo(predecessor, this); });
  1189. if(setLastInstrInPredecessor)
  1190. {
  1191. predecessor->SetLastInstr(insertBeforeInstr->m_prev);
  1192. }
  1193. }
  1194. bool
  1195. GlobOpt::AreFromSameBytecodeFunc(IR::RegOpnd const* src1, IR::RegOpnd const* dst) const
  1196. {
  1197. Assert(this->func->m_symTable->FindStackSym(src1->m_sym->m_id) == src1->m_sym);
  1198. Assert(this->func->m_symTable->FindStackSym(dst->m_sym->m_id) == dst->m_sym);
  1199. if (dst->m_sym->HasByteCodeRegSlot() && src1->m_sym->HasByteCodeRegSlot())
  1200. {
  1201. return src1->m_sym->GetByteCodeFunc() == dst->m_sym->GetByteCodeFunc();
  1202. }
  1203. return false;
  1204. }
  1205. /*
  1206. * This is for scope object removal along with Heap Arguments optimization.
  1207. * We track several instructions to facilitate the removal of scope object.
  1208. * - LdSlotArr - This instr is tracked to keep track of the formals array (the dest)
  1209. * - InlineeStart - To keep track of the stack syms for the formals of the inlinee.
  1210. */
  1211. void
  1212. GlobOpt::TrackInstrsForScopeObjectRemoval(IR::Instr * instr)
  1213. {
  1214. IR::Opnd* dst = instr->GetDst();
  1215. IR::Opnd* src1 = instr->GetSrc1();
  1216. if (instr->m_opcode == Js::OpCode::Ld_A && src1->IsRegOpnd())
  1217. {
  1218. AssertMsg(!instr->m_func->IsStackArgsEnabled() || !src1->IsScopeObjOpnd(instr->m_func), "There can be no aliasing for scope object.");
  1219. }
  1220. // The following is to track formals array for Stack Arguments optimization with Formals
  1221. if (instr->m_func->IsStackArgsEnabled() && !this->IsLoopPrePass())
  1222. {
  1223. if (instr->m_opcode == Js::OpCode::LdSlotArr)
  1224. {
  1225. if (instr->GetSrc1()->IsScopeObjOpnd(instr->m_func))
  1226. {
  1227. AssertMsg(!instr->m_func->GetJITFunctionBody()->HasImplicitArgIns(), "No mapping is required in this case. So it should already be generating ArgIns.");
  1228. instr->m_func->TrackFormalsArraySym(dst->GetStackSym()->m_id);
  1229. }
  1230. }
  1231. else if (instr->m_opcode == Js::OpCode::InlineeStart)
  1232. {
  1233. Assert(instr->m_func->IsInlined());
  1234. Js::ArgSlot actualsCount = instr->m_func->actualCount - 1;
  1235. Js::ArgSlot formalsCount = instr->m_func->GetJITFunctionBody()->GetInParamsCount() - 1;
  1236. Func * func = instr->m_func;
  1237. Func * inlinerFunc = func->GetParentFunc(); //Inliner's func
  1238. IR::Instr * argOutInstr = instr->GetSrc2()->GetStackSym()->GetInstrDef();
  1239. //The argout immediately before the InlineeStart will be the ArgOut for NewScObject
  1240. //So we don't want to track the stack sym for this argout.- Skipping it here.
  1241. if (instr->m_func->IsInlinedConstructor())
  1242. {
  1243. //PRE might introduce a second defintion for the Src1. So assert for the opcode only when it has single definition.
  1244. Assert(argOutInstr->GetSrc1()->GetStackSym()->GetInstrDef() == nullptr ||
  1245. argOutInstr->GetSrc1()->GetStackSym()->GetInstrDef()->m_opcode == Js::OpCode::NewScObjectNoCtor);
  1246. argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef();
  1247. }
  1248. if (formalsCount < actualsCount)
  1249. {
  1250. Js::ArgSlot extraActuals = actualsCount - formalsCount;
  1251. //Skipping extra actuals passed
  1252. for (Js::ArgSlot i = 0; i < extraActuals; i++)
  1253. {
  1254. argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef();
  1255. }
  1256. }
  1257. StackSym * undefinedSym = nullptr;
  1258. for (Js::ArgSlot param = formalsCount; param > 0; param--)
  1259. {
  1260. StackSym * argOutSym = nullptr;
  1261. if (argOutInstr->GetSrc1())
  1262. {
  1263. if (argOutInstr->GetSrc1()->IsRegOpnd())
  1264. {
  1265. argOutSym = argOutInstr->GetSrc1()->GetStackSym();
  1266. }
  1267. else
  1268. {
  1269. // We will always have ArgOut instr - so the source operand will not be removed.
  1270. argOutSym = StackSym::New(inlinerFunc);
  1271. IR::Opnd * srcOpnd = argOutInstr->GetSrc1();
  1272. IR::Opnd * dstOpnd = IR::RegOpnd::New(argOutSym, TyVar, inlinerFunc);
  1273. IR::Instr * assignInstr = IR::Instr::New(Js::OpCode::Ld_A, dstOpnd, srcOpnd, inlinerFunc);
  1274. instr->InsertBefore(assignInstr);
  1275. }
  1276. }
  1277. Assert(!func->HasStackSymForFormal(param - 1));
  1278. if (param <= actualsCount)
  1279. {
  1280. Assert(argOutSym);
  1281. func->TrackStackSymForFormalIndex(param - 1, argOutSym);
  1282. argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef();
  1283. }
  1284. else
  1285. {
  1286. /*When param is out of range of actuals count, load undefined*/
  1287. // TODO: saravind: This will insert undefined for each of the param not having an actual. - Clean up this by having a sym for undefined on func ?
  1288. Assert(formalsCount > actualsCount);
  1289. if (undefinedSym == nullptr)
  1290. {
  1291. undefinedSym = StackSym::New(inlinerFunc);
  1292. IR::Opnd * srcOpnd = IR::AddrOpnd::New(inlinerFunc->GetScriptContextInfo()->GetUndefinedAddr(), IR::AddrOpndKindDynamicMisc, inlinerFunc);
  1293. IR::Opnd * dstOpnd = IR::RegOpnd::New(undefinedSym, TyVar, inlinerFunc);
  1294. IR::Instr * assignUndefined = IR::Instr::New(Js::OpCode::Ld_A, dstOpnd, srcOpnd, inlinerFunc);
  1295. instr->InsertBefore(assignUndefined);
  1296. }
  1297. func->TrackStackSymForFormalIndex(param - 1, undefinedSym);
  1298. }
  1299. }
  1300. }
  1301. }
  1302. }
  1303. void
  1304. GlobOpt::OptArguments(IR::Instr *instr)
  1305. {
  1306. IR::Opnd* dst = instr->GetDst();
  1307. IR::Opnd* src1 = instr->GetSrc1();
  1308. IR::Opnd* src2 = instr->GetSrc2();
  1309. TrackInstrsForScopeObjectRemoval(instr);
  1310. if (!TrackArgumentsObject())
  1311. {
  1312. return;
  1313. }
  1314. if (instr->HasAnyLoadHeapArgsOpCode())
  1315. {
  1316. #ifdef ENABLE_DEBUG_CONFIG_OPTIONS
  1317. if (instr->m_func->IsStackArgsEnabled())
  1318. {
  1319. if (instr->GetSrc1()->IsRegOpnd() && instr->m_func->GetJITFunctionBody()->GetInParamsCount() > 1)
  1320. {
  1321. StackSym * scopeObjSym = instr->GetSrc1()->GetStackSym();
  1322. Assert(scopeObjSym);
  1323. Assert(scopeObjSym->GetInstrDef()->m_opcode == Js::OpCode::InitCachedScope || scopeObjSym->GetInstrDef()->m_opcode == Js::OpCode::NewScopeObject);
  1324. Assert(instr->m_func->GetScopeObjSym() == scopeObjSym);
  1325. if (PHASE_VERBOSE_TRACE1(Js::StackArgFormalsOptPhase))
  1326. {
  1327. Output::Print(_u("StackArgFormals : %s (%d) :Setting scopeObjSym in forward pass. \n"), instr->m_func->GetJITFunctionBody()->GetDisplayName(), instr->m_func->GetJITFunctionBody()->GetFunctionNumber());
  1328. Output::Flush();
  1329. }
  1330. }
  1331. }
  1332. #endif
  1333. if (instr->m_func->GetJITFunctionBody()->GetInParamsCount() != 1 && !instr->m_func->IsStackArgsEnabled())
  1334. {
  1335. CannotAllocateArgumentsObjectOnStack();
  1336. }
  1337. else
  1338. {
  1339. CurrentBlockData()->TrackArgumentsSym(dst->AsRegOpnd());
  1340. }
  1341. return;
  1342. }
  1343. // Keep track of arguments objects and its aliases
  1344. // LdHeapArguments loads the arguments object and Ld_A tracks the aliases.
  1345. if ((instr->m_opcode == Js::OpCode::Ld_A || instr->m_opcode == Js::OpCode::BytecodeArgOutCapture) && (src1->IsRegOpnd() && CurrentBlockData()->IsArgumentsOpnd(src1)))
  1346. {
  1347. // In the debug mode, we don't want to optimize away the aliases. Since we may have to show them on the inspection.
  1348. if (((!AreFromSameBytecodeFunc(src1->AsRegOpnd(), dst->AsRegOpnd()) || this->currentBlock->loop) && instr->m_opcode != Js::OpCode::BytecodeArgOutCapture) || this->func->IsJitInDebugMode())
  1349. {
  1350. CannotAllocateArgumentsObjectOnStack();
  1351. return;
  1352. }
  1353. if(!dst->AsRegOpnd()->GetStackSym()->m_nonEscapingArgObjAlias)
  1354. {
  1355. CurrentBlockData()->TrackArgumentsSym(dst->AsRegOpnd());
  1356. }
  1357. return;
  1358. }
  1359. if (!CurrentBlockData()->TestAnyArgumentsSym())
  1360. {
  1361. // There are no syms to track yet, don't start tracking arguments sym.
  1362. return;
  1363. }
  1364. // Avoid loop prepass
  1365. if (this->currentBlock->loop && this->IsLoopPrePass())
  1366. {
  1367. return;
  1368. }
  1369. SymID id = 0;
  1370. switch(instr->m_opcode)
  1371. {
  1372. case Js::OpCode::LdElemI_A:
  1373. case Js::OpCode::TypeofElem:
  1374. {
  1375. Assert(src1->IsIndirOpnd());
  1376. IR::RegOpnd *indexOpnd = src1->AsIndirOpnd()->GetIndexOpnd();
  1377. if (indexOpnd && CurrentBlockData()->IsArgumentsSymID(indexOpnd->m_sym->m_id))
  1378. {
  1379. // Pathological test cases such as a[arguments]
  1380. CannotAllocateArgumentsObjectOnStack();
  1381. return;
  1382. }
  1383. IR::RegOpnd *baseOpnd = src1->AsIndirOpnd()->GetBaseOpnd();
  1384. id = baseOpnd->m_sym->m_id;
  1385. if (CurrentBlockData()->IsArgumentsSymID(id))
  1386. {
  1387. instr->usesStackArgumentsObject = true;
  1388. }
  1389. break;
  1390. }
  1391. case Js::OpCode::LdLen_A:
  1392. {
  1393. Assert(src1->IsRegOpnd());
  1394. if(CurrentBlockData()->IsArgumentsOpnd(src1))
  1395. {
  1396. instr->usesStackArgumentsObject = true;
  1397. }
  1398. break;
  1399. }
  1400. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  1401. {
  1402. if (CurrentBlockData()->IsArgumentsOpnd(src1))
  1403. {
  1404. instr->usesStackArgumentsObject = true;
  1405. }
  1406. if (CurrentBlockData()->IsArgumentsOpnd(src1) &&
  1407. src1->AsRegOpnd()->m_sym->GetInstrDef()->m_opcode == Js::OpCode::BytecodeArgOutCapture)
  1408. {
  1409. // Apply inlining results in such usage - this is to ignore this sym that is def'd by ByteCodeArgOutCapture
  1410. // It's needed because we do not have block level merging of arguments object and this def due to inlining can turn off stack args opt.
  1411. IR::Instr* builtinStart = instr->GetNextRealInstr();
  1412. if (builtinStart->m_opcode == Js::OpCode::InlineBuiltInStart)
  1413. {
  1414. IR::Opnd* builtinOpnd = builtinStart->GetSrc1();
  1415. if (builtinStart->GetSrc1()->IsAddrOpnd())
  1416. {
  1417. Assert(builtinOpnd->AsAddrOpnd()->m_isFunction);
  1418. Js::BuiltinFunction builtinFunction = Js::JavascriptLibrary::GetBuiltInForFuncInfo(((FixedFieldInfo*)builtinOpnd->AsAddrOpnd()->m_metadata)->GetLocalFuncId());
  1419. if (builtinFunction == Js::BuiltinFunction::JavascriptFunction_Apply)
  1420. {
  1421. CurrentBlockData()->ClearArgumentsSym(src1->AsRegOpnd());
  1422. }
  1423. }
  1424. else if (builtinOpnd->IsRegOpnd())
  1425. {
  1426. if (builtinOpnd->AsRegOpnd()->m_sym->m_builtInIndex == Js::BuiltinFunction::JavascriptFunction_Apply)
  1427. {
  1428. CurrentBlockData()->ClearArgumentsSym(src1->AsRegOpnd());
  1429. }
  1430. }
  1431. }
  1432. }
  1433. break;
  1434. }
  1435. case Js::OpCode::BailOnNotStackArgs:
  1436. case Js::OpCode::ArgOut_A_FromStackArgs:
  1437. case Js::OpCode::BytecodeArgOutUse:
  1438. {
  1439. if (src1 && CurrentBlockData()->IsArgumentsOpnd(src1))
  1440. {
  1441. instr->usesStackArgumentsObject = true;
  1442. }
  1443. break;
  1444. }
  1445. default:
  1446. {
  1447. // Super conservative here, if we see the arguments or any of its alias being used in any
  1448. // other opcode just don't do this optimization. Revisit this to optimize further if we see any common
  1449. // case is missed.
  1450. if (src1)
  1451. {
  1452. if (src1->IsRegOpnd() || src1->IsSymOpnd() || src1->IsIndirOpnd())
  1453. {
  1454. if (CurrentBlockData()->IsArgumentsOpnd(src1))
  1455. {
  1456. #ifdef PERF_HINT
  1457. if (PHASE_TRACE1(Js::PerfHintPhase))
  1458. {
  1459. WritePerfHint(PerfHints::HeapArgumentsCreated, instr->m_func, instr->GetByteCodeOffset());
  1460. }
  1461. #endif
  1462. CannotAllocateArgumentsObjectOnStack();
  1463. return;
  1464. }
  1465. }
  1466. }
  1467. if (src2)
  1468. {
  1469. if (src2->IsRegOpnd() || src2->IsSymOpnd() || src2->IsIndirOpnd())
  1470. {
  1471. if (CurrentBlockData()->IsArgumentsOpnd(src2))
  1472. {
  1473. #ifdef PERF_HINT
  1474. if (PHASE_TRACE1(Js::PerfHintPhase))
  1475. {
  1476. WritePerfHint(PerfHints::HeapArgumentsCreated, instr->m_func, instr->GetByteCodeOffset());
  1477. }
  1478. #endif
  1479. CannotAllocateArgumentsObjectOnStack();
  1480. return;
  1481. }
  1482. }
  1483. }
  1484. // We should look at dst last to correctly handle cases where it's the same as one of the src operands.
  1485. if (dst)
  1486. {
  1487. if (dst->IsIndirOpnd() || dst->IsSymOpnd())
  1488. {
  1489. if (CurrentBlockData()->IsArgumentsOpnd(dst))
  1490. {
  1491. #ifdef PERF_HINT
  1492. if (PHASE_TRACE1(Js::PerfHintPhase))
  1493. {
  1494. WritePerfHint(PerfHints::HeapArgumentsModification, instr->m_func, instr->GetByteCodeOffset());
  1495. }
  1496. #endif
  1497. CannotAllocateArgumentsObjectOnStack();
  1498. return;
  1499. }
  1500. }
  1501. else if (dst->IsRegOpnd())
  1502. {
  1503. if (this->currentBlock->loop && CurrentBlockData()->IsArgumentsOpnd(dst))
  1504. {
  1505. #ifdef PERF_HINT
  1506. if (PHASE_TRACE1(Js::PerfHintPhase))
  1507. {
  1508. WritePerfHint(PerfHints::HeapArgumentsModification, instr->m_func, instr->GetByteCodeOffset());
  1509. }
  1510. #endif
  1511. CannotAllocateArgumentsObjectOnStack();
  1512. return;
  1513. }
  1514. CurrentBlockData()->ClearArgumentsSym(dst->AsRegOpnd());
  1515. }
  1516. }
  1517. }
  1518. break;
  1519. }
  1520. return;
  1521. }
  1522. void
  1523. GlobOpt::MarkArgumentsUsedForBranch(IR::Instr * instr)
  1524. {
  1525. // If it's a conditional branch instruction and the operand used for branching is one of the arguments
  1526. // to the function, tag the m_argUsedForBranch of the functionBody so that it can be used later for inlining decisions.
  1527. if (instr->IsBranchInstr() && !instr->AsBranchInstr()->IsUnconditional())
  1528. {
  1529. IR::BranchInstr * bInstr = instr->AsBranchInstr();
  1530. IR::Opnd *src1 = bInstr->GetSrc1();
  1531. IR::Opnd *src2 = bInstr->GetSrc2();
  1532. // These are used because we don't want to rely on src1 or src2 to always be the register/constant
  1533. IR::RegOpnd *regOpnd = nullptr;
  1534. if (!src2 && (instr->m_opcode == Js::OpCode::BrFalse_A || instr->m_opcode == Js::OpCode::BrTrue_A) && src1->IsRegOpnd())
  1535. {
  1536. regOpnd = src1->AsRegOpnd();
  1537. }
  1538. // We need to check for (0===arg) and (arg===0); this is especially important since some minifiers
  1539. // change all instances of one to the other.
  1540. else if (src2 && src2->IsConstOpnd() && src1->IsRegOpnd())
  1541. {
  1542. regOpnd = src1->AsRegOpnd();
  1543. }
  1544. else if (src2 && src2->IsRegOpnd() && src1->IsConstOpnd())
  1545. {
  1546. regOpnd = src2->AsRegOpnd();
  1547. }
  1548. if (regOpnd != nullptr)
  1549. {
  1550. if (regOpnd->m_sym->IsSingleDef())
  1551. {
  1552. IR::Instr * defInst = regOpnd->m_sym->GetInstrDef();
  1553. IR::Opnd *defSym = defInst->GetSrc1();
  1554. if (defSym && defSym->IsSymOpnd() && defSym->AsSymOpnd()->m_sym->IsStackSym()
  1555. && defSym->AsSymOpnd()->m_sym->AsStackSym()->IsParamSlotSym())
  1556. {
  1557. uint16 param = defSym->AsSymOpnd()->m_sym->AsStackSym()->GetParamSlotNum();
  1558. // We only support functions with 13 arguments to ensure optimal size of callSiteInfo
  1559. if (param < Js::Constants::MaximumArgumentCountForConstantArgumentInlining)
  1560. {
  1561. this->func->GetJITOutput()->SetArgUsedForBranch((uint8)param);
  1562. }
  1563. }
  1564. }
  1565. }
  1566. }
  1567. }
  1568. const InductionVariable*
  1569. GlobOpt::GetInductionVariable(SymID sym, Loop *loop)
  1570. {
  1571. if (loop->inductionVariables)
  1572. {
  1573. for (auto it = loop->inductionVariables->GetIterator(); it.IsValid(); it.MoveNext())
  1574. {
  1575. InductionVariable* iv = &it.CurrentValueReference();
  1576. if (!iv->IsChangeDeterminate() || !iv->IsChangeUnidirectional())
  1577. {
  1578. continue;
  1579. }
  1580. if (iv->Sym()->m_id == sym)
  1581. {
  1582. return iv;
  1583. }
  1584. }
  1585. }
  1586. return nullptr;
  1587. }
  1588. bool
  1589. GlobOpt::IsSymIDInductionVariable(SymID sym, Loop *loop)
  1590. {
  1591. return GetInductionVariable(sym, loop) != nullptr;
  1592. }
  1593. SymID
  1594. GlobOpt::GetVarSymID(StackSym *sym)
  1595. {
  1596. if (sym && sym->m_type != TyVar)
  1597. {
  1598. sym = sym->GetVarEquivSym(nullptr);
  1599. }
  1600. if (!sym)
  1601. {
  1602. return Js::Constants::InvalidSymID;
  1603. }
  1604. return sym->m_id;
  1605. }
  1606. bool
  1607. GlobOpt::IsAllowedForMemOpt(IR::Instr* instr, bool isMemset, IR::RegOpnd *baseOpnd, IR::Opnd *indexOpnd)
  1608. {
  1609. Assert(instr);
  1610. if (!baseOpnd || !indexOpnd)
  1611. {
  1612. return false;
  1613. }
  1614. Loop* loop = this->currentBlock->loop;
  1615. const ValueType baseValueType(baseOpnd->GetValueType());
  1616. const ValueType indexValueType(indexOpnd->GetValueType());
  1617. // Validate the array and index types
  1618. if (
  1619. !indexValueType.IsInt() ||
  1620. !(
  1621. baseValueType.IsTypedIntOrFloatArray() ||
  1622. baseValueType.IsArray()
  1623. )
  1624. )
  1625. {
  1626. #if DBG_DUMP
  1627. wchar indexValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  1628. indexValueType.ToString(indexValueTypeStr);
  1629. wchar baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  1630. baseValueType.ToString(baseValueTypeStr);
  1631. TRACE_MEMOP_VERBOSE(loop, instr, _u("Index[%s] or Array[%s] value type is invalid"), indexValueTypeStr, baseValueTypeStr);
  1632. #endif
  1633. return false;
  1634. }
  1635. // The following is conservative and works around a bug in induction variable analysis.
  1636. if (baseOpnd->IsArrayRegOpnd())
  1637. {
  1638. IR::ArrayRegOpnd *baseArrayOp = baseOpnd->AsArrayRegOpnd();
  1639. bool hasBoundChecksRemoved = (
  1640. baseArrayOp->EliminatedLowerBoundCheck() &&
  1641. baseArrayOp->EliminatedUpperBoundCheck() &&
  1642. !instr->extractedUpperBoundCheckWithoutHoisting &&
  1643. !instr->loadedArrayHeadSegment &&
  1644. !instr->loadedArrayHeadSegmentLength
  1645. );
  1646. if (!hasBoundChecksRemoved)
  1647. {
  1648. TRACE_MEMOP_VERBOSE(loop, instr, _u("Missing bounds check optimization"));
  1649. return false;
  1650. }
  1651. }
  1652. else
  1653. {
  1654. return false;
  1655. }
  1656. if (!baseValueType.IsTypedArray())
  1657. {
  1658. // Check if the instr can kill the value type of the array
  1659. JsArrayKills arrayKills = CheckJsArrayKills(instr);
  1660. if (arrayKills.KillsValueType(baseValueType))
  1661. {
  1662. TRACE_MEMOP_VERBOSE(loop, instr, _u("The array (s%d) can lose its value type"), GetVarSymID(baseOpnd->GetStackSym()));
  1663. return false;
  1664. }
  1665. }
  1666. // Process the Index Operand
  1667. if (!this->OptIsInvariant(baseOpnd, this->currentBlock, loop, CurrentBlockData()->FindValue(baseOpnd->m_sym), false, true))
  1668. {
  1669. TRACE_MEMOP_VERBOSE(loop, instr, _u("Base (s%d) is not invariant"), GetVarSymID(baseOpnd->GetStackSym()));
  1670. return false;
  1671. }
  1672. // Validate the index
  1673. Assert(indexOpnd->GetStackSym());
  1674. SymID indexSymID = GetVarSymID(indexOpnd->GetStackSym());
  1675. const InductionVariable* iv = GetInductionVariable(indexSymID, loop);
  1676. if (!iv)
  1677. {
  1678. // If the index is not an induction variable return
  1679. TRACE_MEMOP_VERBOSE(loop, instr, _u("Index (s%d) is not an induction variable"), indexSymID);
  1680. return false;
  1681. }
  1682. Assert(iv->IsChangeDeterminate() && iv->IsChangeUnidirectional());
  1683. const IntConstantBounds & bounds = iv->ChangeBounds();
  1684. if (loop->memOpInfo)
  1685. {
  1686. // Only accept induction variables that increments by 1
  1687. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 };
  1688. inductionVariableChangeInfo = loop->memOpInfo->inductionVariableChangeInfoMap->Lookup(indexSymID, inductionVariableChangeInfo);
  1689. if (
  1690. (bounds.LowerBound() != 1 && bounds.LowerBound() != -1) ||
  1691. (bounds.UpperBound() != bounds.LowerBound()) ||
  1692. inductionVariableChangeInfo.unroll > 1 // Must be 0 (not seen yet) or 1 (already seen)
  1693. )
  1694. {
  1695. TRACE_MEMOP_VERBOSE(loop, instr, _u("The index does not change by 1: %d><%d, unroll=%d"), bounds.LowerBound(), bounds.UpperBound(), inductionVariableChangeInfo.unroll);
  1696. return false;
  1697. }
  1698. // Check if the index is the same in all MemOp optimization in this loop
  1699. if (!loop->memOpInfo->candidates->Empty())
  1700. {
  1701. Loop::MemOpCandidate* previousCandidate = loop->memOpInfo->candidates->Head();
  1702. // All MemOp operations within the same loop must use the same index
  1703. if (previousCandidate->index != indexSymID)
  1704. {
  1705. TRACE_MEMOP_VERBOSE(loop, instr, _u("The index is not the same as other MemOp in the loop"));
  1706. return false;
  1707. }
  1708. }
  1709. }
  1710. return true;
  1711. }
  1712. bool
  1713. GlobOpt::CollectMemcopyLdElementI(IR::Instr *instr, Loop *loop)
  1714. {
  1715. Assert(instr->GetSrc1()->IsIndirOpnd());
  1716. IR::IndirOpnd *src1 = instr->GetSrc1()->AsIndirOpnd();
  1717. IR::Opnd *indexOpnd = src1->GetIndexOpnd();
  1718. IR::RegOpnd *baseOpnd = src1->GetBaseOpnd()->AsRegOpnd();
  1719. SymID baseSymID = GetVarSymID(baseOpnd->GetStackSym());
  1720. if (!IsAllowedForMemOpt(instr, false, baseOpnd, indexOpnd))
  1721. {
  1722. return false;
  1723. }
  1724. SymID inductionSymID = GetVarSymID(indexOpnd->GetStackSym());
  1725. Assert(IsSymIDInductionVariable(inductionSymID, loop));
  1726. loop->EnsureMemOpVariablesInitialized();
  1727. bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID);
  1728. IR::Opnd * dst = instr->GetDst();
  1729. if (!dst->IsRegOpnd() || !dst->AsRegOpnd()->GetStackSym()->IsSingleDef())
  1730. {
  1731. return false;
  1732. }
  1733. Loop::MemCopyCandidate* memcopyInfo = memcopyInfo = JitAnewStruct(this->func->GetTopFunc()->m_fg->alloc, Loop::MemCopyCandidate);
  1734. memcopyInfo->ldBase = baseSymID;
  1735. memcopyInfo->ldCount = 1;
  1736. memcopyInfo->count = 0;
  1737. memcopyInfo->bIndexAlreadyChanged = isIndexPreIncr;
  1738. memcopyInfo->base = Js::Constants::InvalidSymID; //need to find the stElem first
  1739. memcopyInfo->index = inductionSymID;
  1740. memcopyInfo->transferSym = dst->AsRegOpnd()->GetStackSym();
  1741. loop->memOpInfo->candidates->Prepend(memcopyInfo);
  1742. return true;
  1743. }
  1744. bool
  1745. GlobOpt::CollectMemsetStElementI(IR::Instr *instr, Loop *loop)
  1746. {
  1747. Assert(instr->GetDst()->IsIndirOpnd());
  1748. IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd();
  1749. IR::Opnd *indexOp = dst->GetIndexOpnd();
  1750. IR::RegOpnd *baseOp = dst->GetBaseOpnd()->AsRegOpnd();
  1751. if (!IsAllowedForMemOpt(instr, true, baseOp, indexOp))
  1752. {
  1753. return false;
  1754. }
  1755. SymID baseSymID = GetVarSymID(baseOp->GetStackSym());
  1756. IR::Opnd *srcDef = instr->GetSrc1();
  1757. StackSym *srcSym = nullptr;
  1758. if (srcDef->IsRegOpnd())
  1759. {
  1760. IR::RegOpnd* opnd = srcDef->AsRegOpnd();
  1761. if (this->OptIsInvariant(opnd, this->currentBlock, loop, CurrentBlockData()->FindValue(opnd->m_sym), true, true))
  1762. {
  1763. srcSym = opnd->GetStackSym();
  1764. }
  1765. }
  1766. BailoutConstantValue constant = {TyIllegal, 0};
  1767. if (srcDef->IsFloatConstOpnd())
  1768. {
  1769. constant.InitFloatConstValue(srcDef->AsFloatConstOpnd()->m_value);
  1770. }
  1771. else if (srcDef->IsIntConstOpnd())
  1772. {
  1773. constant.InitIntConstValue(srcDef->AsIntConstOpnd()->GetValue(), srcDef->AsIntConstOpnd()->GetType());
  1774. }
  1775. else if (srcDef->IsAddrOpnd())
  1776. {
  1777. constant.InitVarConstValue(srcDef->AsAddrOpnd()->m_address);
  1778. }
  1779. else if(!srcSym)
  1780. {
  1781. TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Source is not an invariant"));
  1782. return false;
  1783. }
  1784. // Process the Index Operand
  1785. Assert(indexOp->GetStackSym());
  1786. SymID inductionSymID = GetVarSymID(indexOp->GetStackSym());
  1787. Assert(IsSymIDInductionVariable(inductionSymID, loop));
  1788. loop->EnsureMemOpVariablesInitialized();
  1789. bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID);
  1790. Loop::MemSetCandidate* memsetInfo = JitAnewStruct(this->func->GetTopFunc()->m_fg->alloc, Loop::MemSetCandidate);
  1791. memsetInfo->base = baseSymID;
  1792. memsetInfo->index = inductionSymID;
  1793. memsetInfo->constant = constant;
  1794. memsetInfo->srcSym = srcSym;
  1795. memsetInfo->count = 1;
  1796. memsetInfo->bIndexAlreadyChanged = isIndexPreIncr;
  1797. loop->memOpInfo->candidates->Prepend(memsetInfo);
  1798. return true;
  1799. }
  1800. bool GlobOpt::CollectMemcopyStElementI(IR::Instr *instr, Loop *loop)
  1801. {
  1802. if (!loop->memOpInfo || loop->memOpInfo->candidates->Empty())
  1803. {
  1804. // There is no ldElem matching this stElem
  1805. return false;
  1806. }
  1807. Assert(instr->GetDst()->IsIndirOpnd());
  1808. IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd();
  1809. IR::Opnd *indexOp = dst->GetIndexOpnd();
  1810. IR::RegOpnd *baseOp = dst->GetBaseOpnd()->AsRegOpnd();
  1811. SymID baseSymID = GetVarSymID(baseOp->GetStackSym());
  1812. if (!instr->GetSrc1()->IsRegOpnd())
  1813. {
  1814. return false;
  1815. }
  1816. IR::RegOpnd* src1 = instr->GetSrc1()->AsRegOpnd();
  1817. if (!src1->GetIsDead())
  1818. {
  1819. // This must be the last use of the register.
  1820. // It will invalidate `var m = a[i]; b[i] = m;` but this is not a very interesting case.
  1821. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Source (s%d) is still alive after StElemI"), baseSymID);
  1822. return false;
  1823. }
  1824. if (!IsAllowedForMemOpt(instr, false, baseOp, indexOp))
  1825. {
  1826. return false;
  1827. }
  1828. SymID srcSymID = GetVarSymID(src1->GetStackSym());
  1829. // Prepare the memcopyCandidate entry
  1830. Loop::MemOpCandidate* previousCandidate = loop->memOpInfo->candidates->Head();
  1831. if (!previousCandidate->IsMemCopy())
  1832. {
  1833. return false;
  1834. }
  1835. Loop::MemCopyCandidate* memcopyInfo = previousCandidate->AsMemCopy();
  1836. // The previous candidate has to have been created by the matching ldElem
  1837. if (
  1838. memcopyInfo->base != Js::Constants::InvalidSymID ||
  1839. GetVarSymID(memcopyInfo->transferSym) != srcSymID
  1840. )
  1841. {
  1842. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("No matching LdElem found (s%d)"), baseSymID);
  1843. return false;
  1844. }
  1845. Assert(indexOp->GetStackSym());
  1846. SymID inductionSymID = GetVarSymID(indexOp->GetStackSym());
  1847. Assert(IsSymIDInductionVariable(inductionSymID, loop));
  1848. bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID);
  1849. if (isIndexPreIncr != memcopyInfo->bIndexAlreadyChanged)
  1850. {
  1851. // The index changed between the load and the store
  1852. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Index value changed between ldElem and stElem"));
  1853. return false;
  1854. }
  1855. // Consider: Can we remove the count field?
  1856. memcopyInfo->count++;
  1857. memcopyInfo->base = baseSymID;
  1858. return true;
  1859. }
  1860. bool
  1861. GlobOpt::CollectMemOpLdElementI(IR::Instr *instr, Loop *loop)
  1862. {
  1863. Assert(instr->m_opcode == Js::OpCode::LdElemI_A);
  1864. return (!PHASE_OFF(Js::MemCopyPhase, this->func) && CollectMemcopyLdElementI(instr, loop));
  1865. }
  1866. bool
  1867. GlobOpt::CollectMemOpStElementI(IR::Instr *instr, Loop *loop)
  1868. {
  1869. Assert(instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict);
  1870. Assert(instr->GetSrc1());
  1871. return (!PHASE_OFF(Js::MemSetPhase, this->func) && CollectMemsetStElementI(instr, loop)) ||
  1872. (!PHASE_OFF(Js::MemCopyPhase, this->func) && CollectMemcopyStElementI(instr, loop));
  1873. }
  1874. bool
  1875. GlobOpt::CollectMemOpInfo(IR::Instr *instrBegin, IR::Instr *instr, Value *src1Val, Value *src2Val)
  1876. {
  1877. Assert(this->currentBlock->loop);
  1878. Loop *loop = this->currentBlock->loop;
  1879. if (!loop->blockList.HasTwo())
  1880. {
  1881. // We support memcopy and memset for loops which have only two blocks.
  1882. return false;
  1883. }
  1884. if (loop->GetLoopFlags().isInterpreted && !loop->GetLoopFlags().memopMinCountReached)
  1885. {
  1886. TRACE_MEMOP_VERBOSE(loop, instr, _u("minimum loop count not reached"))
  1887. loop->doMemOp = false;
  1888. return false;
  1889. }
  1890. Assert(loop->doMemOp);
  1891. bool isIncr = true, isChangedByOne = false;
  1892. switch (instr->m_opcode)
  1893. {
  1894. case Js::OpCode::StElemI_A:
  1895. case Js::OpCode::StElemI_A_Strict:
  1896. if (!CollectMemOpStElementI(instr, loop))
  1897. {
  1898. loop->doMemOp = false;
  1899. return false;
  1900. }
  1901. break;
  1902. case Js::OpCode::LdElemI_A:
  1903. if (!CollectMemOpLdElementI(instr, loop))
  1904. {
  1905. loop->doMemOp = false;
  1906. return false;
  1907. }
  1908. break;
  1909. case Js::OpCode::Decr_A:
  1910. isIncr = false;
  1911. case Js::OpCode::Incr_A:
  1912. isChangedByOne = true;
  1913. goto MemOpCheckInductionVariable;
  1914. case Js::OpCode::Sub_I4:
  1915. case Js::OpCode::Sub_A:
  1916. isIncr = false;
  1917. case Js::OpCode::Add_A:
  1918. case Js::OpCode::Add_I4:
  1919. {
  1920. MemOpCheckInductionVariable:
  1921. StackSym *sym = instr->GetSrc1()->GetStackSym();
  1922. if (!sym)
  1923. {
  1924. sym = instr->GetSrc2()->GetStackSym();
  1925. }
  1926. SymID inductionSymID = GetVarSymID(sym);
  1927. if (IsSymIDInductionVariable(inductionSymID, this->currentBlock->loop))
  1928. {
  1929. if (!isChangedByOne)
  1930. {
  1931. IR::Opnd *src1, *src2;
  1932. src1 = instr->GetSrc1();
  1933. src2 = instr->GetSrc2();
  1934. if (src2->IsRegOpnd())
  1935. {
  1936. Value *val = CurrentBlockData()->FindValue(src2->AsRegOpnd()->m_sym);
  1937. if (val)
  1938. {
  1939. ValueInfo *vi = val->GetValueInfo();
  1940. int constValue;
  1941. if (vi && vi->TryGetIntConstantValue(&constValue))
  1942. {
  1943. if (constValue == 1)
  1944. {
  1945. isChangedByOne = true;
  1946. }
  1947. }
  1948. }
  1949. }
  1950. else if (src2->IsIntConstOpnd())
  1951. {
  1952. if (src2->AsIntConstOpnd()->GetValue() == 1)
  1953. {
  1954. isChangedByOne = true;
  1955. }
  1956. }
  1957. }
  1958. loop->EnsureMemOpVariablesInitialized();
  1959. if (!isChangedByOne)
  1960. {
  1961. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { Js::Constants::InvalidLoopUnrollFactor, 0 };
  1962. if (!loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID))
  1963. {
  1964. loop->memOpInfo->inductionVariableChangeInfoMap->Add(inductionSymID, inductionVariableChangeInfo);
  1965. }
  1966. else
  1967. {
  1968. loop->memOpInfo->inductionVariableChangeInfoMap->Item(inductionSymID, inductionVariableChangeInfo);
  1969. }
  1970. }
  1971. else
  1972. {
  1973. if (!loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID))
  1974. {
  1975. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 1, isIncr };
  1976. loop->memOpInfo->inductionVariableChangeInfoMap->Add(inductionSymID, inductionVariableChangeInfo);
  1977. }
  1978. else
  1979. {
  1980. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 };
  1981. inductionVariableChangeInfo = loop->memOpInfo->inductionVariableChangeInfoMap->Lookup(inductionSymID, inductionVariableChangeInfo);
  1982. inductionVariableChangeInfo.unroll++;
  1983. inductionVariableChangeInfo.isIncremental = isIncr;
  1984. loop->memOpInfo->inductionVariableChangeInfoMap->Item(inductionSymID, inductionVariableChangeInfo);
  1985. }
  1986. }
  1987. break;
  1988. }
  1989. // Fallthrough if not an induction variable
  1990. }
  1991. default:
  1992. FOREACH_INSTR_IN_RANGE(chkInstr, instrBegin->m_next, instr)
  1993. {
  1994. if (IsInstrInvalidForMemOp(chkInstr, loop, src1Val, src2Val))
  1995. {
  1996. loop->doMemOp = false;
  1997. return false;
  1998. }
  1999. // Make sure this instruction doesn't use the memcopy transfer sym before it is checked by StElemI
  2000. if (loop->memOpInfo && !loop->memOpInfo->candidates->Empty())
  2001. {
  2002. Loop::MemOpCandidate* prevCandidate = loop->memOpInfo->candidates->Head();
  2003. if (prevCandidate->IsMemCopy())
  2004. {
  2005. Loop::MemCopyCandidate* memcopyCandidate = prevCandidate->AsMemCopy();
  2006. if (memcopyCandidate->base == Js::Constants::InvalidSymID)
  2007. {
  2008. if (chkInstr->HasSymUse(memcopyCandidate->transferSym))
  2009. {
  2010. loop->doMemOp = false;
  2011. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, chkInstr, _u("Found illegal use of LdElemI value(s%d)"), GetVarSymID(memcopyCandidate->transferSym));
  2012. return false;
  2013. }
  2014. }
  2015. }
  2016. }
  2017. }
  2018. NEXT_INSTR_IN_RANGE;
  2019. }
  2020. return true;
  2021. }
  2022. bool
  2023. GlobOpt::IsInstrInvalidForMemOp(IR::Instr *instr, Loop *loop, Value *src1Val, Value *src2Val)
  2024. {
  2025. // List of instruction that are valid with memop (ie: instr that gets removed if memop is emitted)
  2026. if (
  2027. this->currentBlock != loop->GetHeadBlock() &&
  2028. !instr->IsLabelInstr() &&
  2029. instr->IsRealInstr() &&
  2030. instr->m_opcode != Js::OpCode::IncrLoopBodyCount &&
  2031. instr->m_opcode != Js::OpCode::StLoopBodyCount &&
  2032. instr->m_opcode != Js::OpCode::Ld_A &&
  2033. instr->m_opcode != Js::OpCode::Ld_I4 &&
  2034. !(instr->IsBranchInstr() && instr->AsBranchInstr()->IsUnconditional())
  2035. )
  2036. {
  2037. TRACE_MEMOP_VERBOSE(loop, instr, _u("Instruction not accepted for memop"));
  2038. return true;
  2039. }
  2040. // Check prev instr because it could have been added by an optimization and we won't see it here.
  2041. if (OpCodeAttr::FastFldInstr(instr->m_opcode) || (instr->m_prev && OpCodeAttr::FastFldInstr(instr->m_prev->m_opcode)))
  2042. {
  2043. // Refuse any operations interacting with Fields
  2044. TRACE_MEMOP_VERBOSE(loop, instr, _u("Field interaction detected"));
  2045. return true;
  2046. }
  2047. if (Js::OpCodeUtil::GetOpCodeLayout(instr->m_opcode) == Js::OpLayoutType::ElementSlot)
  2048. {
  2049. // Refuse any operations interacting with slots
  2050. TRACE_MEMOP_VERBOSE(loop, instr, _u("Slot interaction detected"));
  2051. return true;
  2052. }
  2053. if (this->MayNeedBailOnImplicitCall(instr, src1Val, src2Val))
  2054. {
  2055. TRACE_MEMOP_VERBOSE(loop, instr, _u("Implicit call bailout detected"));
  2056. return true;
  2057. }
  2058. return false;
  2059. }
  2060. void
  2061. GlobOpt::TryReplaceLdLen(IR::Instr *& instr)
  2062. {
  2063. // Change LdLen on objects other than arrays, strings, and 'arguments' to LdFld. Otherwise, convert the SymOpnd to a RegOpnd here.
  2064. if (instr->m_opcode == Js::OpCode::LdLen_A && instr->GetSrc1() && instr->GetSrc1()->IsSymOpnd())
  2065. {
  2066. IR::SymOpnd * opnd = instr->GetSrc1()->AsSymOpnd();
  2067. Sym *sym = opnd->m_sym;
  2068. Assert(sym->IsPropertySym());
  2069. PropertySym *originalPropertySym = sym->AsPropertySym();
  2070. IR::RegOpnd* newopnd = IR::RegOpnd::New(originalPropertySym->m_stackSym, IRType::TyVar, instr->m_func);
  2071. ValueInfo *const objectValueInfo = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym)->GetValueInfo();
  2072. // things we'd emit a fast path for
  2073. if (
  2074. objectValueInfo->IsLikelyAnyArray() ||
  2075. objectValueInfo->HasHadStringTag() ||
  2076. objectValueInfo->IsLikelyString() ||
  2077. newopnd->IsArgumentsObject() ||
  2078. (CurrentBlockData()->argObjSyms && CurrentBlockData()->IsArgumentsOpnd(newopnd))
  2079. )
  2080. {
  2081. // We need to properly transfer over the information from the old operand, which is
  2082. // a SymOpnd, to the new one, which is a RegOpnd. Unfortunately, the types mean the
  2083. // normal copy methods won't work here, so we're going to directly copy data.
  2084. newopnd->SetIsJITOptimizedReg(opnd->GetIsJITOptimizedReg());
  2085. newopnd->SetValueType(objectValueInfo->Type());
  2086. newopnd->SetIsDead(opnd->GetIsDead());
  2087. instr->ReplaceSrc1(newopnd);
  2088. }
  2089. else
  2090. {
  2091. // otherwise, change the instruction to an LdFld here.
  2092. instr->m_opcode = Js::OpCode::LdFld;
  2093. }
  2094. }
  2095. }
  2096. IR::Instr *
  2097. GlobOpt::OptInstr(IR::Instr *&instr, bool* isInstrRemoved)
  2098. {
  2099. Assert(instr->m_func->IsTopFunc() || instr->m_func->isGetterSetter || instr->m_func->callSiteIdInParentFunc != UINT16_MAX);
  2100. IR::Opnd *src1, *src2;
  2101. Value *src1Val = nullptr, *src2Val = nullptr, *dstVal = nullptr;
  2102. Value *src1IndirIndexVal = nullptr, *dstIndirIndexVal = nullptr;
  2103. IR::Instr *instrPrev = instr->m_prev;
  2104. IR::Instr *instrNext = instr->m_next;
  2105. if (instr->IsLabelInstr() && this->func->HasTry() && this->func->DoOptimizeTry())
  2106. {
  2107. this->currentRegion = instr->AsLabelInstr()->GetRegion();
  2108. Assert(this->currentRegion);
  2109. }
  2110. if(PrepareForIgnoringIntOverflow(instr))
  2111. {
  2112. if(!IsLoopPrePass())
  2113. {
  2114. *isInstrRemoved = true;
  2115. currentBlock->RemoveInstr(instr);
  2116. }
  2117. return instrNext;
  2118. }
  2119. if (!instr->IsRealInstr() || instr->IsByteCodeUsesInstr() || instr->m_opcode == Js::OpCode::Conv_Bool)
  2120. {
  2121. return instrNext;
  2122. }
  2123. if (instr->m_opcode == Js::OpCode::Yield)
  2124. {
  2125. // TODO[generators][ianhall]: Can this and the FillBailOutInfo call below be moved to after Src1 and Src2 so that Yield can be optimized right up to the actual yield?
  2126. CurrentBlockData()->KillStateForGeneratorYield();
  2127. }
  2128. if (!IsLoopPrePass())
  2129. {
  2130. // Change LdLen on objects other than arrays, strings, and 'arguments' to LdFld.
  2131. this->TryReplaceLdLen(instr);
  2132. }
  2133. // Consider: Do we ever get post-op bailout here, and if so is the FillBailOutInfo call in the right place?
  2134. if (instr->HasBailOutInfo() && !this->IsLoopPrePass())
  2135. {
  2136. this->FillBailOutInfo(this->currentBlock, instr);
  2137. }
  2138. this->instrCountSinceLastCleanUp++;
  2139. instr = this->PreOptPeep(instr);
  2140. this->OptArguments(instr);
  2141. //StackArguments Optimization - We bail out if the index is out of range of actuals.
  2142. if ((instr->m_opcode == Js::OpCode::LdElemI_A || instr->m_opcode == Js::OpCode::TypeofElem) &&
  2143. instr->DoStackArgsOpt(this->func) && !this->IsLoopPrePass())
  2144. {
  2145. GenerateBailAtOperation(&instr, IR::BailOnStackArgsOutOfActualsRange);
  2146. }
  2147. #if DBG
  2148. PropertySym *propertySymUseBefore = nullptr;
  2149. Assert(this->byteCodeUses == nullptr);
  2150. this->byteCodeUsesBeforeOpt->ClearAll();
  2151. GlobOpt::TrackByteCodeSymUsed(instr, this->byteCodeUsesBeforeOpt, &propertySymUseBefore);
  2152. Assert(noImplicitCallUsesToInsert->Count() == 0);
  2153. #endif
  2154. this->ignoredIntOverflowForCurrentInstr = false;
  2155. this->ignoredNegativeZeroForCurrentInstr = false;
  2156. src1 = instr->GetSrc1();
  2157. src2 = instr->GetSrc2();
  2158. if (src1)
  2159. {
  2160. src1Val = this->OptSrc(src1, &instr, &src1IndirIndexVal);
  2161. GOPT_TRACE_VALUENUMBER(_u("[src1] "), instr->GetSrc1(), _u("%d"), src1Val ? src1Val->GetValueNumber() : -1);
  2162. instr = this->SetTypeCheckBailOut(instr->GetSrc1(), instr, nullptr);
  2163. if (src2)
  2164. {
  2165. src2Val = this->OptSrc(src2, &instr);
  2166. GOPT_TRACE_VALUENUMBER(_u("[src2] "), instr->GetSrc2(), _u("%d"), src2Val ? src2Val->GetValueNumber() : -1);
  2167. }
  2168. }
  2169. if(instr->GetDst() && instr->GetDst()->IsIndirOpnd())
  2170. {
  2171. this->OptSrc(instr->GetDst(), &instr, &dstIndirIndexVal);
  2172. }
  2173. MarkArgumentsUsedForBranch(instr);
  2174. CSEOptimize(this->currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal);
  2175. OptimizeChecks(instr);
  2176. OptArraySrc(&instr, &src1Val, &src2Val);
  2177. OptNewScObject(&instr, src1Val);
  2178. instr = this->OptPeep(instr, src1Val, src2Val);
  2179. if (instr->m_opcode == Js::OpCode::Nop ||
  2180. (instr->m_opcode == Js::OpCode::CheckThis &&
  2181. instr->GetSrc1()->IsRegOpnd() &&
  2182. instr->GetSrc1()->AsRegOpnd()->m_sym->m_isSafeThis))
  2183. {
  2184. instrNext = instr->m_next;
  2185. InsertNoImplicitCallUses(instr);
  2186. if (this->byteCodeUses)
  2187. {
  2188. this->InsertByteCodeUses(instr);
  2189. }
  2190. *isInstrRemoved = true;
  2191. this->currentBlock->RemoveInstr(instr);
  2192. return instrNext;
  2193. }
  2194. else if (instr->m_opcode == Js::OpCode::GetNewScObject && !this->IsLoopPrePass() && src1Val->GetValueInfo()->IsPrimitive())
  2195. {
  2196. // Constructor returned (src1) a primitive value, so fold this into "dst = Ld_A src2", where src2 is the new object that
  2197. // was passed into the constructor as its 'this' parameter
  2198. instr->FreeSrc1();
  2199. instr->SetSrc1(instr->UnlinkSrc2());
  2200. instr->m_opcode = Js::OpCode::Ld_A;
  2201. src1Val = src2Val;
  2202. src2Val = nullptr;
  2203. }
  2204. else if ((instr->m_opcode == Js::OpCode::TryCatch && this->func->DoOptimizeTry()) || (instr->m_opcode == Js::OpCode::TryFinally && this->func->DoOptimizeTry()))
  2205. {
  2206. ProcessTryHandler(instr);
  2207. }
  2208. else if (instr->m_opcode == Js::OpCode::BrOnException || instr->m_opcode == Js::OpCode::BrOnNoException)
  2209. {
  2210. if (this->ProcessExceptionHandlingEdges(instr))
  2211. {
  2212. *isInstrRemoved = true;
  2213. return instrNext;
  2214. }
  2215. }
  2216. bool isAlreadyTypeSpecialized = false;
  2217. if (!IsLoopPrePass() && instr->HasBailOutInfo())
  2218. {
  2219. if (instr->GetBailOutKind() == IR::BailOutExpectingInteger)
  2220. {
  2221. isAlreadyTypeSpecialized = TypeSpecializeBailoutExpectedInteger(instr, src1Val, &dstVal);
  2222. }
  2223. else if (instr->GetBailOutKind() == IR::BailOutExpectingString)
  2224. {
  2225. if (instr->GetSrc1()->IsRegOpnd())
  2226. {
  2227. if (!src1Val || !src1Val->GetValueInfo()->IsLikelyString())
  2228. {
  2229. // Disable SwitchOpt if the source is definitely not a string - This may be realized only in Globopt
  2230. Assert(IsSwitchOptEnabled());
  2231. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingString);
  2232. }
  2233. }
  2234. }
  2235. }
  2236. bool forceInvariantHoisting = false;
  2237. const bool ignoreIntOverflowInRangeForInstr = instr->ignoreIntOverflowInRange; // Save it since the instr can change
  2238. if (!isAlreadyTypeSpecialized)
  2239. {
  2240. bool redoTypeSpec;
  2241. instr = this->TypeSpecialization(instr, &src1Val, &src2Val, &dstVal, &redoTypeSpec, &forceInvariantHoisting);
  2242. if(redoTypeSpec && instr->m_opcode != Js::OpCode::Nop)
  2243. {
  2244. forceInvariantHoisting = false;
  2245. instr = this->TypeSpecialization(instr, &src1Val, &src2Val, &dstVal, &redoTypeSpec, &forceInvariantHoisting);
  2246. Assert(!redoTypeSpec);
  2247. }
  2248. if (instr->m_opcode == Js::OpCode::Nop)
  2249. {
  2250. InsertNoImplicitCallUses(instr);
  2251. if (this->byteCodeUses)
  2252. {
  2253. this->InsertByteCodeUses(instr);
  2254. }
  2255. instrNext = instr->m_next;
  2256. *isInstrRemoved = true;
  2257. this->currentBlock->RemoveInstr(instr);
  2258. return instrNext;
  2259. }
  2260. }
  2261. if (ignoreIntOverflowInRangeForInstr)
  2262. {
  2263. VerifyIntSpecForIgnoringIntOverflow(instr);
  2264. }
  2265. // Track calls after any pre-op bailouts have been inserted before the call, because they will need to restore out params.
  2266. this->TrackCalls(instr);
  2267. if (instr->GetSrc1())
  2268. {
  2269. this->UpdateObjPtrValueType(instr->GetSrc1(), instr);
  2270. }
  2271. IR::Opnd *dst = instr->GetDst();
  2272. if (dst)
  2273. {
  2274. // Copy prop dst uses and mark live/available type syms before tracking kills.
  2275. CopyPropDstUses(dst, instr, src1Val);
  2276. }
  2277. // Track mark temp object before we process the dst so we can generate pre-op bailout
  2278. instr = this->TrackMarkTempObject(instrPrev->m_next, instr);
  2279. bool removed = OptTagChecks(instr);
  2280. if (removed)
  2281. {
  2282. *isInstrRemoved = true;
  2283. return instrNext;
  2284. }
  2285. dstVal = this->OptDst(&instr, dstVal, src1Val, src2Val, dstIndirIndexVal, src1IndirIndexVal);
  2286. if (dst)
  2287. {
  2288. GOPT_TRACE_VALUENUMBER(_u("[dst] "), instr->GetDst(), _u("%d\n"), dstVal ? dstVal->GetValueNumber() : -1);
  2289. }
  2290. dst = instr->GetDst();
  2291. instrNext = instr->m_next;
  2292. if (dst)
  2293. {
  2294. if (this->func->HasTry() && this->func->DoOptimizeTry())
  2295. {
  2296. this->InsertToVarAtDefInTryRegion(instr, dst);
  2297. }
  2298. instr = this->SetTypeCheckBailOut(dst, instr, nullptr);
  2299. this->UpdateObjPtrValueType(dst, instr);
  2300. }
  2301. BVSparse<JitArenaAllocator> instrByteCodeStackSymUsedAfter(this->alloc);
  2302. PropertySym *propertySymUseAfter = nullptr;
  2303. if (this->byteCodeUses != nullptr)
  2304. {
  2305. GlobOpt::TrackByteCodeSymUsed(instr, &instrByteCodeStackSymUsedAfter, &propertySymUseAfter);
  2306. }
  2307. #if DBG
  2308. else
  2309. {
  2310. GlobOpt::TrackByteCodeSymUsed(instr, &instrByteCodeStackSymUsedAfter, &propertySymUseAfter);
  2311. instrByteCodeStackSymUsedAfter.Equal(this->byteCodeUsesBeforeOpt);
  2312. Assert(propertySymUseAfter == propertySymUseBefore);
  2313. }
  2314. #endif
  2315. bool isHoisted = false;
  2316. if (this->currentBlock->loop && !this->IsLoopPrePass())
  2317. {
  2318. isHoisted = this->TryHoistInvariant(instr, this->currentBlock, dstVal, src1Val, src2Val, true, false, forceInvariantHoisting);
  2319. }
  2320. src1 = instr->GetSrc1();
  2321. if (!this->IsLoopPrePass() && src1)
  2322. {
  2323. // instr const, nonConst => canonicalize by swapping operands
  2324. // This simplifies lowering. (somewhat machine dependent)
  2325. // Note that because of Var overflows, src1 may not have been constant prop'd to an IntConst
  2326. this->PreLowerCanonicalize(instr, &src1Val, &src2Val);
  2327. }
  2328. if (!PHASE_OFF(Js::MemOpPhase, this->func) &&
  2329. !isHoisted &&
  2330. !(instr->IsJitProfilingInstr()) &&
  2331. this->currentBlock->loop && !IsLoopPrePass() &&
  2332. !func->IsJitInDebugMode() &&
  2333. (func->HasProfileInfo() && !func->GetReadOnlyProfileInfo()->IsMemOpDisabled()) &&
  2334. this->currentBlock->loop->doMemOp)
  2335. {
  2336. CollectMemOpInfo(instrPrev, instr, src1Val, src2Val);
  2337. }
  2338. InsertNoImplicitCallUses(instr);
  2339. if (this->byteCodeUses != nullptr)
  2340. {
  2341. // Optimization removed some uses from the instruction.
  2342. // Need to insert fake uses so we can get the correct live register to restore in bailout.
  2343. this->byteCodeUses->Minus(&instrByteCodeStackSymUsedAfter);
  2344. if (this->propertySymUse == propertySymUseAfter)
  2345. {
  2346. this->propertySymUse = nullptr;
  2347. }
  2348. this->InsertByteCodeUses(instr);
  2349. }
  2350. if (!this->IsLoopPrePass() && !isHoisted && this->IsImplicitCallBailOutCurrentlyNeeded(instr, src1Val, src2Val))
  2351. {
  2352. IR::BailOutKind kind = IR::BailOutOnImplicitCalls;
  2353. if(instr->HasBailOutInfo())
  2354. {
  2355. Assert(instr->GetBailOutInfo()->bailOutOffset == instr->GetByteCodeOffset());
  2356. const IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  2357. if((bailOutKind & ~IR::BailOutKindBits) != IR::BailOutOnImplicitCallsPreOp)
  2358. {
  2359. Assert(!(bailOutKind & ~IR::BailOutKindBits));
  2360. instr->SetBailOutKind(bailOutKind + IR::BailOutOnImplicitCallsPreOp);
  2361. }
  2362. }
  2363. else if (instr->forcePreOpBailOutIfNeeded || this->isRecursiveCallOnLandingPad)
  2364. {
  2365. // We can't have a byte code reg slot as dst to generate a
  2366. // pre-op implicit call after we have processed the dst.
  2367. // Consider: This might miss an opportunity to use a copy prop sym to restore
  2368. // some other byte code reg if the dst is that copy prop that we already killed.
  2369. Assert(!instr->GetDst()
  2370. || !instr->GetDst()->IsRegOpnd()
  2371. || instr->GetDst()->AsRegOpnd()->GetIsJITOptimizedReg()
  2372. || !instr->GetDst()->AsRegOpnd()->m_sym->HasByteCodeRegSlot());
  2373. this->GenerateBailAtOperation(&instr, IR::BailOutOnImplicitCallsPreOp);
  2374. }
  2375. else
  2376. {
  2377. // Capture value of the bailout after the operation is done.
  2378. this->GenerateBailAfterOperation(&instr, kind);
  2379. }
  2380. }
  2381. if (CurrentBlockData()->capturedValuesCandidate && !this->IsLoopPrePass())
  2382. {
  2383. this->CommitCapturedValuesCandidate();
  2384. }
  2385. #if DBG
  2386. if (CONFIG_FLAG(ValidateIntRanges) && !IsLoopPrePass())
  2387. {
  2388. if (instr->ShouldEmitIntRangeCheck())
  2389. {
  2390. this->EmitIntRangeChecks(instr);
  2391. }
  2392. }
  2393. #endif
  2394. return instrNext;
  2395. }
  2396. bool
  2397. GlobOpt::IsNonNumericRegOpnd(IR::RegOpnd *opnd, bool inGlobOpt) const
  2398. {
  2399. if (opnd == nullptr)
  2400. {
  2401. return false;
  2402. }
  2403. if (opnd->m_sym->m_isNotNumber)
  2404. {
  2405. return true;
  2406. }
  2407. if (!inGlobOpt)
  2408. {
  2409. return false;
  2410. }
  2411. if (opnd->GetValueType().IsNumber() || currentBlock->globOptData.IsTypeSpecialized(opnd->m_sym))
  2412. {
  2413. if (!this->IsLoopPrePass())
  2414. {
  2415. return false;
  2416. }
  2417. Value * opndValue = this->currentBlock->globOptData.FindValue(opnd->m_sym);
  2418. ValueInfo * opndValueInfo = opndValue ? opndValue->GetValueInfo() : nullptr;
  2419. if (!opndValueInfo)
  2420. {
  2421. return true;
  2422. }
  2423. if (this->prePassLoop->preservesNumberValue->Test(opnd->m_sym->m_id))
  2424. {
  2425. return false;
  2426. }
  2427. return !this->IsSafeToTransferInPrepass(opnd->m_sym, opndValueInfo);
  2428. }
  2429. return true;
  2430. }
  2431. bool
  2432. GlobOpt::OptTagChecks(IR::Instr *instr)
  2433. {
  2434. if (PHASE_OFF(Js::OptTagChecksPhase, this->func) || !this->DoTagChecks())
  2435. {
  2436. return false;
  2437. }
  2438. StackSym *stackSym = nullptr;
  2439. IR::SymOpnd *symOpnd = nullptr;
  2440. IR::RegOpnd *regOpnd = nullptr;
  2441. switch(instr->m_opcode)
  2442. {
  2443. case Js::OpCode::LdFld:
  2444. case Js::OpCode::LdMethodFld:
  2445. case Js::OpCode::CheckFixedFld:
  2446. case Js::OpCode::CheckPropertyGuardAndLoadType:
  2447. symOpnd = instr->GetSrc1()->AsSymOpnd();
  2448. stackSym = symOpnd->m_sym->AsPropertySym()->m_stackSym;
  2449. break;
  2450. case Js::OpCode::BailOnNotObject:
  2451. case Js::OpCode::BailOnNotArray:
  2452. if (instr->GetSrc1()->IsRegOpnd())
  2453. {
  2454. regOpnd = instr->GetSrc1()->AsRegOpnd();
  2455. stackSym = regOpnd->m_sym;
  2456. }
  2457. break;
  2458. case Js::OpCode::StFld:
  2459. symOpnd = instr->GetDst()->AsSymOpnd();
  2460. stackSym = symOpnd->m_sym->AsPropertySym()->m_stackSym;
  2461. break;
  2462. }
  2463. if (stackSym)
  2464. {
  2465. Value *value = CurrentBlockData()->FindValue(stackSym);
  2466. if (value)
  2467. {
  2468. ValueInfo *valInfo = value->GetValueInfo();
  2469. if (valInfo->GetSymStore() && valInfo->GetSymStore()->IsStackSym() && valInfo->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable())
  2470. {
  2471. return false;
  2472. }
  2473. ValueType valueType = value->GetValueInfo()->Type();
  2474. if (instr->m_opcode == Js::OpCode::BailOnNotObject)
  2475. {
  2476. if (valueType.CanBeTaggedValue())
  2477. {
  2478. // We're not adding new information to the value other than changing the value type. Preserve any existing
  2479. // information and just change the value type.
  2480. ChangeValueType(nullptr, value, valueType.SetCanBeTaggedValue(false), true /*preserveSubClassInfo*/);
  2481. return false;
  2482. }
  2483. if (!this->IsLoopPrePass())
  2484. {
  2485. if (this->byteCodeUses)
  2486. {
  2487. this->InsertByteCodeUses(instr);
  2488. }
  2489. this->currentBlock->RemoveInstr(instr);
  2490. }
  2491. return true;
  2492. }
  2493. if (valueType.CanBeTaggedValue() &&
  2494. !valueType.HasBeenNumber() &&
  2495. !this->IsLoopPrePass())
  2496. {
  2497. ValueType newValueType = valueType.SetCanBeTaggedValue(false);
  2498. // Split out the tag check as a separate instruction.
  2499. IR::Instr *bailOutInstr;
  2500. bailOutInstr = IR::BailOutInstr::New(Js::OpCode::BailOnNotObject, IR::BailOutOnTaggedValue, instr, instr->m_func);
  2501. if (!this->IsLoopPrePass())
  2502. {
  2503. FillBailOutInfo(this->currentBlock, bailOutInstr);
  2504. }
  2505. IR::RegOpnd *srcOpnd = regOpnd;
  2506. if (!srcOpnd)
  2507. {
  2508. srcOpnd = IR::RegOpnd::New(stackSym, stackSym->GetType(), instr->m_func);
  2509. AnalysisAssert(symOpnd);
  2510. if (symOpnd->GetIsJITOptimizedReg())
  2511. {
  2512. srcOpnd->SetIsJITOptimizedReg(true);
  2513. }
  2514. }
  2515. bailOutInstr->SetSrc1(srcOpnd);
  2516. bailOutInstr->GetSrc1()->SetValueType(valueType);
  2517. bailOutInstr->SetByteCodeOffset(instr);
  2518. instr->InsertBefore(bailOutInstr);
  2519. if (this->currentBlock->loop)
  2520. {
  2521. // Try hoisting the BailOnNotObject instr.
  2522. // But since this isn't the current instr being optimized, we need to play tricks with
  2523. // the byteCodeUse fields...
  2524. TrackByteCodeUsesForInstrAddedInOptInstr(bailOutInstr, [&]()
  2525. {
  2526. TryHoistInvariant(bailOutInstr, this->currentBlock, nullptr, value, nullptr, true, false, false, IR::BailOutOnTaggedValue);
  2527. });
  2528. }
  2529. if (symOpnd)
  2530. {
  2531. symOpnd->SetPropertyOwnerValueType(newValueType);
  2532. }
  2533. else
  2534. {
  2535. regOpnd->SetValueType(newValueType);
  2536. }
  2537. ChangeValueType(nullptr, value, newValueType, false);
  2538. }
  2539. }
  2540. }
  2541. return false;
  2542. }
  2543. bool
  2544. GlobOpt::TypeSpecializeBailoutExpectedInteger(IR::Instr* instr, Value* src1Val, Value** dstVal)
  2545. {
  2546. bool isAlreadyTypeSpecialized = false;
  2547. if(instr->GetSrc1()->IsRegOpnd())
  2548. {
  2549. if (!src1Val || !src1Val->GetValueInfo()->IsLikelyInt() || instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber)
  2550. {
  2551. Assert(IsSwitchOptEnabledForIntTypeSpec());
  2552. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingInteger);
  2553. }
  2554. // Attach the BailOutExpectingInteger to FromVar and Remove the bail out info on the Ld_A (Begin Switch) instr.
  2555. this->ToTypeSpecUse(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, TyInt32, IR::BailOutExpectingInteger, false, instr);
  2556. //TypeSpecialize the dst of Ld_A
  2557. TypeSpecializeIntDst(instr, instr->m_opcode, src1Val, src1Val, nullptr, IR::BailOutInvalid, INT32_MIN, INT32_MAX, dstVal);
  2558. isAlreadyTypeSpecialized = true;
  2559. }
  2560. instr->ClearBailOutInfo();
  2561. return isAlreadyTypeSpecialized;
  2562. }
  2563. Value*
  2564. GlobOpt::OptDst(
  2565. IR::Instr ** pInstr,
  2566. Value *dstVal,
  2567. Value *src1Val,
  2568. Value *src2Val,
  2569. Value *dstIndirIndexVal,
  2570. Value *src1IndirIndexVal)
  2571. {
  2572. IR::Instr *&instr = *pInstr;
  2573. IR::Opnd *opnd = instr->GetDst();
  2574. if (opnd)
  2575. {
  2576. if (opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd())
  2577. {
  2578. this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd());
  2579. }
  2580. if (opnd->IsIndirOpnd() && !this->IsLoopPrePass())
  2581. {
  2582. IR::RegOpnd *baseOpnd = opnd->AsIndirOpnd()->GetBaseOpnd();
  2583. const ValueType baseValueType(baseOpnd->GetValueType());
  2584. if ((
  2585. baseValueType.IsLikelyNativeArray() ||
  2586. #ifdef _M_IX86
  2587. (
  2588. !AutoSystemInfo::Data.SSE2Available() &&
  2589. baseValueType.IsLikelyObject() &&
  2590. (
  2591. baseValueType.GetObjectType() == ObjectType::Float32Array ||
  2592. baseValueType.GetObjectType() == ObjectType::Float64Array
  2593. )
  2594. )
  2595. #else
  2596. false
  2597. #endif
  2598. ) &&
  2599. instr->GetSrc1()->IsVar())
  2600. {
  2601. if(instr->m_opcode == Js::OpCode::StElemC)
  2602. {
  2603. // StElemC has different code that handles native array conversion or missing value stores. Add a bailout
  2604. // for those cases.
  2605. Assert(baseValueType.IsLikelyNativeArray());
  2606. Assert(!instr->HasBailOutInfo());
  2607. GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly);
  2608. }
  2609. else if(instr->HasBailOutInfo())
  2610. {
  2611. // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast
  2612. // path. Note that the removed bailouts should not be necessary for correctness. Bailout on native array
  2613. // conversion will be handled automatically as normal.
  2614. IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  2615. if(bailOutKind & IR::BailOutOnArrayAccessHelperCall)
  2616. {
  2617. bailOutKind -= IR::BailOutOnArrayAccessHelperCall;
  2618. }
  2619. if(bailOutKind == IR::BailOutOnImplicitCallsPreOp)
  2620. {
  2621. bailOutKind -= IR::BailOutOnImplicitCallsPreOp;
  2622. }
  2623. if(bailOutKind)
  2624. {
  2625. instr->SetBailOutKind(bailOutKind);
  2626. }
  2627. else
  2628. {
  2629. instr->ClearBailOutInfo();
  2630. }
  2631. }
  2632. }
  2633. }
  2634. }
  2635. this->ProcessKills(instr);
  2636. if (opnd)
  2637. {
  2638. if (dstVal == nullptr)
  2639. {
  2640. dstVal = ValueNumberDst(pInstr, src1Val, src2Val);
  2641. }
  2642. if (this->IsLoopPrePass())
  2643. {
  2644. // Keep track of symbols defined in the loop.
  2645. if (opnd->IsRegOpnd())
  2646. {
  2647. StackSym *symDst = opnd->AsRegOpnd()->m_sym;
  2648. rootLoopPrePass->symsDefInLoop->Set(symDst->m_id);
  2649. }
  2650. }
  2651. else if (dstVal)
  2652. {
  2653. opnd->SetValueType(dstVal->GetValueInfo()->Type());
  2654. if(currentBlock->loop &&
  2655. !IsLoopPrePass() &&
  2656. (instr->m_opcode == Js::OpCode::Ld_A || instr->m_opcode == Js::OpCode::Ld_I4) &&
  2657. instr->GetSrc1()->IsRegOpnd() &&
  2658. !func->IsJitInDebugMode() &&
  2659. func->DoGlobOptsForGeneratorFunc())
  2660. {
  2661. // Look for the following patterns:
  2662. //
  2663. // Pattern 1:
  2664. // s1[liveOnBackEdge] = s3[dead]
  2665. //
  2666. // Pattern 2:
  2667. // s3 = operation(s1[liveOnBackEdge], s2)
  2668. // s1[liveOnBackEdge] = s3
  2669. //
  2670. // In both patterns, s1 and s3 have the same value by the end. Prefer to use s1 as the sym store instead of s3
  2671. // since s1 is live on back-edge, as otherwise, their lifetimes overlap, requiring two registers to hold the
  2672. // value instead of one.
  2673. do
  2674. {
  2675. IR::RegOpnd *const src = instr->GetSrc1()->AsRegOpnd();
  2676. StackSym *srcVarSym = src->m_sym;
  2677. if(srcVarSym->IsTypeSpec())
  2678. {
  2679. srcVarSym = srcVarSym->GetVarEquivSym(nullptr);
  2680. Assert(srcVarSym);
  2681. }
  2682. if(dstVal->GetValueInfo()->GetSymStore() != srcVarSym)
  2683. {
  2684. break;
  2685. }
  2686. IR::RegOpnd *const dst = opnd->AsRegOpnd();
  2687. StackSym *dstVarSym = dst->m_sym;
  2688. if(dstVarSym->IsTypeSpec())
  2689. {
  2690. dstVarSym = dstVarSym->GetVarEquivSym(nullptr);
  2691. Assert(dstVarSym);
  2692. }
  2693. if(!currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(dstVarSym->m_id))
  2694. {
  2695. break;
  2696. }
  2697. Value *const srcValue = CurrentBlockData()->FindValue(srcVarSym);
  2698. if(srcValue->GetValueNumber() != dstVal->GetValueNumber())
  2699. {
  2700. break;
  2701. }
  2702. if(!src->GetIsDead())
  2703. {
  2704. IR::Instr *const prevInstr = instr->GetPrevRealInstrOrLabel();
  2705. IR::Opnd *const prevDst = prevInstr->GetDst();
  2706. if(!prevDst ||
  2707. !src->IsEqualInternal(prevDst) ||
  2708. !(
  2709. (prevInstr->GetSrc1() && dst->IsEqual(prevInstr->GetSrc1())) ||
  2710. (prevInstr->GetSrc2() && dst->IsEqual(prevInstr->GetSrc2()))
  2711. ))
  2712. {
  2713. break;
  2714. }
  2715. }
  2716. this->SetSymStoreDirect(dstVal->GetValueInfo(), dstVarSym);
  2717. } while(false);
  2718. }
  2719. }
  2720. this->ValueNumberObjectType(opnd, instr);
  2721. }
  2722. this->CSEAddInstr(this->currentBlock, *pInstr, dstVal, src1Val, src2Val, dstIndirIndexVal, src1IndirIndexVal);
  2723. return dstVal;
  2724. }
  2725. void
  2726. GlobOpt::CopyPropDstUses(IR::Opnd *opnd, IR::Instr *instr, Value *src1Val)
  2727. {
  2728. if (opnd->IsSymOpnd())
  2729. {
  2730. IR::SymOpnd *symOpnd = opnd->AsSymOpnd();
  2731. if (symOpnd->m_sym->IsPropertySym())
  2732. {
  2733. PropertySym * originalPropertySym = symOpnd->m_sym->AsPropertySym();
  2734. Value *const objectValue = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym);
  2735. symOpnd->SetPropertyOwnerValueType(objectValue ? objectValue->GetValueInfo()->Type() : ValueType::Uninitialized);
  2736. this->CopyPropPropertySymObj(symOpnd, instr);
  2737. }
  2738. }
  2739. }
  2740. void
  2741. GlobOpt::SetLoopFieldInitialValue(Loop *loop, IR::Instr *instr, PropertySym *propertySym, PropertySym *originalPropertySym)
  2742. {
  2743. Value *initialValue = nullptr;
  2744. StackSym *symStore;
  2745. if (loop->allFieldsKilled || loop->fieldKilled->Test(originalPropertySym->m_id) || loop->fieldKilled->Test(propertySym->m_id))
  2746. {
  2747. return;
  2748. }
  2749. // Value already exists
  2750. if (CurrentBlockData()->FindValue(propertySym))
  2751. {
  2752. return;
  2753. }
  2754. // If this initial value was already added, we would find in the current value table.
  2755. Assert(!loop->initialValueFieldMap.TryGetValue(propertySym, &initialValue));
  2756. // If propertySym is live in landingPad, we don't need an initial value.
  2757. if (loop->landingPad->globOptData.liveFields->Test(propertySym->m_id))
  2758. {
  2759. return;
  2760. }
  2761. StackSym * objectSym = propertySym->m_stackSym;
  2762. Value *landingPadObjPtrVal, *currentObjPtrVal;
  2763. landingPadObjPtrVal = loop->landingPad->globOptData.FindValue(objectSym);
  2764. currentObjPtrVal = CurrentBlockData()->FindValue(objectSym);
  2765. auto CanSetInitialValue = [&]() -> bool {
  2766. if (!currentObjPtrVal)
  2767. {
  2768. return false;
  2769. }
  2770. if (landingPadObjPtrVal)
  2771. {
  2772. return currentObjPtrVal->GetValueNumber() == landingPadObjPtrVal->GetValueNumber();
  2773. }
  2774. else
  2775. {
  2776. if (!objectSym->IsSingleDef())
  2777. {
  2778. return false;
  2779. }
  2780. IR::Instr * defInstr = objectSym->GetInstrDef();
  2781. IR::Opnd * src1 = defInstr->GetSrc1();
  2782. while (!(src1 && src1->IsSymOpnd() && src1->AsSymOpnd()->m_sym->IsPropertySym()))
  2783. {
  2784. if (src1 && src1->IsRegOpnd() && src1->AsRegOpnd()->GetStackSym()->IsSingleDef())
  2785. {
  2786. defInstr = src1->AsRegOpnd()->GetStackSym()->GetInstrDef();
  2787. src1 = defInstr->GetSrc1();
  2788. }
  2789. else
  2790. {
  2791. return false;
  2792. }
  2793. }
  2794. return true;
  2795. // Todo: allow other kinds of operands as src1 of instr def of the object sym of the current propertySym
  2796. // SymOpnd, but not PropertySymOpnd - LdSlotArr, some LdSlots (?)
  2797. // nullptr - NewScObject
  2798. }
  2799. };
  2800. if (!CanSetInitialValue())
  2801. {
  2802. // objPtr has a different value in the landing pad.
  2803. return;
  2804. }
  2805. // The opnd's value type has not yet been initialized. Since the property sym doesn't have a value, it effectively has an
  2806. // Uninitialized value type. Use the profiled value type from the instruction.
  2807. const ValueType profiledValueType =
  2808. instr->IsProfiledInstr() ? instr->AsProfiledInstr()->u.FldInfo().valueType : ValueType::Uninitialized;
  2809. Assert(!profiledValueType.IsDefinite()); // Hence the values created here don't need to be tracked for kills
  2810. initialValue = this->NewGenericValue(profiledValueType, propertySym);
  2811. symStore = StackSym::New(this->func);
  2812. initialValue->GetValueInfo()->SetSymStore(symStore);
  2813. loop->initialValueFieldMap.Add(propertySym, initialValue->Copy(this->alloc, initialValue->GetValueNumber()));
  2814. // Copy the initial value into the landing pad, but without a symStore
  2815. Value *landingPadInitialValue = Value::New(this->alloc, initialValue->GetValueNumber(),
  2816. ValueInfo::New(this->alloc, initialValue->GetValueInfo()->Type()));
  2817. loop->landingPad->globOptData.SetValue(landingPadInitialValue, propertySym);
  2818. loop->landingPad->globOptData.liveFields->Set(propertySym->m_id);
  2819. #if DBG_DUMP
  2820. if (PHASE_TRACE(Js::FieldPREPhase, this->func))
  2821. {
  2822. Output::Print(_u("** TRACE: Field PRE initial value for loop head #%d. Val:%d symStore:"),
  2823. loop->GetHeadBlock()->GetBlockNum(), initialValue->GetValueNumber());
  2824. symStore->Dump();
  2825. Output::Print(_u("\n Instr: "));
  2826. instr->Dump();
  2827. Output::Flush();
  2828. }
  2829. #endif
  2830. // Add initial value to all the previous blocks in the loop.
  2831. FOREACH_BLOCK_BACKWARD_IN_RANGE(block, this->currentBlock->GetPrev(), loop->GetHeadBlock())
  2832. {
  2833. if (block->GetDataUseCount() == 0)
  2834. {
  2835. // All successor blocks have been processed, no point in adding the value.
  2836. continue;
  2837. }
  2838. Value *newValue = initialValue->Copy(this->alloc, initialValue->GetValueNumber());
  2839. block->globOptData.SetValue(newValue, propertySym);
  2840. block->globOptData.liveFields->Set(propertySym->m_id);
  2841. block->globOptData.SetValue(newValue, symStore);
  2842. block->globOptData.liveVarSyms->Set(symStore->m_id);
  2843. } NEXT_BLOCK_BACKWARD_IN_RANGE;
  2844. CurrentBlockData()->SetValue(initialValue, symStore);
  2845. CurrentBlockData()->liveVarSyms->Set(symStore->m_id);
  2846. CurrentBlockData()->liveFields->Set(propertySym->m_id);
  2847. }
  2848. // Examine src, apply copy prop and value number it
  2849. Value*
  2850. GlobOpt::OptSrc(IR::Opnd *opnd, IR::Instr * *pInstr, Value **indirIndexValRef, IR::IndirOpnd *parentIndirOpnd)
  2851. {
  2852. IR::Instr * &instr = *pInstr;
  2853. Assert(!indirIndexValRef || !*indirIndexValRef);
  2854. Assert(
  2855. parentIndirOpnd
  2856. ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd()
  2857. : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd());
  2858. Sym *sym;
  2859. Value *val;
  2860. PropertySym *originalPropertySym = nullptr;
  2861. switch(opnd->GetKind())
  2862. {
  2863. case IR::OpndKindIntConst:
  2864. val = this->GetIntConstantValue(opnd->AsIntConstOpnd()->AsInt32(), instr);
  2865. opnd->SetValueType(val->GetValueInfo()->Type());
  2866. return val;
  2867. case IR::OpndKindInt64Const:
  2868. val = this->GetIntConstantValue(opnd->AsInt64ConstOpnd()->GetValue(), instr);
  2869. opnd->SetValueType(val->GetValueInfo()->Type());
  2870. return val;
  2871. case IR::OpndKindFloatConst:
  2872. {
  2873. const FloatConstType floatValue = opnd->AsFloatConstOpnd()->m_value;
  2874. int32 int32Value;
  2875. if(Js::JavascriptNumber::TryGetInt32Value(floatValue, &int32Value))
  2876. {
  2877. val = GetIntConstantValue(int32Value, instr);
  2878. }
  2879. else
  2880. {
  2881. val = NewFloatConstantValue(floatValue);
  2882. }
  2883. opnd->SetValueType(val->GetValueInfo()->Type());
  2884. return val;
  2885. }
  2886. case IR::OpndKindAddr:
  2887. {
  2888. IR::AddrOpnd *addrOpnd = opnd->AsAddrOpnd();
  2889. if (addrOpnd->m_isFunction)
  2890. {
  2891. AssertMsg(!PHASE_OFF(Js::FixedMethodsPhase, instr->m_func), "Fixed function address operand with fixed method calls phase disabled?");
  2892. val = NewFixedFunctionValue((Js::JavascriptFunction *)addrOpnd->m_address, addrOpnd);
  2893. opnd->SetValueType(val->GetValueInfo()->Type());
  2894. return val;
  2895. }
  2896. else if (addrOpnd->IsVar() && Js::TaggedInt::Is(addrOpnd->m_address))
  2897. {
  2898. val = this->GetIntConstantValue(Js::TaggedInt::ToInt32(addrOpnd->m_address), instr);
  2899. opnd->SetValueType(val->GetValueInfo()->Type());
  2900. return val;
  2901. }
  2902. val = this->GetVarConstantValue(addrOpnd);
  2903. return val;
  2904. }
  2905. case IR::OpndKindSym:
  2906. {
  2907. // Clear the opnd's value type up-front, so that this code cannot accidentally use the value type set from a previous
  2908. // OptSrc on the same instruction (for instance, from an earlier loop prepass). The value type will be set from the
  2909. // value if available, before returning from this function.
  2910. opnd->SetValueType(ValueType::Uninitialized);
  2911. sym = opnd->AsSymOpnd()->m_sym;
  2912. // Don't create a new value for ArgSlots and don't copy prop them away.
  2913. if (sym->IsStackSym() && sym->AsStackSym()->IsArgSlotSym())
  2914. {
  2915. return nullptr;
  2916. }
  2917. // Unless we have profile info, don't create a new value for ArgSlots and don't copy prop them away.
  2918. if (sym->IsStackSym() && sym->AsStackSym()->IsParamSlotSym())
  2919. {
  2920. if (!instr->m_func->IsLoopBody() && instr->m_func->HasProfileInfo())
  2921. {
  2922. // Skip "this" pointer.
  2923. int paramSlotNum = sym->AsStackSym()->GetParamSlotNum() - 2;
  2924. if (paramSlotNum >= 0)
  2925. {
  2926. const auto parameterType = instr->m_func->GetReadOnlyProfileInfo()->GetParameterInfo(static_cast<Js::ArgSlot>(paramSlotNum));
  2927. val = NewGenericValue(parameterType);
  2928. opnd->SetValueType(val->GetValueInfo()->Type());
  2929. return val;
  2930. }
  2931. }
  2932. return nullptr;
  2933. }
  2934. if (!sym->IsPropertySym())
  2935. {
  2936. break;
  2937. }
  2938. originalPropertySym = sym->AsPropertySym();
  2939. // Don't give a value to 'arguments' property sym to prevent field copy prop of 'arguments'
  2940. if (originalPropertySym->AsPropertySym()->m_propertyId == Js::PropertyIds::arguments &&
  2941. originalPropertySym->AsPropertySym()->m_fieldKind == PropertyKindData)
  2942. {
  2943. if (opnd->AsSymOpnd()->IsPropertySymOpnd())
  2944. {
  2945. this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd());
  2946. }
  2947. return nullptr;
  2948. }
  2949. Value *const objectValue = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym);
  2950. opnd->AsSymOpnd()->SetPropertyOwnerValueType(
  2951. objectValue ? objectValue->GetValueInfo()->Type() : ValueType::Uninitialized);
  2952. sym = this->CopyPropPropertySymObj(opnd->AsSymOpnd(), instr);
  2953. if (!DoFieldCopyProp())
  2954. {
  2955. if (opnd->AsSymOpnd()->IsPropertySymOpnd())
  2956. {
  2957. this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd());
  2958. }
  2959. return nullptr;
  2960. }
  2961. switch (instr->m_opcode)
  2962. {
  2963. // These need the symbolic reference to the field, don't copy prop the value of the field
  2964. case Js::OpCode::DeleteFld:
  2965. case Js::OpCode::DeleteRootFld:
  2966. case Js::OpCode::DeleteFldStrict:
  2967. case Js::OpCode::DeleteRootFldStrict:
  2968. case Js::OpCode::ScopedDeleteFld:
  2969. case Js::OpCode::ScopedDeleteFldStrict:
  2970. case Js::OpCode::LdMethodFromFlags:
  2971. case Js::OpCode::BrOnNoProperty:
  2972. case Js::OpCode::BrOnHasProperty:
  2973. case Js::OpCode::LdMethodFldPolyInlineMiss:
  2974. case Js::OpCode::StSlotChkUndecl:
  2975. case Js::OpCode::ScopedLdInst:
  2976. return nullptr;
  2977. };
  2978. if (instr->CallsGetter())
  2979. {
  2980. return nullptr;
  2981. }
  2982. if (this->IsLoopPrePass() && this->DoFieldPRE(this->rootLoopPrePass))
  2983. {
  2984. if (!this->prePassLoop->allFieldsKilled && !this->prePassLoop->fieldKilled->Test(sym->m_id))
  2985. {
  2986. this->SetLoopFieldInitialValue(this->rootLoopPrePass, instr, sym->AsPropertySym(), originalPropertySym);
  2987. }
  2988. if (this->IsPREInstrCandidateLoad(instr->m_opcode))
  2989. {
  2990. // Foreach property sym, remember the first instruction that loads it.
  2991. // Can this be done in one call?
  2992. if (!this->prePassInstrMap->ContainsKey(sym->m_id))
  2993. {
  2994. this->prePassInstrMap->AddNew(sym->m_id, instr->CopyWithoutDst());
  2995. }
  2996. }
  2997. }
  2998. break;
  2999. }
  3000. case IR::OpndKindReg:
  3001. // Clear the opnd's value type up-front, so that this code cannot accidentally use the value type set from a previous
  3002. // OptSrc on the same instruction (for instance, from an earlier loop prepass). The value type will be set from the
  3003. // value if available, before returning from this function.
  3004. opnd->SetValueType(ValueType::Uninitialized);
  3005. sym = opnd->AsRegOpnd()->m_sym;
  3006. CurrentBlockData()->MarkTempLastUse(instr, opnd->AsRegOpnd());
  3007. if (sym->AsStackSym()->IsTypeSpec())
  3008. {
  3009. sym = sym->AsStackSym()->GetVarEquivSym(this->func);
  3010. }
  3011. break;
  3012. case IR::OpndKindIndir:
  3013. this->OptimizeIndirUses(opnd->AsIndirOpnd(), &instr, indirIndexValRef);
  3014. return nullptr;
  3015. default:
  3016. return nullptr;
  3017. }
  3018. val = CurrentBlockData()->FindValue(sym);
  3019. if (val)
  3020. {
  3021. Assert(CurrentBlockData()->IsLive(sym) || (sym->IsPropertySym()));
  3022. if (instr)
  3023. {
  3024. opnd = this->CopyProp(opnd, instr, val, parentIndirOpnd);
  3025. }
  3026. // Check if we freed the operand.
  3027. if (opnd == nullptr)
  3028. {
  3029. return nullptr;
  3030. }
  3031. // In a loop prepass, determine stack syms that are used before they are defined in the root loop for which the prepass
  3032. // is being done. This information is used to do type specialization conversions in the landing pad where appropriate.
  3033. if(IsLoopPrePass() &&
  3034. sym->IsStackSym() &&
  3035. !rootLoopPrePass->symsUsedBeforeDefined->Test(sym->m_id) &&
  3036. rootLoopPrePass->landingPad->globOptData.IsLive(sym) && !isAsmJSFunc) // no typespec in asmjs and hence skipping this
  3037. {
  3038. Value *const landingPadValue = rootLoopPrePass->landingPad->globOptData.FindValue(sym);
  3039. if(landingPadValue && val->GetValueNumber() == landingPadValue->GetValueNumber())
  3040. {
  3041. rootLoopPrePass->symsUsedBeforeDefined->Set(sym->m_id);
  3042. ValueInfo *landingPadValueInfo = landingPadValue->GetValueInfo();
  3043. if(landingPadValueInfo->IsLikelyNumber())
  3044. {
  3045. rootLoopPrePass->likelyNumberSymsUsedBeforeDefined->Set(sym->m_id);
  3046. if(DoAggressiveIntTypeSpec() ? landingPadValueInfo->IsLikelyInt() : landingPadValueInfo->IsInt())
  3047. {
  3048. // Can only force int conversions in the landing pad based on likely-int values if aggressive int type
  3049. // specialization is enabled.
  3050. rootLoopPrePass->likelyIntSymsUsedBeforeDefined->Set(sym->m_id);
  3051. }
  3052. }
  3053. }
  3054. }
  3055. }
  3056. else if ((instr->TransfersSrcValue() || OpCodeAttr::CanCSE(instr->m_opcode)) && (opnd == instr->GetSrc1() || opnd == instr->GetSrc2()))
  3057. {
  3058. if (sym->IsPropertySym())
  3059. {
  3060. val = this->CreateFieldSrcValue(sym->AsPropertySym(), originalPropertySym, &opnd, instr);
  3061. }
  3062. else
  3063. {
  3064. val = this->NewGenericValue(ValueType::Uninitialized, opnd);
  3065. }
  3066. }
  3067. if (opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd())
  3068. {
  3069. TryOptimizeInstrWithFixedDataProperty(&instr);
  3070. this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd());
  3071. }
  3072. if (val)
  3073. {
  3074. ValueType valueType(val->GetValueInfo()->Type());
  3075. // This block uses per-instruction profile information on array types to optimize using the best available profile
  3076. // information and to prevent infinite bailouts by ensuring array type information is updated on bailouts.
  3077. if (valueType.IsLikelyArray() && !valueType.IsDefinite() && !valueType.IsObject() && instr->IsProfiledInstr())
  3078. {
  3079. // See if we have profile data for the array type
  3080. IR::ProfiledInstr *const profiledInstr = instr->AsProfiledInstr();
  3081. ValueType profiledArrayType;
  3082. bool useAggressiveSpecialization = true;
  3083. switch(instr->m_opcode)
  3084. {
  3085. case Js::OpCode::LdElemI_A:
  3086. if(instr->GetSrc1()->IsIndirOpnd() && opnd == instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd())
  3087. {
  3088. profiledArrayType = profiledInstr->u.ldElemInfo->GetArrayType();
  3089. useAggressiveSpecialization = !profiledInstr->u.ldElemInfo->IsAggressiveSpecializationDisabled();
  3090. }
  3091. break;
  3092. case Js::OpCode::StElemI_A:
  3093. case Js::OpCode::StElemI_A_Strict:
  3094. case Js::OpCode::StElemC:
  3095. if(instr->GetDst()->IsIndirOpnd() && opnd == instr->GetDst()->AsIndirOpnd()->GetBaseOpnd())
  3096. {
  3097. profiledArrayType = profiledInstr->u.stElemInfo->GetArrayType();
  3098. useAggressiveSpecialization = !profiledInstr->u.stElemInfo->IsAggressiveSpecializationDisabled();
  3099. }
  3100. break;
  3101. case Js::OpCode::LdLen_A:
  3102. if(instr->GetSrc1()->IsRegOpnd() && opnd == instr->GetSrc1())
  3103. {
  3104. profiledArrayType = profiledInstr->u.LdLenInfo().GetArrayType();
  3105. useAggressiveSpecialization = !profiledInstr->u.LdLenInfo().IsAggressiveSpecializationDisabled();
  3106. }
  3107. break;
  3108. case Js::OpCode::IsIn:
  3109. if (instr->GetSrc2()->IsRegOpnd() && opnd == instr->GetSrc2())
  3110. {
  3111. profiledArrayType = profiledInstr->u.ldElemInfo->GetArrayType();
  3112. useAggressiveSpecialization = !profiledInstr->u.ldElemInfo->IsAggressiveSpecializationDisabled();
  3113. }
  3114. break;
  3115. }
  3116. if (profiledArrayType.IsLikelyObject())
  3117. {
  3118. // Ideally we want to use the most specialized type seen by this path, but when that causes bailouts use the least specialized type instead.
  3119. if (useAggressiveSpecialization &&
  3120. profiledArrayType.GetObjectType() == valueType.GetObjectType() &&
  3121. !valueType.IsLikelyNativeIntArray() &&
  3122. (
  3123. profiledArrayType.HasIntElements() || (valueType.HasVarElements() && profiledArrayType.HasFloatElements())
  3124. ))
  3125. {
  3126. // use the more specialized type profiled by the instruction.
  3127. valueType = profiledArrayType.SetHasNoMissingValues(valueType.HasNoMissingValues());
  3128. ChangeValueType(this->currentBlock, CurrentBlockData()->FindValue(opnd->AsRegOpnd()->m_sym), valueType, false);
  3129. }
  3130. else if (!useAggressiveSpecialization &&
  3131. (profiledArrayType.GetObjectType() != valueType.GetObjectType() ||
  3132. (
  3133. valueType.IsLikelyNativeArray() &&
  3134. (
  3135. profiledArrayType.HasVarElements() || (valueType.HasIntElements() && profiledArrayType.HasFloatElements())
  3136. )
  3137. )
  3138. ))
  3139. {
  3140. // Merge array type we pulled from profile with type propagated by dataflow.
  3141. if (profiledArrayType.IsLikelyArray())
  3142. {
  3143. valueType = valueType.Merge(profiledArrayType).SetHasNoMissingValues(valueType.HasNoMissingValues());
  3144. }
  3145. else
  3146. {
  3147. valueType = valueType.Merge(profiledArrayType);
  3148. }
  3149. ChangeValueType(this->currentBlock, CurrentBlockData()->FindValue(opnd->AsRegOpnd()->m_sym), valueType, false, true);
  3150. }
  3151. }
  3152. }
  3153. opnd->SetValueType(valueType);
  3154. if(!IsLoopPrePass() && opnd->IsSymOpnd() && valueType.IsDefinite())
  3155. {
  3156. if (opnd->AsSymOpnd()->m_sym->IsPropertySym())
  3157. {
  3158. // A property sym can only be guaranteed to have a definite value type when implicit calls are disabled from the
  3159. // point where the sym was defined with the definite value type. Insert an instruction to indicate to the
  3160. // dead-store pass that implicit calls need to be kept disabled until after this instruction.
  3161. Assert(DoFieldCopyProp());
  3162. CaptureNoImplicitCallUses(opnd, false, instr);
  3163. }
  3164. }
  3165. }
  3166. else
  3167. {
  3168. opnd->SetValueType(ValueType::Uninitialized);
  3169. }
  3170. return val;
  3171. }
  3172. /*
  3173. * GlobOpt::TryOptimizeInstrWithFixedDataProperty
  3174. * Converts Ld[Root]Fld instr to
  3175. * * CheckFixedFld
  3176. * * Dst = Ld_A <int Constant value>
  3177. * This API assumes that the source operand is a Sym/PropertySym kind.
  3178. */
  3179. void
  3180. GlobOpt::TryOptimizeInstrWithFixedDataProperty(IR::Instr ** const pInstr)
  3181. {
  3182. Assert(pInstr);
  3183. IR::Instr * &instr = *pInstr;
  3184. IR::Opnd * src1 = instr->GetSrc1();
  3185. Assert(src1 && src1->IsSymOpnd() && src1->AsSymOpnd()->IsPropertySymOpnd());
  3186. if(PHASE_OFF(Js::UseFixedDataPropsPhase, instr->m_func))
  3187. {
  3188. return;
  3189. }
  3190. if (!this->IsLoopPrePass() && !this->isRecursiveCallOnLandingPad &&
  3191. OpCodeAttr::CanLoadFixedFields(instr->m_opcode))
  3192. {
  3193. instr->TryOptimizeInstrWithFixedDataProperty(&instr, this);
  3194. }
  3195. }
  3196. // Constant prop if possible, otherwise if this value already resides in another
  3197. // symbol, reuse this previous symbol. This should help register allocation.
  3198. IR::Opnd *
  3199. GlobOpt::CopyProp(IR::Opnd *opnd, IR::Instr *instr, Value *val, IR::IndirOpnd *parentIndirOpnd)
  3200. {
  3201. Assert(
  3202. parentIndirOpnd
  3203. ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd()
  3204. : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd());
  3205. if (this->IsLoopPrePass())
  3206. {
  3207. // Transformations are not legal in prepass...
  3208. return opnd;
  3209. }
  3210. if (!this->func->DoGlobOptsForGeneratorFunc())
  3211. {
  3212. // Don't copy prop in generator functions because non-bytecode temps that span a yield
  3213. // cannot be saved and restored by the current bail-out mechanics utilized by generator
  3214. // yield/resume.
  3215. // TODO[generators][ianhall]: Enable copy-prop at least for in between yields.
  3216. return opnd;
  3217. }
  3218. if (instr->m_opcode == Js::OpCode::CheckFixedFld || instr->m_opcode == Js::OpCode::CheckPropertyGuardAndLoadType)
  3219. {
  3220. // Don't copy prop into CheckFixedFld or CheckPropertyGuardAndLoadType
  3221. return opnd;
  3222. }
  3223. // Don't copy-prop link operands of ExtendedArgs
  3224. if (instr->m_opcode == Js::OpCode::ExtendArg_A && opnd == instr->GetSrc2())
  3225. {
  3226. return opnd;
  3227. }
  3228. // Don't copy-prop operand of SIMD instr with ExtendedArg operands. Each instr should have its exclusive EA sequence.
  3229. if (
  3230. Js::IsSimd128Opcode(instr->m_opcode) &&
  3231. instr->GetSrc1() != nullptr &&
  3232. instr->GetSrc1()->IsRegOpnd() &&
  3233. instr->GetSrc2() == nullptr
  3234. )
  3235. {
  3236. StackSym *sym = instr->GetSrc1()->GetStackSym();
  3237. if (sym && sym->IsSingleDef() && sym->GetInstrDef()->m_opcode == Js::OpCode::ExtendArg_A)
  3238. {
  3239. return opnd;
  3240. }
  3241. }
  3242. ValueInfo *valueInfo = val->GetValueInfo();
  3243. if (this->func->HasFinally())
  3244. {
  3245. // s0 = undefined was added on functions with early exit in try-finally functions, that can get copy-proped and case incorrect results
  3246. if (instr->m_opcode == Js::OpCode::ArgOut_A_Inline && valueInfo->GetSymStore() &&
  3247. valueInfo->GetSymStore()->m_id == 0)
  3248. {
  3249. // We don't want to copy-prop s0 (return symbol) into inlinee code
  3250. return opnd;
  3251. }
  3252. }
  3253. // Constant prop?
  3254. int32 intConstantValue;
  3255. int64 int64ConstantValue;
  3256. if (valueInfo->TryGetIntConstantValue(&intConstantValue))
  3257. {
  3258. if (PHASE_OFF(Js::ConstPropPhase, this->func))
  3259. {
  3260. return opnd;
  3261. }
  3262. if ((
  3263. instr->m_opcode == Js::OpCode::StElemI_A ||
  3264. instr->m_opcode == Js::OpCode::StElemI_A_Strict ||
  3265. instr->m_opcode == Js::OpCode::StElemC
  3266. ) && instr->GetSrc1() == opnd)
  3267. {
  3268. // Disabling prop to src of native array store, because we were losing the chance to type specialize.
  3269. // Is it possible to type specialize this src if we allow constants, etc., to be prop'd here?
  3270. if (instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyNativeArray())
  3271. {
  3272. return opnd;
  3273. }
  3274. }
  3275. if(opnd != instr->GetSrc1() && opnd != instr->GetSrc2())
  3276. {
  3277. if(PHASE_OFF(Js::IndirCopyPropPhase, instr->m_func))
  3278. {
  3279. return opnd;
  3280. }
  3281. // Const-prop an indir opnd's constant index into its offset
  3282. IR::Opnd *srcs[] = { instr->GetSrc1(), instr->GetSrc2(), instr->GetDst() };
  3283. for(int i = 0; i < sizeof(srcs) / sizeof(srcs[0]); ++i)
  3284. {
  3285. const auto src = srcs[i];
  3286. if(!src || !src->IsIndirOpnd())
  3287. {
  3288. continue;
  3289. }
  3290. const auto indir = src->AsIndirOpnd();
  3291. if ((int64)indir->GetOffset() + intConstantValue > INT32_MAX)
  3292. {
  3293. continue;
  3294. }
  3295. if(opnd == indir->GetIndexOpnd())
  3296. {
  3297. Assert(indir->GetScale() == 0);
  3298. GOPT_TRACE_OPND(opnd, _u("Constant prop indir index into offset (value: %d)\n"), intConstantValue);
  3299. this->CaptureByteCodeSymUses(instr);
  3300. indir->SetOffset(indir->GetOffset() + intConstantValue);
  3301. indir->SetIndexOpnd(nullptr);
  3302. }
  3303. }
  3304. return opnd;
  3305. }
  3306. if (Js::TaggedInt::IsOverflow(intConstantValue))
  3307. {
  3308. return opnd;
  3309. }
  3310. IR::Opnd *constOpnd;
  3311. if (opnd->IsVar())
  3312. {
  3313. IR::AddrOpnd *addrOpnd = IR::AddrOpnd::New(Js::TaggedInt::ToVarUnchecked((int)intConstantValue), IR::AddrOpndKindConstantVar, instr->m_func);
  3314. GOPT_TRACE_OPND(opnd, _u("Constant prop %d (value:%d)\n"), addrOpnd->m_address, intConstantValue);
  3315. constOpnd = addrOpnd;
  3316. }
  3317. else
  3318. {
  3319. // Note: Jit loop body generates some i32 operands...
  3320. Assert(opnd->IsInt32() || opnd->IsInt64() || opnd->IsUInt32());
  3321. IRType opndType;
  3322. IntConstType constVal;
  3323. if (opnd->IsUInt32())
  3324. {
  3325. // avoid sign extension
  3326. constVal = (uint32)intConstantValue;
  3327. opndType = TyUint32;
  3328. }
  3329. else
  3330. {
  3331. constVal = intConstantValue;
  3332. opndType = TyInt32;
  3333. }
  3334. IR::IntConstOpnd *intOpnd = IR::IntConstOpnd::New(constVal, opndType, instr->m_func);
  3335. GOPT_TRACE_OPND(opnd, _u("Constant prop %d (value:%d)\n"), intOpnd->GetImmediateValue(instr->m_func), intConstantValue);
  3336. constOpnd = intOpnd;
  3337. }
  3338. #if ENABLE_DEBUG_CONFIG_OPTIONS
  3339. //Need to update DumpFieldCopyPropTestTrace for every new opcode that is added for fieldcopyprop
  3340. if(Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FieldCopyPropPhase))
  3341. {
  3342. instr->DumpFieldCopyPropTestTrace(this->isRecursiveCallOnLandingPad);
  3343. }
  3344. #endif
  3345. this->CaptureByteCodeSymUses(instr);
  3346. opnd = instr->ReplaceSrc(opnd, constOpnd);
  3347. switch (instr->m_opcode)
  3348. {
  3349. case Js::OpCode::LdSlot:
  3350. case Js::OpCode::LdSlotArr:
  3351. case Js::OpCode::LdFld:
  3352. case Js::OpCode::LdFldForTypeOf:
  3353. case Js::OpCode::LdRootFldForTypeOf:
  3354. case Js::OpCode::LdFldForCallApplyTarget:
  3355. case Js::OpCode::LdRootFld:
  3356. case Js::OpCode::LdMethodFld:
  3357. case Js::OpCode::LdRootMethodFld:
  3358. case Js::OpCode::LdMethodFromFlags:
  3359. case Js::OpCode::ScopedLdMethodFld:
  3360. case Js::OpCode::ScopedLdFld:
  3361. case Js::OpCode::ScopedLdFldForTypeOf:
  3362. instr->m_opcode = Js::OpCode::Ld_A;
  3363. case Js::OpCode::Ld_A:
  3364. {
  3365. IR::Opnd * dst = instr->GetDst();
  3366. if (dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->IsSingleDef())
  3367. {
  3368. dst->AsRegOpnd()->m_sym->SetIsIntConst((int)intConstantValue);
  3369. }
  3370. break;
  3371. }
  3372. case Js::OpCode::ArgOut_A:
  3373. case Js::OpCode::ArgOut_A_Inline:
  3374. case Js::OpCode::ArgOut_A_FixupForStackArgs:
  3375. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  3376. if (instr->GetDst()->IsRegOpnd())
  3377. {
  3378. Assert(instr->GetDst()->AsRegOpnd()->m_sym->m_isSingleDef);
  3379. instr->GetDst()->AsRegOpnd()->m_sym->AsStackSym()->SetIsIntConst((int)intConstantValue);
  3380. }
  3381. else
  3382. {
  3383. instr->GetDst()->AsSymOpnd()->m_sym->AsStackSym()->SetIsIntConst((int)intConstantValue);
  3384. }
  3385. break;
  3386. case Js::OpCode::TypeofElem:
  3387. instr->m_opcode = Js::OpCode::Typeof;
  3388. break;
  3389. case Js::OpCode::StSlotChkUndecl:
  3390. if (instr->GetSrc2() == opnd)
  3391. {
  3392. // Src2 here should refer to the same location as the Dst operand, which we need to keep live
  3393. // due to the implicit read for ChkUndecl.
  3394. instr->m_opcode = Js::OpCode::StSlot;
  3395. instr->FreeSrc2();
  3396. opnd = nullptr;
  3397. }
  3398. break;
  3399. }
  3400. return opnd;
  3401. }
  3402. else if (valueInfo->TryGetIntConstantValue(&int64ConstantValue, false))
  3403. {
  3404. if (PHASE_OFF(Js::ConstPropPhase, this->func) || !PHASE_ON(Js::Int64ConstPropPhase, this->func))
  3405. {
  3406. return opnd;
  3407. }
  3408. Assert(this->func->GetJITFunctionBody()->IsWasmFunction());
  3409. if (this->func->GetJITFunctionBody()->IsWasmFunction() && opnd->IsInt64())
  3410. {
  3411. IR::Int64ConstOpnd *intOpnd = IR::Int64ConstOpnd::New(int64ConstantValue, opnd->GetType(), instr->m_func);
  3412. GOPT_TRACE_OPND(opnd, _u("Constant prop %lld (value:%lld)\n"), intOpnd->GetImmediateValue(instr->m_func), int64ConstantValue);
  3413. this->CaptureByteCodeSymUses(instr);
  3414. opnd = instr->ReplaceSrc(opnd, intOpnd);
  3415. }
  3416. return opnd;
  3417. }
  3418. Sym *opndSym = nullptr;
  3419. if (opnd->IsRegOpnd())
  3420. {
  3421. IR::RegOpnd *regOpnd = opnd->AsRegOpnd();
  3422. opndSym = regOpnd->m_sym;
  3423. }
  3424. else if (opnd->IsSymOpnd())
  3425. {
  3426. IR::SymOpnd *symOpnd = opnd->AsSymOpnd();
  3427. opndSym = symOpnd->m_sym;
  3428. }
  3429. if (!opndSym)
  3430. {
  3431. return opnd;
  3432. }
  3433. if (PHASE_OFF(Js::CopyPropPhase, this->func))
  3434. {
  3435. this->SetSymStoreDirect(valueInfo, opndSym);
  3436. return opnd;
  3437. }
  3438. StackSym *copySym = CurrentBlockData()->GetCopyPropSym(opndSym, val);
  3439. if (copySym != nullptr)
  3440. {
  3441. Assert(!opndSym->IsStackSym() || copySym->GetSymSize() == opndSym->AsStackSym()->GetSymSize());
  3442. // Copy prop.
  3443. return CopyPropReplaceOpnd(instr, opnd, copySym, parentIndirOpnd);
  3444. }
  3445. else
  3446. {
  3447. if (valueInfo->GetSymStore() && instr->m_opcode == Js::OpCode::Ld_A && instr->GetDst()->IsRegOpnd()
  3448. && valueInfo->GetSymStore() == instr->GetDst()->AsRegOpnd()->m_sym)
  3449. {
  3450. // Avoid resetting symStore after fieldHoisting:
  3451. // t1 = LdFld field <- set symStore to fieldHoistSym
  3452. // fieldHoistSym = Ld_A t1 <- we're looking at t1 now, but want to copy-prop fieldHoistSym forward
  3453. return opnd;
  3454. }
  3455. this->SetSymStoreDirect(valueInfo, opndSym);
  3456. }
  3457. return opnd;
  3458. }
  3459. IR::Opnd *
  3460. GlobOpt::CopyPropReplaceOpnd(IR::Instr * instr, IR::Opnd * opnd, StackSym * copySym, IR::IndirOpnd *parentIndirOpnd)
  3461. {
  3462. Assert(
  3463. parentIndirOpnd
  3464. ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd()
  3465. : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd());
  3466. Assert(CurrentBlockData()->IsLive(copySym));
  3467. IR::RegOpnd *regOpnd;
  3468. StackSym *newSym = copySym;
  3469. GOPT_TRACE_OPND(opnd, _u("Copy prop s%d\n"), newSym->m_id);
  3470. #if ENABLE_DEBUG_CONFIG_OPTIONS
  3471. //Need to update DumpFieldCopyPropTestTrace for every new opcode that is added for fieldcopyprop
  3472. if(Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FieldCopyPropPhase))
  3473. {
  3474. instr->DumpFieldCopyPropTestTrace(this->isRecursiveCallOnLandingPad);
  3475. }
  3476. #endif
  3477. this->CaptureByteCodeSymUses(instr);
  3478. if (opnd->IsRegOpnd())
  3479. {
  3480. regOpnd = opnd->AsRegOpnd();
  3481. regOpnd->m_sym = newSym;
  3482. regOpnd->SetIsJITOptimizedReg(true);
  3483. // The dead bit on the opnd is specific to the sym it is referencing. Since we replaced the sym, the bit is reset.
  3484. regOpnd->SetIsDead(false);
  3485. if(parentIndirOpnd)
  3486. {
  3487. return regOpnd;
  3488. }
  3489. }
  3490. else
  3491. {
  3492. // If this is an object type specialized field load inside a loop, and it produces a type value which wasn't live
  3493. // before, make sure the type check is left in the loop, because it may be the last type check in the loop protecting
  3494. // other fields which are not hoistable and are lexically upstream in the loop. If the check is not ultimately
  3495. // needed, the dead store pass will remove it.
  3496. if (this->currentBlock->loop != nullptr && opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd())
  3497. {
  3498. IR::PropertySymOpnd* propertySymOpnd = opnd->AsPropertySymOpnd();
  3499. if (CheckIfPropOpEmitsTypeCheck(instr, propertySymOpnd))
  3500. {
  3501. // We only set guarded properties in the dead store pass, so they shouldn't be set here yet. If they were
  3502. // we would need to move them from this operand to the operand which is being copy propagated.
  3503. Assert(propertySymOpnd->GetGuardedPropOps() == nullptr);
  3504. // We're creating a copy of this operand to be reused in the same spot in the flow, so we can copy all
  3505. // flow sensitive fields. However, we will do only a type check here (no property access) and only for
  3506. // the sake of downstream instructions, so the flags pertaining to this property access are irrelevant.
  3507. IR::PropertySymOpnd* checkObjTypeOpnd = CreateOpndForTypeCheckOnly(propertySymOpnd, instr->m_func);
  3508. IR::Instr* checkObjTypeInstr = IR::Instr::New(Js::OpCode::CheckObjType, instr->m_func);
  3509. checkObjTypeInstr->SetSrc1(checkObjTypeOpnd);
  3510. checkObjTypeInstr->SetByteCodeOffset(instr);
  3511. instr->InsertBefore(checkObjTypeInstr);
  3512. // Since we inserted this instruction before the one that is being processed in natural flow, we must process
  3513. // it for object type spec explicitly here.
  3514. FinishOptPropOp(checkObjTypeInstr, checkObjTypeOpnd);
  3515. Assert(!propertySymOpnd->IsTypeChecked());
  3516. checkObjTypeInstr = this->SetTypeCheckBailOut(checkObjTypeOpnd, checkObjTypeInstr, nullptr);
  3517. Assert(checkObjTypeInstr->HasBailOutInfo());
  3518. if (this->currentBlock->loop && !this->IsLoopPrePass())
  3519. {
  3520. // Try hoisting this checkObjType.
  3521. // But since this isn't the current instr being optimized, we need to play tricks with
  3522. // the byteCodeUse fields...
  3523. TrackByteCodeUsesForInstrAddedInOptInstr(checkObjTypeInstr, [&]()
  3524. {
  3525. TryHoistInvariant(checkObjTypeInstr, this->currentBlock, NULL, CurrentBlockData()->FindValue(copySym), NULL, true);
  3526. });
  3527. }
  3528. }
  3529. }
  3530. if (opnd->IsSymOpnd() && opnd->GetIsDead())
  3531. {
  3532. // Take the property sym out of the live fields set
  3533. this->EndFieldLifetime(opnd->AsSymOpnd());
  3534. }
  3535. regOpnd = IR::RegOpnd::New(newSym, opnd->GetType(), instr->m_func);
  3536. regOpnd->SetIsJITOptimizedReg(true);
  3537. instr->ReplaceSrc(opnd, regOpnd);
  3538. }
  3539. switch (instr->m_opcode)
  3540. {
  3541. case Js::OpCode::Ld_A:
  3542. if (instr->GetDst()->IsRegOpnd() && instr->GetSrc1()->IsRegOpnd() &&
  3543. instr->GetDst()->AsRegOpnd()->GetStackSym() == instr->GetSrc1()->AsRegOpnd()->GetStackSym())
  3544. {
  3545. this->InsertByteCodeUses(instr, true);
  3546. instr->m_opcode = Js::OpCode::Nop;
  3547. }
  3548. break;
  3549. case Js::OpCode::LdSlot:
  3550. case Js::OpCode::LdSlotArr:
  3551. if (instr->GetDst()->IsRegOpnd() && instr->GetSrc1()->IsRegOpnd() &&
  3552. instr->GetDst()->AsRegOpnd()->GetStackSym() == instr->GetSrc1()->AsRegOpnd()->GetStackSym())
  3553. {
  3554. this->InsertByteCodeUses(instr, true);
  3555. instr->m_opcode = Js::OpCode::Nop;
  3556. }
  3557. else
  3558. {
  3559. instr->m_opcode = Js::OpCode::Ld_A;
  3560. }
  3561. break;
  3562. case Js::OpCode::StSlotChkUndecl:
  3563. if (instr->GetSrc2()->IsRegOpnd())
  3564. {
  3565. // Src2 here should refer to the same location as the Dst operand, which we need to keep live
  3566. // due to the implicit read for ChkUndecl.
  3567. instr->m_opcode = Js::OpCode::StSlot;
  3568. instr->FreeSrc2();
  3569. return nullptr;
  3570. }
  3571. break;
  3572. case Js::OpCode::LdFld:
  3573. case Js::OpCode::LdFldForTypeOf:
  3574. case Js::OpCode::LdRootFldForTypeOf:
  3575. case Js::OpCode::LdFldForCallApplyTarget:
  3576. case Js::OpCode::LdRootFld:
  3577. case Js::OpCode::LdMethodFld:
  3578. case Js::OpCode::LdRootMethodFld:
  3579. case Js::OpCode::ScopedLdMethodFld:
  3580. case Js::OpCode::ScopedLdFld:
  3581. case Js::OpCode::ScopedLdFldForTypeOf:
  3582. instr->m_opcode = Js::OpCode::Ld_A;
  3583. break;
  3584. case Js::OpCode::LdMethodFromFlags:
  3585. // The bailout is checked on the loop top and we don't need to check bailout again in loop.
  3586. instr->m_opcode = Js::OpCode::Ld_A;
  3587. instr->ClearBailOutInfo();
  3588. break;
  3589. case Js::OpCode::TypeofElem:
  3590. instr->m_opcode = Js::OpCode::Typeof;
  3591. break;
  3592. }
  3593. CurrentBlockData()->MarkTempLastUse(instr, regOpnd);
  3594. return regOpnd;
  3595. }
  3596. ValueNumber
  3597. GlobOpt::NewValueNumber()
  3598. {
  3599. ValueNumber valueNumber = this->currentValue++;
  3600. if (valueNumber == 0)
  3601. {
  3602. Js::Throw::OutOfMemory();
  3603. }
  3604. return valueNumber;
  3605. }
  3606. Value *GlobOpt::NewValue(ValueInfo *const valueInfo)
  3607. {
  3608. return NewValue(NewValueNumber(), valueInfo);
  3609. }
  3610. Value *GlobOpt::NewValue(const ValueNumber valueNumber, ValueInfo *const valueInfo)
  3611. {
  3612. Assert(valueInfo);
  3613. return Value::New(alloc, valueNumber, valueInfo);
  3614. }
  3615. Value *GlobOpt::CopyValue(Value const *const value)
  3616. {
  3617. return CopyValue(value, NewValueNumber());
  3618. }
  3619. Value *GlobOpt::CopyValue(Value const *const value, const ValueNumber valueNumber)
  3620. {
  3621. Assert(value);
  3622. return value->Copy(alloc, valueNumber);
  3623. }
  3624. Value *
  3625. GlobOpt::NewGenericValue(const ValueType valueType)
  3626. {
  3627. return NewGenericValue(valueType, static_cast<IR::Opnd *>(nullptr));
  3628. }
  3629. Value *
  3630. GlobOpt::NewGenericValue(const ValueType valueType, IR::Opnd *const opnd)
  3631. {
  3632. // Shouldn't assign a likely-int value to something that is definitely not an int
  3633. Assert(!(valueType.IsLikelyInt() && opnd && opnd->IsNotInt()));
  3634. ValueInfo *valueInfo = ValueInfo::New(this->alloc, valueType);
  3635. Value *val = NewValue(valueInfo);
  3636. TrackNewValueForKills(val);
  3637. CurrentBlockData()->InsertNewValue(val, opnd);
  3638. return val;
  3639. }
  3640. Value *
  3641. GlobOpt::NewGenericValue(const ValueType valueType, Sym *const sym)
  3642. {
  3643. ValueInfo *valueInfo = ValueInfo::New(this->alloc, valueType);
  3644. Value *val = NewValue(valueInfo);
  3645. TrackNewValueForKills(val);
  3646. CurrentBlockData()->SetValue(val, sym);
  3647. return val;
  3648. }
  3649. Value *
  3650. GlobOpt::GetIntConstantValue(const int32 intConst, IR::Instr * instr, IR::Opnd *const opnd)
  3651. {
  3652. Value *value = nullptr;
  3653. Value *const cachedValue = this->intConstantToValueMap->Lookup(intConst, nullptr);
  3654. if(cachedValue)
  3655. {
  3656. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  3657. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  3658. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  3659. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  3660. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  3661. // Otherwise, we have to create a new value with a new value number.
  3662. Sym *const symStore = cachedValue->GetValueInfo()->GetSymStore();
  3663. if (symStore && CurrentBlockData()->IsLive(symStore))
  3664. {
  3665. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  3666. int32 symStoreIntConstantValue;
  3667. if (symStoreValue &&
  3668. symStoreValue->GetValueNumber() == cachedValue->GetValueNumber() &&
  3669. symStoreValue->GetValueInfo()->TryGetIntConstantValue(&symStoreIntConstantValue) &&
  3670. symStoreIntConstantValue == intConst)
  3671. {
  3672. value = symStoreValue;
  3673. }
  3674. }
  3675. }
  3676. if (!value)
  3677. {
  3678. value = NewIntConstantValue(intConst, instr, !Js::TaggedInt::IsOverflow(intConst));
  3679. }
  3680. return CurrentBlockData()->InsertNewValue(value, opnd);
  3681. }
  3682. Value *
  3683. GlobOpt::GetIntConstantValue(const int64 intConst, IR::Instr * instr, IR::Opnd *const opnd)
  3684. {
  3685. Assert(instr->m_func->GetJITFunctionBody()->IsWasmFunction());
  3686. Value *value = nullptr;
  3687. Value *const cachedValue = this->int64ConstantToValueMap->Lookup(intConst, nullptr);
  3688. if (cachedValue)
  3689. {
  3690. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  3691. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  3692. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  3693. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  3694. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  3695. // Otherwise, we have to create a new value with a new value number.
  3696. Sym *const symStore = cachedValue->GetValueInfo()->GetSymStore();
  3697. if (symStore && this->currentBlock->globOptData.IsLive(symStore))
  3698. {
  3699. Value *const symStoreValue = this->currentBlock->globOptData.FindValue(symStore);
  3700. int64 symStoreIntConstantValue;
  3701. if (symStoreValue &&
  3702. symStoreValue->GetValueNumber() == cachedValue->GetValueNumber() &&
  3703. symStoreValue->GetValueInfo()->TryGetInt64ConstantValue(&symStoreIntConstantValue, false) &&
  3704. symStoreIntConstantValue == intConst)
  3705. {
  3706. value = symStoreValue;
  3707. }
  3708. }
  3709. }
  3710. if (!value)
  3711. {
  3712. value = NewInt64ConstantValue(intConst, instr);
  3713. }
  3714. return this->currentBlock->globOptData.InsertNewValue(value, opnd);
  3715. }
  3716. Value *
  3717. GlobOpt::NewInt64ConstantValue(const int64 intConst, IR::Instr* instr)
  3718. {
  3719. Value * value = NewValue(Int64ConstantValueInfo::New(this->alloc, intConst));
  3720. this->int64ConstantToValueMap->Item(intConst, value);
  3721. if (!value->GetValueInfo()->GetSymStore() &&
  3722. (instr->m_opcode == Js::OpCode::LdC_A_I4 || instr->m_opcode == Js::OpCode::Ld_I4))
  3723. {
  3724. StackSym * sym = instr->GetDst()->GetStackSym();
  3725. Assert(sym && !sym->IsTypeSpec());
  3726. this->currentBlock->globOptData.SetValue(value, sym);
  3727. this->currentBlock->globOptData.liveVarSyms->Set(sym->m_id);
  3728. }
  3729. return value;
  3730. }
  3731. Value *
  3732. GlobOpt::NewIntConstantValue(const int32 intConst, IR::Instr * instr, bool isTaggable)
  3733. {
  3734. Value * value = NewValue(IntConstantValueInfo::New(this->alloc, intConst));
  3735. this->intConstantToValueMap->Item(intConst, value);
  3736. if (isTaggable &&
  3737. !PHASE_OFF(Js::HoistConstIntPhase, this->func))
  3738. {
  3739. // When creating a new int constant value, make sure it gets a symstore. If the int const doesn't have a symstore,
  3740. // any downstream instruction using the same int will have to create a new value (object) for the int.
  3741. // This gets in the way of CSE.
  3742. value = HoistConstantLoadAndPropagateValueBackward(Js::TaggedInt::ToVarUnchecked(intConst), instr, value);
  3743. if (!value->GetValueInfo()->GetSymStore() &&
  3744. (instr->m_opcode == Js::OpCode::LdC_A_I4 || instr->m_opcode == Js::OpCode::Ld_I4))
  3745. {
  3746. StackSym * sym = instr->GetDst()->GetStackSym();
  3747. Assert(sym);
  3748. if (sym->IsTypeSpec())
  3749. {
  3750. Assert(sym->IsInt32());
  3751. StackSym * varSym = sym->GetVarEquivSym(instr->m_func);
  3752. CurrentBlockData()->SetValue(value, varSym);
  3753. CurrentBlockData()->liveInt32Syms->Set(varSym->m_id);
  3754. }
  3755. else
  3756. {
  3757. CurrentBlockData()->SetValue(value, sym);
  3758. CurrentBlockData()->liveVarSyms->Set(sym->m_id);
  3759. }
  3760. }
  3761. }
  3762. return value;
  3763. }
  3764. ValueInfo *
  3765. GlobOpt::NewIntRangeValueInfo(const int32 min, const int32 max, const bool wasNegativeZeroPreventedByBailout)
  3766. {
  3767. return ValueInfo::NewIntRangeValueInfo(this->alloc, min, max, wasNegativeZeroPreventedByBailout);
  3768. }
  3769. ValueInfo *GlobOpt::NewIntRangeValueInfo(
  3770. const ValueInfo *const originalValueInfo,
  3771. const int32 min,
  3772. const int32 max) const
  3773. {
  3774. Assert(originalValueInfo);
  3775. ValueInfo *valueInfo;
  3776. if(min == max)
  3777. {
  3778. // Since int constant values are const-propped, negative zero tracking does not track them, and so it's okay to ignore
  3779. // 'wasNegativeZeroPreventedByBailout'
  3780. valueInfo = IntConstantValueInfo::New(alloc, min);
  3781. }
  3782. else
  3783. {
  3784. valueInfo =
  3785. IntRangeValueInfo::New(
  3786. alloc,
  3787. min,
  3788. max,
  3789. min <= 0 && max >= 0 && originalValueInfo->WasNegativeZeroPreventedByBailout());
  3790. }
  3791. valueInfo->SetSymStore(originalValueInfo->GetSymStore());
  3792. return valueInfo;
  3793. }
  3794. Value *
  3795. GlobOpt::NewIntRangeValue(
  3796. const int32 min,
  3797. const int32 max,
  3798. const bool wasNegativeZeroPreventedByBailout,
  3799. IR::Opnd *const opnd)
  3800. {
  3801. ValueInfo *valueInfo = this->NewIntRangeValueInfo(min, max, wasNegativeZeroPreventedByBailout);
  3802. Value *val = NewValue(valueInfo);
  3803. if (opnd)
  3804. {
  3805. GOPT_TRACE_OPND(opnd, _u("Range %d (0x%X) to %d (0x%X)\n"), min, min, max, max);
  3806. }
  3807. CurrentBlockData()->InsertNewValue(val, opnd);
  3808. return val;
  3809. }
  3810. IntBoundedValueInfo *GlobOpt::NewIntBoundedValueInfo(
  3811. const ValueInfo *const originalValueInfo,
  3812. const IntBounds *const bounds) const
  3813. {
  3814. Assert(originalValueInfo);
  3815. bounds->Verify();
  3816. IntBoundedValueInfo *const valueInfo =
  3817. IntBoundedValueInfo::New(
  3818. originalValueInfo->Type(),
  3819. bounds,
  3820. (
  3821. bounds->ConstantLowerBound() <= 0 &&
  3822. bounds->ConstantUpperBound() >= 0 &&
  3823. originalValueInfo->WasNegativeZeroPreventedByBailout()
  3824. ),
  3825. alloc);
  3826. valueInfo->SetSymStore(originalValueInfo->GetSymStore());
  3827. return valueInfo;
  3828. }
  3829. Value *GlobOpt::NewIntBoundedValue(
  3830. const ValueType valueType,
  3831. const IntBounds *const bounds,
  3832. const bool wasNegativeZeroPreventedByBailout,
  3833. IR::Opnd *const opnd)
  3834. {
  3835. Value *const value = NewValue(IntBoundedValueInfo::New(valueType, bounds, wasNegativeZeroPreventedByBailout, alloc));
  3836. CurrentBlockData()->InsertNewValue(value, opnd);
  3837. return value;
  3838. }
  3839. Value *
  3840. GlobOpt::NewFloatConstantValue(const FloatConstType floatValue, IR::Opnd *const opnd)
  3841. {
  3842. FloatConstantValueInfo *valueInfo = FloatConstantValueInfo::New(this->alloc, floatValue);
  3843. Value *val = NewValue(valueInfo);
  3844. CurrentBlockData()->InsertNewValue(val, opnd);
  3845. return val;
  3846. }
  3847. Value *
  3848. GlobOpt::GetVarConstantValue(IR::AddrOpnd *addrOpnd)
  3849. {
  3850. bool isVar = addrOpnd->IsVar();
  3851. bool isString = isVar && addrOpnd->m_localAddress && JITJavascriptString::Is(addrOpnd->m_localAddress);
  3852. Value *val = nullptr;
  3853. Value *cachedValue = nullptr;
  3854. if(this->addrConstantToValueMap->TryGetValue(addrOpnd->m_address, &cachedValue))
  3855. {
  3856. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  3857. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  3858. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  3859. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  3860. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  3861. // Otherwise, we have to create a new value with a new value number.
  3862. Sym *symStore = cachedValue->GetValueInfo()->GetSymStore();
  3863. if(symStore && CurrentBlockData()->IsLive(symStore))
  3864. {
  3865. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  3866. if(symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber())
  3867. {
  3868. ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo();
  3869. if(symStoreValueInfo->IsVarConstant() && symStoreValueInfo->AsVarConstant()->VarValue() == addrOpnd->m_address)
  3870. {
  3871. val = symStoreValue;
  3872. }
  3873. }
  3874. }
  3875. }
  3876. else if (isString)
  3877. {
  3878. JITJavascriptString* jsString = JITJavascriptString::FromVar(addrOpnd->m_localAddress);
  3879. Js::InternalString internalString(jsString->GetString(), jsString->GetLength());
  3880. if (this->stringConstantToValueMap->TryGetValue(internalString, &cachedValue))
  3881. {
  3882. Sym *symStore = cachedValue->GetValueInfo()->GetSymStore();
  3883. if (symStore && CurrentBlockData()->IsLive(symStore))
  3884. {
  3885. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  3886. if (symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber())
  3887. {
  3888. ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo();
  3889. if (symStoreValueInfo->IsVarConstant())
  3890. {
  3891. JITJavascriptString * cachedString = JITJavascriptString::FromVar(symStoreValue->GetValueInfo()->AsVarConstant()->VarValue(true));
  3892. Js::InternalString cachedInternalString(cachedString->GetString(), cachedString->GetLength());
  3893. if (Js::InternalStringComparer::Equals(internalString, cachedInternalString))
  3894. {
  3895. val = symStoreValue;
  3896. }
  3897. }
  3898. }
  3899. }
  3900. }
  3901. }
  3902. if(!val)
  3903. {
  3904. val = NewVarConstantValue(addrOpnd, isString);
  3905. }
  3906. addrOpnd->SetValueType(val->GetValueInfo()->Type());
  3907. return val;
  3908. }
  3909. Value *
  3910. GlobOpt::NewVarConstantValue(IR::AddrOpnd *addrOpnd, bool isString)
  3911. {
  3912. VarConstantValueInfo *valueInfo = VarConstantValueInfo::New(this->alloc, addrOpnd->m_address, addrOpnd->GetValueType(), false, addrOpnd->m_localAddress);
  3913. Value * value = NewValue(valueInfo);
  3914. this->addrConstantToValueMap->Item(addrOpnd->m_address, value);
  3915. if (isString)
  3916. {
  3917. JITJavascriptString* jsString = JITJavascriptString::FromVar(addrOpnd->m_localAddress);
  3918. Js::InternalString internalString(jsString->GetString(), jsString->GetLength());
  3919. this->stringConstantToValueMap->Item(internalString, value);
  3920. }
  3921. return value;
  3922. }
  3923. Value *
  3924. GlobOpt::HoistConstantLoadAndPropagateValueBackward(Js::Var varConst, IR::Instr * origInstr, Value * value)
  3925. {
  3926. if (this->IsLoopPrePass() ||
  3927. ((this->currentBlock == this->func->m_fg->blockList) &&
  3928. origInstr->TransfersSrcValue()))
  3929. {
  3930. return value;
  3931. }
  3932. // Only hoisting taggable int const loads for now. Could be extended to other constants (floats, strings, addr opnds) if we see some benefit.
  3933. Assert(Js::TaggedInt::Is(varConst));
  3934. // Insert a load of the constant at the top of the function
  3935. StackSym * dstSym = StackSym::New(this->func);
  3936. IR::RegOpnd * constRegOpnd = IR::RegOpnd::New(dstSym, TyVar, this->func);
  3937. IR::Instr * loadInstr = IR::Instr::NewConstantLoad(constRegOpnd, (intptr_t)varConst, ValueType::GetInt(true), this->func);
  3938. this->func->m_fg->blockList->GetFirstInstr()->InsertAfter(loadInstr);
  3939. // Type-spec the load (Support for floats needs to be added when we start hoisting float constants).
  3940. bool typeSpecedToInt = false;
  3941. if (Js::TaggedInt::Is(varConst) && !IsTypeSpecPhaseOff(this->func))
  3942. {
  3943. typeSpecedToInt = true;
  3944. loadInstr->m_opcode = Js::OpCode::Ld_I4;
  3945. ToInt32Dst(loadInstr, loadInstr->GetDst()->AsRegOpnd(), this->currentBlock);
  3946. loadInstr->GetDst()->GetStackSym()->SetIsConst();
  3947. }
  3948. else
  3949. {
  3950. CurrentBlockData()->liveVarSyms->Set(dstSym->m_id);
  3951. }
  3952. // Add the value (object) to the current block's symToValueMap and propagate the value backward to all relevant blocks so it is available on merges.
  3953. value = CurrentBlockData()->InsertNewValue(value, constRegOpnd);
  3954. BVSparse<JitArenaAllocator>* GlobOptBlockData::*bv;
  3955. bv = typeSpecedToInt ? &GlobOptBlockData::liveInt32Syms : &GlobOptBlockData::liveVarSyms; // Will need to be expanded when we start hoisting float constants.
  3956. if (this->currentBlock != this->func->m_fg->blockList)
  3957. {
  3958. for (InvariantBlockBackwardIterator it(this, this->currentBlock, this->func->m_fg->blockList, nullptr);
  3959. it.IsValid();
  3960. it.MoveNext())
  3961. {
  3962. BasicBlock * block = it.Block();
  3963. (block->globOptData.*bv)->Set(dstSym->m_id);
  3964. if (!block->globOptData.FindValue(dstSym))
  3965. {
  3966. Value *const valueCopy = CopyValue(value, value->GetValueNumber());
  3967. block->globOptData.SetValue(valueCopy, dstSym);
  3968. }
  3969. }
  3970. }
  3971. return value;
  3972. }
  3973. Value *
  3974. GlobOpt::NewFixedFunctionValue(Js::JavascriptFunction *function, IR::AddrOpnd *addrOpnd)
  3975. {
  3976. Assert(function != nullptr);
  3977. Value *val = nullptr;
  3978. Value *cachedValue = nullptr;
  3979. if(this->addrConstantToValueMap->TryGetValue(addrOpnd->m_address, &cachedValue))
  3980. {
  3981. // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since
  3982. // values are cloned for each block, we can't use the same value object. We also can't have two values with the same
  3983. // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast
  3984. // way to determine if a value with the same value number exists for this block. So the best we can do with a global
  3985. // cache is to check the sym-store's value in the current block to see if it has a value with the same number.
  3986. // Otherwise, we have to create a new value with a new value number.
  3987. Sym *symStore = cachedValue->GetValueInfo()->GetSymStore();
  3988. if(symStore && CurrentBlockData()->IsLive(symStore))
  3989. {
  3990. Value *const symStoreValue = CurrentBlockData()->FindValue(symStore);
  3991. if(symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber())
  3992. {
  3993. ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo();
  3994. if(symStoreValueInfo->IsVarConstant())
  3995. {
  3996. VarConstantValueInfo *const symStoreVarConstantValueInfo = symStoreValueInfo->AsVarConstant();
  3997. if(symStoreVarConstantValueInfo->VarValue() == addrOpnd->m_address &&
  3998. symStoreVarConstantValueInfo->IsFunction())
  3999. {
  4000. val = symStoreValue;
  4001. }
  4002. }
  4003. }
  4004. }
  4005. }
  4006. if(!val)
  4007. {
  4008. VarConstantValueInfo *valueInfo = VarConstantValueInfo::New(this->alloc, function, addrOpnd->GetValueType(), true, addrOpnd->m_localAddress);
  4009. val = NewValue(valueInfo);
  4010. this->addrConstantToValueMap->AddNew(addrOpnd->m_address, val);
  4011. }
  4012. CurrentBlockData()->InsertNewValue(val, addrOpnd);
  4013. return val;
  4014. }
  4015. StackSym *GlobOpt::GetTaggedIntConstantStackSym(const int32 intConstantValue) const
  4016. {
  4017. Assert(!Js::TaggedInt::IsOverflow(intConstantValue));
  4018. return intConstantToStackSymMap->Lookup(intConstantValue, nullptr);
  4019. }
  4020. StackSym *GlobOpt::GetOrCreateTaggedIntConstantStackSym(const int32 intConstantValue) const
  4021. {
  4022. StackSym *stackSym = GetTaggedIntConstantStackSym(intConstantValue);
  4023. if(stackSym)
  4024. {
  4025. return stackSym;
  4026. }
  4027. stackSym = StackSym::New(TyVar,func);
  4028. intConstantToStackSymMap->Add(intConstantValue, stackSym);
  4029. return stackSym;
  4030. }
  4031. Sym *
  4032. GlobOpt::SetSymStore(ValueInfo *valueInfo, Sym *sym)
  4033. {
  4034. if (sym->IsStackSym())
  4035. {
  4036. StackSym *stackSym = sym->AsStackSym();
  4037. if (stackSym->IsTypeSpec())
  4038. {
  4039. stackSym = stackSym->GetVarEquivSym(this->func);
  4040. sym = stackSym;
  4041. }
  4042. }
  4043. if (valueInfo->GetSymStore() == nullptr || valueInfo->GetSymStore()->IsPropertySym())
  4044. {
  4045. SetSymStoreDirect(valueInfo, sym);
  4046. }
  4047. return sym;
  4048. }
  4049. void
  4050. GlobOpt::SetSymStoreDirect(ValueInfo * valueInfo, Sym * sym)
  4051. {
  4052. Sym * prevSymStore = valueInfo->GetSymStore();
  4053. CurrentBlockData()->SetChangedSym(prevSymStore);
  4054. valueInfo->SetSymStore(sym);
  4055. }
  4056. // Figure out the Value of this dst.
  4057. Value *
  4058. GlobOpt::ValueNumberDst(IR::Instr **pInstr, Value *src1Val, Value *src2Val)
  4059. {
  4060. IR::Instr *&instr = *pInstr;
  4061. IR::Opnd *dst = instr->GetDst();
  4062. Value *dstVal = nullptr;
  4063. Sym *sym;
  4064. if (instr->CallsSetter())
  4065. {
  4066. return nullptr;
  4067. }
  4068. if (dst == nullptr)
  4069. {
  4070. return nullptr;
  4071. }
  4072. switch (dst->GetKind())
  4073. {
  4074. case IR::OpndKindSym:
  4075. sym = dst->AsSymOpnd()->m_sym;
  4076. break;
  4077. case IR::OpndKindReg:
  4078. sym = dst->AsRegOpnd()->m_sym;
  4079. if (OpCodeAttr::TempNumberProducing(instr->m_opcode))
  4080. {
  4081. CurrentBlockData()->isTempSrc->Set(sym->m_id);
  4082. }
  4083. else if (OpCodeAttr::TempNumberTransfer(instr->m_opcode))
  4084. {
  4085. IR::Opnd *src1 = instr->GetSrc1();
  4086. if (src1->IsRegOpnd() && CurrentBlockData()->isTempSrc->Test(src1->AsRegOpnd()->m_sym->m_id))
  4087. {
  4088. StackSym *src1Sym = src1->AsRegOpnd()->m_sym;
  4089. // isTempSrc is used for marking isTempLastUse, which is used to generate AddLeftDead()
  4090. // calls instead of the normal Add helpers. It tells the runtime that concats can use string
  4091. // builders.
  4092. // We need to be careful in the case where src1 points to a string builder and is getting aliased.
  4093. // Clear the bit on src and dst of the transfer instr in this case, unless we can prove src1
  4094. // isn't pointing at a string builder, like if it is single def and the def instr is not an Add,
  4095. // but TempProducing.
  4096. if (src1Sym->IsSingleDef() && src1Sym->m_instrDef->m_opcode != Js::OpCode::Add_A
  4097. && OpCodeAttr::TempNumberProducing(src1Sym->m_instrDef->m_opcode))
  4098. {
  4099. CurrentBlockData()->isTempSrc->Set(sym->m_id);
  4100. }
  4101. else
  4102. {
  4103. CurrentBlockData()->isTempSrc->Clear(src1->AsRegOpnd()->m_sym->m_id);
  4104. CurrentBlockData()->isTempSrc->Clear(sym->m_id);
  4105. }
  4106. }
  4107. else
  4108. {
  4109. CurrentBlockData()->isTempSrc->Clear(sym->m_id);
  4110. }
  4111. }
  4112. else
  4113. {
  4114. CurrentBlockData()->isTempSrc->Clear(sym->m_id);
  4115. }
  4116. break;
  4117. case IR::OpndKindIndir:
  4118. return nullptr;
  4119. default:
  4120. return nullptr;
  4121. }
  4122. int32 min1, max1, min2, max2, newMin, newMax;
  4123. ValueInfo *src1ValueInfo = (src1Val ? src1Val->GetValueInfo() : nullptr);
  4124. ValueInfo *src2ValueInfo = (src2Val ? src2Val->GetValueInfo() : nullptr);
  4125. switch (instr->m_opcode)
  4126. {
  4127. case Js::OpCode::Conv_PrimStr:
  4128. AssertMsg(instr->GetDst()->GetValueType().IsString(),
  4129. "Creator of this instruction should have set the type");
  4130. if (this->IsLoopPrePass() || src1ValueInfo == nullptr || !src1ValueInfo->IsPrimitive())
  4131. {
  4132. break;
  4133. }
  4134. instr->m_opcode = Js::OpCode::Conv_Str;
  4135. // fall-through
  4136. case Js::OpCode::Conv_Str:
  4137. // This opcode is commented out since we don't track regex information in GlobOpt now.
  4138. //case Js::OpCode::Coerce_Regex:
  4139. case Js::OpCode::Coerce_Str:
  4140. AssertMsg(instr->GetDst()->GetValueType().IsString(),
  4141. "Creator of this instruction should have set the type");
  4142. // fall-through
  4143. case Js::OpCode::Coerce_StrOrRegex:
  4144. // We don't set the ValueType of src1 for Coerce_StrOrRegex, hence skip the ASSERT
  4145. if (this->IsLoopPrePass() || src1ValueInfo == nullptr || !src1ValueInfo->IsString())
  4146. {
  4147. break;
  4148. }
  4149. instr->m_opcode = Js::OpCode::Ld_A;
  4150. // fall-through
  4151. case Js::OpCode::BytecodeArgOutCapture:
  4152. case Js::OpCode::InitConst:
  4153. case Js::OpCode::LdAsmJsFunc:
  4154. case Js::OpCode::Ld_A:
  4155. case Js::OpCode::Ld_I4:
  4156. // Propagate sym attributes across the reg copy.
  4157. if (!this->IsLoopPrePass() && instr->GetSrc1()->IsRegOpnd())
  4158. {
  4159. if (dst->AsRegOpnd()->m_sym->IsSingleDef())
  4160. {
  4161. dst->AsRegOpnd()->m_sym->CopySymAttrs(instr->GetSrc1()->AsRegOpnd()->m_sym);
  4162. }
  4163. }
  4164. if (instr->IsProfiledInstr())
  4165. {
  4166. const ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType);
  4167. if(!(
  4168. profiledValueType.IsLikelyInt() &&
  4169. (
  4170. (dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber) ||
  4171. (instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber)
  4172. )
  4173. ))
  4174. {
  4175. if(!src1ValueInfo)
  4176. {
  4177. dstVal = this->NewGenericValue(profiledValueType, dst);
  4178. }
  4179. else if(src1ValueInfo->IsUninitialized())
  4180. {
  4181. if(IsLoopPrePass())
  4182. {
  4183. dstVal = this->NewGenericValue(profiledValueType, dst);
  4184. }
  4185. else
  4186. {
  4187. // Assuming the profile data gives more precise value types based on the path it took at runtime, we
  4188. // can improve the original value type.
  4189. src1ValueInfo->Type() = profiledValueType;
  4190. instr->GetSrc1()->SetValueType(profiledValueType);
  4191. }
  4192. }
  4193. }
  4194. }
  4195. if (dstVal == nullptr)
  4196. {
  4197. // Ld_A is just transferring the value
  4198. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4199. }
  4200. break;
  4201. case Js::OpCode::ExtendArg_A:
  4202. {
  4203. // SIMD_JS
  4204. // We avoid transforming EAs to Lds to keep the IR shape consistent and avoid CSEing of EAs.
  4205. // CSEOptimize only assigns a Value to the EA dst, and doesn't turn it to a Ld. If this happened, we shouldn't assign a new Value here.
  4206. if (DoCSE())
  4207. {
  4208. IR::Opnd * currDst = instr->GetDst();
  4209. Value * currDstVal = CurrentBlockData()->FindValue(currDst->GetStackSym());
  4210. if (currDstVal != nullptr)
  4211. {
  4212. return currDstVal;
  4213. }
  4214. }
  4215. break;
  4216. }
  4217. case Js::OpCode::CheckFixedFld:
  4218. AssertMsg(false, "CheckFixedFld doesn't have a dst, so we should never get here");
  4219. break;
  4220. case Js::OpCode::LdSlot:
  4221. case Js::OpCode::LdSlotArr:
  4222. case Js::OpCode::LdFld:
  4223. case Js::OpCode::LdFldForTypeOf:
  4224. case Js::OpCode::LdFldForCallApplyTarget:
  4225. // Do not transfer value type on LdRootFldForTypeOf to prevent copy-prop to LdRootFld in case the field doesn't exist since LdRootFldForTypeOf does not throw.
  4226. // Same goes for ScopedLdFldForTypeOf as we'll end up loading the property from the root object if the property is not in the scope chain.
  4227. //case Js::OpCode::LdRootFldForTypeOf:
  4228. //case Js::OpCode::ScopedLdFldForTypeOf:
  4229. case Js::OpCode::LdRootFld:
  4230. case Js::OpCode::LdMethodFld:
  4231. case Js::OpCode::LdRootMethodFld:
  4232. case Js::OpCode::ScopedLdMethodFld:
  4233. case Js::OpCode::LdMethodFromFlags:
  4234. case Js::OpCode::ScopedLdFld:
  4235. if (instr->IsProfiledInstr())
  4236. {
  4237. ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType);
  4238. if(!(profiledValueType.IsLikelyInt() && dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber))
  4239. {
  4240. if(!src1ValueInfo)
  4241. {
  4242. dstVal = this->NewGenericValue(profiledValueType, dst);
  4243. }
  4244. else if(src1ValueInfo->IsUninitialized())
  4245. {
  4246. if(IsLoopPrePass() && (!dst->IsRegOpnd() || !dst->AsRegOpnd()->m_sym->IsSingleDef()))
  4247. {
  4248. dstVal = this->NewGenericValue(profiledValueType, dst);
  4249. }
  4250. else
  4251. {
  4252. // Assuming the profile data gives more precise value types based on the path it took at runtime, we
  4253. // can improve the original value type.
  4254. src1ValueInfo->Type() = profiledValueType;
  4255. instr->GetSrc1()->SetValueType(profiledValueType);
  4256. }
  4257. }
  4258. }
  4259. }
  4260. if (dstVal == nullptr)
  4261. {
  4262. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4263. }
  4264. if(!this->IsLoopPrePass())
  4265. {
  4266. // We cannot transfer value if the field hasn't been copy prop'd because we don't generate
  4267. // an implicit call bailout between those values if we don't have "live fields" unless, we are hoisting the field.
  4268. ValueInfo *dstValueInfo = (dstVal ? dstVal->GetValueInfo() : nullptr);
  4269. // Update symStore if it isn't a stackSym
  4270. if (dstVal && (!dstValueInfo->GetSymStore() || !dstValueInfo->GetSymStore()->IsStackSym()))
  4271. {
  4272. Assert(dst->IsRegOpnd());
  4273. this->SetSymStoreDirect(dstValueInfo, dst->AsRegOpnd()->m_sym);
  4274. }
  4275. if (src1Val != dstVal)
  4276. {
  4277. CurrentBlockData()->SetValue(dstVal, instr->GetSrc1());
  4278. }
  4279. }
  4280. break;
  4281. case Js::OpCode::LdC_A_R8:
  4282. case Js::OpCode::LdC_A_I4:
  4283. case Js::OpCode::ArgIn_A:
  4284. dstVal = src1Val;
  4285. break;
  4286. case Js::OpCode::LdStr:
  4287. if (src1Val == nullptr)
  4288. {
  4289. src1Val = NewGenericValue(ValueType::String, dst);
  4290. }
  4291. dstVal = src1Val;
  4292. break;
  4293. // LdElemUndef only assign undef if the field doesn't exist.
  4294. // So we don't actually know what the value is, so we can't really copy prop it.
  4295. //case Js::OpCode::LdElemUndef:
  4296. case Js::OpCode::StSlot:
  4297. case Js::OpCode::StSlotChkUndecl:
  4298. case Js::OpCode::StFld:
  4299. case Js::OpCode::StRootFld:
  4300. case Js::OpCode::StFldStrict:
  4301. case Js::OpCode::StRootFldStrict:
  4302. case Js::OpCode::InitFld:
  4303. case Js::OpCode::InitComputedProperty:
  4304. if (DoFieldCopyProp())
  4305. {
  4306. if (src1Val == nullptr)
  4307. {
  4308. // src1 may have no value if it's not a valid var, e.g., NULL for let/const initialization.
  4309. // Consider creating generic values for such things.
  4310. return nullptr;
  4311. }
  4312. AssertMsg(!src2Val, "Bad src Values...");
  4313. Assert(sym->IsPropertySym());
  4314. SymID symId = sym->m_id;
  4315. Assert(instr->m_opcode == Js::OpCode::StSlot || instr->m_opcode == Js::OpCode::StSlotChkUndecl || !CurrentBlockData()->liveFields->Test(symId));
  4316. CurrentBlockData()->liveFields->Set(symId);
  4317. if (!this->IsLoopPrePass() && dst->GetIsDead())
  4318. {
  4319. // Take the property sym out of the live fields set (with special handling for loops).
  4320. this->EndFieldLifetime(dst->AsSymOpnd());
  4321. }
  4322. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4323. }
  4324. else
  4325. {
  4326. return nullptr;
  4327. }
  4328. break;
  4329. case Js::OpCode::Conv_Num:
  4330. if(src1ValueInfo->IsNumber())
  4331. {
  4332. dstVal = ValueNumberTransferDst(instr, src1Val);
  4333. }
  4334. else
  4335. {
  4336. return NewGenericValue(src1ValueInfo->Type().ToDefiniteAnyNumber().SetCanBeTaggedValue(true), dst);
  4337. }
  4338. break;
  4339. case Js::OpCode::Not_A:
  4340. {
  4341. if (!src1Val || !src1ValueInfo->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec()))
  4342. {
  4343. min1 = INT32_MIN;
  4344. max1 = INT32_MAX;
  4345. }
  4346. this->PropagateIntRangeForNot(min1, max1, &newMin, &newMax);
  4347. return CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val);
  4348. }
  4349. case Js::OpCode::Xor_A:
  4350. case Js::OpCode::Or_A:
  4351. case Js::OpCode::And_A:
  4352. case Js::OpCode::Shl_A:
  4353. case Js::OpCode::Shr_A:
  4354. case Js::OpCode::ShrU_A:
  4355. {
  4356. if (!src1Val || !src1ValueInfo->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec()))
  4357. {
  4358. min1 = INT32_MIN;
  4359. max1 = INT32_MAX;
  4360. }
  4361. if (!src2Val || !src2ValueInfo->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec()))
  4362. {
  4363. min2 = INT32_MIN;
  4364. max2 = INT32_MAX;
  4365. }
  4366. if (instr->m_opcode == Js::OpCode::ShrU_A &&
  4367. min1 < 0 &&
  4368. IntConstantBounds(min2, max2).And_0x1f().Contains(0))
  4369. {
  4370. // Src1 may be too large to represent as a signed int32, and src2 may be zero.
  4371. // Since the result can therefore be too large to represent as a signed int32,
  4372. // include Number in the value type.
  4373. return CreateDstUntransferredValue(
  4374. ValueType::AnyNumber.SetCanBeTaggedValue(true), instr, src1Val, src2Val);
  4375. }
  4376. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  4377. return CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val);
  4378. }
  4379. case Js::OpCode::Incr_A:
  4380. case Js::OpCode::Decr_A:
  4381. {
  4382. ValueType valueType;
  4383. if(src1Val)
  4384. {
  4385. valueType = src1Val->GetValueInfo()->Type().ToDefiniteAnyNumber();
  4386. }
  4387. else
  4388. {
  4389. valueType = ValueType::Number;
  4390. }
  4391. return CreateDstUntransferredValue(valueType.SetCanBeTaggedValue(true), instr, src1Val, src2Val);
  4392. }
  4393. case Js::OpCode::Add_A:
  4394. {
  4395. ValueType valueType;
  4396. if (src1Val && src1ValueInfo->IsLikelyNumber() && src2Val && src2ValueInfo->IsLikelyNumber())
  4397. {
  4398. if(src1ValueInfo->IsLikelyInt() && src2ValueInfo->IsLikelyInt())
  4399. {
  4400. // When doing aggressiveIntType, just assume the result is likely going to be int
  4401. // if both input is int.
  4402. const bool isLikelyTagged = src1ValueInfo->IsLikelyTaggedInt() && src2ValueInfo->IsLikelyTaggedInt();
  4403. if(src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber())
  4404. {
  4405. // If both of them are numbers then we can definitely say that the result is a number.
  4406. valueType = ValueType::GetNumberAndLikelyInt(isLikelyTagged);
  4407. }
  4408. else
  4409. {
  4410. // This is only likely going to be int but can be a string as well.
  4411. valueType = ValueType::GetInt(isLikelyTagged).ToLikely();
  4412. }
  4413. }
  4414. else
  4415. {
  4416. // We can only be certain of any thing if both of them are numbers.
  4417. // Otherwise, the result could be string.
  4418. if (src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber())
  4419. {
  4420. if (src1ValueInfo->IsFloat() || src2ValueInfo->IsFloat())
  4421. {
  4422. // If one of them is a float, the result probably is a float instead of just int
  4423. // but should always be a number.
  4424. valueType = ValueType::Float.SetCanBeTaggedValue(true);
  4425. }
  4426. else
  4427. {
  4428. // Could be int, could be number
  4429. valueType = ValueType::Number.SetCanBeTaggedValue(true);
  4430. }
  4431. }
  4432. else if (src1ValueInfo->IsLikelyFloat() || src2ValueInfo->IsLikelyFloat())
  4433. {
  4434. // Result is likely a float (but can be anything)
  4435. valueType = ValueType::Float.ToLikely();
  4436. }
  4437. else
  4438. {
  4439. // Otherwise it is a likely int or float (but can be anything)
  4440. valueType = ValueType::Number.ToLikely();
  4441. }
  4442. }
  4443. }
  4444. else if((src1Val && src1ValueInfo->IsString()) || (src2Val && src2ValueInfo->IsString()))
  4445. {
  4446. // String + anything should always result in a string
  4447. valueType = ValueType::String;
  4448. }
  4449. else if((src1Val && src1ValueInfo->IsNotString() && src1ValueInfo->IsPrimitive())
  4450. && (src2Val && src2ValueInfo->IsNotString() && src2ValueInfo->IsPrimitive()))
  4451. {
  4452. // If src1 and src2 are not strings and primitive, add should yield a number.
  4453. valueType = ValueType::Number.SetCanBeTaggedValue(true);
  4454. }
  4455. else if((src1Val && src1ValueInfo->IsLikelyString()) || (src2Val && src2ValueInfo->IsLikelyString()))
  4456. {
  4457. // likelystring + anything should always result in a likelystring
  4458. valueType = ValueType::String.ToLikely();
  4459. }
  4460. else
  4461. {
  4462. // Number or string. Could make the value a merge of Number and String, but Uninitialized is more useful at the moment.
  4463. Assert(valueType.IsUninitialized());
  4464. }
  4465. return CreateDstUntransferredValue(valueType, instr, src1Val, src2Val);
  4466. }
  4467. case Js::OpCode::Div_A:
  4468. {
  4469. ValueType divValueType = GetDivValueType(instr, src1Val, src2Val, false);
  4470. if (divValueType.IsLikelyInt() || divValueType.IsFloat())
  4471. {
  4472. return CreateDstUntransferredValue(divValueType.SetCanBeTaggedValue(true), instr, src1Val, src2Val);
  4473. }
  4474. }
  4475. // fall-through
  4476. case Js::OpCode::Sub_A:
  4477. case Js::OpCode::Mul_A:
  4478. case Js::OpCode::Rem_A:
  4479. {
  4480. ValueType valueType;
  4481. if( src1Val &&
  4482. src1ValueInfo->IsLikelyInt() &&
  4483. src2Val &&
  4484. src2ValueInfo->IsLikelyInt() &&
  4485. instr->m_opcode != Js::OpCode::Div_A)
  4486. {
  4487. const bool isLikelyTagged =
  4488. src1ValueInfo->IsLikelyTaggedInt() && (src2ValueInfo->IsLikelyTaggedInt() || instr->m_opcode == Js::OpCode::Rem_A);
  4489. if(src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber())
  4490. {
  4491. valueType = ValueType::GetNumberAndLikelyInt(isLikelyTagged);
  4492. }
  4493. else
  4494. {
  4495. valueType = ValueType::GetInt(isLikelyTagged).ToLikely();
  4496. }
  4497. }
  4498. else if ((src1Val && src1ValueInfo->IsLikelyFloat()) || (src2Val && src2ValueInfo->IsLikelyFloat()))
  4499. {
  4500. // This should ideally be NewNumberAndLikelyFloatValue since we know the result is a number but not sure if it will
  4501. // be a float value. However, that Number/LikelyFloat value type doesn't exist currently and all the necessary
  4502. // checks are done for float values (tagged int checks, etc.) so it's sufficient to just create a float value here.
  4503. valueType = ValueType::Float.SetCanBeTaggedValue(true);
  4504. }
  4505. else
  4506. {
  4507. valueType = ValueType::Number.SetCanBeTaggedValue(true);
  4508. }
  4509. return CreateDstUntransferredValue(valueType, instr, src1Val, src2Val);
  4510. }
  4511. case Js::OpCode::CallI:
  4512. Assert(dst->IsRegOpnd());
  4513. return NewGenericValue(dst->AsRegOpnd()->GetValueType(), dst);
  4514. case Js::OpCode::LdElemI_A:
  4515. {
  4516. dstVal = ValueNumberLdElemDst(pInstr, src1Val);
  4517. const ValueType baseValueType(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType());
  4518. if( (
  4519. baseValueType.IsLikelyNativeArray() ||
  4520. #ifdef _M_IX86
  4521. (
  4522. !AutoSystemInfo::Data.SSE2Available() &&
  4523. baseValueType.IsLikelyObject() &&
  4524. (
  4525. baseValueType.GetObjectType() == ObjectType::Float32Array ||
  4526. baseValueType.GetObjectType() == ObjectType::Float64Array
  4527. )
  4528. )
  4529. #else
  4530. false
  4531. #endif
  4532. ) &&
  4533. instr->GetDst()->IsVar() &&
  4534. instr->HasBailOutInfo())
  4535. {
  4536. // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast
  4537. // path. Note that the removed bailouts should not be necessary for correctness.
  4538. IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  4539. if(bailOutKind & IR::BailOutOnArrayAccessHelperCall)
  4540. {
  4541. bailOutKind -= IR::BailOutOnArrayAccessHelperCall;
  4542. }
  4543. if(bailOutKind == IR::BailOutOnImplicitCallsPreOp)
  4544. {
  4545. bailOutKind -= IR::BailOutOnImplicitCallsPreOp;
  4546. }
  4547. if(bailOutKind)
  4548. {
  4549. instr->SetBailOutKind(bailOutKind);
  4550. }
  4551. else
  4552. {
  4553. instr->ClearBailOutInfo();
  4554. }
  4555. }
  4556. return dstVal;
  4557. }
  4558. case Js::OpCode::LdMethodElem:
  4559. // Not worth profiling this, just assume it's likely object (should be likely function but ValueType does not track
  4560. // functions currently, so using ObjectType::Object instead)
  4561. dstVal = NewGenericValue(ValueType::GetObject(ObjectType::Object).ToLikely(), dst);
  4562. if(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyNativeArray() && instr->HasBailOutInfo())
  4563. {
  4564. // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast
  4565. // path. Note that the removed bailouts should not be necessary for correctness.
  4566. IR::BailOutKind bailOutKind = instr->GetBailOutKind();
  4567. if(bailOutKind & IR::BailOutOnArrayAccessHelperCall)
  4568. {
  4569. bailOutKind -= IR::BailOutOnArrayAccessHelperCall;
  4570. }
  4571. if(bailOutKind == IR::BailOutOnImplicitCallsPreOp)
  4572. {
  4573. bailOutKind -= IR::BailOutOnImplicitCallsPreOp;
  4574. }
  4575. if(bailOutKind)
  4576. {
  4577. instr->SetBailOutKind(bailOutKind);
  4578. }
  4579. else
  4580. {
  4581. instr->ClearBailOutInfo();
  4582. }
  4583. }
  4584. return dstVal;
  4585. case Js::OpCode::StElemI_A:
  4586. case Js::OpCode::StElemI_A_Strict:
  4587. dstVal = this->ValueNumberTransferDst(instr, src1Val);
  4588. break;
  4589. case Js::OpCode::LdLen_A:
  4590. if (instr->IsProfiledInstr())
  4591. {
  4592. const ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType);
  4593. if(!(profiledValueType.IsLikelyInt() && dst->AsRegOpnd()->m_sym->m_isNotNumber))
  4594. {
  4595. return this->NewGenericValue(profiledValueType, dst);
  4596. }
  4597. }
  4598. break;
  4599. case Js::OpCode::BrOnEmpty:
  4600. case Js::OpCode::BrOnNotEmpty:
  4601. Assert(dst->IsRegOpnd());
  4602. Assert(dst->GetValueType().IsString());
  4603. return this->NewGenericValue(ValueType::String, dst);
  4604. case Js::OpCode::IsInst:
  4605. case Js::OpCode::LdTrue:
  4606. case Js::OpCode::LdFalse:
  4607. return this->NewGenericValue(ValueType::Boolean, dst);
  4608. case Js::OpCode::LdUndef:
  4609. return this->NewGenericValue(ValueType::Undefined, dst);
  4610. case Js::OpCode::LdC_A_Null:
  4611. return this->NewGenericValue(ValueType::Null, dst);
  4612. case Js::OpCode::LdThis:
  4613. if (!PHASE_OFF(Js::OptTagChecksPhase, this->func) &&
  4614. (src1ValueInfo == nullptr || src1ValueInfo->IsUninitialized()))
  4615. {
  4616. return this->NewGenericValue(ValueType::GetObject(ObjectType::Object).ToLikely().SetCanBeTaggedValue(false), dst);
  4617. }
  4618. break;
  4619. case Js::OpCode::Typeof:
  4620. case Js::OpCode::TypeofElem:
  4621. return this->NewGenericValue(ValueType::String, dst);
  4622. case Js::OpCode::InitLocalClosure:
  4623. Assert(instr->GetDst());
  4624. Assert(instr->GetDst()->IsRegOpnd());
  4625. IR::RegOpnd *regOpnd = instr->GetDst()->AsRegOpnd();
  4626. StackSym *opndStackSym = regOpnd->m_sym;
  4627. Assert(opndStackSym != nullptr);
  4628. ObjectSymInfo *objectSymInfo = opndStackSym->m_objectInfo;
  4629. Assert(objectSymInfo != nullptr);
  4630. for (PropertySym *localVarSlotList = objectSymInfo->m_propertySymList; localVarSlotList; localVarSlotList = localVarSlotList->m_nextInStackSymList)
  4631. {
  4632. this->slotSyms->Set(localVarSlotList->m_id);
  4633. }
  4634. break;
  4635. }
  4636. if (dstVal == nullptr)
  4637. {
  4638. return this->NewGenericValue(dst->GetValueType(), dst);
  4639. }
  4640. return CurrentBlockData()->SetValue(dstVal, dst);
  4641. }
  4642. Value *
  4643. GlobOpt::ValueNumberLdElemDst(IR::Instr **pInstr, Value *srcVal)
  4644. {
  4645. IR::Instr *&instr = *pInstr;
  4646. IR::Opnd *dst = instr->GetDst();
  4647. Value *dstVal = nullptr;
  4648. int32 newMin, newMax;
  4649. ValueInfo *srcValueInfo = (srcVal ? srcVal->GetValueInfo() : nullptr);
  4650. ValueType profiledElementType;
  4651. if (instr->IsProfiledInstr())
  4652. {
  4653. profiledElementType = instr->AsProfiledInstr()->u.ldElemInfo->GetElementType();
  4654. if(!(profiledElementType.IsLikelyInt() && dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber) &&
  4655. srcVal &&
  4656. srcValueInfo->IsUninitialized())
  4657. {
  4658. if(IsLoopPrePass())
  4659. {
  4660. dstVal = NewGenericValue(profiledElementType, dst);
  4661. }
  4662. else
  4663. {
  4664. // Assuming the profile data gives more precise value types based on the path it took at runtime, we
  4665. // can improve the original value type.
  4666. srcValueInfo->Type() = profiledElementType;
  4667. instr->GetSrc1()->SetValueType(profiledElementType);
  4668. }
  4669. }
  4670. }
  4671. IR::IndirOpnd *src = instr->GetSrc1()->AsIndirOpnd();
  4672. const ValueType baseValueType(src->GetBaseOpnd()->GetValueType());
  4673. if (instr->DoStackArgsOpt(this->func) ||
  4674. !(
  4675. baseValueType.IsLikelyOptimizedTypedArray() ||
  4676. (baseValueType.IsLikelyNativeArray() && instr->IsProfiledInstr()) // Specialized native array lowering for LdElem requires that it is profiled.
  4677. ) ||
  4678. (!this->DoTypedArrayTypeSpec() && baseValueType.IsLikelyOptimizedTypedArray()) ||
  4679. // Don't do type spec on native array with a history of accessing gaps, as this is a bailout
  4680. (!this->DoNativeArrayTypeSpec() && baseValueType.IsLikelyNativeArray()) ||
  4681. !ShouldExpectConventionalArrayIndexValue(src))
  4682. {
  4683. if(DoTypedArrayTypeSpec() && !IsLoopPrePass())
  4684. {
  4685. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access.\n"));
  4686. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  4687. {
  4688. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  4689. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  4690. baseValueType.ToString(baseValueTypeStr);
  4691. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not type specialize, because %s.\n"),
  4692. this->func->GetJITFunctionBody()->GetDisplayName(),
  4693. this->func->GetDebugNumberSet(debugStringBuffer),
  4694. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  4695. baseValueTypeStr,
  4696. instr->DoStackArgsOpt(this->func) ? _u("instruction uses the arguments object") :
  4697. baseValueType.IsLikelyOptimizedTypedArray() ? _u("index is negative or likely not int") : _u("of array type"));
  4698. Output::Flush();
  4699. }
  4700. }
  4701. if(!dstVal)
  4702. {
  4703. if(srcVal)
  4704. {
  4705. dstVal = this->ValueNumberTransferDst(instr, srcVal);
  4706. }
  4707. else
  4708. {
  4709. dstVal = NewGenericValue(profiledElementType, dst);
  4710. }
  4711. }
  4712. return dstVal;
  4713. }
  4714. Assert(instr->GetSrc1()->IsIndirOpnd());
  4715. IRType toType = TyVar;
  4716. IR::BailOutKind bailOutKind = IR::BailOutConventionalTypedArrayAccessOnly;
  4717. switch(baseValueType.GetObjectType())
  4718. {
  4719. case ObjectType::Int8Array:
  4720. case ObjectType::Int8VirtualArray:
  4721. case ObjectType::Int8MixedArray:
  4722. newMin = Int8ConstMin;
  4723. newMax = Int8ConstMax;
  4724. goto IntArrayCommon;
  4725. case ObjectType::Uint8Array:
  4726. case ObjectType::Uint8VirtualArray:
  4727. case ObjectType::Uint8MixedArray:
  4728. case ObjectType::Uint8ClampedArray:
  4729. case ObjectType::Uint8ClampedVirtualArray:
  4730. case ObjectType::Uint8ClampedMixedArray:
  4731. newMin = Uint8ConstMin;
  4732. newMax = Uint8ConstMax;
  4733. goto IntArrayCommon;
  4734. case ObjectType::Int16Array:
  4735. case ObjectType::Int16VirtualArray:
  4736. case ObjectType::Int16MixedArray:
  4737. newMin = Int16ConstMin;
  4738. newMax = Int16ConstMax;
  4739. goto IntArrayCommon;
  4740. case ObjectType::Uint16Array:
  4741. case ObjectType::Uint16VirtualArray:
  4742. case ObjectType::Uint16MixedArray:
  4743. newMin = Uint16ConstMin;
  4744. newMax = Uint16ConstMax;
  4745. goto IntArrayCommon;
  4746. case ObjectType::Int32Array:
  4747. case ObjectType::Int32VirtualArray:
  4748. case ObjectType::Int32MixedArray:
  4749. case ObjectType::Uint32Array: // int-specialized loads from uint32 arrays will bail out on values that don't fit in an int32
  4750. case ObjectType::Uint32VirtualArray:
  4751. case ObjectType::Uint32MixedArray:
  4752. Int32Array:
  4753. newMin = Int32ConstMin;
  4754. newMax = Int32ConstMax;
  4755. goto IntArrayCommon;
  4756. IntArrayCommon:
  4757. Assert(dst->IsRegOpnd());
  4758. // If int type spec is disabled, it is ok to load int values as they can help float type spec, and merging int32 with float64 => float64.
  4759. // But if float type spec is also disabled, we'll have problems because float64 merged with var => float64...
  4760. if (!this->DoAggressiveIntTypeSpec() && !this->DoFloatTypeSpec())
  4761. {
  4762. if (!dstVal)
  4763. {
  4764. if (srcVal)
  4765. {
  4766. dstVal = this->ValueNumberTransferDst(instr, srcVal);
  4767. }
  4768. else
  4769. {
  4770. dstVal = NewGenericValue(profiledElementType, dst);
  4771. }
  4772. }
  4773. return dstVal;
  4774. }
  4775. if (!this->IsLoopPrePass())
  4776. {
  4777. if (instr->HasBailOutInfo())
  4778. {
  4779. const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind();
  4780. Assert(
  4781. (
  4782. !(oldBailOutKind & ~IR::BailOutKindBits) ||
  4783. (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp
  4784. ) &&
  4785. !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject)));
  4786. if (bailOutKind == IR::BailOutConventionalTypedArrayAccessOnly)
  4787. {
  4788. // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head
  4789. // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction
  4790. // bails out for the right reason.
  4791. instr->SetBailOutKind(
  4792. bailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall)));
  4793. }
  4794. else
  4795. {
  4796. // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit
  4797. // calls to occur, so it must be merged in to eliminate generating the helper call
  4798. Assert(bailOutKind == IR::BailOutConventionalNativeArrayAccessOnly);
  4799. instr->SetBailOutKind(oldBailOutKind | bailOutKind);
  4800. }
  4801. }
  4802. else
  4803. {
  4804. GenerateBailAtOperation(&instr, bailOutKind);
  4805. }
  4806. }
  4807. TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, nullptr, nullptr, bailOutKind, newMin, newMax, &dstVal);
  4808. toType = TyInt32;
  4809. break;
  4810. case ObjectType::Float32Array:
  4811. case ObjectType::Float32VirtualArray:
  4812. case ObjectType::Float32MixedArray:
  4813. case ObjectType::Float64Array:
  4814. case ObjectType::Float64VirtualArray:
  4815. case ObjectType::Float64MixedArray:
  4816. Float64Array:
  4817. Assert(dst->IsRegOpnd());
  4818. // If float type spec is disabled, don't load float64 values
  4819. if (!this->DoFloatTypeSpec())
  4820. {
  4821. if (!dstVal)
  4822. {
  4823. if (srcVal)
  4824. {
  4825. dstVal = this->ValueNumberTransferDst(instr, srcVal);
  4826. }
  4827. else
  4828. {
  4829. dstVal = NewGenericValue(profiledElementType, dst);
  4830. }
  4831. }
  4832. return dstVal;
  4833. }
  4834. if (!this->IsLoopPrePass())
  4835. {
  4836. if (instr->HasBailOutInfo())
  4837. {
  4838. const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind();
  4839. Assert(
  4840. (
  4841. !(oldBailOutKind & ~IR::BailOutKindBits) ||
  4842. (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp
  4843. ) &&
  4844. !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject)));
  4845. if (bailOutKind == IR::BailOutConventionalTypedArrayAccessOnly)
  4846. {
  4847. // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head
  4848. // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction
  4849. // bails out for the right reason.
  4850. instr->SetBailOutKind(
  4851. bailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall)));
  4852. }
  4853. else
  4854. {
  4855. // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit
  4856. // calls to occur, so it must be merged in to eliminate generating the helper call
  4857. Assert(bailOutKind == IR::BailOutConventionalNativeArrayAccessOnly);
  4858. instr->SetBailOutKind(oldBailOutKind | bailOutKind);
  4859. }
  4860. }
  4861. else
  4862. {
  4863. GenerateBailAtOperation(&instr, bailOutKind);
  4864. }
  4865. }
  4866. TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, &dstVal);
  4867. toType = TyFloat64;
  4868. break;
  4869. default:
  4870. Assert(baseValueType.IsLikelyNativeArray());
  4871. bailOutKind = IR::BailOutConventionalNativeArrayAccessOnly;
  4872. if(baseValueType.HasIntElements())
  4873. {
  4874. goto Int32Array;
  4875. }
  4876. Assert(baseValueType.HasFloatElements());
  4877. goto Float64Array;
  4878. }
  4879. if(!dstVal)
  4880. {
  4881. dstVal = NewGenericValue(profiledElementType, dst);
  4882. }
  4883. Assert(toType != TyVar);
  4884. GOPT_TRACE_INSTR(instr, _u("Type specialized array access.\n"));
  4885. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  4886. {
  4887. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  4888. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  4889. baseValueType.ToString(baseValueTypeStr);
  4890. char dstValTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  4891. dstVal->GetValueInfo()->Type().ToString(dstValTypeStr);
  4892. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, type specialized to %s producing %S"),
  4893. this->func->GetJITFunctionBody()->GetDisplayName(),
  4894. this->func->GetDebugNumberSet(debugStringBuffer),
  4895. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  4896. baseValueTypeStr,
  4897. toType == TyInt32 ? _u("int32") : _u("float64"),
  4898. dstValTypeStr);
  4899. #if DBG_DUMP
  4900. Output::Print(_u(" ("));
  4901. dstVal->Dump();
  4902. Output::Print(_u(").\n"));
  4903. #else
  4904. Output::Print(_u(".\n"));
  4905. #endif
  4906. Output::Flush();
  4907. }
  4908. return dstVal;
  4909. }
  4910. ValueType
  4911. GlobOpt::GetPrepassValueTypeForDst(
  4912. const ValueType desiredValueType,
  4913. IR::Instr *const instr,
  4914. Value *const src1Value,
  4915. Value *const src2Value,
  4916. bool const isValueInfoPrecise,
  4917. bool const isSafeToTransferInPrepass) const
  4918. {
  4919. // Values with definite types can be created in the loop prepass only when it is guaranteed that the value type will be the
  4920. // same on any iteration of the loop. The heuristics currently used are:
  4921. // - If the source sym is not live on the back-edge, then it acquires a new value for each iteration of the loop, so
  4922. // that value type can be definite
  4923. // - Consider: A better solution for this is to track values that originate in this loop, which can have definite value
  4924. // types. That catches more cases, should look into that in the future.
  4925. // - If the source sym has a constant value that doesn't change for the duration of the function
  4926. // - The operation always results in a definite value type. For instance, signed bitwise operations always result in an
  4927. // int32, conv_num and ++ always result in a number, etc.
  4928. // - For operations that always result in an int32, the resulting int range is precise only if the source syms pass
  4929. // the above heuristics. Otherwise, the range must be expanded to the full int32 range.
  4930. Assert(IsLoopPrePass());
  4931. Assert(instr);
  4932. if(!isValueInfoPrecise)
  4933. {
  4934. if(!desiredValueType.IsDefinite())
  4935. {
  4936. return isSafeToTransferInPrepass ? desiredValueType : desiredValueType.SetCanBeTaggedValue(true);
  4937. }
  4938. // If the desired value type is not precise, the value type of the destination is derived from the value types of the
  4939. // sources. Since the value type of a source sym is not definite, the destination value type also cannot be definite.
  4940. if(desiredValueType.IsInt() && OpCodeAttr::IsInt32(instr->m_opcode))
  4941. {
  4942. // The op always produces an int32, but not always a tagged int
  4943. return ValueType::GetInt(desiredValueType.IsLikelyTaggedInt());
  4944. }
  4945. if(desiredValueType.IsNumber() && OpCodeAttr::ProducesNumber(instr->m_opcode))
  4946. {
  4947. // The op always produces a number, but not always an int
  4948. return desiredValueType.ToDefiniteAnyNumber();
  4949. }
  4950. // Note: ToLikely() also sets CanBeTaggedValue
  4951. return desiredValueType.ToLikely();
  4952. }
  4953. return desiredValueType;
  4954. }
  4955. bool
  4956. GlobOpt::IsPrepassSrcValueInfoPrecise(IR::Instr *const instr, Value *const src1Value, Value *const src2Value, bool * isSafeToTransferInPrepass) const
  4957. {
  4958. return
  4959. (!instr->GetSrc1() || IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Value, isSafeToTransferInPrepass)) &&
  4960. (!instr->GetSrc2() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Value, isSafeToTransferInPrepass));
  4961. }
  4962. bool
  4963. GlobOpt::IsPrepassSrcValueInfoPrecise(IR::Opnd *const src, Value *const srcValue, bool * isSafeToTransferInPrepass) const
  4964. {
  4965. Assert(IsLoopPrePass());
  4966. Assert(src);
  4967. if (isSafeToTransferInPrepass)
  4968. {
  4969. *isSafeToTransferInPrepass = false;
  4970. }
  4971. if (src->IsAddrOpnd() &&
  4972. srcValue->GetValueInfo()->GetSymStore() &&
  4973. srcValue->GetValueInfo()->GetSymStore()->IsStackSym() &&
  4974. srcValue->GetValueInfo()->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable())
  4975. {
  4976. if (isSafeToTransferInPrepass)
  4977. {
  4978. *isSafeToTransferInPrepass = false;
  4979. }
  4980. return true;
  4981. }
  4982. if (!src->IsRegOpnd() || !srcValue)
  4983. {
  4984. return false;
  4985. }
  4986. ValueInfo *const srcValueInfo = srcValue->GetValueInfo();
  4987. bool isValueInfoDefinite = srcValueInfo->IsDefinite();
  4988. StackSym * srcSym = src->AsRegOpnd()->m_sym;
  4989. bool isSafeToTransfer = IsSafeToTransferInPrepass(srcSym, srcValueInfo);
  4990. if (isSafeToTransferInPrepass)
  4991. {
  4992. *isSafeToTransferInPrepass = isSafeToTransfer;
  4993. }
  4994. return isValueInfoDefinite && isSafeToTransfer;
  4995. }
  4996. bool
  4997. GlobOpt::IsSafeToTransferInPrepass(StackSym * const srcSym, ValueInfo *const srcValueInfo) const
  4998. {
  4999. int32 intConstantValue;
  5000. return
  5001. srcSym->IsFromByteCodeConstantTable() ||
  5002. (
  5003. srcValueInfo->TryGetIntConstantValue(&intConstantValue) &&
  5004. !Js::TaggedInt::IsOverflow(intConstantValue) &&
  5005. GetTaggedIntConstantStackSym(intConstantValue) == srcSym
  5006. ) ||
  5007. !currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(srcSym->m_id) ||
  5008. !currentBlock->loop->IsSymAssignedToInSelfOrParents(srcSym);
  5009. }
  5010. bool
  5011. GlobOpt::SafeToCopyPropInPrepass(StackSym * const originalSym, StackSym * const copySym, Value *const value) const
  5012. {
  5013. Assert(this->currentBlock->globOptData.GetCopyPropSym(originalSym, value) == copySym);
  5014. // In the following example, to copy-prop s2 into s1, it is not enough to check if s1 and s2 are safe to transfer.
  5015. // In fact, both s1 and s2 are safe to transfer, but it is not legal to copy prop s2 into s1.
  5016. //
  5017. // s1 = s2
  5018. // $Loop:
  5019. // s3 = s1
  5020. // s2 = s4
  5021. // Br $Loop
  5022. //
  5023. // In general, requirements for copy-propping in prepass are more restricted than those for transferring values.
  5024. // For copy prop in prepass, if the original sym is live on back-edge, then the copy-prop sym should not be written to
  5025. // in the loop (or its parents)
  5026. ValueInfo* const valueInfo = value->GetValueInfo();
  5027. return IsSafeToTransferInPrepass(originalSym, valueInfo) &&
  5028. IsSafeToTransferInPrepass(copySym, valueInfo) &&
  5029. (!currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(originalSym->m_id) || !currentBlock->loop->IsSymAssignedToInSelfOrParents(copySym));
  5030. }
  5031. Value *GlobOpt::CreateDstUntransferredIntValue(
  5032. const int32 min,
  5033. const int32 max,
  5034. IR::Instr *const instr,
  5035. Value *const src1Value,
  5036. Value *const src2Value)
  5037. {
  5038. Assert(instr);
  5039. Assert(instr->GetDst());
  5040. Assert(OpCodeAttr::ProducesNumber(instr->m_opcode)
  5041. || (instr->m_opcode == Js::OpCode::Add_A && src1Value->GetValueInfo()->IsNumber()
  5042. && src2Value->GetValueInfo()->IsNumber()));
  5043. ValueType valueType(ValueType::GetInt(IntConstantBounds(min, max).IsLikelyTaggable()));
  5044. Assert(valueType.IsInt());
  5045. bool isValueInfoPrecise;
  5046. if(IsLoopPrePass())
  5047. {
  5048. isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value);
  5049. valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, isValueInfoPrecise);
  5050. }
  5051. else
  5052. {
  5053. isValueInfoPrecise = true;
  5054. }
  5055. IR::Opnd *const dst = instr->GetDst();
  5056. if(isValueInfoPrecise)
  5057. {
  5058. Assert(valueType == ValueType::GetInt(IntConstantBounds(min, max).IsLikelyTaggable()));
  5059. Assert(!(dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->IsTypeSpec()));
  5060. return NewIntRangeValue(min, max, false, dst);
  5061. }
  5062. return NewGenericValue(valueType, dst);
  5063. }
  5064. Value *
  5065. GlobOpt::CreateDstUntransferredValue(
  5066. const ValueType desiredValueType,
  5067. IR::Instr *const instr,
  5068. Value *const src1Value,
  5069. Value *const src2Value)
  5070. {
  5071. Assert(instr);
  5072. Assert(instr->GetDst());
  5073. Assert(!desiredValueType.IsInt()); // use CreateDstUntransferredIntValue instead
  5074. ValueType valueType(desiredValueType);
  5075. if(IsLoopPrePass())
  5076. {
  5077. valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value));
  5078. }
  5079. return NewGenericValue(valueType, instr->GetDst());
  5080. }
  5081. Value *
  5082. GlobOpt::ValueNumberTransferDst(IR::Instr *const instr, Value * src1Val)
  5083. {
  5084. Value *dstVal = this->IsLoopPrePass() ? this->ValueNumberTransferDstInPrepass(instr, src1Val) : src1Val;
  5085. // Don't copy-prop a temp over a user symbol. This is likely to extend the temp's lifetime, as the user symbol
  5086. // is more likely to already have later references.
  5087. // REVIEW: Enabling this does cause perf issues...
  5088. #if 0
  5089. if (dstVal != src1Val)
  5090. {
  5091. return dstVal;
  5092. }
  5093. Sym *dstSym = dst->GetStackSym();
  5094. if (dstVal && dstSym && dstSym->IsStackSym() && !dstSym->AsStackSym()->m_isBytecodeTmp)
  5095. {
  5096. Sym *dstValSym = dstVal->GetValueInfo()->GetSymStore();
  5097. if (dstValSym && dstValSym->AsStackSym()->m_isBytecodeTmp /* src->GetIsDead()*/)
  5098. {
  5099. dstVal->GetValueInfo()->SetSymStore(dstSym);
  5100. }
  5101. }
  5102. #endif
  5103. return dstVal;
  5104. }
  5105. bool
  5106. GlobOpt::IsSafeToTransferInPrePass(IR::Opnd *src, Value *srcValue)
  5107. {
  5108. if (src->IsRegOpnd())
  5109. {
  5110. StackSym *srcSym = src->AsRegOpnd()->m_sym;
  5111. if (srcSym->IsFromByteCodeConstantTable())
  5112. {
  5113. return true;
  5114. }
  5115. ValueInfo *srcValueInfo = srcValue->GetValueInfo();
  5116. int32 srcIntConstantValue;
  5117. if (srcValueInfo->TryGetIntConstantValue(&srcIntConstantValue) && !Js::TaggedInt::IsOverflow(srcIntConstantValue)
  5118. && GetTaggedIntConstantStackSym(srcIntConstantValue) == srcSym)
  5119. {
  5120. return true;
  5121. }
  5122. }
  5123. return false;
  5124. }
  5125. Value *
  5126. GlobOpt::ValueNumberTransferDstInPrepass(IR::Instr *const instr, Value *const src1Val)
  5127. {
  5128. Value *dstVal = nullptr;
  5129. if (!src1Val)
  5130. {
  5131. return nullptr;
  5132. }
  5133. bool isValueInfoPrecise;
  5134. ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  5135. // TODO: This conflicts with new values created by the type specialization code
  5136. // We should re-enable if we change that code to avoid the new values.
  5137. #if 0
  5138. if (this->IsSafeToTransferInPrePass(instr->GetSrc1(), src1Val))
  5139. {
  5140. return src1Val;
  5141. }
  5142. if (this->IsPREInstrCandidateLoad(instr->m_opcode) && instr->GetDst())
  5143. {
  5144. StackSym *dstSym = instr->GetDst()->AsRegOpnd()->m_sym;
  5145. for (Loop *curLoop = this->currentBlock->loop; curLoop; curLoop = curLoop->parent)
  5146. {
  5147. if (curLoop->fieldPRESymStore->Test(dstSym->m_id))
  5148. {
  5149. return src1Val;
  5150. }
  5151. }
  5152. }
  5153. if (instr->GetDst()->IsRegOpnd())
  5154. {
  5155. StackSym *stackSym = instr->GetDst()->AsRegOpnd()->m_sym;
  5156. if (stackSym->IsSingleDef() || this->IsLive(stackSym, this->prePassLoop->landingPad))
  5157. {
  5158. IntConstantBounds src1IntConstantBounds;
  5159. if (src1ValueInfo->TryGetIntConstantBounds(&src1IntConstantBounds) &&
  5160. !(
  5161. src1IntConstantBounds.LowerBound() == INT32_MIN &&
  5162. src1IntConstantBounds.UpperBound() == INT32_MAX
  5163. ))
  5164. {
  5165. const ValueType valueType(
  5166. GetPrepassValueTypeForDst(src1ValueInfo->Type(), instr, src1Val, nullptr, &isValueInfoPrecise));
  5167. if (isValueInfoPrecise)
  5168. {
  5169. return src1Val;
  5170. }
  5171. }
  5172. else
  5173. {
  5174. return src1Val;
  5175. }
  5176. }
  5177. }
  5178. #endif
  5179. // Src1's value could change later in the loop, so the value wouldn't be the same for each
  5180. // iteration. Since we don't iterate over loops "while (!changed)", go conservative on the
  5181. // first pass when transferring a value that is live on the back-edge.
  5182. // In prepass we are going to copy the value but with a different value number
  5183. // for aggressive int type spec.
  5184. bool isSafeToTransferInPrepass = false;
  5185. isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Val, nullptr, &isSafeToTransferInPrepass);
  5186. const ValueType valueType(GetPrepassValueTypeForDst(src1ValueInfo->Type(), instr, src1Val, nullptr, isValueInfoPrecise, isSafeToTransferInPrepass));
  5187. if(isValueInfoPrecise || isSafeToTransferInPrepass)
  5188. {
  5189. Assert(valueType == src1ValueInfo->Type());
  5190. if (!PHASE_OFF1(Js::AVTInPrePassPhase))
  5191. {
  5192. dstVal = src1Val;
  5193. }
  5194. else
  5195. {
  5196. dstVal = CopyValue(src1Val);
  5197. TrackCopiedValueForKills(dstVal);
  5198. }
  5199. }
  5200. else if (valueType == src1ValueInfo->Type() && src1ValueInfo->IsGeneric()) // this else branch is probably not needed
  5201. {
  5202. Assert(valueType == src1ValueInfo->Type());
  5203. dstVal = CopyValue(src1Val);
  5204. TrackCopiedValueForKills(dstVal);
  5205. }
  5206. else
  5207. {
  5208. dstVal = NewGenericValue(valueType);
  5209. dstVal->GetValueInfo()->SetSymStore(src1ValueInfo->GetSymStore());
  5210. }
  5211. return dstVal;
  5212. }
  5213. void
  5214. GlobOpt::PropagateIntRangeForNot(int32 minimum, int32 maximum, int32 *pNewMin, int32* pNewMax)
  5215. {
  5216. int32 tmp;
  5217. Int32Math::Not(minimum, pNewMin);
  5218. *pNewMax = *pNewMin;
  5219. Int32Math::Not(maximum, &tmp);
  5220. *pNewMin = min(*pNewMin, tmp);
  5221. *pNewMax = max(*pNewMax, tmp);
  5222. }
  5223. void
  5224. GlobOpt::PropagateIntRangeBinary(IR::Instr *instr, int32 min1, int32 max1,
  5225. int32 min2, int32 max2, int32 *pNewMin, int32* pNewMax)
  5226. {
  5227. int32 min, max, tmp, tmp2;
  5228. min = INT32_MIN;
  5229. max = INT32_MAX;
  5230. switch (instr->m_opcode)
  5231. {
  5232. case Js::OpCode::Xor_A:
  5233. case Js::OpCode::Or_A:
  5234. // Find range with highest high order bit
  5235. tmp = ::max((uint32)min1, (uint32)max1);
  5236. tmp2 = ::max((uint32)min2, (uint32)max2);
  5237. if ((uint32)tmp > (uint32)tmp2)
  5238. {
  5239. max = tmp;
  5240. }
  5241. else
  5242. {
  5243. max = tmp2;
  5244. }
  5245. if (max < 0)
  5246. {
  5247. min = INT32_MIN; // REVIEW: conservative...
  5248. max = INT32_MAX;
  5249. }
  5250. else
  5251. {
  5252. // Turn values like 0x1010 into 0x1111
  5253. max = 1 << Math::Log2(max);
  5254. max = (uint32)(max << 1) - 1;
  5255. min = 0;
  5256. }
  5257. break;
  5258. case Js::OpCode::And_A:
  5259. if (min1 == INT32_MIN && min2 == INT32_MIN)
  5260. {
  5261. // Shortcut
  5262. break;
  5263. }
  5264. // Find range with lowest higher bit
  5265. tmp = ::max((uint32)min1, (uint32)max1);
  5266. tmp2 = ::max((uint32)min2, (uint32)max2);
  5267. if ((uint32)tmp < (uint32)tmp2)
  5268. {
  5269. min = min1;
  5270. max = max1;
  5271. }
  5272. else
  5273. {
  5274. min = min2;
  5275. max = max2;
  5276. }
  5277. // To compute max, look if min has higher high bit
  5278. if ((uint32)min > (uint32)max)
  5279. {
  5280. max = min;
  5281. }
  5282. // If max is negative, max let's assume it could be -1, so result in MAX_INT
  5283. if (max < 0)
  5284. {
  5285. max = INT32_MAX;
  5286. }
  5287. // If min is positive, the resulting min is zero
  5288. if (min >= 0)
  5289. {
  5290. min = 0;
  5291. }
  5292. else
  5293. {
  5294. min = INT32_MIN;
  5295. }
  5296. break;
  5297. case Js::OpCode::Shl_A:
  5298. {
  5299. // Shift count
  5300. if (min2 != max2 && ((uint32)min2 > 0x1F || (uint32)max2 > 0x1F))
  5301. {
  5302. min2 = 0;
  5303. max2 = 0x1F;
  5304. }
  5305. else
  5306. {
  5307. min2 &= 0x1F;
  5308. max2 &= 0x1F;
  5309. }
  5310. int32 min1FreeTopBitCount = min1 ? (sizeof(int32) * 8) - (Math::Log2(min1) + 1) : (sizeof(int32) * 8);
  5311. int32 max1FreeTopBitCount = max1 ? (sizeof(int32) * 8) - (Math::Log2(max1) + 1) : (sizeof(int32) * 8);
  5312. if (min1FreeTopBitCount <= max2 || max1FreeTopBitCount <= max2)
  5313. {
  5314. // If the shift is going to touch the sign bit return the max range
  5315. min = INT32_MIN;
  5316. max = INT32_MAX;
  5317. }
  5318. else
  5319. {
  5320. // Compute max
  5321. // Turn values like 0x1010 into 0x1111
  5322. if (min1)
  5323. {
  5324. min1 = 1 << Math::Log2(min1);
  5325. min1 = (min1 << 1) - 1;
  5326. }
  5327. if (max1)
  5328. {
  5329. max1 = 1 << Math::Log2(max1);
  5330. max1 = (uint32)(max1 << 1) - 1;
  5331. }
  5332. if (max1 > 0)
  5333. {
  5334. int32 nrTopBits = (sizeof(int32) * 8) - Math::Log2(max1);
  5335. if (nrTopBits < ::min(max2, 30))
  5336. max = INT32_MAX;
  5337. else
  5338. max = ::max((max1 << ::min(max2, 30)) & ~0x80000000, (min1 << min2) & ~0x80000000);
  5339. }
  5340. else
  5341. {
  5342. max = (max1 << min2) & ~0x80000000;
  5343. }
  5344. // Compute min
  5345. if (min1 < 0)
  5346. {
  5347. min = ::min(min1 << max2, max1 << max2);
  5348. }
  5349. else
  5350. {
  5351. min = ::min(min1 << min2, max1 << max2);
  5352. }
  5353. // Turn values like 0x1110 into 0x1000
  5354. if (min)
  5355. {
  5356. min = 1 << Math::Log2(min);
  5357. }
  5358. }
  5359. }
  5360. break;
  5361. case Js::OpCode::Shr_A:
  5362. // Shift count
  5363. if (min2 != max2 && ((uint32)min2 > 0x1F || (uint32)max2 > 0x1F))
  5364. {
  5365. min2 = 0;
  5366. max2 = 0x1F;
  5367. }
  5368. else
  5369. {
  5370. min2 &= 0x1F;
  5371. max2 &= 0x1F;
  5372. }
  5373. // Compute max
  5374. if (max1 < 0)
  5375. {
  5376. max = max1 >> max2;
  5377. }
  5378. else
  5379. {
  5380. max = max1 >> min2;
  5381. }
  5382. // Compute min
  5383. if (min1 < 0)
  5384. {
  5385. min = min1 >> min2;
  5386. }
  5387. else
  5388. {
  5389. min = min1 >> max2;
  5390. }
  5391. break;
  5392. case Js::OpCode::ShrU_A:
  5393. // shift count is constant zero
  5394. if ((min2 == max2) && (max2 & 0x1f) == 0)
  5395. {
  5396. // We can't encode uint32 result, so it has to be used as int32 only or the original value is positive.
  5397. Assert(instr->ignoreIntOverflow || min1 >= 0);
  5398. // We can transfer the signed int32 range.
  5399. min = min1;
  5400. max = max1;
  5401. break;
  5402. }
  5403. const IntConstantBounds src2NewBounds = IntConstantBounds(min2, max2).And_0x1f();
  5404. // Zero is only allowed if result is always a signed int32 or always used as a signed int32
  5405. Assert(min1 >= 0 || instr->ignoreIntOverflow || !src2NewBounds.Contains(0));
  5406. min2 = src2NewBounds.LowerBound();
  5407. max2 = src2NewBounds.UpperBound();
  5408. Assert(min2 <= max2);
  5409. // zero shift count is only allowed if result is used as int32 and/or value is positive
  5410. Assert(min2 > 0 || instr->ignoreIntOverflow || min1 >= 0);
  5411. uint32 umin1 = (uint32)min1;
  5412. uint32 umax1 = (uint32)max1;
  5413. if (umin1 > umax1)
  5414. {
  5415. uint32 temp = umax1;
  5416. umax1 = umin1;
  5417. umin1 = temp;
  5418. }
  5419. Assert(min2 >= 0 && max2 < 32);
  5420. // Compute max
  5421. if (min1 < 0)
  5422. {
  5423. umax1 = UINT32_MAX;
  5424. }
  5425. max = umax1 >> min2;
  5426. // Compute min
  5427. if (min1 <= 0 && max1 >=0)
  5428. {
  5429. min = 0;
  5430. }
  5431. else
  5432. {
  5433. min = umin1 >> max2;
  5434. }
  5435. // We should be able to fit uint32 range as int32
  5436. Assert(instr->ignoreIntOverflow || (min >= 0 && max >= 0) );
  5437. if (min > max)
  5438. {
  5439. // can only happen if shift count can be zero
  5440. Assert(min2 == 0 && (instr->ignoreIntOverflow || min1 >= 0));
  5441. min = Int32ConstMin;
  5442. max = Int32ConstMax;
  5443. }
  5444. break;
  5445. }
  5446. *pNewMin = min;
  5447. *pNewMax = max;
  5448. }
  5449. IR::Instr *
  5450. GlobOpt::TypeSpecialization(
  5451. IR::Instr *instr,
  5452. Value **pSrc1Val,
  5453. Value **pSrc2Val,
  5454. Value **pDstVal,
  5455. bool *redoTypeSpecRef,
  5456. bool *const forceInvariantHoistingRef)
  5457. {
  5458. Value *&src1Val = *pSrc1Val;
  5459. Value *&src2Val = *pSrc2Val;
  5460. *redoTypeSpecRef = false;
  5461. Assert(!*forceInvariantHoistingRef);
  5462. this->ignoredIntOverflowForCurrentInstr = false;
  5463. this->ignoredNegativeZeroForCurrentInstr = false;
  5464. // - Int32 values that can't be tagged are created as float constant values instead because a JavascriptNumber var is needed
  5465. // for that value at runtime. For the purposes of type specialization, recover the int32 values so that they will be
  5466. // treated as ints.
  5467. // - If int overflow does not matter for the instruction, we can additionally treat uint32 values as int32 values because
  5468. // the value resulting from the operation will eventually be converted to int32 anyway
  5469. Value *const src1OriginalVal = src1Val;
  5470. Value *const src2OriginalVal = src2Val;
  5471. if(!instr->ShouldCheckForIntOverflow())
  5472. {
  5473. if(src1Val && src1Val->GetValueInfo()->IsFloatConstant())
  5474. {
  5475. int32 int32Value;
  5476. bool isInt32;
  5477. if(Js::JavascriptNumber::TryGetInt32OrUInt32Value(
  5478. src1Val->GetValueInfo()->AsFloatConstant()->FloatValue(),
  5479. &int32Value,
  5480. &isInt32))
  5481. {
  5482. src1Val = GetIntConstantValue(int32Value, instr);
  5483. if(!isInt32)
  5484. {
  5485. this->ignoredIntOverflowForCurrentInstr = true;
  5486. }
  5487. }
  5488. }
  5489. if(src2Val && src2Val->GetValueInfo()->IsFloatConstant())
  5490. {
  5491. int32 int32Value;
  5492. bool isInt32;
  5493. if(Js::JavascriptNumber::TryGetInt32OrUInt32Value(
  5494. src2Val->GetValueInfo()->AsFloatConstant()->FloatValue(),
  5495. &int32Value,
  5496. &isInt32))
  5497. {
  5498. src2Val = GetIntConstantValue(int32Value, instr);
  5499. if(!isInt32)
  5500. {
  5501. this->ignoredIntOverflowForCurrentInstr = true;
  5502. }
  5503. }
  5504. }
  5505. }
  5506. const AutoRestoreVal autoRestoreSrc1Val(src1OriginalVal, &src1Val);
  5507. const AutoRestoreVal autoRestoreSrc2Val(src2OriginalVal, &src2Val);
  5508. if (src1Val && instr->GetSrc2() == nullptr)
  5509. {
  5510. // Unary
  5511. // Note make sure that native array StElemI gets to TypeSpecializeStElem. Do this for typed arrays, too?
  5512. int32 intConstantValue;
  5513. if (!this->IsLoopPrePass() &&
  5514. !instr->IsBranchInstr() &&
  5515. src1Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) &&
  5516. !(
  5517. // Nothing to fold for element stores. Go into type specialization to see if they can at least be specialized.
  5518. instr->m_opcode == Js::OpCode::StElemI_A ||
  5519. instr->m_opcode == Js::OpCode::StElemI_A_Strict ||
  5520. instr->m_opcode == Js::OpCode::StElemC ||
  5521. instr->m_opcode == Js::OpCode::MultiBr ||
  5522. instr->m_opcode == Js::OpCode::InlineArrayPop
  5523. ))
  5524. {
  5525. if (OptConstFoldUnary(&instr, intConstantValue, src1Val == src1OriginalVal, pDstVal))
  5526. {
  5527. return instr;
  5528. }
  5529. }
  5530. else if (this->TypeSpecializeUnary(
  5531. &instr,
  5532. &src1Val,
  5533. pDstVal,
  5534. src1OriginalVal,
  5535. redoTypeSpecRef,
  5536. forceInvariantHoistingRef))
  5537. {
  5538. return instr;
  5539. }
  5540. else if(*redoTypeSpecRef)
  5541. {
  5542. return instr;
  5543. }
  5544. }
  5545. else if (instr->GetSrc2() && !instr->IsBranchInstr())
  5546. {
  5547. // Binary
  5548. if (!this->IsLoopPrePass())
  5549. {
  5550. if (GetIsAsmJSFunc())
  5551. {
  5552. if (CONFIG_FLAG(WasmFold))
  5553. {
  5554. bool success = instr->GetSrc1()->IsInt64() ?
  5555. this->OptConstFoldBinaryWasm<int64>(&instr, src1Val, src2Val, pDstVal) :
  5556. this->OptConstFoldBinaryWasm<int>(&instr, src1Val, src2Val, pDstVal);
  5557. if (success)
  5558. {
  5559. return instr;
  5560. }
  5561. }
  5562. }
  5563. else
  5564. {
  5565. // OptConstFoldBinary doesn't do type spec, so only deal with things we are sure are int (IntConstant and IntRange)
  5566. // and not just likely ints TypeSpecializeBinary will deal with type specializing them and fold them again
  5567. IntConstantBounds src1IntConstantBounds, src2IntConstantBounds;
  5568. if (src1Val && src1Val->GetValueInfo()->TryGetIntConstantBounds(&src1IntConstantBounds))
  5569. {
  5570. if (src2Val && src2Val->GetValueInfo()->TryGetIntConstantBounds(&src2IntConstantBounds))
  5571. {
  5572. if (this->OptConstFoldBinary(&instr, src1IntConstantBounds, src2IntConstantBounds, pDstVal))
  5573. {
  5574. return instr;
  5575. }
  5576. }
  5577. }
  5578. }
  5579. }
  5580. }
  5581. if (instr->GetSrc2() && this->TypeSpecializeBinary(&instr, pSrc1Val, pSrc2Val, pDstVal, src1OriginalVal, src2OriginalVal, redoTypeSpecRef))
  5582. {
  5583. if (!this->IsLoopPrePass() &&
  5584. instr->m_opcode != Js::OpCode::Nop &&
  5585. instr->m_opcode != Js::OpCode::Br && // We may have const fold a branch
  5586. // Cannot const-peep if the result of the operation is required for a bailout check
  5587. !(instr->HasBailOutInfo() && instr->GetBailOutKind() & IR::BailOutOnResultConditions))
  5588. {
  5589. if (src1Val && src1Val->GetValueInfo()->HasIntConstantValue())
  5590. {
  5591. if (this->OptConstPeep(instr, instr->GetSrc1(), pDstVal, src1Val->GetValueInfo()))
  5592. {
  5593. return instr;
  5594. }
  5595. }
  5596. else if (src2Val && src2Val->GetValueInfo()->HasIntConstantValue())
  5597. {
  5598. if (this->OptConstPeep(instr, instr->GetSrc2(), pDstVal, src2Val->GetValueInfo()))
  5599. {
  5600. return instr;
  5601. }
  5602. }
  5603. }
  5604. return instr;
  5605. }
  5606. else if(*redoTypeSpecRef)
  5607. {
  5608. return instr;
  5609. }
  5610. if (instr->IsBranchInstr() && !this->IsLoopPrePass())
  5611. {
  5612. if (this->OptConstFoldBranch(instr, src1Val, src2Val, pDstVal))
  5613. {
  5614. return instr;
  5615. }
  5616. }
  5617. // We didn't type specialize, make sure the srcs are unspecialized
  5618. IR::Opnd *src1 = instr->GetSrc1();
  5619. if (src1)
  5620. {
  5621. instr = this->ToVarUses(instr, src1, false, src1Val);
  5622. IR::Opnd *src2 = instr->GetSrc2();
  5623. if (src2)
  5624. {
  5625. instr = this->ToVarUses(instr, src2, false, src2Val);
  5626. }
  5627. }
  5628. IR::Opnd *dst = instr->GetDst();
  5629. if (dst)
  5630. {
  5631. instr = this->ToVarUses(instr, dst, true, nullptr);
  5632. // Handling for instructions other than built-ins that may require only dst type specialization
  5633. // should be added here.
  5634. if(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode) && !GetIsAsmJSFunc()) // don't need to do typespec for asmjs
  5635. {
  5636. this->TypeSpecializeInlineBuiltInDst(&instr, pDstVal);
  5637. return instr;
  5638. }
  5639. // Clear the int specialized bit on the dst.
  5640. if (dst->IsRegOpnd())
  5641. {
  5642. IR::RegOpnd *dstRegOpnd = dst->AsRegOpnd();
  5643. if (!dstRegOpnd->m_sym->IsTypeSpec())
  5644. {
  5645. this->ToVarRegOpnd(dstRegOpnd, this->currentBlock);
  5646. }
  5647. else if (dstRegOpnd->m_sym->IsInt32())
  5648. {
  5649. this->ToInt32Dst(instr, dstRegOpnd, this->currentBlock);
  5650. }
  5651. else if (dstRegOpnd->m_sym->IsUInt32() && GetIsAsmJSFunc())
  5652. {
  5653. this->ToUInt32Dst(instr, dstRegOpnd, this->currentBlock);
  5654. }
  5655. else if (dstRegOpnd->m_sym->IsFloat64())
  5656. {
  5657. this->ToFloat64Dst(instr, dstRegOpnd, this->currentBlock);
  5658. }
  5659. }
  5660. else if (dst->IsSymOpnd() && dst->AsSymOpnd()->m_sym->IsStackSym())
  5661. {
  5662. this->ToVarStackSym(dst->AsSymOpnd()->m_sym->AsStackSym(), this->currentBlock);
  5663. }
  5664. }
  5665. return instr;
  5666. }
  5667. bool
  5668. GlobOpt::OptConstPeep(IR::Instr *instr, IR::Opnd *constSrc, Value **pDstVal, ValueInfo *valuInfo)
  5669. {
  5670. int32 value;
  5671. IR::Opnd *src;
  5672. IR::Opnd *nonConstSrc = (constSrc == instr->GetSrc1() ? instr->GetSrc2() : instr->GetSrc1());
  5673. // Try to find the value from value info first
  5674. if (valuInfo->TryGetIntConstantValue(&value))
  5675. {
  5676. }
  5677. else if (constSrc->IsAddrOpnd())
  5678. {
  5679. IR::AddrOpnd *addrOpnd = constSrc->AsAddrOpnd();
  5680. #ifdef _M_X64
  5681. Assert(addrOpnd->IsVar() || Math::FitsInDWord((size_t)addrOpnd->m_address));
  5682. #else
  5683. Assert(sizeof(value) == sizeof(addrOpnd->m_address));
  5684. #endif
  5685. if (addrOpnd->IsVar())
  5686. {
  5687. value = Js::TaggedInt::ToInt32(addrOpnd->m_address);
  5688. }
  5689. else
  5690. {
  5691. // We asserted that the address will fit in a DWORD above
  5692. value = ::Math::PointerCastToIntegral<int32>(constSrc->AsAddrOpnd()->m_address);
  5693. }
  5694. }
  5695. else if (constSrc->IsIntConstOpnd())
  5696. {
  5697. value = constSrc->AsIntConstOpnd()->AsInt32();
  5698. }
  5699. else
  5700. {
  5701. return false;
  5702. }
  5703. switch(instr->m_opcode)
  5704. {
  5705. // Can't do all Add_A because of string concats.
  5706. // Sub_A cannot be transformed to a NEG_A because 0 - 0 != -0
  5707. case Js::OpCode::Add_A:
  5708. src = nonConstSrc;
  5709. if (!src->GetValueType().IsInt())
  5710. {
  5711. // 0 + -0 != -0
  5712. // "Foo" + 0 != "Foo
  5713. return false;
  5714. }
  5715. // fall-through
  5716. case Js::OpCode::Add_I4:
  5717. if (value != 0)
  5718. {
  5719. return false;
  5720. }
  5721. if (constSrc == instr->GetSrc1())
  5722. {
  5723. src = instr->GetSrc2();
  5724. }
  5725. else
  5726. {
  5727. src = instr->GetSrc1();
  5728. }
  5729. break;
  5730. case Js::OpCode::Mul_A:
  5731. case Js::OpCode::Mul_I4:
  5732. if (value == 0)
  5733. {
  5734. // -0 * 0 != 0
  5735. return false;
  5736. }
  5737. else if (value == 1)
  5738. {
  5739. src = nonConstSrc;
  5740. }
  5741. else
  5742. {
  5743. return false;
  5744. }
  5745. break;
  5746. case Js::OpCode::Div_A:
  5747. if (value == 1 && constSrc == instr->GetSrc2())
  5748. {
  5749. src = instr->GetSrc1();
  5750. }
  5751. else
  5752. {
  5753. return false;
  5754. }
  5755. break;
  5756. case Js::OpCode::Or_I4:
  5757. if (value == -1)
  5758. {
  5759. src = constSrc;
  5760. }
  5761. else if (value == 0)
  5762. {
  5763. src = nonConstSrc;
  5764. }
  5765. else
  5766. {
  5767. return false;
  5768. }
  5769. break;
  5770. case Js::OpCode::And_I4:
  5771. if (value == -1)
  5772. {
  5773. src = nonConstSrc;
  5774. }
  5775. else if (value == 0)
  5776. {
  5777. src = constSrc;
  5778. }
  5779. else
  5780. {
  5781. return false;
  5782. }
  5783. break;
  5784. case Js::OpCode::Shl_I4:
  5785. case Js::OpCode::ShrU_I4:
  5786. case Js::OpCode::Shr_I4:
  5787. if (value != 0 || constSrc != instr->GetSrc2())
  5788. {
  5789. return false;
  5790. }
  5791. src = instr->GetSrc1();
  5792. break;
  5793. default:
  5794. return false;
  5795. }
  5796. this->CaptureByteCodeSymUses(instr);
  5797. if (src == instr->GetSrc1())
  5798. {
  5799. instr->FreeSrc2();
  5800. }
  5801. else
  5802. {
  5803. Assert(src == instr->GetSrc2());
  5804. instr->ReplaceSrc1(instr->UnlinkSrc2());
  5805. }
  5806. instr->m_opcode = Js::OpCode::Ld_A;
  5807. InvalidateInductionVariables(instr);
  5808. return true;
  5809. }
  5810. Js::Var // TODO: michhol OOP JIT, shouldn't play with Vars
  5811. GlobOpt::GetConstantVar(IR::Opnd *opnd, Value *val)
  5812. {
  5813. ValueInfo *valueInfo = val->GetValueInfo();
  5814. if (valueInfo->IsVarConstant() && valueInfo->IsPrimitive())
  5815. {
  5816. return valueInfo->AsVarConstant()->VarValue();
  5817. }
  5818. if (opnd->IsAddrOpnd())
  5819. {
  5820. IR::AddrOpnd *addrOpnd = opnd->AsAddrOpnd();
  5821. if (addrOpnd->IsVar())
  5822. {
  5823. return addrOpnd->m_address;
  5824. }
  5825. }
  5826. else if (opnd->IsIntConstOpnd())
  5827. {
  5828. if (!Js::TaggedInt::IsOverflow(opnd->AsIntConstOpnd()->AsInt32()))
  5829. {
  5830. return Js::TaggedInt::ToVarUnchecked(opnd->AsIntConstOpnd()->AsInt32());
  5831. }
  5832. }
  5833. else if (opnd->IsRegOpnd() && opnd->AsRegOpnd()->m_sym->IsSingleDef())
  5834. {
  5835. if (valueInfo->IsBoolean())
  5836. {
  5837. IR::Instr * defInstr = opnd->AsRegOpnd()->m_sym->GetInstrDef();
  5838. if (defInstr->m_opcode != Js::OpCode::Ld_A || !defInstr->GetSrc1()->IsAddrOpnd())
  5839. {
  5840. return nullptr;
  5841. }
  5842. Assert(defInstr->GetSrc1()->AsAddrOpnd()->IsVar());
  5843. return defInstr->GetSrc1()->AsAddrOpnd()->m_address;
  5844. }
  5845. else if (valueInfo->IsUndefined())
  5846. {
  5847. return (Js::Var)this->func->GetScriptContextInfo()->GetUndefinedAddr();
  5848. }
  5849. else if (valueInfo->IsNull())
  5850. {
  5851. return (Js::Var)this->func->GetScriptContextInfo()->GetNullAddr();
  5852. }
  5853. }
  5854. return nullptr;
  5855. }
  5856. bool BoolAndIntStaticAndTypeMismatch(Value* src1Val, Value* src2Val, Js::Var src1Var, Js::Var src2Var)
  5857. {
  5858. ValueInfo *src1ValInfo = src1Val->GetValueInfo();
  5859. ValueInfo *src2ValInfo = src2Val->GetValueInfo();
  5860. return (src1ValInfo->IsNumber() && src1Var && src2ValInfo->IsBoolean() && src1Var != Js::TaggedInt::ToVarUnchecked(0) && src1Var != Js::TaggedInt::ToVarUnchecked(1)) ||
  5861. (src2ValInfo->IsNumber() && src2Var && src1ValInfo->IsBoolean() && src2Var != Js::TaggedInt::ToVarUnchecked(0) && src2Var != Js::TaggedInt::ToVarUnchecked(1));
  5862. }
  5863. bool
  5864. GlobOpt::CanProveConditionalBranch(IR::Instr *instr, Value *src1Val, Value *src2Val, Js::Var src1Var, Js::Var src2Var, bool *result)
  5865. {
  5866. auto AreSourcesEqual = [&](Value * val1, Value * val2, bool undefinedCmp) -> bool
  5867. {
  5868. // NaN !== NaN, and objects can have valueOf/toString
  5869. if (val1->IsEqualTo(val2))
  5870. {
  5871. if (val1->GetValueInfo()->IsUndefined())
  5872. {
  5873. return undefinedCmp;
  5874. }
  5875. return val1->GetValueInfo()->IsPrimitive() && val1->GetValueInfo()->IsNotFloat();
  5876. }
  5877. return false;
  5878. };
  5879. // Make sure GetConstantVar only returns primitives.
  5880. // TODO: OOP JIT, enabled these asserts
  5881. //Assert(!src1Var || !Js::JavascriptOperators::IsObject(src1Var));
  5882. //Assert(!src2Var || !Js::JavascriptOperators::IsObject(src2Var));
  5883. int64 left64, right64;
  5884. int32 left, right;
  5885. int32 constVal;
  5886. switch (instr->m_opcode)
  5887. {
  5888. #define BRANCHSIGNED(OPCODE,CMP,TYPE,UNSIGNEDNESS,UNDEFINEDCMP) \
  5889. case Js::OpCode::##OPCODE: \
  5890. if (src1Val && src2Val) \
  5891. { \
  5892. if (src1Val->GetValueInfo()->TryGetIntConstantValue(&left, UNSIGNEDNESS) && \
  5893. src2Val->GetValueInfo()->TryGetIntConstantValue(&right, UNSIGNEDNESS)) \
  5894. { \
  5895. *result = (TYPE)left CMP(TYPE)right; \
  5896. } \
  5897. if (src1Val->GetValueInfo()->TryGetInt64ConstantValue(&left64, UNSIGNEDNESS) && \
  5898. src2Val->GetValueInfo()->TryGetInt64ConstantValue(&right64, UNSIGNEDNESS)) \
  5899. { \
  5900. *result = (TYPE)left64 CMP(TYPE)right64; \
  5901. } \
  5902. else if (AreSourcesEqual(src1Val, src2Val, UNDEFINEDCMP)) \
  5903. { \
  5904. *result = 0 CMP 0; \
  5905. } \
  5906. else \
  5907. { \
  5908. return false; \
  5909. } \
  5910. } \
  5911. else \
  5912. { \
  5913. return false; \
  5914. } \
  5915. break;
  5916. BRANCHSIGNED(BrEq_I4, == , int64, false, true)
  5917. BRANCHSIGNED(BrGe_I4, >= , int64, false, false)
  5918. BRANCHSIGNED(BrGt_I4, > , int64, false, false)
  5919. BRANCHSIGNED(BrLt_I4, < , int64, false, false)
  5920. BRANCHSIGNED(BrLe_I4, <= , int64, false, false)
  5921. BRANCHSIGNED(BrNeq_I4, != , int64, false, false)
  5922. BRANCHSIGNED(BrUnGe_I4, >= , uint64, true, false)
  5923. BRANCHSIGNED(BrUnGt_I4, > , uint64, true, false)
  5924. BRANCHSIGNED(BrUnLt_I4, < , uint64, true, false)
  5925. BRANCHSIGNED(BrUnLe_I4, <= , uint64, true, false)
  5926. #undef BRANCHSIGNED
  5927. #define BRANCH(OPCODE,CMP,VARCMPFUNC,UNDEFINEDCMP) \
  5928. case Js::OpCode::##OPCODE: \
  5929. if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) && \
  5930. src2Val->GetValueInfo()->TryGetIntConstantValue(&right)) \
  5931. { \
  5932. *result = left CMP right; \
  5933. } \
  5934. else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, UNDEFINEDCMP)) \
  5935. { \
  5936. *result = 0 CMP 0; \
  5937. } \
  5938. else if (src1Var && src2Var) \
  5939. { \
  5940. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts)) \
  5941. { \
  5942. return false; \
  5943. } \
  5944. *result = VARCMPFUNC(src1Var, src2Var, this->func->GetScriptContext()); \
  5945. } \
  5946. else \
  5947. { \
  5948. return false; \
  5949. } \
  5950. break;
  5951. BRANCH(BrGe_A, >= , Js::JavascriptOperators::GreaterEqual, /*undefinedEquality*/ false)
  5952. BRANCH(BrNotGe_A, <, !Js::JavascriptOperators::GreaterEqual, false)
  5953. BRANCH(BrLt_A, <, Js::JavascriptOperators::Less, false)
  5954. BRANCH(BrNotLt_A, >= , !Js::JavascriptOperators::Less, false)
  5955. BRANCH(BrGt_A, >, Js::JavascriptOperators::Greater, false)
  5956. BRANCH(BrNotGt_A, <= , !Js::JavascriptOperators::Greater, false)
  5957. BRANCH(BrLe_A, <= , Js::JavascriptOperators::LessEqual, false)
  5958. BRANCH(BrNotLe_A, >, !Js::JavascriptOperators::LessEqual, false)
  5959. #undef BRANCH
  5960. case Js::OpCode::BrEq_A:
  5961. case Js::OpCode::BrNotNeq_A:
  5962. if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) &&
  5963. src2Val->GetValueInfo()->TryGetIntConstantValue(&right))
  5964. {
  5965. *result = left == right;
  5966. }
  5967. else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, true))
  5968. {
  5969. *result = true;
  5970. }
  5971. else if (!src1Var || !src2Var)
  5972. {
  5973. if (BoolAndIntStaticAndTypeMismatch(src1Val, src2Val, src1Var, src2Var))
  5974. {
  5975. *result = false;
  5976. }
  5977. else
  5978. {
  5979. return false;
  5980. }
  5981. }
  5982. else
  5983. {
  5984. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts))
  5985. {
  5986. // TODO: OOP JIT, const folding
  5987. return false;
  5988. }
  5989. *result = Js::JavascriptOperators::Equal(src1Var, src2Var, this->func->GetScriptContext());
  5990. }
  5991. break;
  5992. case Js::OpCode::BrNeq_A:
  5993. case Js::OpCode::BrNotEq_A:
  5994. if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) &&
  5995. src2Val->GetValueInfo()->TryGetIntConstantValue(&right))
  5996. {
  5997. *result = left != right;
  5998. }
  5999. else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, true))
  6000. {
  6001. *result = false;
  6002. }
  6003. else if (!src1Var || !src2Var)
  6004. {
  6005. if (BoolAndIntStaticAndTypeMismatch(src1Val, src2Val, src1Var, src2Var))
  6006. {
  6007. *result = true;
  6008. }
  6009. else
  6010. {
  6011. return false;
  6012. }
  6013. }
  6014. else
  6015. {
  6016. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts))
  6017. {
  6018. // TODO: OOP JIT, const folding
  6019. return false;
  6020. }
  6021. *result = Js::JavascriptOperators::NotEqual(src1Var, src2Var, this->func->GetScriptContext());
  6022. }
  6023. break;
  6024. case Js::OpCode::BrSrEq_A:
  6025. case Js::OpCode::BrSrNotNeq_A:
  6026. if (!src1Var || !src2Var)
  6027. {
  6028. ValueInfo *src1ValInfo = src1Val->GetValueInfo();
  6029. ValueInfo *src2ValInfo = src2Val->GetValueInfo();
  6030. if (
  6031. (src1ValInfo->IsUndefined() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenUndefined()) ||
  6032. (src1ValInfo->IsNull() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNull()) ||
  6033. (src1ValInfo->IsBoolean() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenBoolean()) ||
  6034. (src1ValInfo->IsNumber() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNumber()) ||
  6035. (src1ValInfo->IsString() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenString()) ||
  6036. (src2ValInfo->IsUndefined() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenUndefined()) ||
  6037. (src2ValInfo->IsNull() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNull()) ||
  6038. (src2ValInfo->IsBoolean() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenBoolean()) ||
  6039. (src2ValInfo->IsNumber() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNumber()) ||
  6040. (src2ValInfo->IsString() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenString())
  6041. )
  6042. {
  6043. *result = false;
  6044. }
  6045. else if (AreSourcesEqual(src1Val, src2Val, true))
  6046. {
  6047. *result = true;
  6048. }
  6049. else
  6050. {
  6051. return false;
  6052. }
  6053. }
  6054. else
  6055. {
  6056. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts))
  6057. {
  6058. // TODO: OOP JIT, const folding
  6059. return false;
  6060. }
  6061. *result = Js::JavascriptOperators::StrictEqual(src1Var, src2Var, this->func->GetScriptContext());
  6062. }
  6063. break;
  6064. case Js::OpCode::BrSrNeq_A:
  6065. case Js::OpCode::BrSrNotEq_A:
  6066. if (!src1Var || !src2Var)
  6067. {
  6068. ValueInfo *src1ValInfo = src1Val->GetValueInfo();
  6069. ValueInfo *src2ValInfo = src2Val->GetValueInfo();
  6070. if (
  6071. (src1ValInfo->IsUndefined() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenUndefined()) ||
  6072. (src1ValInfo->IsNull() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNull()) ||
  6073. (src1ValInfo->IsBoolean() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenBoolean()) ||
  6074. (src1ValInfo->IsNumber() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNumber()) ||
  6075. (src1ValInfo->IsString() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenString()) ||
  6076. (src2ValInfo->IsUndefined() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenUndefined()) ||
  6077. (src2ValInfo->IsNull() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNull()) ||
  6078. (src2ValInfo->IsBoolean() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenBoolean()) ||
  6079. (src2ValInfo->IsNumber() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNumber()) ||
  6080. (src2ValInfo->IsString() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenString())
  6081. )
  6082. {
  6083. *result = true;
  6084. }
  6085. else if (AreSourcesEqual(src1Val, src2Val, true))
  6086. {
  6087. *result = false;
  6088. }
  6089. else
  6090. {
  6091. return false;
  6092. }
  6093. }
  6094. else
  6095. {
  6096. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts))
  6097. {
  6098. // TODO: OOP JIT, const folding
  6099. return false;
  6100. }
  6101. *result = Js::JavascriptOperators::NotStrictEqual(src1Var, src2Var, this->func->GetScriptContext());
  6102. }
  6103. break;
  6104. case Js::OpCode::BrFalse_A:
  6105. case Js::OpCode::BrTrue_A:
  6106. {
  6107. ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  6108. if (src1ValueInfo->IsNull() || src1ValueInfo->IsUndefined())
  6109. {
  6110. *result = instr->m_opcode == Js::OpCode::BrFalse_A;
  6111. break;
  6112. }
  6113. if (src1ValueInfo->IsObject() && src1ValueInfo->GetObjectType() > ObjectType::Object)
  6114. {
  6115. // Specific object types that are tracked are equivalent to 'true'
  6116. *result = instr->m_opcode == Js::OpCode::BrTrue_A;
  6117. break;
  6118. }
  6119. if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts))
  6120. {
  6121. // TODO: OOP JIT, const folding
  6122. return false;
  6123. }
  6124. if (!src1Var)
  6125. {
  6126. return false;
  6127. }
  6128. *result = Js::JavascriptConversion::ToBoolean(src1Var, this->func->GetScriptContext());
  6129. if (instr->m_opcode == Js::OpCode::BrFalse_A)
  6130. {
  6131. *result = !(*result);
  6132. }
  6133. break;
  6134. }
  6135. case Js::OpCode::BrFalse_I4:
  6136. {
  6137. constVal = 0;
  6138. if (!src1Val->GetValueInfo()->TryGetIntConstantValue(&constVal))
  6139. {
  6140. return false;
  6141. }
  6142. *result = constVal == 0;
  6143. break;
  6144. }
  6145. case Js::OpCode::BrOnObject_A:
  6146. {
  6147. ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  6148. if (!src1ValueInfo->IsDefinite())
  6149. {
  6150. return false;
  6151. }
  6152. *result = !src1ValueInfo->IsPrimitive();
  6153. break;
  6154. }
  6155. default:
  6156. return false;
  6157. }
  6158. return true;
  6159. }
  6160. bool
  6161. GlobOpt::OptConstFoldBranch(IR::Instr *instr, Value *src1Val, Value*src2Val, Value **pDstVal)
  6162. {
  6163. if (!src1Val)
  6164. {
  6165. return false;
  6166. }
  6167. Js::Var src1Var = this->GetConstantVar(instr->GetSrc1(), src1Val);
  6168. Js::Var src2Var = nullptr;
  6169. if (instr->GetSrc2())
  6170. {
  6171. if (!src2Val)
  6172. {
  6173. return false;
  6174. }
  6175. src2Var = this->GetConstantVar(instr->GetSrc2(), src2Val);
  6176. }
  6177. bool result;
  6178. if (!CanProveConditionalBranch(instr, src1Val, src2Val, src1Var, src2Var, &result))
  6179. {
  6180. return false;
  6181. }
  6182. this->OptConstFoldBr(!!result, instr);
  6183. return true;
  6184. }
  6185. bool
  6186. GlobOpt::OptConstFoldUnary(
  6187. IR::Instr * *pInstr,
  6188. const int32 intConstantValue,
  6189. const bool isUsingOriginalSrc1Value,
  6190. Value **pDstVal)
  6191. {
  6192. IR::Instr * &instr = *pInstr;
  6193. int32 value = 0;
  6194. IR::Opnd *constOpnd;
  6195. bool isInt = true;
  6196. bool doSetDstVal = true;
  6197. FloatConstType fValue = 0.0;
  6198. if (!DoConstFold())
  6199. {
  6200. return false;
  6201. }
  6202. if (instr->GetDst() && !instr->GetDst()->IsRegOpnd())
  6203. {
  6204. return false;
  6205. }
  6206. switch(instr->m_opcode)
  6207. {
  6208. case Js::OpCode::Neg_A:
  6209. if (intConstantValue == 0)
  6210. {
  6211. // Could fold to -0.0
  6212. return false;
  6213. }
  6214. if (Int32Math::Neg(intConstantValue, &value))
  6215. {
  6216. return false;
  6217. }
  6218. break;
  6219. case Js::OpCode::Not_A:
  6220. Int32Math::Not(intConstantValue, &value);
  6221. break;
  6222. case Js::OpCode::Ld_A:
  6223. if (instr->HasBailOutInfo())
  6224. {
  6225. //The profile data for switch expr can be string and in GlobOpt we realize it is an int.
  6226. if(instr->GetBailOutKind() == IR::BailOutExpectingString)
  6227. {
  6228. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingString);
  6229. }
  6230. Assert(instr->GetBailOutKind() == IR::BailOutExpectingInteger);
  6231. instr->ClearBailOutInfo();
  6232. }
  6233. value = intConstantValue;
  6234. if(isUsingOriginalSrc1Value)
  6235. {
  6236. doSetDstVal = false; // Let OptDst do it by copying src1Val
  6237. }
  6238. break;
  6239. case Js::OpCode::Conv_Num:
  6240. case Js::OpCode::LdC_A_I4:
  6241. value = intConstantValue;
  6242. if(isUsingOriginalSrc1Value)
  6243. {
  6244. doSetDstVal = false; // Let OptDst do it by copying src1Val
  6245. }
  6246. break;
  6247. case Js::OpCode::Incr_A:
  6248. if (Int32Math::Inc(intConstantValue, &value))
  6249. {
  6250. return false;
  6251. }
  6252. break;
  6253. case Js::OpCode::Decr_A:
  6254. if (Int32Math::Dec(intConstantValue, &value))
  6255. {
  6256. return false;
  6257. }
  6258. break;
  6259. case Js::OpCode::InlineMathAcos:
  6260. fValue = Js::Math::Acos((double)intConstantValue);
  6261. isInt = false;
  6262. break;
  6263. case Js::OpCode::InlineMathAsin:
  6264. fValue = Js::Math::Asin((double)intConstantValue);
  6265. isInt = false;
  6266. break;
  6267. case Js::OpCode::InlineMathAtan:
  6268. fValue = Js::Math::Atan((double)intConstantValue);
  6269. isInt = false;
  6270. break;
  6271. case Js::OpCode::InlineMathCos:
  6272. fValue = Js::Math::Cos((double)intConstantValue);
  6273. isInt = false;
  6274. break;
  6275. case Js::OpCode::InlineMathExp:
  6276. fValue = Js::Math::Exp((double)intConstantValue);
  6277. isInt = false;
  6278. break;
  6279. case Js::OpCode::InlineMathLog:
  6280. fValue = Js::Math::Log((double)intConstantValue);
  6281. isInt = false;
  6282. break;
  6283. case Js::OpCode::InlineMathSin:
  6284. fValue = Js::Math::Sin((double)intConstantValue);
  6285. isInt = false;
  6286. break;
  6287. case Js::OpCode::InlineMathSqrt:
  6288. fValue = ::sqrt((double)intConstantValue);
  6289. isInt = false;
  6290. break;
  6291. case Js::OpCode::InlineMathTan:
  6292. fValue = ::tan((double)intConstantValue);
  6293. isInt = false;
  6294. break;
  6295. case Js::OpCode::InlineMathFround:
  6296. fValue = (double) (float) intConstantValue;
  6297. isInt = false;
  6298. break;
  6299. case Js::OpCode::InlineMathAbs:
  6300. if (intConstantValue == INT32_MIN)
  6301. {
  6302. if (instr->GetDst()->IsInt32())
  6303. {
  6304. // if dst is an int (e.g. in asm.js), we should coerce it, not convert to float
  6305. value = static_cast<int32>(2147483648U);
  6306. }
  6307. else
  6308. {
  6309. // Rejit with AggressiveIntTypeSpecDisabled for Math.abs(INT32_MIN) because it causes dst
  6310. // to be float type which could be different with previous type spec result in LoopPrePass
  6311. throw Js::RejitException(RejitReason::AggressiveIntTypeSpecDisabled);
  6312. }
  6313. }
  6314. else
  6315. {
  6316. value = ::abs(intConstantValue);
  6317. }
  6318. break;
  6319. case Js::OpCode::InlineMathClz:
  6320. DWORD clz;
  6321. if (_BitScanReverse(&clz, intConstantValue))
  6322. {
  6323. value = 31 - clz;
  6324. }
  6325. else
  6326. {
  6327. value = 32;
  6328. }
  6329. instr->ClearBailOutInfo();
  6330. break;
  6331. case Js::OpCode::Ctz:
  6332. Assert(func->GetJITFunctionBody()->IsWasmFunction());
  6333. Assert(!instr->HasBailOutInfo());
  6334. DWORD ctz;
  6335. if (_BitScanForward(&ctz, intConstantValue))
  6336. {
  6337. value = ctz;
  6338. }
  6339. else
  6340. {
  6341. value = 32;
  6342. }
  6343. break;
  6344. case Js::OpCode::InlineMathFloor:
  6345. value = intConstantValue;
  6346. instr->ClearBailOutInfo();
  6347. break;
  6348. case Js::OpCode::InlineMathCeil:
  6349. value = intConstantValue;
  6350. instr->ClearBailOutInfo();
  6351. break;
  6352. case Js::OpCode::InlineMathRound:
  6353. value = intConstantValue;
  6354. instr->ClearBailOutInfo();
  6355. break;
  6356. case Js::OpCode::ToVar:
  6357. if (Js::TaggedInt::IsOverflow(intConstantValue))
  6358. {
  6359. return false;
  6360. }
  6361. else
  6362. {
  6363. value = intConstantValue;
  6364. instr->ClearBailOutInfo();
  6365. break;
  6366. }
  6367. default:
  6368. return false;
  6369. }
  6370. this->CaptureByteCodeSymUses(instr);
  6371. Assert(!instr->HasBailOutInfo()); // If we are, in fact, successful in constant folding the instruction, there is no point in having the bailoutinfo around anymore.
  6372. // Make sure that it is cleared if it was initially present.
  6373. if (!isInt)
  6374. {
  6375. value = (int32)fValue;
  6376. if (fValue == (double)value)
  6377. {
  6378. isInt = true;
  6379. }
  6380. }
  6381. if (isInt)
  6382. {
  6383. constOpnd = IR::IntConstOpnd::New(value, TyInt32, instr->m_func);
  6384. GOPT_TRACE(_u("Constant folding to %d\n"), value);
  6385. }
  6386. else
  6387. {
  6388. constOpnd = IR::FloatConstOpnd::New(fValue, TyFloat64, instr->m_func);
  6389. GOPT_TRACE(_u("Constant folding to %f\n"), fValue);
  6390. }
  6391. instr->ReplaceSrc1(constOpnd);
  6392. this->OptSrc(constOpnd, &instr);
  6393. IR::Opnd *dst = instr->GetDst();
  6394. Assert(dst->IsRegOpnd());
  6395. StackSym *dstSym = dst->AsRegOpnd()->m_sym;
  6396. if (isInt)
  6397. {
  6398. if (dstSym->IsSingleDef())
  6399. {
  6400. dstSym->SetIsIntConst(value);
  6401. }
  6402. if (doSetDstVal)
  6403. {
  6404. *pDstVal = GetIntConstantValue(value, instr, dst);
  6405. }
  6406. if (IsTypeSpecPhaseOff(this->func))
  6407. {
  6408. instr->m_opcode = Js::OpCode::LdC_A_I4;
  6409. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  6410. }
  6411. else
  6412. {
  6413. instr->m_opcode = Js::OpCode::Ld_I4;
  6414. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  6415. StackSym * currDstSym = instr->GetDst()->AsRegOpnd()->m_sym;
  6416. if (currDstSym->IsSingleDef())
  6417. {
  6418. currDstSym->SetIsIntConst(value);
  6419. }
  6420. }
  6421. }
  6422. else
  6423. {
  6424. *pDstVal = NewFloatConstantValue(fValue, dst);
  6425. if (IsTypeSpecPhaseOff(this->func))
  6426. {
  6427. instr->m_opcode = Js::OpCode::LdC_A_R8;
  6428. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  6429. }
  6430. else
  6431. {
  6432. instr->m_opcode = Js::OpCode::LdC_F8_R8;
  6433. this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  6434. }
  6435. }
  6436. InvalidateInductionVariables(instr);
  6437. return true;
  6438. }
  6439. //------------------------------------------------------------------------------------------------------
  6440. // Type specialization
  6441. //------------------------------------------------------------------------------------------------------
  6442. bool
  6443. GlobOpt::IsWorthSpecializingToInt32DueToSrc(IR::Opnd *const src, Value *const val)
  6444. {
  6445. Assert(src);
  6446. Assert(val);
  6447. ValueInfo *valueInfo = val->GetValueInfo();
  6448. Assert(valueInfo->IsLikelyInt());
  6449. // If it is not known that the operand is definitely an int, the operand is not already type-specialized, and it's not live
  6450. // in the loop landing pad (if we're in a loop), it's probably not worth type-specializing this instruction. The common case
  6451. // where type-specializing this would be bad is where the operations are entirely on properties or array elements, where the
  6452. // ratio of FromVars and ToVars to the number of actual operations is high, and the conversions would dominate the time
  6453. // spent. On the other hand, if we're using a function formal parameter more than once, it would probably be worth
  6454. // type-specializing it, hence the IsDead check on the operands.
  6455. return
  6456. valueInfo->IsInt() ||
  6457. valueInfo->HasIntConstantValue(true) ||
  6458. !src->GetIsDead() ||
  6459. !src->IsRegOpnd() ||
  6460. CurrentBlockData()->IsInt32TypeSpecialized(src->AsRegOpnd()->m_sym) ||
  6461. (this->currentBlock->loop && this->currentBlock->loop->landingPad->globOptData.IsLive(src->AsRegOpnd()->m_sym));
  6462. }
  6463. bool
  6464. GlobOpt::IsWorthSpecializingToInt32DueToDst(IR::Opnd *const dst)
  6465. {
  6466. Assert(dst);
  6467. const auto sym = dst->AsRegOpnd()->m_sym;
  6468. return
  6469. CurrentBlockData()->IsInt32TypeSpecialized(sym) ||
  6470. (this->currentBlock->loop && this->currentBlock->loop->landingPad->globOptData.IsLive(sym));
  6471. }
  6472. bool
  6473. GlobOpt::IsWorthSpecializingToInt32(IR::Instr *const instr, Value *const src1Val, Value *const src2Val)
  6474. {
  6475. Assert(instr);
  6476. const auto src1 = instr->GetSrc1();
  6477. const auto src2 = instr->GetSrc2();
  6478. // In addition to checking each operand and the destination, if for any reason we only have to do a maximum of two
  6479. // conversions instead of the worst-case 3 conversions, it's probably worth specializing.
  6480. if (IsWorthSpecializingToInt32DueToSrc(src1, src1Val) ||
  6481. (src2Val && IsWorthSpecializingToInt32DueToSrc(src2, src2Val)))
  6482. {
  6483. return true;
  6484. }
  6485. IR::Opnd *dst = instr->GetDst();
  6486. if (!dst || IsWorthSpecializingToInt32DueToDst(dst))
  6487. {
  6488. return true;
  6489. }
  6490. if (dst->IsEqual(src1) || (src2Val && (dst->IsEqual(src2) || src1->IsEqual(src2))))
  6491. {
  6492. return true;
  6493. }
  6494. IR::Instr *instrNext = instr->GetNextRealInstrOrLabel();
  6495. // Skip useless Ld_A's
  6496. do
  6497. {
  6498. switch (instrNext->m_opcode)
  6499. {
  6500. case Js::OpCode::Ld_A:
  6501. if (!dst->IsEqual(instrNext->GetSrc1()))
  6502. {
  6503. goto done;
  6504. }
  6505. dst = instrNext->GetDst();
  6506. break;
  6507. case Js::OpCode::LdFld:
  6508. case Js::OpCode::LdRootFld:
  6509. case Js::OpCode::LdRootFldForTypeOf:
  6510. case Js::OpCode::LdFldForTypeOf:
  6511. case Js::OpCode::LdElemI_A:
  6512. case Js::OpCode::ByteCodeUses:
  6513. break;
  6514. default:
  6515. goto done;
  6516. }
  6517. instrNext = instrNext->GetNextRealInstrOrLabel();
  6518. } while (true);
  6519. done:
  6520. // If the next instr could also be type specialized, then it is probably worth it.
  6521. if ((instrNext->GetSrc1() && dst->IsEqual(instrNext->GetSrc1())) || (instrNext->GetSrc2() && dst->IsEqual(instrNext->GetSrc2())))
  6522. {
  6523. switch (instrNext->m_opcode)
  6524. {
  6525. case Js::OpCode::Add_A:
  6526. case Js::OpCode::Sub_A:
  6527. case Js::OpCode::Mul_A:
  6528. case Js::OpCode::Div_A:
  6529. case Js::OpCode::Rem_A:
  6530. case Js::OpCode::Xor_A:
  6531. case Js::OpCode::And_A:
  6532. case Js::OpCode::Or_A:
  6533. case Js::OpCode::Shl_A:
  6534. case Js::OpCode::Shr_A:
  6535. case Js::OpCode::Incr_A:
  6536. case Js::OpCode::Decr_A:
  6537. case Js::OpCode::Neg_A:
  6538. case Js::OpCode::Not_A:
  6539. case Js::OpCode::Conv_Num:
  6540. case Js::OpCode::BrEq_I4:
  6541. case Js::OpCode::BrTrue_I4:
  6542. case Js::OpCode::BrFalse_I4:
  6543. case Js::OpCode::BrGe_I4:
  6544. case Js::OpCode::BrGt_I4:
  6545. case Js::OpCode::BrLt_I4:
  6546. case Js::OpCode::BrLe_I4:
  6547. case Js::OpCode::BrNeq_I4:
  6548. return true;
  6549. }
  6550. }
  6551. return false;
  6552. }
  6553. bool
  6554. GlobOpt::TypeSpecializeNumberUnary(IR::Instr *instr, Value *src1Val, Value **pDstVal)
  6555. {
  6556. Assert(src1Val->GetValueInfo()->IsNumber());
  6557. if (this->IsLoopPrePass())
  6558. {
  6559. return false;
  6560. }
  6561. switch (instr->m_opcode)
  6562. {
  6563. case Js::OpCode::Conv_Num:
  6564. // Optimize Conv_Num away since we know this is a number
  6565. instr->m_opcode = Js::OpCode::Ld_A;
  6566. return false;
  6567. }
  6568. return false;
  6569. }
  6570. bool
  6571. GlobOpt::TypeSpecializeUnary(
  6572. IR::Instr **pInstr,
  6573. Value **pSrc1Val,
  6574. Value **pDstVal,
  6575. Value *const src1OriginalVal,
  6576. bool *redoTypeSpecRef,
  6577. bool *const forceInvariantHoistingRef)
  6578. {
  6579. Assert(pSrc1Val);
  6580. Value *&src1Val = *pSrc1Val;
  6581. Assert(src1Val);
  6582. // We don't need to do typespec for asmjs
  6583. if (IsTypeSpecPhaseOff(this->func) || GetIsAsmJSFunc())
  6584. {
  6585. return false;
  6586. }
  6587. IR::Instr *&instr = *pInstr;
  6588. int32 min, max;
  6589. // Inline built-ins explicitly specify how srcs/dst must be specialized.
  6590. if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  6591. {
  6592. TypeSpecializeInlineBuiltInUnary(pInstr, &src1Val, pDstVal, src1OriginalVal, redoTypeSpecRef);
  6593. return true;
  6594. }
  6595. // Consider: If type spec wasn't completely done, make sure that we don't type-spec the dst 2nd time.
  6596. if(instr->m_opcode == Js::OpCode::LdLen_A && TypeSpecializeLdLen(&instr, &src1Val, pDstVal, forceInvariantHoistingRef))
  6597. {
  6598. return true;
  6599. }
  6600. if (!src1Val->GetValueInfo()->GetIntValMinMax(&min, &max, this->DoAggressiveIntTypeSpec()))
  6601. {
  6602. src1Val = src1OriginalVal;
  6603. if (src1Val->GetValueInfo()->IsLikelyFloat())
  6604. {
  6605. // Try to type specialize to float
  6606. return this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal);
  6607. }
  6608. else if (src1Val->GetValueInfo()->IsNumber())
  6609. {
  6610. return TypeSpecializeNumberUnary(instr, src1Val, pDstVal);
  6611. }
  6612. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  6613. }
  6614. return this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, min, max, src1OriginalVal, redoTypeSpecRef);
  6615. }
  6616. // Returns true if the built-in requested type specialization, and no further action needed,
  6617. // otherwise returns false.
  6618. void
  6619. GlobOpt::TypeSpecializeInlineBuiltInUnary(IR::Instr **pInstr, Value **pSrc1Val, Value **pDstVal, Value *const src1OriginalVal, bool *redoTypeSpecRef)
  6620. {
  6621. IR::Instr *&instr = *pInstr;
  6622. Assert(pSrc1Val);
  6623. Value *&src1Val = *pSrc1Val;
  6624. Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  6625. Js::BuiltinFunction builtInId = Js::JavascriptLibrary::GetBuiltInInlineCandidateId(instr->m_opcode); // From actual instr, not profile based.
  6626. Assert(builtInId != Js::BuiltinFunction::None);
  6627. // Consider using different bailout for float/int FromVars, so that when the arg cannot be converted to number we don't disable
  6628. // type spec for other parts of the big function but rather just don't inline that built-in instr.
  6629. // E.g. could do that if the value is not likelyInt/likelyFloat.
  6630. Js::BuiltInFlags builtInFlags = Js::JavascriptLibrary::GetFlagsForBuiltIn(builtInId);
  6631. bool areAllArgsAlwaysFloat = (builtInFlags & Js::BuiltInFlags::BIF_Args) == Js::BuiltInFlags::BIF_TypeSpecUnaryToFloat;
  6632. if (areAllArgsAlwaysFloat)
  6633. {
  6634. // InlineMathAcos, InlineMathAsin, InlineMathAtan, InlineMathCos, InlineMathExp, InlineMathLog, InlineMathSin, InlineMathSqrt, InlineMathTan.
  6635. Assert(this->DoFloatTypeSpec());
  6636. // Type-spec the src.
  6637. src1Val = src1OriginalVal;
  6638. bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, /* skipDst = */ true);
  6639. AssertMsg(retVal, "For inline built-ins the args have to be type-specialized to float, but something failed during the process.");
  6640. // Type-spec the dst.
  6641. this->TypeSpecializeFloatDst(instr, nullptr, src1Val, nullptr, pDstVal);
  6642. }
  6643. else if (instr->m_opcode == Js::OpCode::InlineMathAbs)
  6644. {
  6645. // Consider the case when the value is unknown - because of bailout in abs we may disable type spec for the whole function which is too much.
  6646. // First, try int.
  6647. int minVal, maxVal;
  6648. bool shouldTypeSpecToInt = src1Val->GetValueInfo()->GetIntValMinMax(&minVal, &maxVal, /* doAggressiveIntTypeSpec = */ true);
  6649. if (shouldTypeSpecToInt)
  6650. {
  6651. Assert(this->DoAggressiveIntTypeSpec());
  6652. bool retVal = this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, minVal, maxVal, src1OriginalVal, redoTypeSpecRef, true);
  6653. AssertMsg(retVal, "For inline built-ins the args have to be type-specialized (int), but something failed during the process.");
  6654. if (!this->IsLoopPrePass())
  6655. {
  6656. // Create bailout for INT_MIN which does not have corresponding int value on the positive side.
  6657. // Check int range: if we know the range is out of overflow, we do not need the bail out at all.
  6658. if (minVal == INT32_MIN)
  6659. {
  6660. GenerateBailAtOperation(&instr, IR::BailOnIntMin);
  6661. }
  6662. }
  6663. // Account for ::abs(INT_MIN) == INT_MIN (which is less than 0).
  6664. maxVal = ::max(
  6665. ::abs(Int32Math::NearestInRangeTo(minVal, INT_MIN + 1, INT_MAX)),
  6666. ::abs(Int32Math::NearestInRangeTo(maxVal, INT_MIN + 1, INT_MAX)));
  6667. minVal = minVal >= 0 ? minVal : 0;
  6668. this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, nullptr, IR::BailOutInvalid, minVal, maxVal, pDstVal);
  6669. }
  6670. else
  6671. {
  6672. // If we couldn't do int, do float.
  6673. Assert(this->DoFloatTypeSpec());
  6674. src1Val = src1OriginalVal;
  6675. bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, true);
  6676. AssertMsg(retVal, "For inline built-ins the args have to be type-specialized (float), but something failed during the process.");
  6677. this->TypeSpecializeFloatDst(instr, nullptr, src1Val, nullptr, pDstVal);
  6678. }
  6679. }
  6680. else if (instr->m_opcode == Js::OpCode::InlineMathFloor || instr->m_opcode == Js::OpCode::InlineMathCeil || instr->m_opcode == Js::OpCode::InlineMathRound)
  6681. {
  6682. // Type specialize src to float
  6683. src1Val = src1OriginalVal;
  6684. bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, /* skipDst = */ true);
  6685. AssertMsg(retVal, "For inline Math.floor and Math.ceil the src has to be type-specialized to float, but something failed during the process.");
  6686. // Type specialize dst to int
  6687. this->TypeSpecializeIntDst(
  6688. instr,
  6689. instr->m_opcode,
  6690. nullptr,
  6691. src1Val,
  6692. nullptr,
  6693. IR::BailOutInvalid,
  6694. INT32_MIN,
  6695. INT32_MAX,
  6696. pDstVal);
  6697. }
  6698. else if(instr->m_opcode == Js::OpCode::InlineArrayPop)
  6699. {
  6700. IR::Opnd *const thisOpnd = instr->GetSrc1();
  6701. Assert(thisOpnd);
  6702. // Ensure src1 (Array) is a var
  6703. this->ToVarUses(instr, thisOpnd, false, src1Val);
  6704. if(!this->IsLoopPrePass() && thisOpnd->GetValueType().IsLikelyNativeArray())
  6705. {
  6706. // We bail out, if there is illegal access or a mismatch in the Native array type that is optimized for, during the run time.
  6707. GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly);
  6708. }
  6709. if(!instr->GetDst())
  6710. {
  6711. return;
  6712. }
  6713. // Try Type Specializing the element (return item from Pop) based on the array's profile data.
  6714. if(thisOpnd->GetValueType().IsLikelyNativeIntArray())
  6715. {
  6716. this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, nullptr, nullptr, IR::BailOutInvalid, INT32_MIN, INT32_MAX, pDstVal);
  6717. }
  6718. else if(thisOpnd->GetValueType().IsLikelyNativeFloatArray())
  6719. {
  6720. this->TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, pDstVal);
  6721. }
  6722. else
  6723. {
  6724. // We reached here so the Element is not yet type specialized. Ensure element is a var
  6725. if(instr->GetDst()->IsRegOpnd())
  6726. {
  6727. this->ToVarRegOpnd(instr->GetDst()->AsRegOpnd(), currentBlock);
  6728. }
  6729. }
  6730. }
  6731. else if (instr->m_opcode == Js::OpCode::InlineMathClz)
  6732. {
  6733. Assert(this->DoAggressiveIntTypeSpec());
  6734. Assert(this->DoLossyIntTypeSpec());
  6735. //Type specialize to int
  6736. bool retVal = this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, INT32_MIN, INT32_MAX, src1OriginalVal, redoTypeSpecRef);
  6737. AssertMsg(retVal, "For clz32, the arg has to be type-specialized to int.");
  6738. }
  6739. else
  6740. {
  6741. AssertMsg(FALSE, "Unsupported built-in!");
  6742. }
  6743. }
  6744. void
  6745. GlobOpt::TypeSpecializeInlineBuiltInBinary(IR::Instr **pInstr, Value *src1Val, Value* src2Val, Value **pDstVal, Value *const src1OriginalVal, Value *const src2OriginalVal)
  6746. {
  6747. IR::Instr *&instr = *pInstr;
  6748. Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  6749. switch(instr->m_opcode)
  6750. {
  6751. case Js::OpCode::InlineMathAtan2:
  6752. {
  6753. Js::BuiltinFunction builtInId = Js::JavascriptLibrary::GetBuiltInInlineCandidateId(instr->m_opcode); // From actual instr, not profile based.
  6754. Js::BuiltInFlags builtInFlags = Js::JavascriptLibrary::GetFlagsForBuiltIn(builtInId);
  6755. bool areAllArgsAlwaysFloat = (builtInFlags & Js::BuiltInFlags::BIF_TypeSpecAllToFloat) != 0;
  6756. Assert(areAllArgsAlwaysFloat);
  6757. Assert(this->DoFloatTypeSpec());
  6758. // Type-spec the src1, src2 and dst.
  6759. src1Val = src1OriginalVal;
  6760. src2Val = src2OriginalVal;
  6761. bool retVal = this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  6762. AssertMsg(retVal, "For pow and atnan2 the args have to be type-specialized to float, but something failed during the process.");
  6763. break;
  6764. }
  6765. case Js::OpCode::InlineMathPow:
  6766. {
  6767. #ifndef _M_ARM32_OR_ARM64
  6768. if (src2Val->GetValueInfo()->IsLikelyInt())
  6769. {
  6770. bool lossy = false;
  6771. this->ToInt32(instr, instr->GetSrc2(), this->currentBlock, src2Val, nullptr, lossy);
  6772. IR::Opnd* src1 = instr->GetSrc1();
  6773. int32 valueMin, valueMax;
  6774. if (src1Val->GetValueInfo()->IsLikelyInt() &&
  6775. this->DoPowIntIntTypeSpec() &&
  6776. src2Val->GetValueInfo()->GetIntValMinMax(&valueMin, &valueMax, this->DoAggressiveIntTypeSpec()) &&
  6777. valueMin >= 0)
  6778. {
  6779. this->ToInt32(instr, src1, this->currentBlock, src1Val, nullptr, lossy);
  6780. this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, src2Val, IR::BailOutInvalid, INT32_MIN, INT32_MAX, pDstVal);
  6781. if(!this->IsLoopPrePass())
  6782. {
  6783. GenerateBailAtOperation(&instr, IR::BailOutOnPowIntIntOverflow);
  6784. }
  6785. }
  6786. else
  6787. {
  6788. this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, IR::BailOutPrimitiveButString);
  6789. TypeSpecializeFloatDst(instr, nullptr, src1Val, src2Val, pDstVal);
  6790. }
  6791. }
  6792. else
  6793. {
  6794. #endif
  6795. this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  6796. #ifndef _M_ARM32_OR_ARM64
  6797. }
  6798. #endif
  6799. break;
  6800. }
  6801. case Js::OpCode::InlineMathImul:
  6802. {
  6803. Assert(this->DoAggressiveIntTypeSpec());
  6804. Assert(this->DoLossyIntTypeSpec());
  6805. //Type specialize to int
  6806. bool retVal = this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, INT32_MIN, INT32_MAX, false /* skipDst */);
  6807. AssertMsg(retVal, "For imul, the args have to be type-specialized to int but something failed during the process.");
  6808. break;
  6809. }
  6810. case Js::OpCode::InlineMathMin:
  6811. case Js::OpCode::InlineMathMax:
  6812. {
  6813. if(src1Val->GetValueInfo()->IsLikelyInt() && src2Val->GetValueInfo()->IsLikelyInt())
  6814. {
  6815. // Compute resulting range info
  6816. int32 min1 = INT32_MIN;
  6817. int32 max1 = INT32_MAX;
  6818. int32 min2 = INT32_MIN;
  6819. int32 max2 = INT32_MAX;
  6820. int32 newMin, newMax;
  6821. Assert(this->DoAggressiveIntTypeSpec());
  6822. src1Val->GetValueInfo()->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec());
  6823. src2Val->GetValueInfo()->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec());
  6824. if (instr->m_opcode == Js::OpCode::InlineMathMin)
  6825. {
  6826. newMin = min(min1, min2);
  6827. newMax = min(max1, max2);
  6828. }
  6829. else
  6830. {
  6831. Assert(instr->m_opcode == Js::OpCode::InlineMathMax);
  6832. newMin = max(min1, min2);
  6833. newMax = max(max1, max2);
  6834. }
  6835. // Type specialize to int
  6836. bool retVal = this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, newMin, newMax, false /* skipDst */);
  6837. AssertMsg(retVal, "For min and max, the args have to be type-specialized to int if any one of the sources is an int, but something failed during the process.");
  6838. }
  6839. // Couldn't type specialize to int, type specialize to float
  6840. else
  6841. {
  6842. Assert(this->DoFloatTypeSpec());
  6843. src1Val = src1OriginalVal;
  6844. src2Val = src2OriginalVal;
  6845. bool retVal = this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  6846. AssertMsg(retVal, "For min and max, the args have to be type-specialized to float if any one of the sources is a float, but something failed during the process.");
  6847. }
  6848. break;
  6849. }
  6850. case Js::OpCode::InlineArrayPush:
  6851. {
  6852. IR::Opnd *const thisOpnd = instr->GetSrc1();
  6853. Assert(thisOpnd);
  6854. if(instr->GetDst() && instr->GetDst()->IsRegOpnd())
  6855. {
  6856. // Set the dst as live here, as the built-ins return early from the TypeSpecialization functions - before the dst is marked as live.
  6857. // Also, we are not specializing the dst separately and we are skipping the dst to be handled when we specialize the instruction above.
  6858. this->ToVarRegOpnd(instr->GetDst()->AsRegOpnd(), currentBlock);
  6859. }
  6860. // Ensure src1 (Array) is a var
  6861. this->ToVarUses(instr, thisOpnd, false, src1Val);
  6862. if(!this->IsLoopPrePass())
  6863. {
  6864. if(thisOpnd->GetValueType().IsLikelyNativeArray())
  6865. {
  6866. // We bail out, if there is illegal access or a mismatch in the Native array type that is optimized for, during run time.
  6867. GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly);
  6868. }
  6869. else
  6870. {
  6871. GenerateBailAtOperation(&instr, IR::BailOutOnImplicitCallsPreOp);
  6872. }
  6873. }
  6874. // Try Type Specializing the element based on the array's profile data.
  6875. if(thisOpnd->GetValueType().IsLikelyNativeFloatArray())
  6876. {
  6877. src1Val = src1OriginalVal;
  6878. src2Val = src2OriginalVal;
  6879. }
  6880. if((thisOpnd->GetValueType().IsLikelyNativeIntArray() && this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, INT32_MIN, INT32_MAX, true))
  6881. || (thisOpnd->GetValueType().IsLikelyNativeFloatArray() && this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal)))
  6882. {
  6883. break;
  6884. }
  6885. // The Element is not yet type specialized. Ensure element is a var
  6886. this->ToVarUses(instr, instr->GetSrc2(), false, src2Val);
  6887. break;
  6888. }
  6889. }
  6890. }
  6891. void
  6892. GlobOpt::TypeSpecializeInlineBuiltInDst(IR::Instr **pInstr, Value **pDstVal)
  6893. {
  6894. IR::Instr *&instr = *pInstr;
  6895. Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  6896. if (instr->m_opcode == Js::OpCode::InlineMathRandom)
  6897. {
  6898. Assert(this->DoFloatTypeSpec());
  6899. // Type specialize dst to float
  6900. this->TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, pDstVal);
  6901. }
  6902. }
  6903. bool
  6904. GlobOpt::TryTypeSpecializeUnaryToFloatHelper(IR::Instr** pInstr, Value** pSrc1Val, Value* const src1OriginalVal, Value **pDstVal)
  6905. {
  6906. // It has been determined that this instruction cannot be int-specialized. We need to determine whether to attempt to
  6907. // float-specialize the instruction, or leave it unspecialized.
  6908. #if !INT32VAR
  6909. Value*& src1Val = *pSrc1Val;
  6910. if(src1Val->GetValueInfo()->IsLikelyUntaggedInt())
  6911. {
  6912. // An input range is completely outside the range of an int31. Even if the operation may overflow, it is
  6913. // unlikely to overflow on these operations, so we leave it unspecialized on 64-bit platforms. However, on
  6914. // 32-bit platforms, the value is untaggable and will be a JavascriptNumber, which is significantly slower to
  6915. // use in an unspecialized operation compared to a tagged int. So, try to float-specialize the instruction.
  6916. src1Val = src1OriginalVal;
  6917. return this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal);
  6918. }
  6919. #endif
  6920. return false;
  6921. }
  6922. bool
  6923. GlobOpt::TypeSpecializeIntBinary(IR::Instr **pInstr, Value *src1Val, Value *src2Val, Value **pDstVal, int32 min, int32 max, bool skipDst /* = false */)
  6924. {
  6925. // Consider moving the code for int type spec-ing binary functions here.
  6926. IR::Instr *&instr = *pInstr;
  6927. bool lossy = false;
  6928. if(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  6929. {
  6930. if(instr->m_opcode == Js::OpCode::InlineArrayPush)
  6931. {
  6932. int32 intConstantValue;
  6933. bool isIntConstMissingItem = src2Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue);
  6934. if(isIntConstMissingItem)
  6935. {
  6936. isIntConstMissingItem = Js::SparseArraySegment<int>::IsMissingItem(&intConstantValue);
  6937. }
  6938. // Don't specialize if the element is not likelyInt or an IntConst which is a missing item value.
  6939. if(!(src2Val->GetValueInfo()->IsLikelyInt()) || isIntConstMissingItem)
  6940. {
  6941. return false;
  6942. }
  6943. // We don't want to specialize both the source operands, though it is a binary instr.
  6944. IR::Opnd * elementOpnd = instr->GetSrc2();
  6945. this->ToInt32(instr, elementOpnd, this->currentBlock, src2Val, nullptr, lossy);
  6946. }
  6947. else
  6948. {
  6949. IR::Opnd *src1 = instr->GetSrc1();
  6950. this->ToInt32(instr, src1, this->currentBlock, src1Val, nullptr, lossy);
  6951. IR::Opnd *src2 = instr->GetSrc2();
  6952. this->ToInt32(instr, src2, this->currentBlock, src2Val, nullptr, lossy);
  6953. }
  6954. if(!skipDst)
  6955. {
  6956. IR::Opnd *dst = instr->GetDst();
  6957. if (dst)
  6958. {
  6959. TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, src2Val, IR::BailOutInvalid, min, max, pDstVal);
  6960. }
  6961. }
  6962. return true;
  6963. }
  6964. else
  6965. {
  6966. AssertMsg(false, "Yet to move code for other binary functions here");
  6967. return false;
  6968. }
  6969. }
  6970. bool
  6971. GlobOpt::TypeSpecializeIntUnary(
  6972. IR::Instr **pInstr,
  6973. Value **pSrc1Val,
  6974. Value **pDstVal,
  6975. int32 min,
  6976. int32 max,
  6977. Value *const src1OriginalVal,
  6978. bool *redoTypeSpecRef,
  6979. bool skipDst /* = false */)
  6980. {
  6981. IR::Instr *&instr = *pInstr;
  6982. Assert(pSrc1Val);
  6983. Value *&src1Val = *pSrc1Val;
  6984. bool isTransfer = false;
  6985. Js::OpCode opcode;
  6986. int32 newMin, newMax;
  6987. bool lossy = false;
  6988. IR::BailOutKind bailOutKind = IR::BailOutInvalid;
  6989. bool ignoredIntOverflow = this->ignoredIntOverflowForCurrentInstr;
  6990. bool ignoredNegativeZero = false;
  6991. bool checkTypeSpecWorth = false;
  6992. if(instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber)
  6993. {
  6994. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  6995. }
  6996. AddSubConstantInfo addSubConstantInfo;
  6997. switch(instr->m_opcode)
  6998. {
  6999. case Js::OpCode::Ld_A:
  7000. if (instr->GetSrc1()->IsRegOpnd())
  7001. {
  7002. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  7003. if (CurrentBlockData()->IsInt32TypeSpecialized(sym) == false)
  7004. {
  7005. // Type specializing an Ld_A isn't worth it, unless the src
  7006. // is already type specialized.
  7007. return false;
  7008. }
  7009. }
  7010. newMin = min;
  7011. newMax = max;
  7012. opcode = Js::OpCode::Ld_I4;
  7013. isTransfer = true;
  7014. break;
  7015. case Js::OpCode::Conv_Num:
  7016. newMin = min;
  7017. newMax = max;
  7018. opcode = Js::OpCode::Ld_I4;
  7019. isTransfer = true;
  7020. break;
  7021. case Js::OpCode::LdC_A_I4:
  7022. newMin = newMax = instr->GetSrc1()->AsIntConstOpnd()->AsInt32();
  7023. opcode = Js::OpCode::Ld_I4;
  7024. break;
  7025. case Js::OpCode::Neg_A:
  7026. if (min <= 0 && max >= 0)
  7027. {
  7028. if(instr->ShouldCheckForNegativeZero())
  7029. {
  7030. // -0 matters since the sym is not a local, or is used in a way in which -0 would differ from +0
  7031. if(!DoAggressiveIntTypeSpec())
  7032. {
  7033. // May result in -0
  7034. // Consider adding a dynamic check for src1 == 0
  7035. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7036. }
  7037. if(min == 0 && max == 0)
  7038. {
  7039. // Always results in -0
  7040. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7041. }
  7042. bailOutKind |= IR::BailOutOnNegativeZero;
  7043. }
  7044. else
  7045. {
  7046. ignoredNegativeZero = true;
  7047. }
  7048. }
  7049. if (Int32Math::Neg(min, &newMax))
  7050. {
  7051. if(instr->ShouldCheckForIntOverflow())
  7052. {
  7053. if(!DoAggressiveIntTypeSpec())
  7054. {
  7055. // May overflow
  7056. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7057. }
  7058. if(min == max)
  7059. {
  7060. // Always overflows
  7061. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7062. }
  7063. bailOutKind |= IR::BailOutOnOverflow;
  7064. newMax = INT32_MAX;
  7065. }
  7066. else
  7067. {
  7068. ignoredIntOverflow = true;
  7069. }
  7070. }
  7071. if (Int32Math::Neg(max, &newMin))
  7072. {
  7073. if(instr->ShouldCheckForIntOverflow())
  7074. {
  7075. if(!DoAggressiveIntTypeSpec())
  7076. {
  7077. // May overflow
  7078. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7079. }
  7080. bailOutKind |= IR::BailOutOnOverflow;
  7081. newMin = INT32_MAX;
  7082. }
  7083. else
  7084. {
  7085. ignoredIntOverflow = true;
  7086. }
  7087. }
  7088. if(!instr->ShouldCheckForIntOverflow() && newMin > newMax)
  7089. {
  7090. // When ignoring overflow, the range needs to account for overflow. Since MIN_INT is the only int32 value that
  7091. // overflows on Neg, and the value resulting from overflow is also MIN_INT, if calculating only the new min or new
  7092. // max overflowed but not both, then the new min will be greater than the new max. In that case we need to consider
  7093. // the full range of int32s as possible resulting values.
  7094. newMin = INT32_MIN;
  7095. newMax = INT32_MAX;
  7096. }
  7097. opcode = Js::OpCode::Neg_I4;
  7098. checkTypeSpecWorth = true;
  7099. break;
  7100. case Js::OpCode::Not_A:
  7101. if(!DoLossyIntTypeSpec())
  7102. {
  7103. return false;
  7104. }
  7105. this->PropagateIntRangeForNot(min, max, &newMin, &newMax);
  7106. opcode = Js::OpCode::Not_I4;
  7107. lossy = true;
  7108. break;
  7109. case Js::OpCode::Incr_A:
  7110. do // while(false)
  7111. {
  7112. const auto CannotOverflowBasedOnRelativeBounds = [&]()
  7113. {
  7114. const ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  7115. return
  7116. (src1ValueInfo->IsInt() || DoAggressiveIntTypeSpec()) &&
  7117. src1ValueInfo->IsIntBounded() &&
  7118. src1ValueInfo->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(1);
  7119. };
  7120. if (Int32Math::Inc(min, &newMin))
  7121. {
  7122. if(CannotOverflowBasedOnRelativeBounds())
  7123. {
  7124. newMin = INT32_MAX;
  7125. }
  7126. else if(instr->ShouldCheckForIntOverflow())
  7127. {
  7128. // Always overflows
  7129. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7130. }
  7131. else
  7132. {
  7133. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow
  7134. // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints,
  7135. // we use the full range of int32s.
  7136. ignoredIntOverflow = true;
  7137. newMin = INT32_MIN;
  7138. newMax = INT32_MAX;
  7139. break;
  7140. }
  7141. }
  7142. if (Int32Math::Inc(max, &newMax))
  7143. {
  7144. if(CannotOverflowBasedOnRelativeBounds())
  7145. {
  7146. newMax = INT32_MAX;
  7147. }
  7148. else if(instr->ShouldCheckForIntOverflow())
  7149. {
  7150. if(!DoAggressiveIntTypeSpec())
  7151. {
  7152. // May overflow
  7153. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7154. }
  7155. bailOutKind |= IR::BailOutOnOverflow;
  7156. newMax = INT32_MAX;
  7157. }
  7158. else
  7159. {
  7160. // See comment about ignoring overflow above
  7161. ignoredIntOverflow = true;
  7162. newMin = INT32_MIN;
  7163. newMax = INT32_MAX;
  7164. break;
  7165. }
  7166. }
  7167. } while(false);
  7168. if(!ignoredIntOverflow && instr->GetSrc1()->IsRegOpnd())
  7169. {
  7170. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min == max, 1);
  7171. }
  7172. opcode = Js::OpCode::Add_I4;
  7173. if (!this->IsLoopPrePass())
  7174. {
  7175. instr->SetSrc2(IR::IntConstOpnd::New(1, TyInt32, instr->m_func));
  7176. }
  7177. checkTypeSpecWorth = true;
  7178. break;
  7179. case Js::OpCode::Decr_A:
  7180. do // while(false)
  7181. {
  7182. const auto CannotOverflowBasedOnRelativeBounds = [&]()
  7183. {
  7184. const ValueInfo *const src1ValueInfo = src1Val->GetValueInfo();
  7185. return
  7186. (src1ValueInfo->IsInt() || DoAggressiveIntTypeSpec()) &&
  7187. src1ValueInfo->IsIntBounded() &&
  7188. src1ValueInfo->AsIntBounded()->Bounds()->SubCannotOverflowBasedOnRelativeBounds(1);
  7189. };
  7190. if (Int32Math::Dec(max, &newMax))
  7191. {
  7192. if(CannotOverflowBasedOnRelativeBounds())
  7193. {
  7194. newMax = INT32_MIN;
  7195. }
  7196. else if(instr->ShouldCheckForIntOverflow())
  7197. {
  7198. // Always overflows
  7199. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7200. }
  7201. else
  7202. {
  7203. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow
  7204. // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints, we
  7205. // use the full range of int32s.
  7206. ignoredIntOverflow = true;
  7207. newMin = INT32_MIN;
  7208. newMax = INT32_MAX;
  7209. break;
  7210. }
  7211. }
  7212. if (Int32Math::Dec(min, &newMin))
  7213. {
  7214. if(CannotOverflowBasedOnRelativeBounds())
  7215. {
  7216. newMin = INT32_MIN;
  7217. }
  7218. else if(instr->ShouldCheckForIntOverflow())
  7219. {
  7220. if(!DoAggressiveIntTypeSpec())
  7221. {
  7222. // May overflow
  7223. return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal);
  7224. }
  7225. bailOutKind |= IR::BailOutOnOverflow;
  7226. newMin = INT32_MIN;
  7227. }
  7228. else
  7229. {
  7230. // See comment about ignoring overflow above
  7231. ignoredIntOverflow = true;
  7232. newMin = INT32_MIN;
  7233. newMax = INT32_MAX;
  7234. break;
  7235. }
  7236. }
  7237. } while(false);
  7238. if(!ignoredIntOverflow && instr->GetSrc1()->IsRegOpnd())
  7239. {
  7240. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min == max, -1);
  7241. }
  7242. opcode = Js::OpCode::Sub_I4;
  7243. if (!this->IsLoopPrePass())
  7244. {
  7245. instr->SetSrc2(IR::IntConstOpnd::New(1, TyInt32, instr->m_func));
  7246. }
  7247. checkTypeSpecWorth = true;
  7248. break;
  7249. case Js::OpCode::BrFalse_A:
  7250. case Js::OpCode::BrTrue_A:
  7251. {
  7252. if(DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrFalse(instr, src1Val, min, max))
  7253. {
  7254. return true;
  7255. }
  7256. bool specialize = true;
  7257. if (!src1Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc1()->IsRegOpnd())
  7258. {
  7259. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  7260. if (CurrentBlockData()->IsInt32TypeSpecialized(sym) == false)
  7261. {
  7262. // Type specializing a BrTrue_A/BrFalse_A isn't worth it, unless the src
  7263. // is already type specialized
  7264. specialize = false;
  7265. }
  7266. }
  7267. if(instr->m_opcode == Js::OpCode::BrTrue_A)
  7268. {
  7269. UpdateIntBoundsForNotEqualBranch(src1Val, nullptr, 0);
  7270. opcode = Js::OpCode::BrTrue_I4;
  7271. }
  7272. else
  7273. {
  7274. UpdateIntBoundsForEqualBranch(src1Val, nullptr, 0);
  7275. opcode = Js::OpCode::BrFalse_I4;
  7276. }
  7277. if(!specialize)
  7278. {
  7279. return false;
  7280. }
  7281. newMin = 2; newMax = 1; // We'll assert if we make a range where min > max
  7282. break;
  7283. }
  7284. case Js::OpCode::MultiBr:
  7285. newMin = min;
  7286. newMax = max;
  7287. opcode = instr->m_opcode;
  7288. break;
  7289. case Js::OpCode::StElemI_A:
  7290. case Js::OpCode::StElemI_A_Strict:
  7291. case Js::OpCode::StElemC:
  7292. if(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyAnyArrayWithNativeFloatValues())
  7293. {
  7294. src1Val = src1OriginalVal;
  7295. }
  7296. return TypeSpecializeStElem(pInstr, src1Val, pDstVal);
  7297. case Js::OpCode::NewScArray:
  7298. case Js::OpCode::NewScArrayWithMissingValues:
  7299. case Js::OpCode::InitFld:
  7300. case Js::OpCode::InitRootFld:
  7301. case Js::OpCode::StSlot:
  7302. case Js::OpCode::StSlotChkUndecl:
  7303. #if !FLOATVAR
  7304. case Js::OpCode::StSlotBoxTemp:
  7305. #endif
  7306. case Js::OpCode::StFld:
  7307. case Js::OpCode::StRootFld:
  7308. case Js::OpCode::StFldStrict:
  7309. case Js::OpCode::StRootFldStrict:
  7310. case Js::OpCode::ArgOut_A:
  7311. case Js::OpCode::ArgOut_A_Inline:
  7312. case Js::OpCode::ArgOut_A_FixupForStackArgs:
  7313. case Js::OpCode::ArgOut_A_Dynamic:
  7314. case Js::OpCode::ArgOut_A_FromStackArgs:
  7315. case Js::OpCode::ArgOut_A_SpreadArg:
  7316. // For this one we need to implement type specialization
  7317. //case Js::OpCode::ArgOut_A_InlineBuiltIn:
  7318. case Js::OpCode::Ret:
  7319. case Js::OpCode::LdElemUndef:
  7320. case Js::OpCode::LdElemUndefScoped:
  7321. return false;
  7322. default:
  7323. if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  7324. {
  7325. newMin = min;
  7326. newMax = max;
  7327. opcode = instr->m_opcode;
  7328. break; // Note: we must keep checkTypeSpecWorth = false to make sure we never return false from this function.
  7329. }
  7330. return false;
  7331. }
  7332. // If this instruction is in a range of instructions where int overflow does not matter, we will still specialize it (won't
  7333. // leave it unspecialized based on heuristics), since it is most likely worth specializing, and the dst value needs to be
  7334. // guaranteed to be an int
  7335. if(checkTypeSpecWorth &&
  7336. !ignoredIntOverflow &&
  7337. !ignoredNegativeZero &&
  7338. instr->ShouldCheckForIntOverflow() &&
  7339. !IsWorthSpecializingToInt32(instr, src1Val))
  7340. {
  7341. // Even though type specialization is being skipped since it may not be worth it, the proper value should still be
  7342. // maintained so that the result may be type specialized later. An int value is not created for the dst in any of
  7343. // the following cases.
  7344. // - A bailout check is necessary to specialize this instruction. The bailout check is what guarantees the result to be
  7345. // an int, but since we're not going to specialize this instruction, there won't be a bailout check.
  7346. // - Aggressive int type specialization is disabled and we're in a loop prepass. We're conservative on dst values in
  7347. // that case, especially if the dst sym is live on the back-edge.
  7348. if(bailOutKind == IR::BailOutInvalid &&
  7349. instr->GetDst() &&
  7350. (DoAggressiveIntTypeSpec() || !this->IsLoopPrePass()))
  7351. {
  7352. *pDstVal = CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, nullptr);
  7353. }
  7354. if(instr->GetSrc2())
  7355. {
  7356. instr->FreeSrc2();
  7357. }
  7358. return false;
  7359. }
  7360. this->ignoredIntOverflowForCurrentInstr = ignoredIntOverflow;
  7361. this->ignoredNegativeZeroForCurrentInstr = ignoredNegativeZero;
  7362. {
  7363. // Try CSE again before modifying the IR, in case some attributes are required for successful CSE
  7364. Value *src1IndirIndexVal = nullptr;
  7365. Value *src2Val = nullptr;
  7366. if(CSEOptimize(currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal, true /* intMathExprOnly */))
  7367. {
  7368. *redoTypeSpecRef = true;
  7369. return false;
  7370. }
  7371. }
  7372. const Js::OpCode originalOpCode = instr->m_opcode;
  7373. if (!this->IsLoopPrePass())
  7374. {
  7375. // No re-write on prepass
  7376. instr->m_opcode = opcode;
  7377. }
  7378. Value *src1ValueToSpecialize = src1Val;
  7379. if(lossy)
  7380. {
  7381. // Lossy conversions to int32 must be done based on the original source values. For instance, if one of the values is a
  7382. // float constant with a value that fits in a uint32 but not an int32, and the instruction can ignore int overflow, the
  7383. // source value for the purposes of int specialization would have been changed to an int constant value by ignoring
  7384. // overflow. If we were to specialize the sym using the int constant value, it would be treated as a lossless
  7385. // conversion, but since there may be subsequent uses of the same float constant value that may not ignore overflow,
  7386. // this must be treated as a lossy conversion by specializing the sym using the original float constant value.
  7387. src1ValueToSpecialize = src1OriginalVal;
  7388. }
  7389. // Make sure the srcs are specialized
  7390. IR::Opnd *src1 = instr->GetSrc1();
  7391. this->ToInt32(instr, src1, this->currentBlock, src1ValueToSpecialize, nullptr, lossy);
  7392. if(bailOutKind != IR::BailOutInvalid && !this->IsLoopPrePass())
  7393. {
  7394. GenerateBailAtOperation(&instr, bailOutKind);
  7395. }
  7396. if (!skipDst)
  7397. {
  7398. IR::Opnd *dst = instr->GetDst();
  7399. if (dst)
  7400. {
  7401. AssertMsg(!(isTransfer && !this->IsLoopPrePass()) || min == newMin && max == newMax, "If this is just a copy, old/new min/max should be the same");
  7402. TypeSpecializeIntDst(
  7403. instr,
  7404. originalOpCode,
  7405. isTransfer ? src1Val : nullptr,
  7406. src1Val,
  7407. nullptr,
  7408. bailOutKind,
  7409. newMin,
  7410. newMax,
  7411. pDstVal,
  7412. addSubConstantInfo.HasInfo() ? &addSubConstantInfo : nullptr);
  7413. }
  7414. }
  7415. if(bailOutKind == IR::BailOutInvalid)
  7416. {
  7417. GOPT_TRACE(_u("Type specialized to INT\n"));
  7418. #if ENABLE_DEBUG_CONFIG_OPTIONS
  7419. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  7420. {
  7421. Output::Print(_u("Type specialized to INT: "));
  7422. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  7423. }
  7424. #endif
  7425. }
  7426. else
  7427. {
  7428. GOPT_TRACE(_u("Type specialized to INT with bailout on:\n"));
  7429. if(bailOutKind & IR::BailOutOnOverflow)
  7430. {
  7431. GOPT_TRACE(_u(" Overflow\n"));
  7432. #if ENABLE_DEBUG_CONFIG_OPTIONS
  7433. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  7434. {
  7435. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Overflow");
  7436. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  7437. }
  7438. #endif
  7439. }
  7440. if(bailOutKind & IR::BailOutOnNegativeZero)
  7441. {
  7442. GOPT_TRACE(_u(" Zero\n"));
  7443. #if ENABLE_DEBUG_CONFIG_OPTIONS
  7444. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  7445. {
  7446. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Zero");
  7447. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  7448. }
  7449. #endif
  7450. }
  7451. }
  7452. return true;
  7453. }
  7454. void
  7455. GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, int32 newMin, int32 newMax, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo)
  7456. {
  7457. this->TypeSpecializeIntDst(instr, originalOpCode, valToTransfer, src1Value, src2Value, bailOutKind, ValueType::GetInt(IntConstantBounds(newMin, newMax).IsLikelyTaggable()), newMin, newMax, pDstVal, addSubConstantInfo);
  7458. }
  7459. void
  7460. GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, ValueType valueType, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo)
  7461. {
  7462. this->TypeSpecializeIntDst(instr, originalOpCode, valToTransfer, src1Value, src2Value, bailOutKind, valueType, 0, 0, pDstVal, addSubConstantInfo);
  7463. }
  7464. void
  7465. GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, ValueType valueType, int32 newMin, int32 newMax, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo)
  7466. {
  7467. Assert(valueType.IsInt() || (valueType.IsNumber() && valueType.IsLikelyInt() && newMin == 0 && newMax == 0));
  7468. Assert(!valToTransfer || valToTransfer == src1Value);
  7469. Assert(!addSubConstantInfo || addSubConstantInfo->HasInfo());
  7470. IR::Opnd *dst = instr->GetDst();
  7471. Assert(dst);
  7472. bool isValueInfoPrecise;
  7473. if(IsLoopPrePass())
  7474. {
  7475. isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value);
  7476. valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, isValueInfoPrecise);
  7477. }
  7478. else
  7479. {
  7480. isValueInfoPrecise = true;
  7481. }
  7482. // If dst has a circular reference in a loop, it probably won't get specialized. Don't mark the dst as type-specialized on
  7483. // the pre-pass. With aggressive int spec though, it will take care of bailing out if necessary so there's no need to assume
  7484. // that the dst will be a var even if it's live on the back-edge. Also if the op always produces an int32, then there's no
  7485. // ambiguity in the dst's value type even in the prepass.
  7486. if (!DoAggressiveIntTypeSpec() && this->IsLoopPrePass() && !valueType.IsInt())
  7487. {
  7488. if (dst->IsRegOpnd())
  7489. {
  7490. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  7491. }
  7492. return;
  7493. }
  7494. const IntBounds *dstBounds = nullptr;
  7495. if(addSubConstantInfo && !addSubConstantInfo->SrcValueIsLikelyConstant() && DoTrackRelativeIntBounds())
  7496. {
  7497. Assert(!ignoredIntOverflowForCurrentInstr);
  7498. // Track bounds for add or sub with a constant. For instance, consider (b = a + 2). The value of 'b' should track that
  7499. // it is equal to (the value of 'a') + 2. Additionally, the value of 'b' should inherit the bounds of 'a', offset by
  7500. // the constant value.
  7501. if(!valueType.IsInt() || !isValueInfoPrecise)
  7502. {
  7503. newMin = INT32_MIN;
  7504. newMax = INT32_MAX;
  7505. }
  7506. dstBounds =
  7507. IntBounds::Add(
  7508. addSubConstantInfo->SrcValue(),
  7509. addSubConstantInfo->Offset(),
  7510. isValueInfoPrecise,
  7511. IntConstantBounds(newMin, newMax),
  7512. alloc);
  7513. }
  7514. // Src1's value could change later in the loop, so the value wouldn't be the same for each
  7515. // iteration. Since we don't iterate over loops "while (!changed)", go conservative on the
  7516. // pre-pass.
  7517. if (valToTransfer)
  7518. {
  7519. // If this is just a copy, no need for creating a new value.
  7520. Assert(!addSubConstantInfo);
  7521. *pDstVal = this->ValueNumberTransferDst(instr, valToTransfer);
  7522. CurrentBlockData()->InsertNewValue(*pDstVal, dst);
  7523. }
  7524. else if (valueType.IsInt() && isValueInfoPrecise)
  7525. {
  7526. bool wasNegativeZeroPreventedByBailout = false;
  7527. if(newMin <= 0 && newMax >= 0)
  7528. {
  7529. switch(originalOpCode)
  7530. {
  7531. case Js::OpCode::Add_A:
  7532. // -0 + -0 == -0
  7533. Assert(src1Value);
  7534. Assert(src2Value);
  7535. wasNegativeZeroPreventedByBailout =
  7536. src1Value->GetValueInfo()->WasNegativeZeroPreventedByBailout() &&
  7537. src2Value->GetValueInfo()->WasNegativeZeroPreventedByBailout();
  7538. break;
  7539. case Js::OpCode::Sub_A:
  7540. // -0 - 0 == -0
  7541. Assert(src1Value);
  7542. wasNegativeZeroPreventedByBailout = src1Value->GetValueInfo()->WasNegativeZeroPreventedByBailout();
  7543. break;
  7544. case Js::OpCode::Neg_A:
  7545. case Js::OpCode::Mul_A:
  7546. case Js::OpCode::Div_A:
  7547. case Js::OpCode::Rem_A:
  7548. wasNegativeZeroPreventedByBailout = !!(bailOutKind & IR::BailOutOnNegativeZero);
  7549. break;
  7550. }
  7551. }
  7552. *pDstVal =
  7553. dstBounds
  7554. ? NewIntBoundedValue(valueType, dstBounds, wasNegativeZeroPreventedByBailout, nullptr)
  7555. : NewIntRangeValue(newMin, newMax, wasNegativeZeroPreventedByBailout, nullptr);
  7556. }
  7557. else
  7558. {
  7559. *pDstVal = dstBounds ? NewIntBoundedValue(valueType, dstBounds, false, nullptr) : NewGenericValue(valueType);
  7560. }
  7561. if(addSubConstantInfo || updateInductionVariableValueNumber)
  7562. {
  7563. TrackIntSpecializedAddSubConstant(instr, addSubConstantInfo, *pDstVal, !!dstBounds);
  7564. }
  7565. CurrentBlockData()->SetValue(*pDstVal, dst);
  7566. AssertMsg(dst->IsRegOpnd(), "What else?");
  7567. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  7568. }
  7569. bool
  7570. GlobOpt::TypeSpecializeBinary(IR::Instr **pInstr, Value **pSrc1Val, Value **pSrc2Val, Value **pDstVal, Value *const src1OriginalVal, Value *const src2OriginalVal, bool *redoTypeSpecRef)
  7571. {
  7572. IR::Instr *&instr = *pInstr;
  7573. int32 min1 = INT32_MIN, max1 = INT32_MAX, min2 = INT32_MIN, max2 = INT32_MAX, newMin, newMax, tmp;
  7574. Js::OpCode opcode;
  7575. Value *&src1Val = *pSrc1Val;
  7576. Value *&src2Val = *pSrc2Val;
  7577. // We don't need to do typespec for asmjs
  7578. if (IsTypeSpecPhaseOff(this->func) || GetIsAsmJSFunc())
  7579. {
  7580. return false;
  7581. }
  7582. if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  7583. {
  7584. this->TypeSpecializeInlineBuiltInBinary(pInstr, src1Val, src2Val, pDstVal, src1OriginalVal, src2OriginalVal);
  7585. return true;
  7586. }
  7587. if (src1Val)
  7588. {
  7589. src1Val->GetValueInfo()->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec());
  7590. }
  7591. if (src2Val)
  7592. {
  7593. src2Val->GetValueInfo()->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec());
  7594. }
  7595. // Type specialize binary operators to int32
  7596. bool src1Lossy = true;
  7597. bool src2Lossy = true;
  7598. IR::BailOutKind bailOutKind = IR::BailOutInvalid;
  7599. bool ignoredIntOverflow = this->ignoredIntOverflowForCurrentInstr;
  7600. bool ignoredNegativeZero = false;
  7601. bool skipSrc2 = false;
  7602. bool skipDst = false;
  7603. bool needsBoolConv = false;
  7604. AddSubConstantInfo addSubConstantInfo;
  7605. switch (instr->m_opcode)
  7606. {
  7607. case Js::OpCode::Or_A:
  7608. if (!DoLossyIntTypeSpec())
  7609. {
  7610. return false;
  7611. }
  7612. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7613. opcode = Js::OpCode::Or_I4;
  7614. break;
  7615. case Js::OpCode::And_A:
  7616. if (!DoLossyIntTypeSpec())
  7617. {
  7618. return false;
  7619. }
  7620. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7621. opcode = Js::OpCode::And_I4;
  7622. break;
  7623. case Js::OpCode::Xor_A:
  7624. if (!DoLossyIntTypeSpec())
  7625. {
  7626. return false;
  7627. }
  7628. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7629. opcode = Js::OpCode::Xor_I4;
  7630. break;
  7631. case Js::OpCode::Shl_A:
  7632. if (!DoLossyIntTypeSpec())
  7633. {
  7634. return false;
  7635. }
  7636. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7637. opcode = Js::OpCode::Shl_I4;
  7638. break;
  7639. case Js::OpCode::Shr_A:
  7640. if (!DoLossyIntTypeSpec())
  7641. {
  7642. return false;
  7643. }
  7644. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7645. opcode = Js::OpCode::Shr_I4;
  7646. break;
  7647. case Js::OpCode::ShrU_A:
  7648. if (!DoLossyIntTypeSpec())
  7649. {
  7650. return false;
  7651. }
  7652. if (min1 < 0 && IntConstantBounds(min2, max2).And_0x1f().Contains(0))
  7653. {
  7654. // Src1 may be too large to represent as a signed int32, and src2 may be zero. Unless the resulting value is only
  7655. // used as a signed int32 (hence allowing us to ignore the result's sign), don't specialize the instruction.
  7656. if (!instr->ignoreIntOverflow)
  7657. return false;
  7658. ignoredIntOverflow = true;
  7659. }
  7660. this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax);
  7661. opcode = Js::OpCode::ShrU_I4;
  7662. break;
  7663. case Js::OpCode::BrUnLe_A:
  7664. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7665. // int already, so require that both sources are likely int for folding.
  7666. if (DoConstFold() &&
  7667. !IsLoopPrePass() &&
  7668. TryOptConstFoldBrUnsignedGreaterThan(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  7669. {
  7670. return true;
  7671. }
  7672. if (min1 >= 0 && min2 >= 0)
  7673. {
  7674. // Only handle positive values since this is unsigned...
  7675. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7676. // (INT32_MIN, INT32_MAX), so we're good.
  7677. Assert(src1Val);
  7678. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7679. Assert(src2Val);
  7680. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7681. UpdateIntBoundsForLessThanOrEqualBranch(src1Val, src2Val);
  7682. }
  7683. if (!DoLossyIntTypeSpec())
  7684. {
  7685. return false;
  7686. }
  7687. newMin = newMax = 0;
  7688. opcode = Js::OpCode::BrUnLe_I4;
  7689. break;
  7690. case Js::OpCode::BrUnLt_A:
  7691. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7692. // int already, so require that both sources are likely int for folding.
  7693. if (DoConstFold() &&
  7694. !IsLoopPrePass() &&
  7695. TryOptConstFoldBrUnsignedLessThan(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  7696. {
  7697. return true;
  7698. }
  7699. if (min1 >= 0 && min2 >= 0)
  7700. {
  7701. // Only handle positive values since this is unsigned...
  7702. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7703. // (INT32_MIN, INT32_MAX), so we're good.
  7704. Assert(src1Val);
  7705. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7706. Assert(src2Val);
  7707. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7708. UpdateIntBoundsForLessThanBranch(src1Val, src2Val);
  7709. }
  7710. if (!DoLossyIntTypeSpec())
  7711. {
  7712. return false;
  7713. }
  7714. newMin = newMax = 0;
  7715. opcode = Js::OpCode::BrUnLt_I4;
  7716. break;
  7717. case Js::OpCode::BrUnGe_A:
  7718. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7719. // int already, so require that both sources are likely int for folding.
  7720. if (DoConstFold() &&
  7721. !IsLoopPrePass() &&
  7722. TryOptConstFoldBrUnsignedLessThan(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  7723. {
  7724. return true;
  7725. }
  7726. if (min1 >= 0 && min2 >= 0)
  7727. {
  7728. // Only handle positive values since this is unsigned...
  7729. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7730. // (INT32_MIN, INT32_MAX), so we're good.
  7731. Assert(src1Val);
  7732. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7733. Assert(src2Val);
  7734. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7735. UpdateIntBoundsForGreaterThanOrEqualBranch(src1Val, src2Val);
  7736. }
  7737. if (!DoLossyIntTypeSpec())
  7738. {
  7739. return false;
  7740. }
  7741. newMin = newMax = 0;
  7742. opcode = Js::OpCode::BrUnGe_I4;
  7743. break;
  7744. case Js::OpCode::BrUnGt_A:
  7745. // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely
  7746. // int already, so require that both sources are likely int for folding.
  7747. if (DoConstFold() &&
  7748. !IsLoopPrePass() &&
  7749. TryOptConstFoldBrUnsignedGreaterThan(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  7750. {
  7751. return true;
  7752. }
  7753. if (min1 >= 0 && min2 >= 0)
  7754. {
  7755. // Only handle positive values since this is unsigned...
  7756. // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults
  7757. // (INT32_MIN, INT32_MAX), so we're good.
  7758. Assert(src1Val);
  7759. Assert(src1Val->GetValueInfo()->IsLikelyInt());
  7760. Assert(src2Val);
  7761. Assert(src2Val->GetValueInfo()->IsLikelyInt());
  7762. UpdateIntBoundsForGreaterThanBranch(src1Val, src2Val);
  7763. }
  7764. if (!DoLossyIntTypeSpec())
  7765. {
  7766. return false;
  7767. }
  7768. newMin = newMax = 0;
  7769. opcode = Js::OpCode::BrUnGt_I4;
  7770. break;
  7771. case Js::OpCode::CmUnLe_A:
  7772. if (!DoLossyIntTypeSpec())
  7773. {
  7774. return false;
  7775. }
  7776. newMin = 0;
  7777. newMax = 1;
  7778. opcode = Js::OpCode::CmUnLe_I4;
  7779. needsBoolConv = true;
  7780. break;
  7781. case Js::OpCode::CmUnLt_A:
  7782. if (!DoLossyIntTypeSpec())
  7783. {
  7784. return false;
  7785. }
  7786. newMin = 0;
  7787. newMax = 1;
  7788. opcode = Js::OpCode::CmUnLt_I4;
  7789. needsBoolConv = true;
  7790. break;
  7791. case Js::OpCode::CmUnGe_A:
  7792. if (!DoLossyIntTypeSpec())
  7793. {
  7794. return false;
  7795. }
  7796. newMin = 0;
  7797. newMax = 1;
  7798. opcode = Js::OpCode::CmUnGe_I4;
  7799. needsBoolConv = true;
  7800. break;
  7801. case Js::OpCode::CmUnGt_A:
  7802. if (!DoLossyIntTypeSpec())
  7803. {
  7804. return false;
  7805. }
  7806. newMin = 0;
  7807. newMax = 1;
  7808. opcode = Js::OpCode::CmUnGt_I4;
  7809. needsBoolConv = true;
  7810. break;
  7811. case Js::OpCode::Expo_A:
  7812. {
  7813. src1Val = src1OriginalVal;
  7814. src2Val = src2OriginalVal;
  7815. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  7816. }
  7817. case Js::OpCode::Div_A:
  7818. {
  7819. ValueType specializedValueType = GetDivValueType(instr, src1Val, src2Val, true);
  7820. if (specializedValueType.IsFloat())
  7821. {
  7822. // Either result is float or 1/x or cst1/cst2 where cst1%cst2 != 0
  7823. // Note: We should really constant fold cst1%cst2...
  7824. src1Val = src1OriginalVal;
  7825. src2Val = src2OriginalVal;
  7826. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  7827. }
  7828. #ifdef _M_ARM
  7829. if (!AutoSystemInfo::Data.ArmDivAvailable())
  7830. {
  7831. return false;
  7832. }
  7833. #endif
  7834. if (specializedValueType.IsInt())
  7835. {
  7836. if (max2 == 0x80000000 || (min2 == 0 && max2 == 00))
  7837. {
  7838. return false;
  7839. }
  7840. if (min1 == 0x80000000 && min2 <= -1 && max2 >= -1)
  7841. {
  7842. // Prevent integer overflow, as div by zero or MIN_INT / -1 will throw an exception
  7843. // Or we know we are dividing by zero (which is weird to have because the profile data
  7844. // say we got an int)
  7845. bailOutKind = IR::BailOutOnDivOfMinInt;
  7846. }
  7847. src1Lossy = false; // Detect -0 on the sources
  7848. src2Lossy = false;
  7849. opcode = Js::OpCode::Div_I4;
  7850. Assert(!instr->GetSrc1()->IsUnsigned());
  7851. bailOutKind |= IR::BailOnDivResultNotInt;
  7852. if (max2 >= 0 && min2 <= 0)
  7853. {
  7854. // Need to check for divide by zero if the denominator range includes 0
  7855. bailOutKind |= IR::BailOutOnDivByZero;
  7856. }
  7857. if (max1 >= 0 && min1 <= 0)
  7858. {
  7859. // Numerator contains 0 so the result contains 0
  7860. newMin = 0;
  7861. newMax = 0;
  7862. if (min2 < 0)
  7863. {
  7864. // Denominator may be negative, so the result could be negative 0
  7865. if (instr->ShouldCheckForNegativeZero())
  7866. {
  7867. bailOutKind |= IR::BailOutOnNegativeZero;
  7868. }
  7869. else
  7870. {
  7871. ignoredNegativeZero = true;
  7872. }
  7873. }
  7874. }
  7875. else
  7876. {
  7877. // Initialize to invalid value, one of the condition below will update it correctly
  7878. newMin = INT_MAX;
  7879. newMax = INT_MIN;
  7880. }
  7881. // Deal with the positive and negative range separately for both the numerator and the denominator,
  7882. // and integrate to the overall min and max.
  7883. // If the result is positive (positive/positive or negative/negative):
  7884. // The min should be the smallest magnitude numerator (positive_Min1 | negative_Max1)
  7885. // divided by ---------------------------------------------------------------
  7886. // largest magnitude denominator (positive_Max2 | negative_Min2)
  7887. //
  7888. // The max should be the largest magnitude numerator (positive_Max1 | negative_Max1)
  7889. // divided by ---------------------------------------------------------------
  7890. // smallest magnitude denominator (positive_Min2 | negative_Max2)
  7891. // If the result is negative (positive/negative or positive/negative):
  7892. // The min should be the largest magnitude numerator (positive_Max1 | negative_Min1)
  7893. // divided by ---------------------------------------------------------------
  7894. // smallest magnitude denominator (negative_Max2 | positive_Min2)
  7895. //
  7896. // The max should be the smallest magnitude numerator (positive_Min1 | negative_Max1)
  7897. // divided by ---------------------------------------------------------------
  7898. // largest magnitude denominator (negative_Min2 | positive_Max2)
  7899. // Consider: The range can be slightly more precise if we take care of the rounding
  7900. if (max1 > 0)
  7901. {
  7902. // Take only the positive numerator range
  7903. int32 positive_Min1 = max(1, min1);
  7904. int32 positive_Max1 = max1;
  7905. if (max2 > 0)
  7906. {
  7907. // Take only the positive denominator range
  7908. int32 positive_Min2 = max(1, min2);
  7909. int32 positive_Max2 = max2;
  7910. // Positive / Positive
  7911. int32 quadrant1_Min = positive_Min1 <= positive_Max2? 1 : positive_Min1 / positive_Max2;
  7912. int32 quadrant1_Max = positive_Max1 <= positive_Min2? 1 : positive_Max1 / positive_Min2;
  7913. Assert(1 <= quadrant1_Min && quadrant1_Min <= quadrant1_Max);
  7914. // The result should positive
  7915. newMin = min(newMin, quadrant1_Min);
  7916. newMax = max(newMax, quadrant1_Max);
  7917. }
  7918. if (min2 < 0)
  7919. {
  7920. // Take only the negative denominator range
  7921. int32 negative_Min2 = min2;
  7922. int32 negative_Max2 = min(-1, max2);
  7923. // Positive / Negative
  7924. int32 quadrant2_Min = -positive_Max1 >= negative_Max2? -1 : positive_Max1 / negative_Max2;
  7925. int32 quadrant2_Max = -positive_Min1 >= negative_Min2? -1 : positive_Min1 / negative_Min2;
  7926. // The result should negative
  7927. Assert(quadrant2_Min <= quadrant2_Max && quadrant2_Max <= -1);
  7928. newMin = min(newMin, quadrant2_Min);
  7929. newMax = max(newMax, quadrant2_Max);
  7930. }
  7931. }
  7932. if (min1 < 0)
  7933. {
  7934. // Take only the native numerator range
  7935. int32 negative_Min1 = min1;
  7936. int32 negative_Max1 = min(-1, max1);
  7937. if (max2 > 0)
  7938. {
  7939. // Take only the positive denominator range
  7940. int32 positive_Min2 = max(1, min2);
  7941. int32 positive_Max2 = max2;
  7942. // Negative / Positive
  7943. int32 quadrant4_Min = negative_Min1 >= -positive_Min2? -1 : negative_Min1 / positive_Min2;
  7944. int32 quadrant4_Max = negative_Max1 >= -positive_Max2? -1 : negative_Max1 / positive_Max2;
  7945. // The result should negative
  7946. Assert(quadrant4_Min <= quadrant4_Max && quadrant4_Max <= -1);
  7947. newMin = min(newMin, quadrant4_Min);
  7948. newMax = max(newMax, quadrant4_Max);
  7949. }
  7950. if (min2 < 0)
  7951. {
  7952. // Take only the negative denominator range
  7953. int32 negative_Min2 = min2;
  7954. int32 negative_Max2 = min(-1, max2);
  7955. int32 quadrant3_Min;
  7956. int32 quadrant3_Max;
  7957. // Negative / Negative
  7958. if (negative_Max1 == 0x80000000 && negative_Min2 == -1)
  7959. {
  7960. quadrant3_Min = negative_Max1 >= negative_Min2? 1 : (negative_Max1+1) / negative_Min2;
  7961. }
  7962. else
  7963. {
  7964. quadrant3_Min = negative_Max1 >= negative_Min2? 1 : negative_Max1 / negative_Min2;
  7965. }
  7966. if (negative_Min1 == 0x80000000 && negative_Max2 == -1)
  7967. {
  7968. quadrant3_Max = negative_Min1 >= negative_Max2? 1 : (negative_Min1+1) / negative_Max2;
  7969. }
  7970. else
  7971. {
  7972. quadrant3_Max = negative_Min1 >= negative_Max2? 1 : negative_Min1 / negative_Max2;
  7973. }
  7974. // The result should positive
  7975. Assert(1 <= quadrant3_Min && quadrant3_Min <= quadrant3_Max);
  7976. newMin = min(newMin, quadrant3_Min);
  7977. newMax = max(newMax, quadrant3_Max);
  7978. }
  7979. }
  7980. Assert(newMin <= newMax);
  7981. // Continue to int type spec
  7982. break;
  7983. }
  7984. }
  7985. // fall-through
  7986. default:
  7987. {
  7988. const bool involesLargeInt32 =
  7989. (src1Val && src1Val->GetValueInfo()->IsLikelyUntaggedInt()) ||
  7990. (src2Val && src2Val->GetValueInfo()->IsLikelyUntaggedInt());
  7991. const auto trySpecializeToFloat =
  7992. [&](const bool mayOverflow) -> bool
  7993. {
  7994. // It has been determined that this instruction cannot be int-specialized. Need to determine whether to attempt
  7995. // to float-specialize the instruction, or leave it unspecialized.
  7996. if((involesLargeInt32
  7997. #if INT32VAR
  7998. && mayOverflow
  7999. #endif
  8000. ) || (instr->m_opcode == Js::OpCode::Mul_A && !this->DoAggressiveMulIntTypeSpec())
  8001. )
  8002. {
  8003. // An input range is completely outside the range of an int31 and the operation is likely to overflow.
  8004. // Additionally, on 32-bit platforms, the value is untaggable and will be a JavascriptNumber, which is
  8005. // significantly slower to use in an unspecialized operation compared to a tagged int. So, try to
  8006. // float-specialize the instruction.
  8007. src1Val = src1OriginalVal;
  8008. src2Val = src2OriginalVal;
  8009. return TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  8010. }
  8011. return false;
  8012. };
  8013. if (instr->m_opcode != Js::OpCode::ArgOut_A_InlineBuiltIn)
  8014. {
  8015. if ((src1Val && src1Val->GetValueInfo()->IsLikelyFloat()) || (src2Val && src2Val->GetValueInfo()->IsLikelyFloat()))
  8016. {
  8017. // Try to type specialize to float
  8018. src1Val = src1OriginalVal;
  8019. src2Val = src2OriginalVal;
  8020. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  8021. }
  8022. if (src1Val == nullptr ||
  8023. src2Val == nullptr ||
  8024. !src1Val->GetValueInfo()->IsLikelyInt() ||
  8025. !src2Val->GetValueInfo()->IsLikelyInt() ||
  8026. (
  8027. !DoAggressiveIntTypeSpec() &&
  8028. (
  8029. !(src1Val->GetValueInfo()->IsInt() || CurrentBlockData()->IsSwitchInt32TypeSpecialized(instr)) ||
  8030. !src2Val->GetValueInfo()->IsInt()
  8031. )
  8032. ) ||
  8033. (instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber) ||
  8034. (instr->GetSrc2()->IsRegOpnd() && instr->GetSrc2()->AsRegOpnd()->m_sym->m_isNotNumber))
  8035. {
  8036. return trySpecializeToFloat(true);
  8037. }
  8038. }
  8039. // Try to type specialize to int32
  8040. // If one of the values is a float constant with a value that fits in a uint32 but not an int32,
  8041. // and the instruction can ignore int overflow, the source value for the purposes of int specialization
  8042. // would have been changed to an int constant value by ignoring overflow. But, the conversion is still lossy.
  8043. if (!(src1OriginalVal && src1OriginalVal->GetValueInfo()->IsFloatConstant() && src1Val && src1Val->GetValueInfo()->HasIntConstantValue()))
  8044. {
  8045. src1Lossy = false;
  8046. }
  8047. if (!(src2OriginalVal && src2OriginalVal->GetValueInfo()->IsFloatConstant() && src2Val && src2Val->GetValueInfo()->HasIntConstantValue()))
  8048. {
  8049. src2Lossy = false;
  8050. }
  8051. switch(instr->m_opcode)
  8052. {
  8053. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  8054. // If the src is already type-specialized, if we don't type-specialize ArgOut_A_InlineBuiltIn instr, we'll get additional ToVar.
  8055. // So, to avoid that, type-specialize the ArgOut_A_InlineBuiltIn instr.
  8056. // Else we don't need to type-specialize the instr, we are fine with src being Var.
  8057. if (instr->GetSrc1()->IsRegOpnd())
  8058. {
  8059. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  8060. if (CurrentBlockData()->IsInt32TypeSpecialized(sym))
  8061. {
  8062. opcode = instr->m_opcode;
  8063. skipDst = true; // We should keep dst as is, otherwise the link opnd for next ArgOut/InlineBuiltInStart would be broken.
  8064. skipSrc2 = true; // src2 is linkOpnd. We don't need to type-specialize it.
  8065. newMin = min1; newMax = max1; // Values don't matter, these are unused.
  8066. goto LOutsideSwitch; // Continue to int-type-specialize.
  8067. }
  8068. else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  8069. {
  8070. src1Val = src1OriginalVal;
  8071. src2Val = src2OriginalVal;
  8072. return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal);
  8073. }
  8074. }
  8075. return false;
  8076. case Js::OpCode::Add_A:
  8077. do // while(false)
  8078. {
  8079. const auto CannotOverflowBasedOnRelativeBounds = [&](int32 *const constantValueRef)
  8080. {
  8081. Assert(constantValueRef);
  8082. if(min2 == max2 &&
  8083. src1Val->GetValueInfo()->IsIntBounded() &&
  8084. src1Val->GetValueInfo()->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(min2))
  8085. {
  8086. *constantValueRef = min2;
  8087. return true;
  8088. }
  8089. else if(
  8090. min1 == max1 &&
  8091. src2Val->GetValueInfo()->IsIntBounded() &&
  8092. src2Val->GetValueInfo()->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(min1))
  8093. {
  8094. *constantValueRef = min1;
  8095. return true;
  8096. }
  8097. return false;
  8098. };
  8099. if (Int32Math::Add(min1, min2, &newMin))
  8100. {
  8101. int32 constantSrcValue;
  8102. if(CannotOverflowBasedOnRelativeBounds(&constantSrcValue))
  8103. {
  8104. newMin = constantSrcValue >= 0 ? INT32_MAX : INT32_MIN;
  8105. }
  8106. else if(instr->ShouldCheckForIntOverflow())
  8107. {
  8108. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8109. {
  8110. // May overflow
  8111. return trySpecializeToFloat(true);
  8112. }
  8113. bailOutKind |= IR::BailOutOnOverflow;
  8114. newMin = min1 < 0 ? INT32_MIN : INT32_MAX;
  8115. }
  8116. else
  8117. {
  8118. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since
  8119. // overflow causes the value to wrap around, and we don't have a way to specify a lower and upper
  8120. // range of ints, we use the full range of int32s.
  8121. ignoredIntOverflow = true;
  8122. newMin = INT32_MIN;
  8123. newMax = INT32_MAX;
  8124. break;
  8125. }
  8126. }
  8127. if (Int32Math::Add(max1, max2, &newMax))
  8128. {
  8129. int32 constantSrcValue;
  8130. if(CannotOverflowBasedOnRelativeBounds(&constantSrcValue))
  8131. {
  8132. newMax = constantSrcValue >= 0 ? INT32_MAX : INT32_MIN;
  8133. }
  8134. else if(instr->ShouldCheckForIntOverflow())
  8135. {
  8136. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8137. {
  8138. // May overflow
  8139. return trySpecializeToFloat(true);
  8140. }
  8141. bailOutKind |= IR::BailOutOnOverflow;
  8142. newMax = max1 < 0 ? INT32_MIN : INT32_MAX;
  8143. }
  8144. else
  8145. {
  8146. // See comment about ignoring overflow above
  8147. ignoredIntOverflow = true;
  8148. newMin = INT32_MIN;
  8149. newMax = INT32_MAX;
  8150. break;
  8151. }
  8152. }
  8153. if(bailOutKind & IR::BailOutOnOverflow)
  8154. {
  8155. Assert(bailOutKind == IR::BailOutOnOverflow);
  8156. Assert(instr->ShouldCheckForIntOverflow());
  8157. int32 temp;
  8158. if(Int32Math::Add(
  8159. Int32Math::NearestInRangeTo(0, min1, max1),
  8160. Int32Math::NearestInRangeTo(0, min2, max2),
  8161. &temp))
  8162. {
  8163. // Always overflows
  8164. return trySpecializeToFloat(true);
  8165. }
  8166. }
  8167. } while(false);
  8168. if (!this->IsLoopPrePass() && newMin == newMax && bailOutKind == IR::BailOutInvalid)
  8169. {
  8170. // Take care of Add with zero here, since we know we're dealing with 2 numbers.
  8171. this->CaptureByteCodeSymUses(instr);
  8172. IR::Opnd *src;
  8173. bool isAddZero = true;
  8174. int32 intConstantValue;
  8175. if (src1Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) && intConstantValue == 0)
  8176. {
  8177. src = instr->UnlinkSrc2();
  8178. instr->FreeSrc1();
  8179. }
  8180. else if (src2Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) && intConstantValue == 0)
  8181. {
  8182. src = instr->UnlinkSrc1();
  8183. instr->FreeSrc2();
  8184. }
  8185. else
  8186. {
  8187. // This should have been handled by const folding, unless:
  8188. // - A source's value was substituted with a different value here, which is after const folding happened
  8189. // - A value is not definitely int, but once converted to definite int, it would be zero due to a
  8190. // condition in the source code such as if(a === 0). Ideally, we would specialize the sources and
  8191. // remove the add, but doesn't seem too important for now.
  8192. Assert(
  8193. !DoConstFold() ||
  8194. src1Val != src1OriginalVal ||
  8195. src2Val != src2OriginalVal ||
  8196. !src1Val->GetValueInfo()->IsInt() ||
  8197. !src2Val->GetValueInfo()->IsInt());
  8198. isAddZero = false;
  8199. src = nullptr;
  8200. }
  8201. if (isAddZero)
  8202. {
  8203. IR::Instr *newInstr = IR::Instr::New(Js::OpCode::Ld_A, instr->UnlinkDst(), src, instr->m_func);
  8204. newInstr->SetByteCodeOffset(instr);
  8205. instr->m_opcode = Js::OpCode::Nop;
  8206. this->currentBlock->InsertInstrAfter(newInstr, instr);
  8207. return true;
  8208. }
  8209. }
  8210. if(!ignoredIntOverflow)
  8211. {
  8212. if(min2 == max2 &&
  8213. (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val)) &&
  8214. instr->GetSrc1()->IsRegOpnd())
  8215. {
  8216. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min1 == max1, min2);
  8217. }
  8218. else if(
  8219. min1 == max1 &&
  8220. (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Val)) &&
  8221. instr->GetSrc2()->IsRegOpnd())
  8222. {
  8223. addSubConstantInfo.Set(instr->GetSrc2()->AsRegOpnd()->m_sym, src2Val, min2 == max2, min1);
  8224. }
  8225. }
  8226. opcode = Js::OpCode::Add_I4;
  8227. break;
  8228. case Js::OpCode::Sub_A:
  8229. do // while(false)
  8230. {
  8231. const auto CannotOverflowBasedOnRelativeBounds = [&]()
  8232. {
  8233. return
  8234. min2 == max2 &&
  8235. src1Val->GetValueInfo()->IsIntBounded() &&
  8236. src1Val->GetValueInfo()->AsIntBounded()->Bounds()->SubCannotOverflowBasedOnRelativeBounds(min2);
  8237. };
  8238. if (Int32Math::Sub(min1, max2, &newMin))
  8239. {
  8240. if(CannotOverflowBasedOnRelativeBounds())
  8241. {
  8242. Assert(min2 == max2);
  8243. newMin = min2 >= 0 ? INT32_MIN : INT32_MAX;
  8244. }
  8245. else if(instr->ShouldCheckForIntOverflow())
  8246. {
  8247. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8248. {
  8249. // May overflow
  8250. return trySpecializeToFloat(true);
  8251. }
  8252. bailOutKind |= IR::BailOutOnOverflow;
  8253. newMin = min1 < 0 ? INT32_MIN : INT32_MAX;
  8254. }
  8255. else
  8256. {
  8257. // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow
  8258. // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints,
  8259. // we use the full range of int32s.
  8260. ignoredIntOverflow = true;
  8261. newMin = INT32_MIN;
  8262. newMax = INT32_MAX;
  8263. break;
  8264. }
  8265. }
  8266. if (Int32Math::Sub(max1, min2, &newMax))
  8267. {
  8268. if(CannotOverflowBasedOnRelativeBounds())
  8269. {
  8270. Assert(min2 == max2);
  8271. newMax = min2 >= 0 ? INT32_MIN: INT32_MAX;
  8272. }
  8273. else if(instr->ShouldCheckForIntOverflow())
  8274. {
  8275. if(involesLargeInt32 || !DoAggressiveIntTypeSpec())
  8276. {
  8277. // May overflow
  8278. return trySpecializeToFloat(true);
  8279. }
  8280. bailOutKind |= IR::BailOutOnOverflow;
  8281. newMax = max1 < 0 ? INT32_MIN : INT32_MAX;
  8282. }
  8283. else
  8284. {
  8285. // See comment about ignoring overflow above
  8286. ignoredIntOverflow = true;
  8287. newMin = INT32_MIN;
  8288. newMax = INT32_MAX;
  8289. break;
  8290. }
  8291. }
  8292. if(bailOutKind & IR::BailOutOnOverflow)
  8293. {
  8294. Assert(bailOutKind == IR::BailOutOnOverflow);
  8295. Assert(instr->ShouldCheckForIntOverflow());
  8296. int32 temp;
  8297. if(Int32Math::Sub(
  8298. Int32Math::NearestInRangeTo(-1, min1, max1),
  8299. Int32Math::NearestInRangeTo(0, min2, max2),
  8300. &temp))
  8301. {
  8302. // Always overflows
  8303. return trySpecializeToFloat(true);
  8304. }
  8305. }
  8306. } while(false);
  8307. if(!ignoredIntOverflow &&
  8308. min2 == max2 &&
  8309. min2 != INT32_MIN &&
  8310. (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val)) &&
  8311. instr->GetSrc1()->IsRegOpnd())
  8312. {
  8313. addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min1 == max1, -min2);
  8314. }
  8315. opcode = Js::OpCode::Sub_I4;
  8316. break;
  8317. case Js::OpCode::Mul_A:
  8318. {
  8319. bool isConservativeMulInt = !DoAggressiveMulIntTypeSpec() || !DoAggressiveIntTypeSpec();
  8320. // Be conservative about predicting Mul overflow in prepass.
  8321. // Operands that are live on back edge may be denied lossless-conversion to int32 and
  8322. // trigger rejit with AggressiveIntTypeSpec off.
  8323. // Besides multiplying a variable in a loop can overflow in just a few iterations even in simple cases like v *= 2
  8324. // So, make sure we definitely know the source max/min values, otherwise assume the full range.
  8325. if (isConservativeMulInt && IsLoopPrePass())
  8326. {
  8327. if (!IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Val))
  8328. {
  8329. max1 = INT32_MAX;
  8330. min1 = INT32_MIN;
  8331. }
  8332. if (!IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val))
  8333. {
  8334. max2 = INT32_MAX;
  8335. min2 = INT32_MIN;
  8336. }
  8337. }
  8338. if (Int32Math::Mul(min1, min2, &newMin))
  8339. {
  8340. if (involesLargeInt32 || isConservativeMulInt)
  8341. {
  8342. // May overflow
  8343. return trySpecializeToFloat(true);
  8344. }
  8345. bailOutKind |= IR::BailOutOnMulOverflow;
  8346. newMin = (min1 < 0) ^ (min2 < 0) ? INT32_MIN : INT32_MAX;
  8347. }
  8348. newMax = newMin;
  8349. if (Int32Math::Mul(max1, max2, &tmp))
  8350. {
  8351. if (involesLargeInt32 || isConservativeMulInt)
  8352. {
  8353. // May overflow
  8354. return trySpecializeToFloat(true);
  8355. }
  8356. bailOutKind |= IR::BailOutOnMulOverflow;
  8357. tmp = (max1 < 0) ^ (max2 < 0) ? INT32_MIN : INT32_MAX;
  8358. }
  8359. newMin = min(newMin, tmp);
  8360. newMax = max(newMax, tmp);
  8361. if (Int32Math::Mul(min1, max2, &tmp))
  8362. {
  8363. if (involesLargeInt32 || isConservativeMulInt)
  8364. {
  8365. // May overflow
  8366. return trySpecializeToFloat(true);
  8367. }
  8368. bailOutKind |= IR::BailOutOnMulOverflow;
  8369. tmp = (min1 < 0) ^ (max2 < 0) ? INT32_MIN : INT32_MAX;
  8370. }
  8371. newMin = min(newMin, tmp);
  8372. newMax = max(newMax, tmp);
  8373. if (Int32Math::Mul(max1, min2, &tmp))
  8374. {
  8375. if (involesLargeInt32 || isConservativeMulInt)
  8376. {
  8377. // May overflow
  8378. return trySpecializeToFloat(true);
  8379. }
  8380. bailOutKind |= IR::BailOutOnMulOverflow;
  8381. tmp = (max1 < 0) ^ (min2 < 0) ? INT32_MIN : INT32_MAX;
  8382. }
  8383. newMin = min(newMin, tmp);
  8384. newMax = max(newMax, tmp);
  8385. if (bailOutKind & IR::BailOutOnMulOverflow)
  8386. {
  8387. // CSE only if two MULs have the same overflow check behavior.
  8388. // Currently this is set to be ignore int32 overflow, but not 53-bit, or int32 overflow matters.
  8389. if (!instr->ShouldCheckFor32BitOverflow() && instr->ShouldCheckForNon32BitOverflow())
  8390. {
  8391. // If we allow int to overflow then there can be anything in the resulting int
  8392. newMin = IntConstMin;
  8393. newMax = IntConstMax;
  8394. ignoredIntOverflow = true;
  8395. }
  8396. int32 temp, overflowValue;
  8397. if (Int32Math::Mul(
  8398. Int32Math::NearestInRangeTo(0, min1, max1),
  8399. Int32Math::NearestInRangeTo(0, min2, max2),
  8400. &temp,
  8401. &overflowValue))
  8402. {
  8403. Assert(instr->ignoreOverflowBitCount >= 32);
  8404. int overflowMatters = 64 - instr->ignoreOverflowBitCount;
  8405. if (!ignoredIntOverflow ||
  8406. // Use shift to check high bits in case its negative
  8407. ((overflowValue << overflowMatters) >> overflowMatters) != overflowValue
  8408. )
  8409. {
  8410. // Always overflows
  8411. return trySpecializeToFloat(true);
  8412. }
  8413. }
  8414. }
  8415. if (newMin <= 0 && newMax >= 0 && // New range crosses zero
  8416. (min1 < 0 || min2 < 0) && // An operand's range contains a negative integer
  8417. !(min1 > 0 || min2 > 0) && // Neither operand's range contains only positive integers
  8418. !instr->GetSrc1()->IsEqual(instr->GetSrc2())) // The operands don't have the same value
  8419. {
  8420. if (instr->ShouldCheckForNegativeZero())
  8421. {
  8422. // -0 matters since the sym is not a local, or is used in a way in which -0 would differ from +0
  8423. if (!DoAggressiveIntTypeSpec())
  8424. {
  8425. // May result in -0
  8426. return trySpecializeToFloat(false);
  8427. }
  8428. if (((min1 == 0 && max1 == 0) || (min2 == 0 && max2 == 0)) && (max1 < 0 || max2 < 0))
  8429. {
  8430. // Always results in -0
  8431. return trySpecializeToFloat(false);
  8432. }
  8433. bailOutKind |= IR::BailOutOnNegativeZero;
  8434. }
  8435. else
  8436. {
  8437. ignoredNegativeZero = true;
  8438. }
  8439. }
  8440. opcode = Js::OpCode::Mul_I4;
  8441. break;
  8442. }
  8443. case Js::OpCode::Rem_A:
  8444. {
  8445. IR::Opnd* src2 = instr->GetSrc2();
  8446. if (!this->IsLoopPrePass() && min2 == max2 && min1 >= 0)
  8447. {
  8448. int32 value = min2;
  8449. if (value == (1 << Math::Log2(value)) && src2->IsAddrOpnd())
  8450. {
  8451. Assert(src2->AsAddrOpnd()->IsVar());
  8452. instr->m_opcode = Js::OpCode::And_A;
  8453. src2->AsAddrOpnd()->SetAddress(Js::TaggedInt::ToVarUnchecked(value - 1),
  8454. IR::AddrOpndKindConstantVar);
  8455. *pSrc2Val = GetIntConstantValue(value - 1, instr);
  8456. src2Val = *pSrc2Val;
  8457. return this->TypeSpecializeBinary(&instr, pSrc1Val, pSrc2Val, pDstVal, src1OriginalVal, src2Val, redoTypeSpecRef);
  8458. }
  8459. }
  8460. #ifdef _M_ARM
  8461. if (!AutoSystemInfo::Data.ArmDivAvailable())
  8462. {
  8463. return false;
  8464. }
  8465. #endif
  8466. if (min1 < 0)
  8467. {
  8468. // The most negative it can be is min1, unless limited by min2/max2
  8469. int32 negMaxAbs2;
  8470. if (min2 == INT32_MIN)
  8471. {
  8472. negMaxAbs2 = INT32_MIN;
  8473. }
  8474. else
  8475. {
  8476. negMaxAbs2 = -max(abs(min2), abs(max2)) + 1;
  8477. }
  8478. newMin = max(min1, negMaxAbs2);
  8479. }
  8480. else
  8481. {
  8482. newMin = 0;
  8483. }
  8484. bool isModByPowerOf2 = (instr->IsProfiledInstr() && instr->m_func->HasProfileInfo() &&
  8485. instr->m_func->GetReadOnlyProfileInfo()->IsModulusOpByPowerOf2(static_cast<Js::ProfileId>(instr->AsProfiledInstr()->u.profileId)));
  8486. if(isModByPowerOf2)
  8487. {
  8488. Assert(bailOutKind == IR::BailOutInvalid);
  8489. bailOutKind = IR::BailOnModByPowerOf2;
  8490. newMin = 0;
  8491. }
  8492. else
  8493. {
  8494. if (min2 <= 0 && max2 >= 0)
  8495. {
  8496. // Consider: We could handle the zero case with a check and bailout...
  8497. return false;
  8498. }
  8499. if (min1 == 0x80000000 && (min2 <= -1 && max2 >= -1))
  8500. {
  8501. // Prevent integer overflow, as div by zero or MIN_INT / -1 will throw an exception
  8502. return false;
  8503. }
  8504. if (min1 < 0)
  8505. {
  8506. if(instr->ShouldCheckForNegativeZero())
  8507. {
  8508. if (!DoAggressiveIntTypeSpec())
  8509. {
  8510. return false;
  8511. }
  8512. bailOutKind |= IR::BailOutOnNegativeZero;
  8513. }
  8514. else
  8515. {
  8516. ignoredNegativeZero = true;
  8517. }
  8518. }
  8519. }
  8520. {
  8521. int32 absMax2;
  8522. if (min2 == INT32_MIN)
  8523. {
  8524. // abs(INT32_MIN) == INT32_MAX because of overflow
  8525. absMax2 = INT32_MAX;
  8526. }
  8527. else
  8528. {
  8529. absMax2 = max(abs(min2), abs(max2)) - 1;
  8530. }
  8531. newMax = min(absMax2, max(max1, 0));
  8532. newMax = max(newMin, newMax);
  8533. }
  8534. opcode = Js::OpCode::Rem_I4;
  8535. Assert(!instr->GetSrc1()->IsUnsigned());
  8536. break;
  8537. }
  8538. case Js::OpCode::CmEq_A:
  8539. case Js::OpCode::CmSrEq_A:
  8540. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8541. {
  8542. return false;
  8543. }
  8544. newMin = 0;
  8545. newMax = 1;
  8546. opcode = Js::OpCode::CmEq_I4;
  8547. needsBoolConv = true;
  8548. break;
  8549. case Js::OpCode::CmNeq_A:
  8550. case Js::OpCode::CmSrNeq_A:
  8551. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8552. {
  8553. return false;
  8554. }
  8555. newMin = 0;
  8556. newMax = 1;
  8557. opcode = Js::OpCode::CmNeq_I4;
  8558. needsBoolConv = true;
  8559. break;
  8560. case Js::OpCode::CmLe_A:
  8561. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8562. {
  8563. return false;
  8564. }
  8565. newMin = 0;
  8566. newMax = 1;
  8567. opcode = Js::OpCode::CmLe_I4;
  8568. needsBoolConv = true;
  8569. break;
  8570. case Js::OpCode::CmLt_A:
  8571. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8572. {
  8573. return false;
  8574. }
  8575. newMin = 0;
  8576. newMax = 1;
  8577. opcode = Js::OpCode::CmLt_I4;
  8578. needsBoolConv = true;
  8579. break;
  8580. case Js::OpCode::CmGe_A:
  8581. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8582. {
  8583. return false;
  8584. }
  8585. newMin = 0;
  8586. newMax = 1;
  8587. opcode = Js::OpCode::CmGe_I4;
  8588. needsBoolConv = true;
  8589. break;
  8590. case Js::OpCode::CmGt_A:
  8591. if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val))
  8592. {
  8593. return false;
  8594. }
  8595. newMin = 0;
  8596. newMax = 1;
  8597. opcode = Js::OpCode::CmGt_I4;
  8598. needsBoolConv = true;
  8599. break;
  8600. case Js::OpCode::BrSrEq_A:
  8601. case Js::OpCode::BrEq_A:
  8602. case Js::OpCode::BrNotNeq_A:
  8603. case Js::OpCode::BrSrNotNeq_A:
  8604. {
  8605. if(DoConstFold() &&
  8606. !IsLoopPrePass() &&
  8607. TryOptConstFoldBrEqual(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  8608. {
  8609. return true;
  8610. }
  8611. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8612. UpdateIntBoundsForEqualBranch(src1Val, src2Val);
  8613. if(!specialize)
  8614. {
  8615. return false;
  8616. }
  8617. opcode = Js::OpCode::BrEq_I4;
  8618. // We'll get a warning if we don't assign a value to these...
  8619. // We'll assert if we use them and make a range where min > max
  8620. newMin = 2; newMax = 1;
  8621. break;
  8622. }
  8623. case Js::OpCode::BrSrNeq_A:
  8624. case Js::OpCode::BrNeq_A:
  8625. case Js::OpCode::BrSrNotEq_A:
  8626. case Js::OpCode::BrNotEq_A:
  8627. {
  8628. if(DoConstFold() &&
  8629. !IsLoopPrePass() &&
  8630. TryOptConstFoldBrEqual(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  8631. {
  8632. return true;
  8633. }
  8634. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8635. UpdateIntBoundsForNotEqualBranch(src1Val, src2Val);
  8636. if(!specialize)
  8637. {
  8638. return false;
  8639. }
  8640. opcode = Js::OpCode::BrNeq_I4;
  8641. // We'll get a warning if we don't assign a value to these...
  8642. // We'll assert if we use them and make a range where min > max
  8643. newMin = 2; newMax = 1;
  8644. break;
  8645. }
  8646. case Js::OpCode::BrGt_A:
  8647. case Js::OpCode::BrNotLe_A:
  8648. {
  8649. if(DoConstFold() &&
  8650. !IsLoopPrePass() &&
  8651. TryOptConstFoldBrGreaterThan(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  8652. {
  8653. return true;
  8654. }
  8655. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8656. UpdateIntBoundsForGreaterThanBranch(src1Val, src2Val);
  8657. if(!specialize)
  8658. {
  8659. return false;
  8660. }
  8661. opcode = Js::OpCode::BrGt_I4;
  8662. // We'll get a warning if we don't assign a value to these...
  8663. // We'll assert if we use them and make a range where min > max
  8664. newMin = 2; newMax = 1;
  8665. break;
  8666. }
  8667. case Js::OpCode::BrGe_A:
  8668. case Js::OpCode::BrNotLt_A:
  8669. {
  8670. if(DoConstFold() &&
  8671. !IsLoopPrePass() &&
  8672. TryOptConstFoldBrGreaterThanOrEqual(instr, true, src1Val, min1, max1, src2Val, min2, max2))
  8673. {
  8674. return true;
  8675. }
  8676. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8677. UpdateIntBoundsForGreaterThanOrEqualBranch(src1Val, src2Val);
  8678. if(!specialize)
  8679. {
  8680. return false;
  8681. }
  8682. opcode = Js::OpCode::BrGe_I4;
  8683. // We'll get a warning if we don't assign a value to these...
  8684. // We'll assert if we use them and make a range where min > max
  8685. newMin = 2; newMax = 1;
  8686. break;
  8687. }
  8688. case Js::OpCode::BrLt_A:
  8689. case Js::OpCode::BrNotGe_A:
  8690. {
  8691. if(DoConstFold() &&
  8692. !IsLoopPrePass() &&
  8693. TryOptConstFoldBrGreaterThanOrEqual(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  8694. {
  8695. return true;
  8696. }
  8697. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8698. UpdateIntBoundsForLessThanBranch(src1Val, src2Val);
  8699. if(!specialize)
  8700. {
  8701. return false;
  8702. }
  8703. opcode = Js::OpCode::BrLt_I4;
  8704. // We'll get a warning if we don't assign a value to these...
  8705. // We'll assert if we use them and make a range where min > max
  8706. newMin = 2; newMax = 1;
  8707. break;
  8708. }
  8709. case Js::OpCode::BrLe_A:
  8710. case Js::OpCode::BrNotGt_A:
  8711. {
  8712. if(DoConstFold() &&
  8713. !IsLoopPrePass() &&
  8714. TryOptConstFoldBrGreaterThan(instr, false, src1Val, min1, max1, src2Val, min2, max2))
  8715. {
  8716. return true;
  8717. }
  8718. const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val);
  8719. UpdateIntBoundsForLessThanOrEqualBranch(src1Val, src2Val);
  8720. if(!specialize)
  8721. {
  8722. return false;
  8723. }
  8724. opcode = Js::OpCode::BrLe_I4;
  8725. // We'll get a warning if we don't assign a value to these...
  8726. // We'll assert if we use them and make a range where min > max
  8727. newMin = 2; newMax = 1;
  8728. break;
  8729. }
  8730. default:
  8731. return false;
  8732. }
  8733. // If this instruction is in a range of instructions where int overflow does not matter, we will still specialize it
  8734. // (won't leave it unspecialized based on heuristics), since it is most likely worth specializing, and the dst value
  8735. // needs to be guaranteed to be an int
  8736. if(!ignoredIntOverflow &&
  8737. !ignoredNegativeZero &&
  8738. !needsBoolConv &&
  8739. instr->ShouldCheckForIntOverflow() &&
  8740. !IsWorthSpecializingToInt32(instr, src1Val, src2Val))
  8741. {
  8742. // Even though type specialization is being skipped since it may not be worth it, the proper value should still be
  8743. // maintained so that the result may be type specialized later. An int value is not created for the dst in any of
  8744. // the following cases.
  8745. // - A bailout check is necessary to specialize this instruction. The bailout check is what guarantees the result to
  8746. // be an int, but since we're not going to specialize this instruction, there won't be a bailout check.
  8747. // - Aggressive int type specialization is disabled and we're in a loop prepass. We're conservative on dst values in
  8748. // that case, especially if the dst sym is live on the back-edge.
  8749. if(bailOutKind == IR::BailOutInvalid &&
  8750. instr->GetDst() &&
  8751. src1Val->GetValueInfo()->IsInt() &&
  8752. src2Val->GetValueInfo()->IsInt() &&
  8753. (DoAggressiveIntTypeSpec() || !this->IsLoopPrePass()))
  8754. {
  8755. *pDstVal = CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val);
  8756. }
  8757. return false;
  8758. }
  8759. } // case default
  8760. } // switch
  8761. LOutsideSwitch:
  8762. this->ignoredIntOverflowForCurrentInstr = ignoredIntOverflow;
  8763. this->ignoredNegativeZeroForCurrentInstr = ignoredNegativeZero;
  8764. {
  8765. // Try CSE again before modifying the IR, in case some attributes are required for successful CSE
  8766. Value *src1IndirIndexVal = nullptr;
  8767. if(CSEOptimize(currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal, true /* intMathExprOnly */))
  8768. {
  8769. *redoTypeSpecRef = true;
  8770. return false;
  8771. }
  8772. }
  8773. const Js::OpCode originalOpCode = instr->m_opcode;
  8774. if (!this->IsLoopPrePass())
  8775. {
  8776. // No re-write on prepass
  8777. instr->m_opcode = opcode;
  8778. }
  8779. Value *src1ValueToSpecialize = src1Val, *src2ValueToSpecialize = src2Val;
  8780. // Lossy conversions to int32 must be done based on the original source values. For instance, if one of the values is a
  8781. // float constant with a value that fits in a uint32 but not an int32, and the instruction can ignore int overflow, the
  8782. // source value for the purposes of int specialization would have been changed to an int constant value by ignoring
  8783. // overflow. If we were to specialize the sym using the int constant value, it would be treated as a lossless
  8784. // conversion, but since there may be subsequent uses of the same float constant value that may not ignore overflow,
  8785. // this must be treated as a lossy conversion by specializing the sym using the original float constant value.
  8786. if(src1Lossy)
  8787. {
  8788. src1ValueToSpecialize = src1OriginalVal;
  8789. }
  8790. if (src2Lossy)
  8791. {
  8792. src2ValueToSpecialize = src2OriginalVal;
  8793. }
  8794. // Make sure the srcs are specialized
  8795. IR::Opnd* src1 = instr->GetSrc1();
  8796. this->ToInt32(instr, src1, this->currentBlock, src1ValueToSpecialize, nullptr, src1Lossy);
  8797. if (!skipSrc2)
  8798. {
  8799. IR::Opnd* src2 = instr->GetSrc2();
  8800. this->ToInt32(instr, src2, this->currentBlock, src2ValueToSpecialize, nullptr, src2Lossy);
  8801. }
  8802. if(bailOutKind != IR::BailOutInvalid && !this->IsLoopPrePass())
  8803. {
  8804. GenerateBailAtOperation(&instr, bailOutKind);
  8805. }
  8806. if (!skipDst && instr->GetDst())
  8807. {
  8808. if (needsBoolConv)
  8809. {
  8810. IR::RegOpnd *varDst;
  8811. if (this->IsLoopPrePass())
  8812. {
  8813. varDst = instr->GetDst()->AsRegOpnd();
  8814. this->ToVarRegOpnd(varDst, this->currentBlock);
  8815. }
  8816. else
  8817. {
  8818. // Generate:
  8819. // t1.i = CmCC t2.i, t3.i
  8820. // t1.v = Conv_bool t1.i
  8821. //
  8822. // If the only uses of t1 are ints, the conv_bool will get dead-stored
  8823. TypeSpecializeIntDst(instr, originalOpCode, nullptr, src1Val, src2Val, bailOutKind, newMin, newMax, pDstVal);
  8824. IR::RegOpnd *intDst = instr->GetDst()->AsRegOpnd();
  8825. intDst->SetIsJITOptimizedReg(true);
  8826. varDst = IR::RegOpnd::New(intDst->m_sym->GetVarEquivSym(this->func), TyVar, this->func);
  8827. IR::Instr *convBoolInstr = IR::Instr::New(Js::OpCode::Conv_Bool, varDst, intDst, this->func);
  8828. // In some cases (e.g. unsigned compare peep code), a comparison will use variables
  8829. // other than the ones initially intended for it, if we can determine that we would
  8830. // arrive at the same result. This means that we get a ByteCodeUses operation after
  8831. // the actual comparison. Since Inserting the Conv_bool just after the compare, and
  8832. // just before the ByteCodeUses, would cause issues later on with register lifetime
  8833. // calculation, we want to insert the Conv_bool after the whole compare instruction
  8834. // block.
  8835. IR::Instr *putAfter = instr;
  8836. while (putAfter->m_next && putAfter->m_next->IsByteCodeUsesInstrFor(instr))
  8837. {
  8838. putAfter = putAfter->m_next;
  8839. }
  8840. putAfter->InsertAfter(convBoolInstr);
  8841. convBoolInstr->SetByteCodeOffset(instr);
  8842. this->ToVarRegOpnd(varDst, this->currentBlock);
  8843. CurrentBlockData()->liveInt32Syms->Set(varDst->m_sym->m_id);
  8844. CurrentBlockData()->liveLossyInt32Syms->Set(varDst->m_sym->m_id);
  8845. }
  8846. *pDstVal = this->NewGenericValue(ValueType::Boolean, varDst);
  8847. }
  8848. else
  8849. {
  8850. TypeSpecializeIntDst(
  8851. instr,
  8852. originalOpCode,
  8853. nullptr,
  8854. src1Val,
  8855. src2Val,
  8856. bailOutKind,
  8857. newMin,
  8858. newMax,
  8859. pDstVal,
  8860. addSubConstantInfo.HasInfo() ? &addSubConstantInfo : nullptr);
  8861. }
  8862. }
  8863. if(bailOutKind == IR::BailOutInvalid)
  8864. {
  8865. GOPT_TRACE(_u("Type specialized to INT\n"));
  8866. #if ENABLE_DEBUG_CONFIG_OPTIONS
  8867. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  8868. {
  8869. Output::Print(_u("Type specialized to INT: "));
  8870. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  8871. }
  8872. #endif
  8873. }
  8874. else
  8875. {
  8876. GOPT_TRACE(_u("Type specialized to INT with bailout on:\n"));
  8877. if(bailOutKind & (IR::BailOutOnOverflow | IR::BailOutOnMulOverflow) )
  8878. {
  8879. GOPT_TRACE(_u(" Overflow\n"));
  8880. #if ENABLE_DEBUG_CONFIG_OPTIONS
  8881. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  8882. {
  8883. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Overflow");
  8884. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  8885. }
  8886. #endif
  8887. }
  8888. if(bailOutKind & IR::BailOutOnNegativeZero)
  8889. {
  8890. GOPT_TRACE(_u(" Zero\n"));
  8891. #if ENABLE_DEBUG_CONFIG_OPTIONS
  8892. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase))
  8893. {
  8894. Output::Print(_u("Type specialized to INT with bailout (%S): "), "Zero");
  8895. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  8896. }
  8897. #endif
  8898. }
  8899. }
  8900. return true;
  8901. }
  8902. bool
  8903. GlobOpt::IsWorthSpecializingToInt32Branch(IR::Instr const * instr, Value const * src1Val, Value const * src2Val) const
  8904. {
  8905. if (!src1Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc1()->IsRegOpnd())
  8906. {
  8907. StackSym const *sym1 = instr->GetSrc1()->AsRegOpnd()->m_sym;
  8908. if (CurrentBlockData()->IsInt32TypeSpecialized(sym1) == false)
  8909. {
  8910. if (!src2Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc2()->IsRegOpnd())
  8911. {
  8912. StackSym const *sym2 = instr->GetSrc2()->AsRegOpnd()->m_sym;
  8913. if (CurrentBlockData()->IsInt32TypeSpecialized(sym2) == false)
  8914. {
  8915. // Type specializing a Br itself isn't worth it, unless one src
  8916. // is already type specialized
  8917. return false;
  8918. }
  8919. }
  8920. }
  8921. }
  8922. return true;
  8923. }
  8924. bool
  8925. GlobOpt::TryOptConstFoldBrFalse(
  8926. IR::Instr *const instr,
  8927. Value *const srcValue,
  8928. const int32 min,
  8929. const int32 max)
  8930. {
  8931. Assert(instr);
  8932. Assert(instr->m_opcode == Js::OpCode::BrFalse_A || instr->m_opcode == Js::OpCode::BrTrue_A);
  8933. Assert(srcValue);
  8934. if(!(DoAggressiveIntTypeSpec() ? srcValue->GetValueInfo()->IsLikelyInt() : srcValue->GetValueInfo()->IsInt()))
  8935. {
  8936. return false;
  8937. }
  8938. if(ValueInfo::IsEqualTo(srcValue, min, max, nullptr, 0, 0))
  8939. {
  8940. OptConstFoldBr(instr->m_opcode == Js::OpCode::BrFalse_A, instr, srcValue);
  8941. return true;
  8942. }
  8943. if(ValueInfo::IsNotEqualTo(srcValue, min, max, nullptr, 0, 0))
  8944. {
  8945. OptConstFoldBr(instr->m_opcode == Js::OpCode::BrTrue_A, instr, srcValue);
  8946. return true;
  8947. }
  8948. return false;
  8949. }
  8950. bool
  8951. GlobOpt::TryOptConstFoldBrEqual(
  8952. IR::Instr *const instr,
  8953. const bool branchOnEqual,
  8954. Value *const src1Value,
  8955. const int32 min1,
  8956. const int32 max1,
  8957. Value *const src2Value,
  8958. const int32 min2,
  8959. const int32 max2)
  8960. {
  8961. Assert(instr);
  8962. Assert(src1Value);
  8963. Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt());
  8964. Assert(src2Value);
  8965. Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt());
  8966. if(ValueInfo::IsEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  8967. {
  8968. OptConstFoldBr(branchOnEqual, instr, src1Value, src2Value);
  8969. return true;
  8970. }
  8971. if(ValueInfo::IsNotEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  8972. {
  8973. OptConstFoldBr(!branchOnEqual, instr, src1Value, src2Value);
  8974. return true;
  8975. }
  8976. return false;
  8977. }
  8978. bool
  8979. GlobOpt::TryOptConstFoldBrGreaterThan(
  8980. IR::Instr *const instr,
  8981. const bool branchOnGreaterThan,
  8982. Value *const src1Value,
  8983. const int32 min1,
  8984. const int32 max1,
  8985. Value *const src2Value,
  8986. const int32 min2,
  8987. const int32 max2)
  8988. {
  8989. Assert(instr);
  8990. Assert(src1Value);
  8991. Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt());
  8992. Assert(src2Value);
  8993. Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt());
  8994. if(ValueInfo::IsGreaterThan(src1Value, min1, max1, src2Value, min2, max2))
  8995. {
  8996. OptConstFoldBr(branchOnGreaterThan, instr, src1Value, src2Value);
  8997. return true;
  8998. }
  8999. if(ValueInfo::IsLessThanOrEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  9000. {
  9001. OptConstFoldBr(!branchOnGreaterThan, instr, src1Value, src2Value);
  9002. return true;
  9003. }
  9004. return false;
  9005. }
  9006. bool
  9007. GlobOpt::TryOptConstFoldBrGreaterThanOrEqual(
  9008. IR::Instr *const instr,
  9009. const bool branchOnGreaterThanOrEqual,
  9010. Value *const src1Value,
  9011. const int32 min1,
  9012. const int32 max1,
  9013. Value *const src2Value,
  9014. const int32 min2,
  9015. const int32 max2)
  9016. {
  9017. Assert(instr);
  9018. Assert(src1Value);
  9019. Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt());
  9020. Assert(src2Value);
  9021. Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt());
  9022. if(ValueInfo::IsGreaterThanOrEqualTo(src1Value, min1, max1, src2Value, min2, max2))
  9023. {
  9024. OptConstFoldBr(branchOnGreaterThanOrEqual, instr, src1Value, src2Value);
  9025. return true;
  9026. }
  9027. if(ValueInfo::IsLessThan(src1Value, min1, max1, src2Value, min2, max2))
  9028. {
  9029. OptConstFoldBr(!branchOnGreaterThanOrEqual, instr, src1Value, src2Value);
  9030. return true;
  9031. }
  9032. return false;
  9033. }
  9034. bool
  9035. GlobOpt::TryOptConstFoldBrUnsignedLessThan(
  9036. IR::Instr *const instr,
  9037. const bool branchOnLessThan,
  9038. Value *const src1Value,
  9039. const int32 min1,
  9040. const int32 max1,
  9041. Value *const src2Value,
  9042. const int32 min2,
  9043. const int32 max2)
  9044. {
  9045. Assert(DoConstFold());
  9046. Assert(!IsLoopPrePass());
  9047. if(!src1Value ||
  9048. !src2Value ||
  9049. !(
  9050. DoAggressiveIntTypeSpec()
  9051. ? src1Value->GetValueInfo()->IsLikelyInt() && src2Value->GetValueInfo()->IsLikelyInt()
  9052. : src1Value->GetValueInfo()->IsInt() && src2Value->GetValueInfo()->IsInt()
  9053. ))
  9054. {
  9055. return false;
  9056. }
  9057. uint uMin1 = (min1 < 0 ? (max1 < 0 ? min((uint)min1, (uint)max1) : 0) : min1);
  9058. uint uMax1 = max((uint)min1, (uint)max1);
  9059. uint uMin2 = (min2 < 0 ? (max2 < 0 ? min((uint)min2, (uint)max2) : 0) : min2);
  9060. uint uMax2 = max((uint)min2, (uint)max2);
  9061. if (uMax1 < uMin2)
  9062. {
  9063. // Range 1 is always lesser than Range 2
  9064. OptConstFoldBr(branchOnLessThan, instr, src1Value, src2Value);
  9065. return true;
  9066. }
  9067. if (uMin1 >= uMax2)
  9068. {
  9069. // Range 2 is always lesser than Range 1
  9070. OptConstFoldBr(!branchOnLessThan, instr, src1Value, src2Value);
  9071. return true;
  9072. }
  9073. return false;
  9074. }
  9075. bool
  9076. GlobOpt::TryOptConstFoldBrUnsignedGreaterThan(
  9077. IR::Instr *const instr,
  9078. const bool branchOnGreaterThan,
  9079. Value *const src1Value,
  9080. const int32 min1,
  9081. const int32 max1,
  9082. Value *const src2Value,
  9083. const int32 min2,
  9084. const int32 max2)
  9085. {
  9086. Assert(DoConstFold());
  9087. Assert(!IsLoopPrePass());
  9088. if(!src1Value ||
  9089. !src2Value ||
  9090. !(
  9091. DoAggressiveIntTypeSpec()
  9092. ? src1Value->GetValueInfo()->IsLikelyInt() && src2Value->GetValueInfo()->IsLikelyInt()
  9093. : src1Value->GetValueInfo()->IsInt() && src2Value->GetValueInfo()->IsInt()
  9094. ))
  9095. {
  9096. return false;
  9097. }
  9098. uint uMin1 = (min1 < 0 ? (max1 < 0 ? min((uint)min1, (uint)max1) : 0) : min1);
  9099. uint uMax1 = max((uint)min1, (uint)max1);
  9100. uint uMin2 = (min2 < 0 ? (max2 < 0 ? min((uint)min2, (uint)max2) : 0) : min2);
  9101. uint uMax2 = max((uint)min2, (uint)max2);
  9102. if (uMin1 > uMax2)
  9103. {
  9104. // Range 1 is always greater than Range 2
  9105. OptConstFoldBr(branchOnGreaterThan, instr, src1Value, src2Value);
  9106. return true;
  9107. }
  9108. if (uMax1 <= uMin2)
  9109. {
  9110. // Range 2 is always greater than Range 1
  9111. OptConstFoldBr(!branchOnGreaterThan, instr, src1Value, src2Value);
  9112. return true;
  9113. }
  9114. return false;
  9115. }
  9116. void
  9117. GlobOpt::SetPathDependentInfo(const bool conditionToBranch, const PathDependentInfo &info)
  9118. {
  9119. Assert(this->currentBlock->GetSuccList()->Count() == 2);
  9120. IR::Instr * fallthrough = this->currentBlock->GetNext()->GetFirstInstr();
  9121. FOREACH_SLISTBASECOUNTED_ENTRY(FlowEdge*, edge, this->currentBlock->GetSuccList())
  9122. {
  9123. if (conditionToBranch == (edge->GetSucc()->GetFirstInstr() != fallthrough))
  9124. {
  9125. edge->SetPathDependentInfo(info, alloc);
  9126. return;
  9127. }
  9128. }
  9129. NEXT_SLISTBASECOUNTED_ENTRY;
  9130. // In case flowgraph peeps is disabled, we could have conditional branch to next instr
  9131. Assert(this->func->HasTry() || PHASE_OFF(Js::FGPeepsPhase, this->func));
  9132. }
  9133. PathDependentInfoToRestore
  9134. GlobOpt::UpdatePathDependentInfo(PathDependentInfo *const info)
  9135. {
  9136. Assert(info);
  9137. if(!info->HasInfo())
  9138. {
  9139. return PathDependentInfoToRestore();
  9140. }
  9141. decltype(&GlobOpt::UpdateIntBoundsForEqual) UpdateIntBoundsForLeftValue, UpdateIntBoundsForRightValue;
  9142. switch(info->Relationship())
  9143. {
  9144. case PathDependentRelationship::Equal:
  9145. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForEqual;
  9146. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForEqual;
  9147. break;
  9148. case PathDependentRelationship::NotEqual:
  9149. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForNotEqual;
  9150. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForNotEqual;
  9151. break;
  9152. case PathDependentRelationship::GreaterThanOrEqual:
  9153. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForGreaterThanOrEqual;
  9154. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForLessThanOrEqual;
  9155. break;
  9156. case PathDependentRelationship::GreaterThan:
  9157. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForGreaterThan;
  9158. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForLessThan;
  9159. break;
  9160. case PathDependentRelationship::LessThanOrEqual:
  9161. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForLessThanOrEqual;
  9162. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForGreaterThanOrEqual;
  9163. break;
  9164. case PathDependentRelationship::LessThan:
  9165. UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForLessThan;
  9166. UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForGreaterThan;
  9167. break;
  9168. default:
  9169. Assert(false);
  9170. __assume(false);
  9171. }
  9172. ValueInfo *leftValueInfo = info->LeftValue()->GetValueInfo();
  9173. IntConstantBounds leftConstantBounds;
  9174. AssertVerify(leftValueInfo->TryGetIntConstantBounds(&leftConstantBounds, true));
  9175. ValueInfo *rightValueInfo;
  9176. IntConstantBounds rightConstantBounds;
  9177. if(info->RightValue())
  9178. {
  9179. rightValueInfo = info->RightValue()->GetValueInfo();
  9180. AssertVerify(rightValueInfo->TryGetIntConstantBounds(&rightConstantBounds, true));
  9181. }
  9182. else
  9183. {
  9184. rightValueInfo = nullptr;
  9185. rightConstantBounds = IntConstantBounds(info->RightConstantValue(), info->RightConstantValue());
  9186. }
  9187. ValueInfo *const newLeftValueInfo =
  9188. (this->*UpdateIntBoundsForLeftValue)(
  9189. info->LeftValue(),
  9190. leftConstantBounds,
  9191. info->RightValue(),
  9192. rightConstantBounds,
  9193. true);
  9194. if(newLeftValueInfo)
  9195. {
  9196. ChangeValueInfo(nullptr, info->LeftValue(), newLeftValueInfo);
  9197. AssertVerify(newLeftValueInfo->TryGetIntConstantBounds(&leftConstantBounds, true));
  9198. }
  9199. else
  9200. {
  9201. leftValueInfo = nullptr;
  9202. }
  9203. ValueInfo *const newRightValueInfo =
  9204. (this->*UpdateIntBoundsForRightValue)(
  9205. info->RightValue(),
  9206. rightConstantBounds,
  9207. info->LeftValue(),
  9208. leftConstantBounds,
  9209. true);
  9210. if(newRightValueInfo)
  9211. {
  9212. ChangeValueInfo(nullptr, info->RightValue(), newRightValueInfo);
  9213. }
  9214. else
  9215. {
  9216. rightValueInfo = nullptr;
  9217. }
  9218. return PathDependentInfoToRestore(leftValueInfo, rightValueInfo);
  9219. }
  9220. void
  9221. GlobOpt::RestorePathDependentInfo(PathDependentInfo *const info, const PathDependentInfoToRestore infoToRestore)
  9222. {
  9223. Assert(info);
  9224. if(infoToRestore.LeftValueInfo())
  9225. {
  9226. Assert(info->LeftValue());
  9227. ChangeValueInfo(nullptr, info->LeftValue(), infoToRestore.LeftValueInfo());
  9228. }
  9229. if(infoToRestore.RightValueInfo())
  9230. {
  9231. Assert(info->RightValue());
  9232. ChangeValueInfo(nullptr, info->RightValue(), infoToRestore.RightValueInfo());
  9233. }
  9234. }
  9235. bool
  9236. GlobOpt::TypeSpecializeFloatUnary(IR::Instr **pInstr, Value *src1Val, Value **pDstVal, bool skipDst /* = false */)
  9237. {
  9238. IR::Instr *&instr = *pInstr;
  9239. IR::Opnd *src1;
  9240. IR::Opnd *dst;
  9241. Js::OpCode opcode = instr->m_opcode;
  9242. Value *valueToTransfer = nullptr;
  9243. Assert(src1Val && src1Val->GetValueInfo()->IsLikelyNumber() || OpCodeAttr::IsInlineBuiltIn(instr->m_opcode));
  9244. if (!this->DoFloatTypeSpec())
  9245. {
  9246. return false;
  9247. }
  9248. // For inline built-ins we need to do type specialization. Check upfront to avoid duplicating same case labels.
  9249. if (!OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  9250. {
  9251. switch (opcode)
  9252. {
  9253. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  9254. skipDst = true;
  9255. // fall-through
  9256. case Js::OpCode::Ld_A:
  9257. case Js::OpCode::BrTrue_A:
  9258. case Js::OpCode::BrFalse_A:
  9259. if (instr->GetSrc1()->IsRegOpnd())
  9260. {
  9261. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  9262. if (CurrentBlockData()->IsFloat64TypeSpecialized(sym) == false)
  9263. {
  9264. // Type specializing an Ld_A isn't worth it, unless the src
  9265. // is already type specialized
  9266. return false;
  9267. }
  9268. }
  9269. if (instr->m_opcode == Js::OpCode::Ld_A)
  9270. {
  9271. valueToTransfer = src1Val;
  9272. }
  9273. break;
  9274. case Js::OpCode::Neg_A:
  9275. break;
  9276. case Js::OpCode::Conv_Num:
  9277. Assert(src1Val);
  9278. opcode = Js::OpCode::Ld_A;
  9279. valueToTransfer = src1Val;
  9280. if (!src1Val->GetValueInfo()->IsNumber())
  9281. {
  9282. StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  9283. valueToTransfer = NewGenericValue(ValueType::Float, instr->GetDst()->GetStackSym());
  9284. if (CurrentBlockData()->IsFloat64TypeSpecialized(sym) == false)
  9285. {
  9286. // Set the dst as a nonDeadStore. We want to keep the Ld_A to prevent the FromVar from
  9287. // being dead-stored, as it could cause implicit calls.
  9288. dst = instr->GetDst();
  9289. dst->AsRegOpnd()->m_dontDeadStore = true;
  9290. }
  9291. }
  9292. break;
  9293. case Js::OpCode::StElemI_A:
  9294. case Js::OpCode::StElemI_A_Strict:
  9295. case Js::OpCode::StElemC:
  9296. return TypeSpecializeStElem(pInstr, src1Val, pDstVal);
  9297. default:
  9298. return false;
  9299. }
  9300. }
  9301. // Make sure the srcs are specialized
  9302. src1 = instr->GetSrc1();
  9303. // Use original val when calling toFloat64 as this is what we'll use to try hoisting the fromVar if we're in a loop.
  9304. this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, IR::BailOutPrimitiveButString);
  9305. if (!skipDst)
  9306. {
  9307. dst = instr->GetDst();
  9308. if (dst)
  9309. {
  9310. this->TypeSpecializeFloatDst(instr, valueToTransfer, src1Val, nullptr, pDstVal);
  9311. if (!this->IsLoopPrePass())
  9312. {
  9313. instr->m_opcode = opcode;
  9314. }
  9315. }
  9316. }
  9317. GOPT_TRACE_INSTR(instr, _u("Type specialized to FLOAT: "));
  9318. #if ENABLE_DEBUG_CONFIG_OPTIONS
  9319. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FloatTypeSpecPhase))
  9320. {
  9321. Output::Print(_u("Type specialized to FLOAT: "));
  9322. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  9323. }
  9324. #endif
  9325. return true;
  9326. }
  9327. // Unconditionally type-spec dst to float.
  9328. void
  9329. GlobOpt::TypeSpecializeFloatDst(IR::Instr *instr, Value *valToTransfer, Value *const src1Value, Value *const src2Value, Value **pDstVal)
  9330. {
  9331. IR::Opnd* dst = instr->GetDst();
  9332. Assert(dst);
  9333. AssertMsg(dst->IsRegOpnd(), "What else?");
  9334. this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  9335. if(valToTransfer)
  9336. {
  9337. *pDstVal = this->ValueNumberTransferDst(instr, valToTransfer);
  9338. CurrentBlockData()->InsertNewValue(*pDstVal, dst);
  9339. }
  9340. else
  9341. {
  9342. *pDstVal = CreateDstUntransferredValue(ValueType::Float, instr, src1Value, src2Value);
  9343. }
  9344. }
  9345. bool
  9346. GlobOpt::TypeSpecializeLdLen(
  9347. IR::Instr * *const instrRef,
  9348. Value * *const src1ValueRef,
  9349. Value * *const dstValueRef,
  9350. bool *const forceInvariantHoistingRef)
  9351. {
  9352. Assert(instrRef);
  9353. IR::Instr *&instr = *instrRef;
  9354. Assert(instr);
  9355. Assert(instr->m_opcode == Js::OpCode::LdLen_A);
  9356. Assert(src1ValueRef);
  9357. Value *&src1Value = *src1ValueRef;
  9358. Assert(dstValueRef);
  9359. Value *&dstValue = *dstValueRef;
  9360. Assert(forceInvariantHoistingRef);
  9361. bool &forceInvariantHoisting = *forceInvariantHoistingRef;
  9362. if(!DoLdLenIntSpec(instr, instr->GetSrc1()->GetValueType()))
  9363. {
  9364. return false;
  9365. }
  9366. IR::BailOutKind bailOutKind = IR::BailOutOnIrregularLength;
  9367. if(!IsLoopPrePass())
  9368. {
  9369. IR::RegOpnd *const baseOpnd = instr->GetSrc1()->AsRegOpnd();
  9370. if(baseOpnd->IsArrayRegOpnd())
  9371. {
  9372. StackSym *const lengthSym = baseOpnd->AsArrayRegOpnd()->LengthSym();
  9373. if(lengthSym)
  9374. {
  9375. CaptureByteCodeSymUses(instr);
  9376. instr->m_opcode = Js::OpCode::Ld_I4;
  9377. instr->ReplaceSrc1(IR::RegOpnd::New(lengthSym, lengthSym->GetType(), func));
  9378. instr->ClearBailOutInfo();
  9379. // Find the hoisted length value
  9380. Value *const lengthValue = CurrentBlockData()->FindValue(lengthSym);
  9381. Assert(lengthValue);
  9382. src1Value = lengthValue;
  9383. ValueInfo *const lengthValueInfo = lengthValue->GetValueInfo();
  9384. IntConstantBounds lengthConstantBounds;
  9385. AssertVerify(lengthValueInfo->TryGetIntConstantBounds(&lengthConstantBounds));
  9386. Assert(lengthConstantBounds.LowerBound() >= 0);
  9387. if (lengthValueInfo->GetSymStore() == lengthSym)
  9388. {
  9389. // When type specializing the dst below, we will end up inserting lengthSym.u32 as symstore for a var
  9390. // Clear the symstore here, so that we dont end up with problems with copyprop later on
  9391. lengthValueInfo->SetSymStore(nullptr);
  9392. }
  9393. // Int-specialize, and transfer the value to the dst
  9394. TypeSpecializeIntDst(
  9395. instr,
  9396. Js::OpCode::LdLen_A,
  9397. src1Value,
  9398. src1Value,
  9399. nullptr,
  9400. bailOutKind,
  9401. lengthConstantBounds.LowerBound(),
  9402. lengthConstantBounds.UpperBound(),
  9403. &dstValue);
  9404. // Try to force hoisting the Ld_I4 so that the length will have an invariant sym store that can be
  9405. // copy-propped. Invariant hoisting does not automatically hoist Ld_I4.
  9406. forceInvariantHoisting = true;
  9407. return true;
  9408. }
  9409. }
  9410. if (instr->HasBailOutInfo())
  9411. {
  9412. Assert(instr->GetBailOutKind() == IR::BailOutMarkTempObject);
  9413. bailOutKind = IR::BailOutOnIrregularLength | IR::BailOutMarkTempObject;
  9414. instr->SetBailOutKind(bailOutKind);
  9415. }
  9416. else
  9417. {
  9418. Assert(bailOutKind == IR::BailOutOnIrregularLength);
  9419. GenerateBailAtOperation(&instr, bailOutKind);
  9420. }
  9421. }
  9422. TypeSpecializeIntDst(
  9423. instr,
  9424. Js::OpCode::LdLen_A,
  9425. nullptr,
  9426. nullptr,
  9427. nullptr,
  9428. bailOutKind,
  9429. 0,
  9430. INT32_MAX,
  9431. &dstValue);
  9432. return true;
  9433. }
  9434. bool
  9435. GlobOpt::TypeSpecializeFloatBinary(IR::Instr *instr, Value *src1Val, Value *src2Val, Value **pDstVal)
  9436. {
  9437. IR::Opnd *src1;
  9438. IR::Opnd *src2;
  9439. IR::Opnd *dst;
  9440. bool allowUndefinedOrNullSrc1 = true;
  9441. bool allowUndefinedOrNullSrc2 = true;
  9442. bool skipSrc1 = false;
  9443. bool skipSrc2 = false;
  9444. bool skipDst = false;
  9445. if (!this->DoFloatTypeSpec())
  9446. {
  9447. return false;
  9448. }
  9449. // For inline built-ins we need to do type specialization. Check upfront to avoid duplicating same case labels.
  9450. if (!OpCodeAttr::IsInlineBuiltIn(instr->m_opcode))
  9451. {
  9452. switch (instr->m_opcode)
  9453. {
  9454. case Js::OpCode::Sub_A:
  9455. case Js::OpCode::Mul_A:
  9456. case Js::OpCode::Div_A:
  9457. case Js::OpCode::Expo_A:
  9458. // Avoid if one source is known not to be a number.
  9459. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber())
  9460. {
  9461. return false;
  9462. }
  9463. break;
  9464. case Js::OpCode::BrSrEq_A:
  9465. case Js::OpCode::BrSrNeq_A:
  9466. case Js::OpCode::BrEq_A:
  9467. case Js::OpCode::BrNeq_A:
  9468. case Js::OpCode::BrSrNotEq_A:
  9469. case Js::OpCode::BrNotEq_A:
  9470. case Js::OpCode::BrSrNotNeq_A:
  9471. case Js::OpCode::BrNotNeq_A:
  9472. // Avoid if one source is known not to be a number.
  9473. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber())
  9474. {
  9475. return false;
  9476. }
  9477. // Undef == Undef, but +Undef != +Undef
  9478. // 0.0 != null, but 0.0 == +null
  9479. //
  9480. // So Bailout on anything but numbers for both src1 and src2
  9481. allowUndefinedOrNullSrc1 = false;
  9482. allowUndefinedOrNullSrc2 = false;
  9483. break;
  9484. case Js::OpCode::BrGt_A:
  9485. case Js::OpCode::BrGe_A:
  9486. case Js::OpCode::BrLt_A:
  9487. case Js::OpCode::BrLe_A:
  9488. case Js::OpCode::BrNotGt_A:
  9489. case Js::OpCode::BrNotGe_A:
  9490. case Js::OpCode::BrNotLt_A:
  9491. case Js::OpCode::BrNotLe_A:
  9492. // Avoid if one source is known not to be a number.
  9493. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber())
  9494. {
  9495. return false;
  9496. }
  9497. break;
  9498. case Js::OpCode::Add_A:
  9499. // For Add, we need both sources to be Numbers, otherwise it could be a string concat
  9500. if (!src1Val || !src2Val || !(src1Val->GetValueInfo()->IsLikelyNumber() && src2Val->GetValueInfo()->IsLikelyNumber()))
  9501. {
  9502. return false;
  9503. }
  9504. break;
  9505. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  9506. skipSrc2 = true;
  9507. skipDst = true;
  9508. break;
  9509. default:
  9510. return false;
  9511. }
  9512. }
  9513. else
  9514. {
  9515. switch (instr->m_opcode)
  9516. {
  9517. case Js::OpCode::InlineArrayPush:
  9518. bool isFloatConstMissingItem = src2Val->GetValueInfo()->IsFloatConstant();
  9519. if(isFloatConstMissingItem)
  9520. {
  9521. FloatConstType floatValue = src2Val->GetValueInfo()->AsFloatConstant()->FloatValue();
  9522. isFloatConstMissingItem = Js::SparseArraySegment<double>::IsMissingItem(&floatValue);
  9523. }
  9524. // Don't specialize if the element is not likelyNumber - we will surely bailout
  9525. if(!(src2Val->GetValueInfo()->IsLikelyNumber()) || isFloatConstMissingItem)
  9526. {
  9527. return false;
  9528. }
  9529. // Only specialize the Second source - element
  9530. skipSrc1 = true;
  9531. skipDst = true;
  9532. allowUndefinedOrNullSrc2 = false;
  9533. break;
  9534. }
  9535. }
  9536. // Make sure the srcs are specialized
  9537. if(!skipSrc1)
  9538. {
  9539. src1 = instr->GetSrc1();
  9540. this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, (allowUndefinedOrNullSrc1 ? IR::BailOutPrimitiveButString : IR::BailOutNumberOnly));
  9541. }
  9542. if (!skipSrc2)
  9543. {
  9544. src2 = instr->GetSrc2();
  9545. this->ToFloat64(instr, src2, this->currentBlock, src2Val, nullptr, (allowUndefinedOrNullSrc2 ? IR::BailOutPrimitiveButString : IR::BailOutNumberOnly));
  9546. }
  9547. if (!skipDst)
  9548. {
  9549. dst = instr->GetDst();
  9550. if (dst)
  9551. {
  9552. *pDstVal = CreateDstUntransferredValue(ValueType::Float, instr, src1Val, src2Val);
  9553. AssertMsg(dst->IsRegOpnd(), "What else?");
  9554. this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  9555. }
  9556. }
  9557. GOPT_TRACE_INSTR(instr, _u("Type specialized to FLOAT: "));
  9558. #if ENABLE_DEBUG_CONFIG_OPTIONS
  9559. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FloatTypeSpecPhase))
  9560. {
  9561. Output::Print(_u("Type specialized to FLOAT: "));
  9562. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  9563. }
  9564. #endif
  9565. return true;
  9566. }
  9567. bool
  9568. GlobOpt::TypeSpecializeStElem(IR::Instr ** pInstr, Value *src1Val, Value **pDstVal)
  9569. {
  9570. IR::Instr *&instr = *pInstr;
  9571. IR::RegOpnd *baseOpnd = instr->GetDst()->AsIndirOpnd()->GetBaseOpnd();
  9572. ValueType baseValueType(baseOpnd->GetValueType());
  9573. if (instr->DoStackArgsOpt(this->func) ||
  9574. (!this->DoTypedArrayTypeSpec() && baseValueType.IsLikelyOptimizedTypedArray()) ||
  9575. (!this->DoNativeArrayTypeSpec() && baseValueType.IsLikelyNativeArray()) ||
  9576. !(baseValueType.IsLikelyOptimizedTypedArray() || baseValueType.IsLikelyNativeArray()))
  9577. {
  9578. GOPT_TRACE_INSTR(instr, _u("Didn't type specialize array access, because typed array type specialization is disabled, or base is not an optimized typed array.\n"));
  9579. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9580. {
  9581. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9582. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9583. baseValueType.ToString(baseValueTypeStr);
  9584. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because %s.\n"),
  9585. this->func->GetJITFunctionBody()->GetDisplayName(),
  9586. this->func->GetDebugNumberSet(debugStringBuffer),
  9587. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9588. baseValueTypeStr,
  9589. instr->DoStackArgsOpt(this->func) ?
  9590. _u("instruction uses the arguments object") :
  9591. _u("typed array type specialization is disabled, or base is not an optimized typed array"));
  9592. Output::Flush();
  9593. }
  9594. return false;
  9595. }
  9596. Assert(instr->GetSrc1()->IsRegOpnd() || (src1Val && src1Val->GetValueInfo()->HasIntConstantValue()));
  9597. StackSym *sym = instr->GetSrc1()->IsRegOpnd() ? instr->GetSrc1()->AsRegOpnd()->m_sym : nullptr;
  9598. // Only type specialize the source of store element if the source symbol is already type specialized to int or float.
  9599. if (sym)
  9600. {
  9601. if (baseValueType.IsLikelyNativeArray())
  9602. {
  9603. // Gently coerce these src's into native if it seems likely to work.
  9604. // Otherwise we can't use the fast path to store.
  9605. // But don't try to put a float-specialized number into an int array this way.
  9606. if (!(
  9607. CurrentBlockData()->IsInt32TypeSpecialized(sym) ||
  9608. (
  9609. src1Val &&
  9610. (
  9611. DoAggressiveIntTypeSpec()
  9612. ? src1Val->GetValueInfo()->IsLikelyInt()
  9613. : src1Val->GetValueInfo()->IsInt()
  9614. )
  9615. )
  9616. ))
  9617. {
  9618. if (!(
  9619. CurrentBlockData()->IsFloat64TypeSpecialized(sym) ||
  9620. (src1Val && src1Val->GetValueInfo()->IsLikelyNumber())
  9621. ) ||
  9622. baseValueType.HasIntElements())
  9623. {
  9624. return false;
  9625. }
  9626. }
  9627. }
  9628. else if (!CurrentBlockData()->IsInt32TypeSpecialized(sym) && !CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  9629. {
  9630. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because src is not type specialized.\n"));
  9631. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9632. {
  9633. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9634. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9635. baseValueType.ToString(baseValueTypeStr);
  9636. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because src is not specialized.\n"),
  9637. this->func->GetJITFunctionBody()->GetDisplayName(),
  9638. this->func->GetDebugNumberSet(debugStringBuffer),
  9639. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9640. baseValueTypeStr);
  9641. Output::Flush();
  9642. }
  9643. return false;
  9644. }
  9645. }
  9646. int32 src1IntConstantValue;
  9647. if(baseValueType.IsLikelyNativeIntArray() && src1Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&src1IntConstantValue))
  9648. {
  9649. if(Js::SparseArraySegment<int32>::IsMissingItem(&src1IntConstantValue))
  9650. {
  9651. return false;
  9652. }
  9653. }
  9654. // Note: doing ToVarUses to make sure we do get the int32 version of the index before trying to access its value in
  9655. // ShouldExpectConventionalArrayIndexValue. Not sure why that never gave us a problem before.
  9656. Assert(instr->GetDst()->IsIndirOpnd());
  9657. IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd();
  9658. // Make sure we use the int32 version of the index operand symbol, if available. Otherwise, ensure the var symbol is live (by
  9659. // potentially inserting a ToVar).
  9660. this->ToVarUses(instr, dst, /* isDst = */ true, nullptr);
  9661. if (!ShouldExpectConventionalArrayIndexValue(dst))
  9662. {
  9663. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because index is negative or likely not int.\n"));
  9664. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9665. {
  9666. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9667. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9668. baseValueType.ToString(baseValueTypeStr);
  9669. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because index is negative or likely not int.\n"),
  9670. this->func->GetJITFunctionBody()->GetDisplayName(),
  9671. this->func->GetDebugNumberSet(debugStringBuffer),
  9672. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9673. baseValueTypeStr);
  9674. Output::Flush();
  9675. }
  9676. return false;
  9677. }
  9678. IRType toType = TyVar;
  9679. bool isLossyAllowed = true;
  9680. IR::BailOutKind arrayBailOutKind = IR::BailOutConventionalTypedArrayAccessOnly;
  9681. switch(baseValueType.GetObjectType())
  9682. {
  9683. case ObjectType::Int8Array:
  9684. case ObjectType::Uint8Array:
  9685. case ObjectType::Int16Array:
  9686. case ObjectType::Uint16Array:
  9687. case ObjectType::Int32Array:
  9688. case ObjectType::Int8VirtualArray:
  9689. case ObjectType::Uint8VirtualArray:
  9690. case ObjectType::Int16VirtualArray:
  9691. case ObjectType::Uint16VirtualArray:
  9692. case ObjectType::Int32VirtualArray:
  9693. case ObjectType::Int8MixedArray:
  9694. case ObjectType::Uint8MixedArray:
  9695. case ObjectType::Int16MixedArray:
  9696. case ObjectType::Uint16MixedArray:
  9697. case ObjectType::Int32MixedArray:
  9698. Int32Array:
  9699. if (this->DoAggressiveIntTypeSpec() || this->DoFloatTypeSpec())
  9700. {
  9701. toType = TyInt32;
  9702. }
  9703. break;
  9704. case ObjectType::Uint32Array:
  9705. case ObjectType::Uint32VirtualArray:
  9706. case ObjectType::Uint32MixedArray:
  9707. // Uint32Arrays may store values that overflow int32. If the value being stored comes from a symbol that's
  9708. // already losslessly type specialized to int32, we'll use it. Otherwise, if we only have a float64 specialized
  9709. // value, we don't want to force bailout if it doesn't fit in int32. Instead, we'll emit conversion in the
  9710. // lowerer, and handle overflow, if necessary.
  9711. if (!sym || CurrentBlockData()->IsInt32TypeSpecialized(sym))
  9712. {
  9713. toType = TyInt32;
  9714. }
  9715. else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  9716. {
  9717. toType = TyFloat64;
  9718. }
  9719. break;
  9720. case ObjectType::Float32Array:
  9721. case ObjectType::Float64Array:
  9722. case ObjectType::Float32VirtualArray:
  9723. case ObjectType::Float32MixedArray:
  9724. case ObjectType::Float64VirtualArray:
  9725. case ObjectType::Float64MixedArray:
  9726. Float64Array:
  9727. if (this->DoFloatTypeSpec())
  9728. {
  9729. toType = TyFloat64;
  9730. }
  9731. break;
  9732. case ObjectType::Uint8ClampedArray:
  9733. case ObjectType::Uint8ClampedVirtualArray:
  9734. case ObjectType::Uint8ClampedMixedArray:
  9735. // Uint8ClampedArray requires rounding (as opposed to truncation) of floating point values. If source symbol is
  9736. // float type specialized, type specialize this instruction to float as well, and handle rounding in the
  9737. // lowerer.
  9738. if (!sym || CurrentBlockData()->IsInt32TypeSpecialized(sym))
  9739. {
  9740. toType = TyInt32;
  9741. isLossyAllowed = false;
  9742. }
  9743. else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym))
  9744. {
  9745. toType = TyFloat64;
  9746. }
  9747. break;
  9748. default:
  9749. Assert(baseValueType.IsLikelyNativeArray());
  9750. isLossyAllowed = false;
  9751. arrayBailOutKind = IR::BailOutConventionalNativeArrayAccessOnly;
  9752. if(baseValueType.HasIntElements())
  9753. {
  9754. goto Int32Array;
  9755. }
  9756. Assert(baseValueType.HasFloatElements());
  9757. goto Float64Array;
  9758. }
  9759. if (toType != TyVar)
  9760. {
  9761. GOPT_TRACE_INSTR(instr, _u("Type specialized array access.\n"));
  9762. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9763. {
  9764. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9765. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9766. baseValueType.ToString(baseValueTypeStr);
  9767. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, type specialized to %s.\n"),
  9768. this->func->GetJITFunctionBody()->GetDisplayName(),
  9769. this->func->GetDebugNumberSet(debugStringBuffer),
  9770. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9771. baseValueTypeStr,
  9772. toType == TyInt32 ? _u("int32") : _u("float64"));
  9773. Output::Flush();
  9774. }
  9775. IR::BailOutKind bailOutKind = ((toType == TyInt32) ? IR::BailOutIntOnly : IR::BailOutNumberOnly);
  9776. this->ToTypeSpecUse(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, toType, bailOutKind, /* lossy = */ isLossyAllowed);
  9777. if (!this->IsLoopPrePass())
  9778. {
  9779. bool bConvertToBailoutInstr = true;
  9780. // Definite StElemC doesn't need bailout, because it can't fail or cause conversion.
  9781. if (instr->m_opcode == Js::OpCode::StElemC && baseValueType.IsObject())
  9782. {
  9783. if (baseValueType.HasIntElements())
  9784. {
  9785. //Native int array requires a missing element check & bailout
  9786. int32 min = INT32_MIN;
  9787. int32 max = INT32_MAX;
  9788. if (src1Val->GetValueInfo()->GetIntValMinMax(&min, &max, false))
  9789. {
  9790. bConvertToBailoutInstr = ((min <= Js::JavascriptNativeIntArray::MissingItem) && (max >= Js::JavascriptNativeIntArray::MissingItem));
  9791. }
  9792. }
  9793. else
  9794. {
  9795. bConvertToBailoutInstr = false;
  9796. }
  9797. }
  9798. if (bConvertToBailoutInstr)
  9799. {
  9800. if(instr->HasBailOutInfo())
  9801. {
  9802. const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind();
  9803. Assert(
  9804. (
  9805. !(oldBailOutKind & ~IR::BailOutKindBits) ||
  9806. (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp
  9807. ) &&
  9808. !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject)));
  9809. if(arrayBailOutKind == IR::BailOutConventionalTypedArrayAccessOnly)
  9810. {
  9811. // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head
  9812. // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction
  9813. // bails out for the right reason.
  9814. instr->SetBailOutKind(
  9815. arrayBailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall)));
  9816. }
  9817. else
  9818. {
  9819. // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit
  9820. // calls to occur, so it must be merged in to eliminate generating the helper call.
  9821. Assert(arrayBailOutKind == IR::BailOutConventionalNativeArrayAccessOnly);
  9822. instr->SetBailOutKind(oldBailOutKind | arrayBailOutKind);
  9823. }
  9824. }
  9825. else
  9826. {
  9827. GenerateBailAtOperation(&instr, arrayBailOutKind);
  9828. }
  9829. }
  9830. }
  9831. }
  9832. else
  9833. {
  9834. GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because the source was not already specialized.\n"));
  9835. if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func))
  9836. {
  9837. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  9838. char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  9839. baseValueType.ToString(baseValueTypeStr);
  9840. Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not type specialize, because of array type.\n"),
  9841. this->func->GetJITFunctionBody()->GetDisplayName(),
  9842. this->func->GetDebugNumberSet(debugStringBuffer),
  9843. Js::OpCodeUtil::GetOpCodeName(instr->m_opcode),
  9844. baseValueTypeStr);
  9845. Output::Flush();
  9846. }
  9847. }
  9848. return toType != TyVar;
  9849. }
  9850. IR::Instr *
  9851. GlobOpt::ToVarUses(IR::Instr *instr, IR::Opnd *opnd, bool isDst, Value *val)
  9852. {
  9853. Sym *sym;
  9854. switch (opnd->GetKind())
  9855. {
  9856. case IR::OpndKindReg:
  9857. if (!isDst && !CurrentBlockData()->liveVarSyms->Test(opnd->AsRegOpnd()->m_sym->m_id))
  9858. {
  9859. instr = this->ToVar(instr, opnd->AsRegOpnd(), this->currentBlock, val, true);
  9860. }
  9861. break;
  9862. case IR::OpndKindSym:
  9863. sym = opnd->AsSymOpnd()->m_sym;
  9864. if (sym->IsPropertySym() && !CurrentBlockData()->liveVarSyms->Test(sym->AsPropertySym()->m_stackSym->m_id)
  9865. && sym->AsPropertySym()->m_stackSym->IsVar())
  9866. {
  9867. StackSym *propertyBase = sym->AsPropertySym()->m_stackSym;
  9868. IR::RegOpnd *newOpnd = IR::RegOpnd::New(propertyBase, TyVar, instr->m_func);
  9869. instr = this->ToVar(instr, newOpnd, this->currentBlock, CurrentBlockData()->FindValue(propertyBase), true);
  9870. }
  9871. break;
  9872. case IR::OpndKindIndir:
  9873. IR::RegOpnd *baseOpnd = opnd->AsIndirOpnd()->GetBaseOpnd();
  9874. if (!CurrentBlockData()->liveVarSyms->Test(baseOpnd->m_sym->m_id))
  9875. {
  9876. instr = this->ToVar(instr, baseOpnd, this->currentBlock, CurrentBlockData()->FindValue(baseOpnd->m_sym), true);
  9877. }
  9878. IR::RegOpnd *indexOpnd = opnd->AsIndirOpnd()->GetIndexOpnd();
  9879. if (indexOpnd && !indexOpnd->m_sym->IsTypeSpec())
  9880. {
  9881. instr = ToTypeSpecIndex(instr, indexOpnd, opnd->AsIndirOpnd());
  9882. }
  9883. break;
  9884. }
  9885. return instr;
  9886. }
  9887. IR::Instr *
  9888. GlobOpt::ToTypeSpecIndex(IR::Instr * instr, IR::RegOpnd * indexOpnd, IR::IndirOpnd * indirOpnd)
  9889. {
  9890. Assert(indirOpnd != nullptr || indexOpnd == instr->GetSrc1());
  9891. bool isGetterOrSetter = instr->m_opcode == Js::OpCode::InitGetElemI ||
  9892. instr->m_opcode == Js::OpCode::InitSetElemI ||
  9893. instr->m_opcode == Js::OpCode::InitClassMemberGetComputedName ||
  9894. instr->m_opcode == Js::OpCode::InitClassMemberSetComputedName;
  9895. if (!isGetterOrSetter // typespec is disabled for getters, setters
  9896. && (indexOpnd->GetValueType().IsInt()
  9897. ? !IsTypeSpecPhaseOff(func)
  9898. : indexOpnd->GetValueType().IsLikelyInt() && DoAggressiveIntTypeSpec())
  9899. && !GetIsAsmJSFunc()) // typespec is disabled for asmjs
  9900. {
  9901. StackSym *const indexVarSym = indexOpnd->m_sym;
  9902. Value *const indexValue = CurrentBlockData()->FindValue(indexVarSym);
  9903. Assert(indexValue);
  9904. Assert(indexValue->GetValueInfo()->IsLikelyInt());
  9905. ToInt32(instr, indexOpnd, currentBlock, indexValue, indirOpnd, false);
  9906. Assert(indexValue->GetValueInfo()->IsInt() || IsLoopPrePass());
  9907. if (!IsLoopPrePass())
  9908. {
  9909. IR::Opnd * intOpnd = indirOpnd ? indirOpnd->GetIndexOpnd() : instr->GetSrc1();
  9910. if (intOpnd != nullptr)
  9911. {
  9912. Assert(!intOpnd->IsRegOpnd() || intOpnd->AsRegOpnd()->m_sym->IsTypeSpec());
  9913. IntConstantBounds indexConstantBounds;
  9914. AssertVerify(indexValue->GetValueInfo()->TryGetIntConstantBounds(&indexConstantBounds));
  9915. if (ValueInfo::IsGreaterThanOrEqualTo(
  9916. indexValue,
  9917. indexConstantBounds.LowerBound(),
  9918. indexConstantBounds.UpperBound(),
  9919. nullptr,
  9920. 0,
  9921. 0))
  9922. {
  9923. intOpnd->SetType(TyUint32);
  9924. }
  9925. }
  9926. }
  9927. }
  9928. else if (!CurrentBlockData()->liveVarSyms->Test(indexOpnd->m_sym->m_id))
  9929. {
  9930. instr = this->ToVar(instr, indexOpnd, this->currentBlock, CurrentBlockData()->FindValue(indexOpnd->m_sym), true);
  9931. }
  9932. return instr;
  9933. }
  9934. IR::Instr *
  9935. GlobOpt::ToVar(IR::Instr *instr, IR::RegOpnd *regOpnd, BasicBlock *block, Value *value, bool needsUpdate)
  9936. {
  9937. IR::Instr *newInstr;
  9938. StackSym *varSym = regOpnd->m_sym;
  9939. if (IsTypeSpecPhaseOff(this->func))
  9940. {
  9941. return instr;
  9942. }
  9943. if (this->IsLoopPrePass())
  9944. {
  9945. block->globOptData.liveVarSyms->Set(varSym->m_id);
  9946. return instr;
  9947. }
  9948. if (block->globOptData.liveVarSyms->Test(varSym->m_id))
  9949. {
  9950. // Already live, nothing to do
  9951. return instr;
  9952. }
  9953. if (!varSym->IsVar())
  9954. {
  9955. Assert(!varSym->IsTypeSpec());
  9956. // Leave non-vars alone.
  9957. return instr;
  9958. }
  9959. Assert(block->globOptData.IsTypeSpecialized(varSym));
  9960. if (!value)
  9961. {
  9962. value = block->globOptData.FindValue(varSym);
  9963. }
  9964. ValueInfo *valueInfo = value ? value->GetValueInfo() : nullptr;
  9965. if(valueInfo && valueInfo->IsInt())
  9966. {
  9967. // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value
  9968. // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as
  9969. // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the
  9970. // lossy state.
  9971. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  9972. }
  9973. IRType fromType = TyIllegal;
  9974. StackSym *typeSpecSym = nullptr;
  9975. if (block->globOptData.liveInt32Syms->Test(varSym->m_id) && !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id))
  9976. {
  9977. fromType = TyInt32;
  9978. typeSpecSym = varSym->GetInt32EquivSym(this->func);
  9979. Assert(valueInfo);
  9980. Assert(valueInfo->IsInt());
  9981. }
  9982. else if (block->globOptData.liveFloat64Syms->Test(varSym->m_id))
  9983. {
  9984. fromType = TyFloat64;
  9985. typeSpecSym = varSym->GetFloat64EquivSym(this->func);
  9986. // Ensure that all bailout FromVars that generate a value for this type-specialized sym will bail out on any non-number
  9987. // value, even ones that have already been generated before. Float-specialized non-number values cannot be converted
  9988. // back to Var since they will not go back to the original non-number value. The dead-store pass will update the bailout
  9989. // kind on already-generated FromVars based on this bit.
  9990. typeSpecSym->m_requiresBailOnNotNumber = true;
  9991. // A previous float conversion may have used BailOutPrimitiveButString, which does not change the value type to say
  9992. // definitely float, since it can also be a non-string primitive. The convert back to Var though, will cause that
  9993. // bailout kind to be changed to BailOutNumberOnly in the dead-store phase, so from the point of the initial conversion
  9994. // to float, that the value is definitely number. Since we don't know where the FromVar is, change the value type here.
  9995. if(valueInfo)
  9996. {
  9997. if(!valueInfo->IsNumber())
  9998. {
  9999. valueInfo = valueInfo->SpecializeToFloat64(alloc);
  10000. ChangeValueInfo(block, value, valueInfo);
  10001. regOpnd->SetValueType(valueInfo->Type());
  10002. }
  10003. }
  10004. else
  10005. {
  10006. value = NewGenericValue(ValueType::Float);
  10007. valueInfo = value->GetValueInfo();
  10008. block->globOptData.SetValue(value, varSym);
  10009. regOpnd->SetValueType(valueInfo->Type());
  10010. }
  10011. }
  10012. else
  10013. {
  10014. Assert(UNREACHED);
  10015. }
  10016. AssertOrFailFast(valueInfo);
  10017. int32 intConstantValue;
  10018. if (valueInfo->TryGetIntConstantValue(&intConstantValue))
  10019. {
  10020. // Lower will tag or create a number directly
  10021. newInstr = IR::Instr::New(Js::OpCode::LdC_A_I4, regOpnd,
  10022. IR::IntConstOpnd::New(intConstantValue, TyInt32, instr->m_func), instr->m_func);
  10023. }
  10024. else
  10025. {
  10026. IR::RegOpnd * regNew = IR::RegOpnd::New(typeSpecSym, fromType, instr->m_func);
  10027. Js::OpCode opcode = Js::OpCode::ToVar;
  10028. regNew->SetIsJITOptimizedReg(true);
  10029. newInstr = IR::Instr::New(opcode, regOpnd, regNew, instr->m_func);
  10030. }
  10031. newInstr->SetByteCodeOffset(instr);
  10032. newInstr->GetDst()->AsRegOpnd()->SetIsJITOptimizedReg(true);
  10033. ValueType valueType = valueInfo->Type();
  10034. if(fromType == TyInt32)
  10035. {
  10036. #if !INT32VAR // All 32-bit ints are taggable on 64-bit architectures
  10037. IntConstantBounds constantBounds;
  10038. AssertVerify(valueInfo->TryGetIntConstantBounds(&constantBounds));
  10039. if(constantBounds.IsTaggable())
  10040. #endif
  10041. {
  10042. // The value is within the taggable range, so set the opnd value types to TaggedInt to avoid the overflow check
  10043. valueType = ValueType::GetTaggedInt();
  10044. }
  10045. }
  10046. newInstr->GetDst()->SetValueType(valueType);
  10047. newInstr->GetSrc1()->SetValueType(valueType);
  10048. IR::Instr *insertAfterInstr = instr->m_prev;
  10049. if (instr == block->GetLastInstr() &&
  10050. (instr->IsBranchInstr() || instr->m_opcode == Js::OpCode::BailTarget))
  10051. {
  10052. // Don't insert code between the branch and the preceding ByteCodeUses instrs...
  10053. while(insertAfterInstr->m_opcode == Js::OpCode::ByteCodeUses)
  10054. {
  10055. insertAfterInstr = insertAfterInstr->m_prev;
  10056. }
  10057. }
  10058. block->InsertInstrAfter(newInstr, insertAfterInstr);
  10059. block->globOptData.liveVarSyms->Set(varSym->m_id);
  10060. GOPT_TRACE_OPND(regOpnd, _u("Converting to var\n"));
  10061. if (block->loop)
  10062. {
  10063. Assert(!this->IsLoopPrePass());
  10064. this->TryHoistInvariant(newInstr, block, value, value, nullptr, false);
  10065. }
  10066. if (needsUpdate)
  10067. {
  10068. // Make sure that the kill effect of the ToVar instruction is tracked and that the kill of a property
  10069. // type is reflected in the current instruction.
  10070. this->ProcessKills(newInstr);
  10071. this->ValueNumberObjectType(newInstr->GetDst(), newInstr);
  10072. if (instr->GetSrc1() && instr->GetSrc1()->IsSymOpnd() && instr->GetSrc1()->AsSymOpnd()->IsPropertySymOpnd())
  10073. {
  10074. // Reprocess the load source. We need to reset the PropertySymOpnd fields first.
  10075. IR::PropertySymOpnd *propertySymOpnd = instr->GetSrc1()->AsPropertySymOpnd();
  10076. if (propertySymOpnd->IsTypeCheckSeqCandidate())
  10077. {
  10078. propertySymOpnd->SetTypeChecked(false);
  10079. propertySymOpnd->SetTypeAvailable(false);
  10080. propertySymOpnd->SetWriteGuardChecked(false);
  10081. }
  10082. this->FinishOptPropOp(instr, propertySymOpnd);
  10083. instr = this->SetTypeCheckBailOut(instr->GetSrc1(), instr, nullptr);
  10084. }
  10085. }
  10086. return instr;
  10087. }
  10088. IR::Instr *
  10089. GlobOpt::ToInt32(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, bool lossy)
  10090. {
  10091. return this->ToTypeSpecUse(instr, opnd, block, val, indir, TyInt32, IR::BailOutIntOnly, lossy);
  10092. }
  10093. IR::Instr *
  10094. GlobOpt::ToFloat64(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, IR::BailOutKind bailOutKind)
  10095. {
  10096. return this->ToTypeSpecUse(instr, opnd, block, val, indir, TyFloat64, bailOutKind);
  10097. }
  10098. IR::Instr *
  10099. GlobOpt::ToTypeSpecUse(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, IRType toType, IR::BailOutKind bailOutKind, bool lossy, IR::Instr *insertBeforeInstr)
  10100. {
  10101. Assert(bailOutKind != IR::BailOutInvalid);
  10102. IR::Instr *newInstr;
  10103. if (!val && opnd->IsRegOpnd())
  10104. {
  10105. val = block->globOptData.FindValue(opnd->AsRegOpnd()->m_sym);
  10106. }
  10107. ValueInfo *valueInfo = val ? val->GetValueInfo() : nullptr;
  10108. bool needReplaceSrc = false;
  10109. bool updateBlockLastInstr = false;
  10110. if (instr)
  10111. {
  10112. needReplaceSrc = true;
  10113. if (!insertBeforeInstr)
  10114. {
  10115. insertBeforeInstr = instr;
  10116. }
  10117. }
  10118. else if (!insertBeforeInstr)
  10119. {
  10120. // Insert it at the end of the block
  10121. insertBeforeInstr = block->GetLastInstr();
  10122. if (insertBeforeInstr->IsBranchInstr() || insertBeforeInstr->m_opcode == Js::OpCode::BailTarget)
  10123. {
  10124. // Don't insert code between the branch and the preceding ByteCodeUses instrs...
  10125. while(insertBeforeInstr->m_prev->m_opcode == Js::OpCode::ByteCodeUses)
  10126. {
  10127. insertBeforeInstr = insertBeforeInstr->m_prev;
  10128. }
  10129. }
  10130. else
  10131. {
  10132. insertBeforeInstr = insertBeforeInstr->m_next;
  10133. updateBlockLastInstr = true;
  10134. }
  10135. }
  10136. // Int constant values will be propagated into the instruction. For ArgOut_A_InlineBuiltIn, there's no benefit from
  10137. // const-propping, so those are excluded.
  10138. if (opnd->IsRegOpnd() &&
  10139. !(
  10140. valueInfo &&
  10141. (valueInfo->HasIntConstantValue() || valueInfo->IsFloatConstant()) &&
  10142. (!instr || instr->m_opcode != Js::OpCode::ArgOut_A_InlineBuiltIn)
  10143. ))
  10144. {
  10145. IR::RegOpnd *regSrc = opnd->AsRegOpnd();
  10146. StackSym *varSym = regSrc->m_sym;
  10147. Js::OpCode opcode = Js::OpCode::FromVar;
  10148. if (varSym->IsTypeSpec() || !block->globOptData.liveVarSyms->Test(varSym->m_id))
  10149. {
  10150. // Conversion between int32 and float64
  10151. if (varSym->IsTypeSpec())
  10152. {
  10153. varSym = varSym->GetVarEquivSym(this->func);
  10154. }
  10155. opcode = Js::OpCode::Conv_Prim;
  10156. }
  10157. Assert(block->globOptData.liveVarSyms->Test(varSym->m_id) || block->globOptData.IsTypeSpecialized(varSym));
  10158. StackSym *typeSpecSym = nullptr;
  10159. BOOL isLive = FALSE;
  10160. BVSparse<JitArenaAllocator> *livenessBv = nullptr;
  10161. if(valueInfo && valueInfo->IsInt())
  10162. {
  10163. // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value
  10164. // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as
  10165. // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the
  10166. // lossy state.
  10167. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10168. }
  10169. if (toType == TyInt32)
  10170. {
  10171. // Need to determine whether the conversion is actually lossy or lossless. If the value is an int, then it's a
  10172. // lossless conversion despite the type of conversion requested. The liveness of the converted int32 sym needs to be
  10173. // set to reflect the actual type of conversion done. Also, a lossless conversion needs the value to determine
  10174. // whether the conversion may need to bail out.
  10175. Assert(valueInfo);
  10176. if(valueInfo->IsInt())
  10177. {
  10178. lossy = false;
  10179. }
  10180. else
  10181. {
  10182. Assert(IsLoopPrePass() || !block->globOptData.IsInt32TypeSpecialized(varSym));
  10183. }
  10184. livenessBv = block->globOptData.liveInt32Syms;
  10185. isLive = livenessBv->Test(varSym->m_id) && (lossy || !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id));
  10186. if (this->IsLoopPrePass())
  10187. {
  10188. if (!isLive)
  10189. {
  10190. livenessBv->Set(varSym->m_id);
  10191. if (lossy)
  10192. {
  10193. block->globOptData.liveLossyInt32Syms->Set(varSym->m_id);
  10194. }
  10195. else
  10196. {
  10197. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10198. }
  10199. }
  10200. return instr;
  10201. }
  10202. typeSpecSym = varSym->GetInt32EquivSym(this->func);
  10203. if (!isLive)
  10204. {
  10205. if (!opnd->IsVar() ||
  10206. !block->globOptData.liveVarSyms->Test(varSym->m_id) ||
  10207. (block->globOptData.liveFloat64Syms->Test(varSym->m_id) && valueInfo && valueInfo->IsLikelyFloat()))
  10208. {
  10209. Assert(block->globOptData.liveFloat64Syms->Test(varSym->m_id));
  10210. if(!lossy && !valueInfo->IsInt())
  10211. {
  10212. // Shouldn't try to do a lossless conversion from float64 to int32 when the value is not known to be an
  10213. // int. There are cases where we need more than two passes over loops to flush out all dependencies.
  10214. // It's possible for the loop prepass to think that a sym s1 remains an int because it acquires the
  10215. // value of another sym s2 that is an int in the prepass at that time. However, s2 can become a float
  10216. // later in the loop body, in which case s1 would become a float on the second iteration of the loop. By
  10217. // that time, we would have already committed to having s1 live as a lossless int on entry into the
  10218. // loop, and we end up having to compensate by doing a lossless conversion from float to int, which will
  10219. // need a bailout and will most likely bail out.
  10220. //
  10221. // If s2 becomes a var instead of a float, then the compensation is legal although not ideal. After
  10222. // enough bailouts, rejit would be triggered with aggressive int type spec turned off. For the
  10223. // float-to-int conversion though, there's no point in emitting a bailout because we already know that
  10224. // the value is a float and has high probability of bailing out (whereas a var has a chance to be a
  10225. // tagged int), and so currently lossless conversion from float to int with bailout is not supported.
  10226. //
  10227. // So, treating this case as a compile-time bailout. The exception will trigger the jit work item to be
  10228. // restarted with aggressive int type specialization disabled.
  10229. if(bailOutKind == IR::BailOutExpectingInteger)
  10230. {
  10231. Assert(IsSwitchOptEnabledForIntTypeSpec());
  10232. throw Js::RejitException(RejitReason::DisableSwitchOptExpectingInteger);
  10233. }
  10234. else
  10235. {
  10236. Assert(DoAggressiveIntTypeSpec());
  10237. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  10238. {
  10239. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  10240. Output::Print(
  10241. _u("BailOut (compile-time): function: %s (%s) varSym: "),
  10242. this->func->GetJITFunctionBody()->GetDisplayName(),
  10243. this->func->GetDebugNumberSet(debugStringBuffer),
  10244. varSym->m_id);
  10245. #if DBG_DUMP
  10246. varSym->Dump();
  10247. #else
  10248. Output::Print(_u("s%u"), varSym->m_id);
  10249. #endif
  10250. if(varSym->HasByteCodeRegSlot())
  10251. {
  10252. Output::Print(_u(" byteCodeReg: R%u"), varSym->GetByteCodeRegSlot());
  10253. }
  10254. Output::Print(_u(" (lossless conversion from float64 to int32)\n"));
  10255. Output::Flush();
  10256. }
  10257. if(!DoAggressiveIntTypeSpec())
  10258. {
  10259. // Aggressive int type specialization is already off for some reason. Prevent trying to rejit again
  10260. // because it won't help and the same thing will happen again. Just abort jitting this function.
  10261. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  10262. {
  10263. Output::Print(_u(" Aborting JIT because AggressiveIntTypeSpec is already off\n"));
  10264. Output::Flush();
  10265. }
  10266. throw Js::OperationAbortedException();
  10267. }
  10268. throw Js::RejitException(RejitReason::AggressiveIntTypeSpecDisabled);
  10269. }
  10270. }
  10271. if(opnd->IsVar())
  10272. {
  10273. regSrc->SetType(TyFloat64);
  10274. regSrc->m_sym = varSym->GetFloat64EquivSym(this->func);
  10275. opcode = Js::OpCode::Conv_Prim;
  10276. }
  10277. else
  10278. {
  10279. Assert(regSrc->IsFloat64());
  10280. Assert(regSrc->m_sym->IsFloat64());
  10281. Assert(opcode == Js::OpCode::Conv_Prim);
  10282. }
  10283. }
  10284. }
  10285. GOPT_TRACE_OPND(regSrc, _u("Converting to int32\n"));
  10286. }
  10287. else if (toType == TyFloat64)
  10288. {
  10289. // float64
  10290. typeSpecSym = varSym->GetFloat64EquivSym(this->func);
  10291. if(!IsLoopPrePass() && typeSpecSym->m_requiresBailOnNotNumber && block->globOptData.IsFloat64TypeSpecialized(varSym))
  10292. {
  10293. // This conversion is already protected by a BailOutNumberOnly bailout (or at least it will be after the
  10294. // dead-store phase). Since 'requiresBailOnNotNumber' is not flow-based, change the value to definitely float.
  10295. if(valueInfo)
  10296. {
  10297. if(!valueInfo->IsNumber())
  10298. {
  10299. valueInfo = valueInfo->SpecializeToFloat64(alloc);
  10300. ChangeValueInfo(block, val, valueInfo);
  10301. opnd->SetValueType(valueInfo->Type());
  10302. }
  10303. }
  10304. else
  10305. {
  10306. val = NewGenericValue(ValueType::Float);
  10307. valueInfo = val->GetValueInfo();
  10308. block->globOptData.SetValue(val, varSym);
  10309. opnd->SetValueType(valueInfo->Type());
  10310. }
  10311. }
  10312. if(bailOutKind == IR::BailOutNumberOnly)
  10313. {
  10314. if(!IsLoopPrePass())
  10315. {
  10316. // Ensure that all bailout FromVars that generate a value for this type-specialized sym will bail out on any
  10317. // non-number value, even ones that have already been generated before. The dead-store pass will update the
  10318. // bailout kind on already-generated FromVars based on this bit.
  10319. typeSpecSym->m_requiresBailOnNotNumber = true;
  10320. }
  10321. }
  10322. else if(typeSpecSym->m_requiresBailOnNotNumber)
  10323. {
  10324. Assert(bailOutKind == IR::BailOutPrimitiveButString);
  10325. bailOutKind = IR::BailOutNumberOnly;
  10326. }
  10327. livenessBv = block->globOptData.liveFloat64Syms;
  10328. isLive = livenessBv->Test(varSym->m_id);
  10329. if (this->IsLoopPrePass())
  10330. {
  10331. if(!isLive)
  10332. {
  10333. livenessBv->Set(varSym->m_id);
  10334. }
  10335. if (this->OptIsInvariant(opnd, block, this->prePassLoop, val, false, true))
  10336. {
  10337. this->prePassLoop->forceFloat64SymsOnEntry->Set(varSym->m_id);
  10338. }
  10339. else
  10340. {
  10341. Sym *symStore = (valueInfo ? valueInfo->GetSymStore() : NULL);
  10342. if (symStore && symStore != varSym
  10343. && this->OptIsInvariant(symStore, block, this->prePassLoop, block->globOptData.FindValue(symStore), false, true))
  10344. {
  10345. // If symStore is assigned to sym and we want sym to be type-specialized, for symStore to be specialized
  10346. // outside the loop.
  10347. this->prePassLoop->forceFloat64SymsOnEntry->Set(symStore->m_id);
  10348. }
  10349. }
  10350. return instr;
  10351. }
  10352. if (!isLive && regSrc->IsVar())
  10353. {
  10354. if (!block->globOptData.liveVarSyms->Test(varSym->m_id) ||
  10355. (
  10356. block->globOptData.liveInt32Syms->Test(varSym->m_id) &&
  10357. !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id) &&
  10358. valueInfo &&
  10359. valueInfo->IsLikelyInt()
  10360. ))
  10361. {
  10362. Assert(block->globOptData.liveInt32Syms->Test(varSym->m_id));
  10363. Assert(!block->globOptData.liveLossyInt32Syms->Test(varSym->m_id)); // Shouldn't try to convert a lossy int32 to anything
  10364. regSrc->SetType(TyInt32);
  10365. regSrc->m_sym = varSym->GetInt32EquivSym(this->func);
  10366. opcode = Js::OpCode::Conv_Prim;
  10367. }
  10368. }
  10369. GOPT_TRACE_OPND(regSrc, _u("Converting to float64\n"));
  10370. }
  10371. bool needLoad = false;
  10372. if (needReplaceSrc)
  10373. {
  10374. bool wasDead = regSrc->GetIsDead();
  10375. // needReplaceSrc means we are type specializing a use, and need to replace the src on the instr
  10376. if (!isLive)
  10377. {
  10378. needLoad = true;
  10379. // ReplaceSrc will delete it.
  10380. regSrc = regSrc->Copy(instr->m_func)->AsRegOpnd();
  10381. }
  10382. IR::RegOpnd * regNew = IR::RegOpnd::New(typeSpecSym, toType, instr->m_func);
  10383. if(valueInfo)
  10384. {
  10385. regNew->SetValueType(valueInfo->Type());
  10386. regNew->m_wasNegativeZeroPreventedByBailout = valueInfo->WasNegativeZeroPreventedByBailout();
  10387. }
  10388. regNew->SetIsDead(wasDead);
  10389. regNew->SetIsJITOptimizedReg(true);
  10390. this->CaptureByteCodeSymUses(instr);
  10391. if (indir == nullptr)
  10392. {
  10393. instr->ReplaceSrc(opnd, regNew);
  10394. }
  10395. else
  10396. {
  10397. indir->ReplaceIndexOpnd(regNew);
  10398. }
  10399. opnd = regNew;
  10400. if (!needLoad)
  10401. {
  10402. Assert(isLive);
  10403. return instr;
  10404. }
  10405. }
  10406. else
  10407. {
  10408. // We just need to insert a load of a type spec sym
  10409. if(isLive)
  10410. {
  10411. return instr;
  10412. }
  10413. // Insert it before the specified instruction
  10414. instr = insertBeforeInstr;
  10415. }
  10416. IR::RegOpnd *regDst = IR::RegOpnd::New(typeSpecSym, toType, instr->m_func);
  10417. bool isBailout = false;
  10418. bool isHoisted = false;
  10419. bool isInLandingPad = (block->next && !block->next->isDeleted && block->next->isLoopHeader);
  10420. if (isInLandingPad)
  10421. {
  10422. Loop *loop = block->next->loop;
  10423. Assert(loop && loop->landingPad == block);
  10424. Assert(loop->bailOutInfo);
  10425. }
  10426. if (opcode == Js::OpCode::FromVar)
  10427. {
  10428. if (toType == TyInt32)
  10429. {
  10430. Assert(valueInfo);
  10431. if (lossy)
  10432. {
  10433. if (!valueInfo->IsPrimitive() && !block->globOptData.IsTypeSpecialized(varSym))
  10434. {
  10435. // Lossy conversions to int32 on non-primitive values may have implicit calls to toString or valueOf, which
  10436. // may be overridden to have a side effect. The side effect needs to happen every time the conversion is
  10437. // supposed to happen, so the resulting lossy int32 value cannot be reused. Bail out on implicit calls.
  10438. Assert(DoLossyIntTypeSpec());
  10439. bailOutKind = IR::BailOutOnNotPrimitive;
  10440. isBailout = true;
  10441. }
  10442. }
  10443. else if (!valueInfo->IsInt())
  10444. {
  10445. // The operand is likely an int (hence the request to convert to int), so bail out if it's not an int. Only
  10446. // bail out if a lossless conversion to int is requested. Lossy conversions to int such as in (a | 0) don't
  10447. // need to bail out.
  10448. if (bailOutKind == IR::BailOutExpectingInteger)
  10449. {
  10450. Assert(IsSwitchOptEnabledForIntTypeSpec());
  10451. }
  10452. else
  10453. {
  10454. Assert(DoAggressiveIntTypeSpec());
  10455. }
  10456. isBailout = true;
  10457. }
  10458. }
  10459. else if (toType == TyFloat64 &&
  10460. (!valueInfo || !valueInfo->IsNumber()))
  10461. {
  10462. // Bailout if converting vars to float if we can't prove they are floats:
  10463. // x = str + float; -> need to bailout if str is a string
  10464. //
  10465. // x = obj * 0.1;
  10466. // y = obj * 0.2; -> if obj has valueof, we'll only call valueof once on the FromVar conversion...
  10467. Assert(bailOutKind != IR::BailOutInvalid);
  10468. isBailout = true;
  10469. }
  10470. }
  10471. if (isBailout)
  10472. {
  10473. if (isInLandingPad)
  10474. {
  10475. Loop *loop = block->next->loop;
  10476. this->EnsureBailTarget(loop);
  10477. instr = loop->bailOutInfo->bailOutInstr;
  10478. updateBlockLastInstr = false;
  10479. newInstr = IR::BailOutInstr::New(opcode, bailOutKind, loop->bailOutInfo, instr->m_func);
  10480. newInstr->SetDst(regDst);
  10481. newInstr->SetSrc1(regSrc);
  10482. }
  10483. else
  10484. {
  10485. newInstr = IR::BailOutInstr::New(opcode, regDst, regSrc, bailOutKind, instr, instr->m_func);
  10486. }
  10487. }
  10488. else
  10489. {
  10490. newInstr = IR::Instr::New(opcode, regDst, regSrc, instr->m_func);
  10491. }
  10492. newInstr->SetByteCodeOffset(instr);
  10493. instr->InsertBefore(newInstr);
  10494. if (updateBlockLastInstr)
  10495. {
  10496. block->SetLastInstr(newInstr);
  10497. }
  10498. regDst->SetIsJITOptimizedReg(true);
  10499. newInstr->GetSrc1()->AsRegOpnd()->SetIsJITOptimizedReg(true);
  10500. ValueInfo *const oldValueInfo = valueInfo;
  10501. if(valueInfo)
  10502. {
  10503. newInstr->GetSrc1()->SetValueType(valueInfo->Type());
  10504. }
  10505. if(isBailout)
  10506. {
  10507. Assert(opcode == Js::OpCode::FromVar);
  10508. if(toType == TyInt32)
  10509. {
  10510. Assert(valueInfo);
  10511. if(!lossy)
  10512. {
  10513. Assert(bailOutKind == IR::BailOutIntOnly || bailOutKind == IR::BailOutExpectingInteger);
  10514. valueInfo = valueInfo->SpecializeToInt32(alloc, isPerformingLoopBackEdgeCompensation);
  10515. ChangeValueInfo(nullptr, val, valueInfo);
  10516. int32 intConstantValue;
  10517. if(indir && needReplaceSrc && valueInfo->TryGetIntConstantValue(&intConstantValue))
  10518. {
  10519. // A likely-int value can have constant bounds due to conditional branches narrowing its range. Now that
  10520. // the sym has been proven to be an int, the likely-int value, after specialization, will be constant.
  10521. // Replace the index opnd in the indir with an offset.
  10522. Assert(opnd == indir->GetIndexOpnd());
  10523. Assert(indir->GetScale() == 0);
  10524. indir->UnlinkIndexOpnd()->Free(instr->m_func);
  10525. opnd = nullptr;
  10526. indir->SetOffset(intConstantValue);
  10527. }
  10528. }
  10529. }
  10530. else if (toType == TyFloat64)
  10531. {
  10532. if(bailOutKind == IR::BailOutNumberOnly)
  10533. {
  10534. if(valueInfo)
  10535. {
  10536. valueInfo = valueInfo->SpecializeToFloat64(alloc);
  10537. ChangeValueInfo(block, val, valueInfo);
  10538. }
  10539. else
  10540. {
  10541. val = NewGenericValue(ValueType::Float);
  10542. valueInfo = val->GetValueInfo();
  10543. block->globOptData.SetValue(val, varSym);
  10544. }
  10545. }
  10546. }
  10547. else
  10548. {
  10549. Assert(UNREACHED);
  10550. }
  10551. }
  10552. if(valueInfo)
  10553. {
  10554. newInstr->GetDst()->SetValueType(valueInfo->Type());
  10555. if(needReplaceSrc && opnd)
  10556. {
  10557. opnd->SetValueType(valueInfo->Type());
  10558. }
  10559. }
  10560. if (block->loop)
  10561. {
  10562. Assert(!this->IsLoopPrePass());
  10563. isHoisted = this->TryHoistInvariant(newInstr, block, val, val, nullptr, false, lossy, false, bailOutKind);
  10564. }
  10565. if (isBailout)
  10566. {
  10567. if (!isHoisted && !isInLandingPad)
  10568. {
  10569. if(valueInfo)
  10570. {
  10571. // Since this is a pre-op bailout, the old value info should be used for the purposes of bailout. For
  10572. // instance, the value info could be LikelyInt but with a constant range. Once specialized to int, the value
  10573. // info would be an int constant. However, the int constant is only guaranteed if the value is actually an
  10574. // int, which this conversion is verifying, so bailout cannot assume the constant value.
  10575. if(oldValueInfo)
  10576. {
  10577. val->SetValueInfo(oldValueInfo);
  10578. }
  10579. else
  10580. {
  10581. block->globOptData.ClearSymValue(varSym);
  10582. }
  10583. }
  10584. // Fill in bail out info if the FromVar is a bailout instr, and it wasn't hoisted as invariant.
  10585. // If it was hoisted, the invariant code will fill out the bailout info with the loop landing pad bailout info.
  10586. this->FillBailOutInfo(block, newInstr);
  10587. if(valueInfo)
  10588. {
  10589. // Restore the new value info after filling the bailout info
  10590. if(oldValueInfo)
  10591. {
  10592. val->SetValueInfo(valueInfo);
  10593. }
  10594. else
  10595. {
  10596. block->globOptData.SetValue(val, varSym);
  10597. }
  10598. }
  10599. }
  10600. }
  10601. // Now that we've captured the liveness in the bailout info, we can mark this as live.
  10602. // This type specialized sym isn't live if the FromVar bails out.
  10603. livenessBv->Set(varSym->m_id);
  10604. if(toType == TyInt32)
  10605. {
  10606. if(lossy)
  10607. {
  10608. block->globOptData.liveLossyInt32Syms->Set(varSym->m_id);
  10609. }
  10610. else
  10611. {
  10612. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10613. }
  10614. }
  10615. }
  10616. else
  10617. {
  10618. Assert(valueInfo);
  10619. if(opnd->IsRegOpnd() && valueInfo->IsInt())
  10620. {
  10621. // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value
  10622. // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as
  10623. // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the
  10624. // lossy state.
  10625. block->globOptData.liveLossyInt32Syms->Clear(opnd->AsRegOpnd()->m_sym->m_id);
  10626. if(toType == TyInt32)
  10627. {
  10628. lossy = false;
  10629. }
  10630. }
  10631. if (this->IsLoopPrePass())
  10632. {
  10633. if(opnd->IsRegOpnd())
  10634. {
  10635. StackSym *const sym = opnd->AsRegOpnd()->m_sym;
  10636. if(toType == TyInt32)
  10637. {
  10638. Assert(!sym->IsTypeSpec());
  10639. block->globOptData.liveInt32Syms->Set(sym->m_id);
  10640. if(lossy)
  10641. {
  10642. block->globOptData.liveLossyInt32Syms->Set(sym->m_id);
  10643. }
  10644. else
  10645. {
  10646. block->globOptData.liveLossyInt32Syms->Clear(sym->m_id);
  10647. }
  10648. }
  10649. else
  10650. {
  10651. Assert(toType == TyFloat64);
  10652. AnalysisAssert(instr);
  10653. StackSym *const varSym = sym->IsTypeSpec() ? sym->GetVarEquivSym(instr->m_func) : sym;
  10654. block->globOptData.liveFloat64Syms->Set(varSym->m_id);
  10655. }
  10656. }
  10657. return instr;
  10658. }
  10659. if (!needReplaceSrc)
  10660. {
  10661. instr = insertBeforeInstr;
  10662. }
  10663. IR::Opnd *constOpnd;
  10664. int32 intConstantValue;
  10665. if(valueInfo->TryGetIntConstantValue(&intConstantValue))
  10666. {
  10667. if(toType == TyInt32)
  10668. {
  10669. constOpnd = IR::IntConstOpnd::New(intConstantValue, TyInt32, instr->m_func);
  10670. }
  10671. else
  10672. {
  10673. Assert(toType == TyFloat64);
  10674. constOpnd = IR::FloatConstOpnd::New(static_cast<FloatConstType>(intConstantValue), TyFloat64, instr->m_func);
  10675. }
  10676. }
  10677. else if(valueInfo->IsFloatConstant())
  10678. {
  10679. const FloatConstType floatValue = valueInfo->AsFloatConstant()->FloatValue();
  10680. if(toType == TyInt32)
  10681. {
  10682. Assert(lossy);
  10683. constOpnd =
  10684. IR::IntConstOpnd::New(
  10685. Js::JavascriptMath::ToInt32(floatValue),
  10686. TyInt32,
  10687. instr->m_func);
  10688. }
  10689. else
  10690. {
  10691. Assert(toType == TyFloat64);
  10692. constOpnd = IR::FloatConstOpnd::New(floatValue, TyFloat64, instr->m_func);
  10693. }
  10694. }
  10695. else
  10696. {
  10697. Assert(opnd->IsVar());
  10698. Assert(opnd->IsAddrOpnd());
  10699. AssertMsg(opnd->AsAddrOpnd()->IsVar(), "We only expect to see addr that are var before lower.");
  10700. // Don't need to capture uses, we are only replacing an addr opnd
  10701. if(toType == TyInt32)
  10702. {
  10703. constOpnd = IR::IntConstOpnd::New(Js::TaggedInt::ToInt32(opnd->AsAddrOpnd()->m_address), TyInt32, instr->m_func);
  10704. }
  10705. else
  10706. {
  10707. Assert(toType == TyFloat64);
  10708. constOpnd = IR::FloatConstOpnd::New(Js::TaggedInt::ToDouble(opnd->AsAddrOpnd()->m_address), TyFloat64, instr->m_func);
  10709. }
  10710. }
  10711. if (toType == TyInt32)
  10712. {
  10713. if (needReplaceSrc)
  10714. {
  10715. CaptureByteCodeSymUses(instr);
  10716. if(indir)
  10717. {
  10718. Assert(opnd == indir->GetIndexOpnd());
  10719. Assert(indir->GetScale() == 0);
  10720. indir->UnlinkIndexOpnd()->Free(instr->m_func);
  10721. indir->SetOffset(constOpnd->AsIntConstOpnd()->AsInt32());
  10722. }
  10723. else
  10724. {
  10725. instr->ReplaceSrc(opnd, constOpnd);
  10726. }
  10727. }
  10728. else
  10729. {
  10730. StackSym *varSym = opnd->AsRegOpnd()->m_sym;
  10731. if(varSym->IsTypeSpec())
  10732. {
  10733. varSym = varSym->GetVarEquivSym(nullptr);
  10734. Assert(varSym);
  10735. }
  10736. if(block->globOptData.liveInt32Syms->TestAndSet(varSym->m_id))
  10737. {
  10738. Assert(!!block->globOptData.liveLossyInt32Syms->Test(varSym->m_id) == lossy);
  10739. }
  10740. else
  10741. {
  10742. if(lossy)
  10743. {
  10744. block->globOptData.liveLossyInt32Syms->Set(varSym->m_id);
  10745. }
  10746. StackSym *int32Sym = varSym->GetInt32EquivSym(instr->m_func);
  10747. IR::RegOpnd *int32Reg = IR::RegOpnd::New(int32Sym, TyInt32, instr->m_func);
  10748. int32Reg->SetIsJITOptimizedReg(true);
  10749. newInstr = IR::Instr::New(Js::OpCode::Ld_I4, int32Reg, constOpnd, instr->m_func);
  10750. newInstr->SetByteCodeOffset(instr);
  10751. instr->InsertBefore(newInstr);
  10752. if (updateBlockLastInstr)
  10753. {
  10754. block->SetLastInstr(newInstr);
  10755. }
  10756. }
  10757. }
  10758. }
  10759. else
  10760. {
  10761. StackSym *floatSym;
  10762. bool newFloatSym = false;
  10763. StackSym* varSym;
  10764. if (opnd->IsRegOpnd())
  10765. {
  10766. varSym = opnd->AsRegOpnd()->m_sym;
  10767. if (varSym->IsTypeSpec())
  10768. {
  10769. varSym = varSym->GetVarEquivSym(nullptr);
  10770. Assert(varSym);
  10771. }
  10772. floatSym = varSym->GetFloat64EquivSym(instr->m_func);
  10773. }
  10774. else
  10775. {
  10776. varSym = block->globOptData.GetCopyPropSym(nullptr, val);
  10777. if(!varSym)
  10778. {
  10779. // Clear the symstore to ensure it's set below to this new symbol
  10780. this->SetSymStoreDirect(val->GetValueInfo(), nullptr);
  10781. varSym = StackSym::New(TyVar, instr->m_func);
  10782. newFloatSym = true;
  10783. }
  10784. floatSym = varSym->GetFloat64EquivSym(instr->m_func);
  10785. }
  10786. IR::RegOpnd *floatReg = IR::RegOpnd::New(floatSym, TyFloat64, instr->m_func);
  10787. floatReg->SetIsJITOptimizedReg(true);
  10788. // If the value is not live - let's load it.
  10789. if(!block->globOptData.liveFloat64Syms->TestAndSet(varSym->m_id))
  10790. {
  10791. newInstr = IR::Instr::New(Js::OpCode::LdC_F8_R8, floatReg, constOpnd, instr->m_func);
  10792. newInstr->SetByteCodeOffset(instr);
  10793. instr->InsertBefore(newInstr);
  10794. if (updateBlockLastInstr)
  10795. {
  10796. block->SetLastInstr(newInstr);
  10797. }
  10798. if(newFloatSym)
  10799. {
  10800. block->globOptData.SetValue(val, varSym);
  10801. }
  10802. // Src is always invariant, but check if the dst is, and then hoist.
  10803. if (block->loop &&
  10804. (
  10805. (newFloatSym && block->loop->CanHoistInvariants()) ||
  10806. this->OptIsInvariant(floatReg, block, block->loop, val, false, false)
  10807. ))
  10808. {
  10809. Assert(!this->IsLoopPrePass());
  10810. this->OptHoistInvariant(newInstr, block, block->loop, val, val, nullptr, false);
  10811. }
  10812. }
  10813. if (needReplaceSrc)
  10814. {
  10815. CaptureByteCodeSymUses(instr);
  10816. instr->ReplaceSrc(opnd, floatReg);
  10817. }
  10818. }
  10819. return instr;
  10820. }
  10821. return newInstr;
  10822. }
  10823. void
  10824. GlobOpt::ToVarRegOpnd(IR::RegOpnd *dst, BasicBlock *block)
  10825. {
  10826. ToVarStackSym(dst->m_sym, block);
  10827. }
  10828. void
  10829. GlobOpt::ToVarStackSym(StackSym *varSym, BasicBlock *block)
  10830. {
  10831. //added another check for sym , in case of asmjs there is mostly no var syms and hence added a new check to see if it is the primary sym
  10832. Assert(!varSym->IsTypeSpec());
  10833. block->globOptData.liveVarSyms->Set(varSym->m_id);
  10834. block->globOptData.liveInt32Syms->Clear(varSym->m_id);
  10835. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10836. block->globOptData.liveFloat64Syms->Clear(varSym->m_id);
  10837. }
  10838. void
  10839. GlobOpt::ToInt32Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block)
  10840. {
  10841. StackSym *varSym = dst->m_sym;
  10842. Assert(!varSym->IsTypeSpec());
  10843. if (!this->IsLoopPrePass() && varSym->IsVar())
  10844. {
  10845. StackSym *int32Sym = varSym->GetInt32EquivSym(instr->m_func);
  10846. // Use UnlinkDst / SetDst to make sure isSingleDef is tracked properly,
  10847. // since we'll just be hammering the symbol.
  10848. dst = instr->UnlinkDst()->AsRegOpnd();
  10849. dst->m_sym = int32Sym;
  10850. dst->SetType(TyInt32);
  10851. instr->SetDst(dst);
  10852. }
  10853. block->globOptData.liveInt32Syms->Set(varSym->m_id);
  10854. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); // The store makes it lossless
  10855. block->globOptData.liveVarSyms->Clear(varSym->m_id);
  10856. block->globOptData.liveFloat64Syms->Clear(varSym->m_id);
  10857. }
  10858. void
  10859. GlobOpt::ToUInt32Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block)
  10860. {
  10861. // We should be calling only for asmjs function
  10862. Assert(GetIsAsmJSFunc());
  10863. StackSym *varSym = dst->m_sym;
  10864. Assert(!varSym->IsTypeSpec());
  10865. block->globOptData.liveInt32Syms->Set(varSym->m_id);
  10866. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); // The store makes it lossless
  10867. block->globOptData.liveVarSyms->Clear(varSym->m_id);
  10868. block->globOptData.liveFloat64Syms->Clear(varSym->m_id);
  10869. }
  10870. void
  10871. GlobOpt::ToFloat64Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block)
  10872. {
  10873. StackSym *varSym = dst->m_sym;
  10874. Assert(!varSym->IsTypeSpec());
  10875. if (!this->IsLoopPrePass() && varSym->IsVar())
  10876. {
  10877. StackSym *float64Sym = varSym->GetFloat64EquivSym(this->func);
  10878. // Use UnlinkDst / SetDst to make sure isSingleDef is tracked properly,
  10879. // since we'll just be hammering the symbol.
  10880. dst = instr->UnlinkDst()->AsRegOpnd();
  10881. dst->m_sym = float64Sym;
  10882. dst->SetType(TyFloat64);
  10883. instr->SetDst(dst);
  10884. }
  10885. block->globOptData.liveFloat64Syms->Set(varSym->m_id);
  10886. block->globOptData.liveVarSyms->Clear(varSym->m_id);
  10887. block->globOptData.liveInt32Syms->Clear(varSym->m_id);
  10888. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id);
  10889. }
  10890. static void SetIsConstFlag(StackSym* dstSym, int64 value)
  10891. {
  10892. Assert(dstSym);
  10893. dstSym->SetIsInt64Const();
  10894. }
  10895. static void SetIsConstFlag(StackSym* dstSym, int value)
  10896. {
  10897. Assert(dstSym);
  10898. dstSym->SetIsIntConst(value);
  10899. }
  10900. static IR::Opnd* CreateIntConstOpnd(IR::Instr* instr, int64 value)
  10901. {
  10902. return (IR::Opnd*)IR::Int64ConstOpnd::New(value, instr->GetDst()->GetType(), instr->m_func);
  10903. }
  10904. static IR::Opnd* CreateIntConstOpnd(IR::Instr* instr, int value)
  10905. {
  10906. IntConstType constVal;
  10907. if (instr->GetDst()->IsUnsigned())
  10908. {
  10909. // we should zero extend in case of uint
  10910. constVal = (uint32)value;
  10911. }
  10912. else
  10913. {
  10914. constVal = value;
  10915. }
  10916. return (IR::Opnd*)IR::IntConstOpnd::New(constVal, instr->GetDst()->GetType(), instr->m_func);
  10917. }
  10918. template <typename T>
  10919. IR::Opnd* GlobOpt::ReplaceWConst(IR::Instr **pInstr, T value, Value **pDstVal)
  10920. {
  10921. IR::Instr * &instr = *pInstr;
  10922. IR::Opnd * constOpnd = CreateIntConstOpnd(instr, value);
  10923. instr->ReplaceSrc1(constOpnd);
  10924. instr->FreeSrc2();
  10925. this->OptSrc(constOpnd, &instr);
  10926. IR::Opnd *dst = instr->GetDst();
  10927. StackSym *dstSym = dst->AsRegOpnd()->m_sym;
  10928. if (dstSym->IsSingleDef())
  10929. {
  10930. SetIsConstFlag(dstSym, value);
  10931. }
  10932. GOPT_TRACE_INSTR(instr, _u("Constant folding to %d: \n"), value);
  10933. *pDstVal = GetIntConstantValue(value, instr, dst);
  10934. return dst;
  10935. }
  10936. template <typename T>
  10937. bool GlobOpt::OptConstFoldBinaryWasm(
  10938. IR::Instr** pInstr,
  10939. const Value* src1,
  10940. const Value* src2,
  10941. Value **pDstVal)
  10942. {
  10943. IR::Instr* &instr = *pInstr;
  10944. if (!DoConstFold())
  10945. {
  10946. return false;
  10947. }
  10948. T src1IntConstantValue, src2IntConstantValue;
  10949. if (!src1 || !src1->GetValueInfo()->TryGetIntConstantValue(&src1IntConstantValue, false) || //a bit sketchy: false for int32 means likelyInt = false
  10950. !src2 || !src2->GetValueInfo()->TryGetIntConstantValue(&src2IntConstantValue, false) //and unsigned = false for int64
  10951. )
  10952. {
  10953. return false;
  10954. }
  10955. int64 tmpValueOut;
  10956. if (!instr->BinaryCalculatorT<T>(src1IntConstantValue, src2IntConstantValue, &tmpValueOut, func->GetJITFunctionBody()->IsWasmFunction()))
  10957. {
  10958. return false;
  10959. }
  10960. this->CaptureByteCodeSymUses(instr);
  10961. IR::Opnd *dst = (instr->GetDst()->IsInt64()) ? //dst can be int32 for int64 comparison operators
  10962. ReplaceWConst(pInstr, tmpValueOut, pDstVal) :
  10963. ReplaceWConst(pInstr, (int)tmpValueOut, pDstVal);
  10964. instr->m_opcode = Js::OpCode::Ld_I4;
  10965. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  10966. return true;
  10967. }
  10968. bool
  10969. GlobOpt::OptConstFoldBinary(
  10970. IR::Instr * *pInstr,
  10971. const IntConstantBounds &src1IntConstantBounds,
  10972. const IntConstantBounds &src2IntConstantBounds,
  10973. Value **pDstVal)
  10974. {
  10975. IR::Instr * &instr = *pInstr;
  10976. int32 value;
  10977. IR::IntConstOpnd *constOpnd;
  10978. if (!DoConstFold())
  10979. {
  10980. return false;
  10981. }
  10982. int32 src1IntConstantValue = -1;
  10983. int32 src2IntConstantValue = -1;
  10984. int32 src1MaxIntConstantValue = -1;
  10985. int32 src2MaxIntConstantValue = -1;
  10986. int32 src1MinIntConstantValue = -1;
  10987. int32 src2MinIntConstantValue = -1;
  10988. if (instr->IsBranchInstr())
  10989. {
  10990. src1MinIntConstantValue = src1IntConstantBounds.LowerBound();
  10991. src1MaxIntConstantValue = src1IntConstantBounds.UpperBound();
  10992. src2MinIntConstantValue = src2IntConstantBounds.LowerBound();
  10993. src2MaxIntConstantValue = src2IntConstantBounds.UpperBound();
  10994. }
  10995. else if (src1IntConstantBounds.IsConstant() && src2IntConstantBounds.IsConstant())
  10996. {
  10997. src1IntConstantValue = src1IntConstantBounds.LowerBound();
  10998. src2IntConstantValue = src2IntConstantBounds.LowerBound();
  10999. }
  11000. else
  11001. {
  11002. return false;
  11003. }
  11004. IntConstType tmpValueOut;
  11005. if (!instr->BinaryCalculator(src1IntConstantValue, src2IntConstantValue, &tmpValueOut, TyInt32)
  11006. || !Math::FitsInDWord(tmpValueOut))
  11007. {
  11008. return false;
  11009. }
  11010. value = (int32)tmpValueOut;
  11011. this->CaptureByteCodeSymUses(instr);
  11012. constOpnd = IR::IntConstOpnd::New(value, TyInt32, instr->m_func);
  11013. instr->ReplaceSrc1(constOpnd);
  11014. instr->FreeSrc2();
  11015. this->OptSrc(constOpnd, &instr);
  11016. IR::Opnd *dst = instr->GetDst();
  11017. Assert(dst->IsRegOpnd());
  11018. StackSym *dstSym = dst->AsRegOpnd()->m_sym;
  11019. if (dstSym->IsSingleDef())
  11020. {
  11021. dstSym->SetIsIntConst(value);
  11022. }
  11023. GOPT_TRACE_INSTR(instr, _u("Constant folding to %d: \n"), value);
  11024. *pDstVal = GetIntConstantValue(value, instr, dst);
  11025. if (IsTypeSpecPhaseOff(this->func))
  11026. {
  11027. instr->m_opcode = Js::OpCode::LdC_A_I4;
  11028. this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock);
  11029. }
  11030. else
  11031. {
  11032. instr->m_opcode = Js::OpCode::Ld_I4;
  11033. this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock);
  11034. }
  11035. InvalidateInductionVariables(instr);
  11036. return true;
  11037. }
  11038. void
  11039. GlobOpt::OptConstFoldBr(bool test, IR::Instr *instr, Value * src1Val, Value * src2Val)
  11040. {
  11041. GOPT_TRACE_INSTR(instr, _u("Constant folding to branch: "));
  11042. BasicBlock *deadBlock;
  11043. if (src1Val)
  11044. {
  11045. this->ToInt32(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, false);
  11046. }
  11047. if (src2Val)
  11048. {
  11049. this->ToInt32(instr, instr->GetSrc2(), this->currentBlock, src2Val, nullptr, false);
  11050. }
  11051. this->CaptureByteCodeSymUses(instr);
  11052. if (test)
  11053. {
  11054. instr->m_opcode = Js::OpCode::Br;
  11055. instr->FreeSrc1();
  11056. if(instr->GetSrc2())
  11057. {
  11058. instr->FreeSrc2();
  11059. }
  11060. deadBlock = instr->m_next->AsLabelInstr()->GetBasicBlock();
  11061. }
  11062. else
  11063. {
  11064. AssertMsg(instr->m_next->IsLabelInstr(), "Next instr of branch should be a label...");
  11065. if(instr->AsBranchInstr()->IsMultiBranch())
  11066. {
  11067. return;
  11068. }
  11069. deadBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock();
  11070. instr->FreeSrc1();
  11071. if(instr->GetSrc2())
  11072. {
  11073. instr->FreeSrc2();
  11074. }
  11075. instr->m_opcode = Js::OpCode::Nop;
  11076. }
  11077. // Loop back edge: we would have already decremented data use count for the tail block when we processed the loop header.
  11078. if (!(this->currentBlock->loop && this->currentBlock->loop->GetHeadBlock() == deadBlock))
  11079. {
  11080. this->currentBlock->DecrementDataUseCount();
  11081. }
  11082. this->currentBlock->RemoveDeadSucc(deadBlock, this->func->m_fg);
  11083. if (deadBlock->GetPredList()->Count() == 0)
  11084. {
  11085. deadBlock->SetDataUseCount(0);
  11086. }
  11087. }
  11088. void
  11089. GlobOpt::ChangeValueType(
  11090. BasicBlock *const block,
  11091. Value *const value,
  11092. const ValueType newValueType,
  11093. const bool preserveSubclassInfo,
  11094. const bool allowIncompatibleType) const
  11095. {
  11096. Assert(value);
  11097. // Why are we trying to change the value type of the type sym value? Asserting here to make sure we don't deep copy the type sym's value info.
  11098. Assert(!value->GetValueInfo()->IsJsType());
  11099. ValueInfo *const valueInfo = value->GetValueInfo();
  11100. const ValueType valueType(valueInfo->Type());
  11101. if(valueType == newValueType && (preserveSubclassInfo || valueInfo->IsGeneric()))
  11102. {
  11103. return;
  11104. }
  11105. // ArrayValueInfo has information specific to the array type, so make sure that doesn't change
  11106. Assert(
  11107. !preserveSubclassInfo ||
  11108. !valueInfo->IsArrayValueInfo() ||
  11109. newValueType.IsObject() && newValueType.GetObjectType() == valueInfo->GetObjectType());
  11110. Assert(!valueInfo->GetSymStore() || !valueInfo->GetSymStore()->IsStackSym() || !valueInfo->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable());
  11111. ValueInfo *const newValueInfo =
  11112. preserveSubclassInfo
  11113. ? valueInfo->Copy(alloc)
  11114. : valueInfo->CopyWithGenericStructureKind(alloc);
  11115. newValueInfo->Type() = newValueType;
  11116. ChangeValueInfo(block, value, newValueInfo, allowIncompatibleType);
  11117. }
  11118. void
  11119. GlobOpt::ChangeValueInfo(BasicBlock *const block, Value *const value, ValueInfo *const newValueInfo, const bool allowIncompatibleType, const bool compensated) const
  11120. {
  11121. Assert(value);
  11122. Assert(newValueInfo);
  11123. // The value type must be changed to something more specific or something more generic. For instance, it would be changed to
  11124. // something more specific if the current value type is LikelyArray and checks have been done to ensure that it's an array,
  11125. // and it would be changed to something more generic if a call kills the Array value type and it must be treated as
  11126. // LikelyArray going forward.
  11127. // There are cases where we change the type because of different profile information, and because of rejit, these profile information
  11128. // may conflict. Need to allow incompatible type in those cause. However, the old type should be indefinite.
  11129. Assert((allowIncompatibleType && !value->GetValueInfo()->IsDefinite()) ||
  11130. AreValueInfosCompatible(newValueInfo, value->GetValueInfo()));
  11131. // ArrayValueInfo has information specific to the array type, so make sure that doesn't change
  11132. Assert(
  11133. !value->GetValueInfo()->IsArrayValueInfo() ||
  11134. !newValueInfo->IsArrayValueInfo() ||
  11135. newValueInfo->GetObjectType() == value->GetValueInfo()->GetObjectType());
  11136. if(block)
  11137. {
  11138. TrackValueInfoChangeForKills(block, value, newValueInfo, compensated);
  11139. }
  11140. value->SetValueInfo(newValueInfo);
  11141. }
  11142. bool
  11143. GlobOpt::AreValueInfosCompatible(const ValueInfo *const v0, const ValueInfo *const v1) const
  11144. {
  11145. Assert(v0);
  11146. Assert(v1);
  11147. if(v0->IsUninitialized() || v1->IsUninitialized())
  11148. {
  11149. return true;
  11150. }
  11151. const bool doAggressiveIntTypeSpec = DoAggressiveIntTypeSpec();
  11152. if(doAggressiveIntTypeSpec && (v0->IsInt() || v1->IsInt()))
  11153. {
  11154. // Int specialization in some uncommon loop cases involving dependencies, needs to allow specializing values of
  11155. // arbitrary types, even values that are definitely not int, to compensate for aggressive assumptions made by a loop
  11156. // prepass
  11157. return true;
  11158. }
  11159. if ((v0->Type()).IsMixedTypedArrayPair(v1->Type()) || (v1->Type()).IsMixedTypedArrayPair(v0->Type()))
  11160. {
  11161. return true;
  11162. }
  11163. const bool doFloatTypeSpec = DoFloatTypeSpec();
  11164. if(doFloatTypeSpec && (v0->IsFloat() || v1->IsFloat()))
  11165. {
  11166. // Float specialization allows specializing values of arbitrary types, even values that are definitely not float
  11167. return true;
  11168. }
  11169. const bool doArrayMissingValueCheckHoist = DoArrayMissingValueCheckHoist();
  11170. const bool doNativeArrayTypeSpec = DoNativeArrayTypeSpec();
  11171. const auto AreValueTypesCompatible = [=](const ValueType t0, const ValueType t1)
  11172. {
  11173. return
  11174. t0.IsSubsetOf(t1, doAggressiveIntTypeSpec, doFloatTypeSpec, doArrayMissingValueCheckHoist, doNativeArrayTypeSpec) ||
  11175. t1.IsSubsetOf(t0, doAggressiveIntTypeSpec, doFloatTypeSpec, doArrayMissingValueCheckHoist, doNativeArrayTypeSpec);
  11176. };
  11177. const ValueType t0(v0->Type().ToDefinite()), t1(v1->Type().ToDefinite());
  11178. if(t0.IsLikelyObject() && t1.IsLikelyObject())
  11179. {
  11180. // Check compatibility for the primitive portions and the object portions of the value types separately
  11181. if(AreValueTypesCompatible(t0.ToDefiniteObject(), t1.ToDefiniteObject()) &&
  11182. (
  11183. !t0.HasBeenPrimitive() ||
  11184. !t1.HasBeenPrimitive() ||
  11185. AreValueTypesCompatible(t0.ToDefinitePrimitiveSubset(), t1.ToDefinitePrimitiveSubset())
  11186. ))
  11187. {
  11188. return true;
  11189. }
  11190. }
  11191. else if(AreValueTypesCompatible(t0, t1))
  11192. {
  11193. return true;
  11194. }
  11195. const FloatConstantValueInfo *floatConstantValueInfo;
  11196. const ValueInfo *likelyIntValueinfo;
  11197. if(v0->IsFloatConstant() && v1->IsLikelyInt())
  11198. {
  11199. floatConstantValueInfo = v0->AsFloatConstant();
  11200. likelyIntValueinfo = v1;
  11201. }
  11202. else if(v0->IsLikelyInt() && v1->IsFloatConstant())
  11203. {
  11204. floatConstantValueInfo = v1->AsFloatConstant();
  11205. likelyIntValueinfo = v0;
  11206. }
  11207. else
  11208. {
  11209. return false;
  11210. }
  11211. // A float constant value with a value that is actually an int is a subset of a likely-int value.
  11212. // Ideally, we should create an int constant value for this up front, such that IsInt() also returns true. There
  11213. // were other issues with that, should see if that can be done.
  11214. int32 int32Value;
  11215. return
  11216. Js::JavascriptNumber::TryGetInt32Value(floatConstantValueInfo->FloatValue(), &int32Value) &&
  11217. (!likelyIntValueinfo->IsLikelyTaggedInt() || !Js::TaggedInt::IsOverflow(int32Value));
  11218. }
  11219. #if DBG
  11220. void
  11221. GlobOpt::VerifyArrayValueInfoForTracking(
  11222. const ValueInfo *const valueInfo,
  11223. const bool isJsArray,
  11224. const BasicBlock *const block,
  11225. const bool ignoreKnownImplicitCalls) const
  11226. {
  11227. Assert(valueInfo);
  11228. Assert(valueInfo->IsAnyOptimizedArray());
  11229. Assert(isJsArray == valueInfo->IsArrayOrObjectWithArray());
  11230. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11231. Assert(block);
  11232. Loop *implicitCallsLoop;
  11233. if(block->next && !block->next->isDeleted && block->next->isLoopHeader)
  11234. {
  11235. // Since a loop's landing pad does not have user code, determine whether disabling implicit calls is allowed in the
  11236. // landing pad based on the loop for which this block is the landing pad.
  11237. implicitCallsLoop = block->next->loop;
  11238. Assert(implicitCallsLoop);
  11239. Assert(implicitCallsLoop->landingPad == block);
  11240. }
  11241. else
  11242. {
  11243. implicitCallsLoop = block->loop;
  11244. }
  11245. Assert(
  11246. !isJsArray ||
  11247. DoArrayCheckHoist(valueInfo->Type(), implicitCallsLoop) ||
  11248. (
  11249. ignoreKnownImplicitCalls &&
  11250. !(implicitCallsLoop ? ImplicitCallFlagsAllowOpts(implicitCallsLoop) : ImplicitCallFlagsAllowOpts(func))
  11251. ));
  11252. Assert(!(isJsArray && valueInfo->HasNoMissingValues() && !DoArrayMissingValueCheckHoist()));
  11253. Assert(
  11254. !(
  11255. valueInfo->IsArrayValueInfo() &&
  11256. (
  11257. valueInfo->AsArrayValueInfo()->HeadSegmentSym() ||
  11258. valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()
  11259. ) &&
  11260. !DoArraySegmentHoist(valueInfo->Type())
  11261. ));
  11262. #if 0
  11263. // We can't assert here that there is only a head segment length sym if hoisting is allowed in the current block,
  11264. // because we may have propagated the sym forward out of a loop, and hoisting may be allowed inside but not
  11265. // outside the loop.
  11266. Assert(
  11267. isJsArray ||
  11268. !valueInfo->IsArrayValueInfo() ||
  11269. !valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym() ||
  11270. DoTypedArraySegmentLengthHoist(implicitCallsLoop) ||
  11271. ignoreKnownImplicitCalls ||
  11272. (implicitCallsLoop ? ImplicitCallFlagsAllowOpts(implicitCallsLoop) : ImplicitCallFlagsAllowOpts(func))
  11273. );
  11274. #endif
  11275. Assert(
  11276. !(
  11277. isJsArray &&
  11278. valueInfo->IsArrayValueInfo() &&
  11279. valueInfo->AsArrayValueInfo()->LengthSym() &&
  11280. !DoArrayLengthHoist()
  11281. ));
  11282. }
  11283. #endif
  11284. void
  11285. GlobOpt::TrackNewValueForKills(Value *const value)
  11286. {
  11287. Assert(value);
  11288. if(!value->GetValueInfo()->IsAnyOptimizedArray())
  11289. {
  11290. return;
  11291. }
  11292. DoTrackNewValueForKills(value);
  11293. }
  11294. void
  11295. GlobOpt::DoTrackNewValueForKills(Value *const value)
  11296. {
  11297. Assert(value);
  11298. ValueInfo *const valueInfo = value->GetValueInfo();
  11299. Assert(valueInfo->IsAnyOptimizedArray());
  11300. Assert(!valueInfo->IsArrayValueInfo());
  11301. // The value and value info here are new, so it's okay to modify the value info in-place
  11302. Assert(!valueInfo->GetSymStore());
  11303. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11304. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11305. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11306. Loop *implicitCallsLoop;
  11307. if(currentBlock->next && !currentBlock->next->isDeleted && currentBlock->next->isLoopHeader)
  11308. {
  11309. // Since a loop's landing pad does not have user code, determine whether disabling implicit calls is allowed in the
  11310. // landing pad based on the loop for which this block is the landing pad.
  11311. implicitCallsLoop = currentBlock->next->loop;
  11312. Assert(implicitCallsLoop);
  11313. Assert(implicitCallsLoop->landingPad == currentBlock);
  11314. }
  11315. else
  11316. {
  11317. implicitCallsLoop = currentBlock->loop;
  11318. }
  11319. if(isJsArray || isVirtualTypedArray)
  11320. {
  11321. if(!DoArrayCheckHoist(valueInfo->Type(), implicitCallsLoop))
  11322. {
  11323. // Array opts are disabled for this value type, so treat it as an indefinite value type going forward
  11324. valueInfo->Type() = valueInfo->Type().ToLikely();
  11325. return;
  11326. }
  11327. if(isJsArray && valueInfo->HasNoMissingValues() && !DoArrayMissingValueCheckHoist())
  11328. {
  11329. valueInfo->Type() = valueInfo->Type().SetHasNoMissingValues(false);
  11330. }
  11331. }
  11332. #if DBG
  11333. VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock);
  11334. #endif
  11335. if(!isJsArray && !isVirtualTypedArray)
  11336. {
  11337. return;
  11338. }
  11339. // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the
  11340. // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can
  11341. // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can
  11342. // revert the value type to a likely version.
  11343. CurrentBlockData()->valuesToKillOnCalls->Add(value);
  11344. }
  11345. void
  11346. GlobOpt::TrackCopiedValueForKills(Value *const value)
  11347. {
  11348. Assert(value);
  11349. if(!value->GetValueInfo()->IsAnyOptimizedArray())
  11350. {
  11351. return;
  11352. }
  11353. DoTrackCopiedValueForKills(value);
  11354. }
  11355. void
  11356. GlobOpt::DoTrackCopiedValueForKills(Value *const value)
  11357. {
  11358. Assert(value);
  11359. ValueInfo *const valueInfo = value->GetValueInfo();
  11360. Assert(valueInfo->IsAnyOptimizedArray());
  11361. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11362. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11363. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11364. #if DBG
  11365. VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock);
  11366. #endif
  11367. if(!isJsArray && !isVirtualTypedArray && !(valueInfo->IsArrayValueInfo() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()))
  11368. {
  11369. return;
  11370. }
  11371. // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the
  11372. // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can
  11373. // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can
  11374. // revert the value type to a likely version.
  11375. CurrentBlockData()->valuesToKillOnCalls->Add(value);
  11376. }
  11377. void
  11378. GlobOpt::TrackMergedValueForKills(
  11379. Value *const value,
  11380. GlobOptBlockData *const blockData,
  11381. BVSparse<JitArenaAllocator> *const mergedValueTypesTrackedForKills) const
  11382. {
  11383. Assert(value);
  11384. if(!value->GetValueInfo()->IsAnyOptimizedArray())
  11385. {
  11386. return;
  11387. }
  11388. DoTrackMergedValueForKills(value, blockData, mergedValueTypesTrackedForKills);
  11389. }
  11390. void
  11391. GlobOpt::DoTrackMergedValueForKills(
  11392. Value *const value,
  11393. GlobOptBlockData *const blockData,
  11394. BVSparse<JitArenaAllocator> *const mergedValueTypesTrackedForKills) const
  11395. {
  11396. Assert(value);
  11397. Assert(blockData);
  11398. ValueInfo *valueInfo = value->GetValueInfo();
  11399. Assert(valueInfo->IsAnyOptimizedArray());
  11400. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11401. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11402. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11403. #if DBG
  11404. VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock, true);
  11405. #endif
  11406. if(!isJsArray && !isVirtualTypedArray && !(valueInfo->IsArrayValueInfo() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()))
  11407. {
  11408. return;
  11409. }
  11410. // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the
  11411. // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can
  11412. // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can
  11413. // revert the value type to a likely version.
  11414. if(!mergedValueTypesTrackedForKills || !mergedValueTypesTrackedForKills->TestAndSet(value->GetValueNumber()))
  11415. {
  11416. blockData->valuesToKillOnCalls->Add(value);
  11417. }
  11418. }
  11419. void
  11420. GlobOpt::TrackValueInfoChangeForKills(BasicBlock *const block, Value *const value, ValueInfo *const newValueInfo, const bool compensated) const
  11421. {
  11422. Assert(block);
  11423. Assert(value);
  11424. Assert(newValueInfo);
  11425. ValueInfo *const oldValueInfo = value->GetValueInfo();
  11426. #if DBG
  11427. if(oldValueInfo->IsAnyOptimizedArray())
  11428. {
  11429. VerifyArrayValueInfoForTracking(oldValueInfo, oldValueInfo->IsArrayOrObjectWithArray(), block, compensated);
  11430. }
  11431. #endif
  11432. const bool trackOldValueInfo =
  11433. oldValueInfo->IsArrayOrObjectWithArray() ||
  11434. oldValueInfo->IsOptimizedVirtualTypedArray() ||
  11435. (
  11436. oldValueInfo->IsOptimizedTypedArray() &&
  11437. oldValueInfo->IsArrayValueInfo() &&
  11438. oldValueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()
  11439. );
  11440. Assert(trackOldValueInfo == block->globOptData.valuesToKillOnCalls->ContainsKey(value));
  11441. #if DBG
  11442. if(newValueInfo->IsAnyOptimizedArray())
  11443. {
  11444. VerifyArrayValueInfoForTracking(newValueInfo, newValueInfo->IsArrayOrObjectWithArray(), block, compensated);
  11445. }
  11446. #endif
  11447. const bool trackNewValueInfo =
  11448. newValueInfo->IsArrayOrObjectWithArray() ||
  11449. newValueInfo->IsOptimizedVirtualTypedArray() ||
  11450. (
  11451. newValueInfo->IsOptimizedTypedArray() &&
  11452. newValueInfo->IsArrayValueInfo() &&
  11453. newValueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()
  11454. );
  11455. if(trackOldValueInfo == trackNewValueInfo)
  11456. {
  11457. return;
  11458. }
  11459. if(trackNewValueInfo)
  11460. {
  11461. block->globOptData.valuesToKillOnCalls->Add(value);
  11462. }
  11463. else
  11464. {
  11465. block->globOptData.valuesToKillOnCalls->Remove(value);
  11466. }
  11467. }
  11468. void
  11469. GlobOpt::ProcessValueKills(IR::Instr *const instr)
  11470. {
  11471. Assert(instr);
  11472. ValueSet *const valuesToKillOnCalls = CurrentBlockData()->valuesToKillOnCalls;
  11473. if(!IsLoopPrePass() && valuesToKillOnCalls->Count() == 0)
  11474. {
  11475. return;
  11476. }
  11477. const JsArrayKills kills = CheckJsArrayKills(instr);
  11478. Assert(!kills.KillsArrayHeadSegments() || kills.KillsArrayHeadSegmentLengths());
  11479. if(IsLoopPrePass())
  11480. {
  11481. rootLoopPrePass->jsArrayKills = rootLoopPrePass->jsArrayKills.Merge(kills);
  11482. Assert(
  11483. !rootLoopPrePass->parent ||
  11484. rootLoopPrePass->jsArrayKills.AreSubsetOf(rootLoopPrePass->parent->jsArrayKills));
  11485. if(kills.KillsAllArrays())
  11486. {
  11487. rootLoopPrePass->needImplicitCallBailoutChecksForJsArrayCheckHoist = false;
  11488. }
  11489. if(valuesToKillOnCalls->Count() == 0)
  11490. {
  11491. return;
  11492. }
  11493. }
  11494. if(kills.KillsAllArrays())
  11495. {
  11496. Assert(kills.KillsTypedArrayHeadSegmentLengths());
  11497. // - Calls need to kill the value types of values in the following list. For instance, calls can transform a JS array
  11498. // into an ES5 array, so any definitely-array value types need to be killed. Also, VirtualTypeArrays do not have
  11499. // bounds checks; this can be problematic if the array is detached, so check to ensure that it is a virtual array.
  11500. // Update the value types to likley to ensure a bailout that asserts Array type is generated.
  11501. // - Calls also need to kill typed array head segment lengths. A typed array's array buffer may be transferred to a web
  11502. // worker, in which case the typed array's length is set to zero.
  11503. for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext())
  11504. {
  11505. Value *const value = it.CurrentValue();
  11506. ValueInfo *const valueInfo = value->GetValueInfo();
  11507. Assert(
  11508. valueInfo->IsArrayOrObjectWithArray() ||
  11509. valueInfo->IsOptimizedVirtualTypedArray() ||
  11510. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11511. if (valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray())
  11512. {
  11513. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11514. continue;
  11515. }
  11516. ChangeValueInfo(
  11517. nullptr,
  11518. value,
  11519. valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true));
  11520. }
  11521. valuesToKillOnCalls->Clear();
  11522. return;
  11523. }
  11524. if(kills.KillsArraysWithNoMissingValues())
  11525. {
  11526. // Some operations may kill arrays with no missing values in unlikely circumstances. Convert their value types to likely
  11527. // versions so that the checks have to be redone.
  11528. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  11529. {
  11530. Value *const value = it.CurrentValue();
  11531. ValueInfo *const valueInfo = value->GetValueInfo();
  11532. Assert(
  11533. valueInfo->IsArrayOrObjectWithArray() ||
  11534. valueInfo->IsOptimizedVirtualTypedArray() ||
  11535. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11536. if(!valueInfo->IsArrayOrObjectWithArray() || !valueInfo->HasNoMissingValues())
  11537. {
  11538. continue;
  11539. }
  11540. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11541. it.RemoveCurrent();
  11542. }
  11543. }
  11544. else if(kills.KillsObjectArraysWithNoMissingValues())
  11545. {
  11546. // Some operations may kill objects with arrays-with-no-missing-values in unlikely circumstances. Convert their value types to likely
  11547. // versions so that the checks have to be redone.
  11548. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  11549. {
  11550. Value *const value = it.CurrentValue();
  11551. ValueInfo *const valueInfo = value->GetValueInfo();
  11552. Assert(
  11553. valueInfo->IsArrayOrObjectWithArray() ||
  11554. valueInfo->IsOptimizedVirtualTypedArray() ||
  11555. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11556. if(!valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsArray() || !valueInfo->HasNoMissingValues())
  11557. {
  11558. continue;
  11559. }
  11560. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11561. it.RemoveCurrent();
  11562. }
  11563. }
  11564. if(kills.KillsNativeArrays())
  11565. {
  11566. // Some operations may kill native arrays in (what should be) unlikely circumstances. Convert their value types to
  11567. // likely versions so that the checks have to be redone.
  11568. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  11569. {
  11570. Value *const value = it.CurrentValue();
  11571. ValueInfo *const valueInfo = value->GetValueInfo();
  11572. Assert(
  11573. valueInfo->IsArrayOrObjectWithArray() ||
  11574. valueInfo->IsOptimizedVirtualTypedArray() ||
  11575. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11576. if(!valueInfo->IsArrayOrObjectWithArray() || valueInfo->HasVarElements())
  11577. {
  11578. continue;
  11579. }
  11580. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11581. it.RemoveCurrent();
  11582. }
  11583. }
  11584. const bool likelyKillsJsArraysWithNoMissingValues = IsOperationThatLikelyKillsJsArraysWithNoMissingValues(instr);
  11585. if(!kills.KillsArrayHeadSegmentLengths())
  11586. {
  11587. Assert(!kills.KillsArrayHeadSegments());
  11588. if(!likelyKillsJsArraysWithNoMissingValues && !kills.KillsArrayLengths())
  11589. {
  11590. return;
  11591. }
  11592. }
  11593. for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext())
  11594. {
  11595. Value *const value = it.CurrentValue();
  11596. ValueInfo *valueInfo = value->GetValueInfo();
  11597. Assert(
  11598. valueInfo->IsArrayOrObjectWithArray() ||
  11599. valueInfo->IsOptimizedVirtualTypedArray() ||
  11600. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11601. if(!valueInfo->IsArrayOrObjectWithArray())
  11602. {
  11603. continue;
  11604. }
  11605. if(likelyKillsJsArraysWithNoMissingValues && valueInfo->HasNoMissingValues())
  11606. {
  11607. ChangeValueType(nullptr, value, valueInfo->Type().SetHasNoMissingValues(false), true);
  11608. valueInfo = value->GetValueInfo();
  11609. }
  11610. if(!valueInfo->IsArrayValueInfo())
  11611. {
  11612. continue;
  11613. }
  11614. ArrayValueInfo *const arrayValueInfo = valueInfo->AsArrayValueInfo();
  11615. const bool removeHeadSegment = kills.KillsArrayHeadSegments() && arrayValueInfo->HeadSegmentSym();
  11616. const bool removeHeadSegmentLength = kills.KillsArrayHeadSegmentLengths() && arrayValueInfo->HeadSegmentLengthSym();
  11617. const bool removeLength = kills.KillsArrayLengths() && arrayValueInfo->LengthSym();
  11618. if(removeHeadSegment || removeHeadSegmentLength || removeLength)
  11619. {
  11620. ChangeValueInfo(
  11621. nullptr,
  11622. value,
  11623. arrayValueInfo->Copy(alloc, !removeHeadSegment, !removeHeadSegmentLength, !removeLength));
  11624. valueInfo = value->GetValueInfo();
  11625. }
  11626. }
  11627. }
  11628. void
  11629. GlobOpt::ProcessValueKills(BasicBlock *const block, GlobOptBlockData *const blockData)
  11630. {
  11631. Assert(block);
  11632. Assert(blockData);
  11633. ValueSet *const valuesToKillOnCalls = blockData->valuesToKillOnCalls;
  11634. if(!IsLoopPrePass() && valuesToKillOnCalls->Count() == 0)
  11635. {
  11636. return;
  11637. }
  11638. // If the current block or loop has implicit calls, kill all definitely-array value types, as using that info will cause
  11639. // implicit calls to be disabled, resulting in unnecessary bailouts
  11640. const bool killValuesOnImplicitCalls =
  11641. (block->loop ? !this->ImplicitCallFlagsAllowOpts(block->loop) : !this->ImplicitCallFlagsAllowOpts(func));
  11642. if (!killValuesOnImplicitCalls)
  11643. {
  11644. return;
  11645. }
  11646. if(IsLoopPrePass() && block->loop == rootLoopPrePass)
  11647. {
  11648. AnalysisAssert(rootLoopPrePass);
  11649. for (Loop * loop = rootLoopPrePass; loop != nullptr; loop = loop->parent)
  11650. {
  11651. loop->jsArrayKills.SetKillsAllArrays();
  11652. }
  11653. Assert(!rootLoopPrePass->parent || rootLoopPrePass->jsArrayKills.AreSubsetOf(rootLoopPrePass->parent->jsArrayKills));
  11654. if(valuesToKillOnCalls->Count() == 0)
  11655. {
  11656. return;
  11657. }
  11658. }
  11659. for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext())
  11660. {
  11661. Value *const value = it.CurrentValue();
  11662. ValueInfo *const valueInfo = value->GetValueInfo();
  11663. Assert(
  11664. valueInfo->IsArrayOrObjectWithArray() ||
  11665. valueInfo->IsOptimizedVirtualTypedArray() ||
  11666. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11667. if(valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray())
  11668. {
  11669. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11670. continue;
  11671. }
  11672. ChangeValueInfo(
  11673. nullptr,
  11674. value,
  11675. valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true));
  11676. }
  11677. valuesToKillOnCalls->Clear();
  11678. }
  11679. void
  11680. GlobOpt::ProcessValueKillsForLoopHeaderAfterBackEdgeMerge(BasicBlock *const block, GlobOptBlockData *const blockData)
  11681. {
  11682. Assert(block);
  11683. Assert(block->isLoopHeader);
  11684. Assert(blockData);
  11685. ValueSet *const valuesToKillOnCalls = blockData->valuesToKillOnCalls;
  11686. if(valuesToKillOnCalls->Count() == 0)
  11687. {
  11688. return;
  11689. }
  11690. const JsArrayKills loopKills(block->loop->jsArrayKills);
  11691. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  11692. {
  11693. Value *const value = it.CurrentValue();
  11694. ValueInfo *valueInfo = value->GetValueInfo();
  11695. Assert(
  11696. valueInfo->IsArrayOrObjectWithArray() ||
  11697. valueInfo->IsOptimizedVirtualTypedArray() ||
  11698. valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym());
  11699. const bool isJsArray = valueInfo->IsArrayOrObjectWithArray();
  11700. Assert(!isJsArray == valueInfo->IsOptimizedTypedArray());
  11701. const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray();
  11702. if((isJsArray || isVirtualTypedArray) ? loopKills.KillsValueType(valueInfo->Type()) : loopKills.KillsTypedArrayHeadSegmentLengths())
  11703. {
  11704. // Hoisting array checks and other related things for this type is disabled for the loop due to the kill, as
  11705. // compensation code is currently not added on back-edges. When merging values from a back-edge, the array value
  11706. // type cannot be definite, as that may require adding compensation code on the back-edge if the optimization pass
  11707. // chooses to not optimize the array.
  11708. if(isJsArray || isVirtualTypedArray)
  11709. {
  11710. ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false);
  11711. }
  11712. else
  11713. {
  11714. ChangeValueInfo(
  11715. nullptr,
  11716. value,
  11717. valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true));
  11718. }
  11719. it.RemoveCurrent();
  11720. continue;
  11721. }
  11722. if(!isJsArray || !valueInfo->IsArrayValueInfo())
  11723. {
  11724. continue;
  11725. }
  11726. // Similarly, if the loop contains an operation that kills JS array segments, don't make the segment or other related
  11727. // syms available initially inside the loop
  11728. ArrayValueInfo *const arrayValueInfo = valueInfo->AsArrayValueInfo();
  11729. const bool removeHeadSegment = loopKills.KillsArrayHeadSegments() && arrayValueInfo->HeadSegmentSym();
  11730. const bool removeHeadSegmentLength = loopKills.KillsArrayHeadSegmentLengths() && arrayValueInfo->HeadSegmentLengthSym();
  11731. const bool removeLength = loopKills.KillsArrayLengths() && arrayValueInfo->LengthSym();
  11732. if(removeHeadSegment || removeHeadSegmentLength || removeLength)
  11733. {
  11734. ChangeValueInfo(
  11735. nullptr,
  11736. value,
  11737. arrayValueInfo->Copy(alloc, !removeHeadSegment, !removeHeadSegmentLength, !removeLength));
  11738. valueInfo = value->GetValueInfo();
  11739. }
  11740. }
  11741. }
  11742. bool
  11743. GlobOpt::NeedBailOnImplicitCallForLiveValues(BasicBlock const * const block, const bool isForwardPass) const
  11744. {
  11745. if(isForwardPass)
  11746. {
  11747. return block->globOptData.valuesToKillOnCalls->Count() != 0;
  11748. }
  11749. if(block->noImplicitCallUses->IsEmpty())
  11750. {
  11751. Assert(block->noImplicitCallNoMissingValuesUses->IsEmpty());
  11752. Assert(block->noImplicitCallNativeArrayUses->IsEmpty());
  11753. Assert(block->noImplicitCallJsArrayHeadSegmentSymUses->IsEmpty());
  11754. Assert(block->noImplicitCallArrayLengthSymUses->IsEmpty());
  11755. return false;
  11756. }
  11757. return true;
  11758. }
  11759. IR::Instr*
  11760. GlobOpt::CreateBoundsCheckInstr(IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset, Func* func)
  11761. {
  11762. IR::Instr* instr = IR::Instr::New(Js::OpCode::BoundCheck, func);
  11763. return AttachBoundsCheckData(instr, lowerBound, upperBound, offset);
  11764. }
  11765. IR::Instr*
  11766. GlobOpt::CreateBoundsCheckInstr(IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset, IR::BailOutKind bailoutkind, BailOutInfo* bailoutInfo, Func * func)
  11767. {
  11768. IR::Instr* instr = IR::BailOutInstr::New(Js::OpCode::BoundCheck, bailoutkind, bailoutInfo, func);
  11769. return AttachBoundsCheckData(instr, lowerBound, upperBound, offset);
  11770. }
  11771. IR::Instr*
  11772. GlobOpt::AttachBoundsCheckData(IR::Instr* instr, IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset)
  11773. {
  11774. instr->SetSrc1(lowerBound);
  11775. instr->SetSrc2(upperBound);
  11776. if (offset != 0)
  11777. {
  11778. instr->SetDst(IR::IntConstOpnd::New(offset, TyInt32, instr->m_func));
  11779. }
  11780. return instr;
  11781. }
  11782. void
  11783. GlobOpt::OptArraySrc(IR::Instr ** const instrRef, Value ** src1Val, Value ** src2Val)
  11784. {
  11785. Assert(instrRef != nullptr);
  11786. ArraySrcOpt arraySrcOpt(this, instrRef, src1Val, src2Val);
  11787. arraySrcOpt.Optimize();
  11788. }
  11789. void
  11790. GlobOpt::CaptureNoImplicitCallUses(
  11791. IR::Opnd *opnd,
  11792. const bool usesNoMissingValuesInfo,
  11793. IR::Instr *const includeCurrentInstr)
  11794. {
  11795. Assert(!IsLoopPrePass());
  11796. Assert(noImplicitCallUsesToInsert);
  11797. Assert(opnd);
  11798. // The opnd may be deleted later, so make a copy to ensure it is alive for inserting NoImplicitCallUses later
  11799. opnd = opnd->Copy(func);
  11800. if(!usesNoMissingValuesInfo)
  11801. {
  11802. const ValueType valueType(opnd->GetValueType());
  11803. if(valueType.IsArrayOrObjectWithArray() && valueType.HasNoMissingValues())
  11804. {
  11805. // Inserting NoImplicitCallUses for an opnd with a definitely-array-with-no-missing-values value type means that the
  11806. // instruction following it uses the information that the array has no missing values in some way, for instance, it
  11807. // may omit missing value checks. Based on that, the dead-store phase in turn ensures that the necessary bailouts
  11808. // are inserted to ensure that the array still has no missing values until the following instruction. Since
  11809. // 'usesNoMissingValuesInfo' is false, change the value type to indicate to the dead-store phase that the following
  11810. // instruction does not use the no-missing-values information.
  11811. opnd->SetValueType(valueType.SetHasNoMissingValues(false));
  11812. }
  11813. }
  11814. if(includeCurrentInstr)
  11815. {
  11816. IR::Instr *const noImplicitCallUses =
  11817. IR::PragmaInstr::New(Js::OpCode::NoImplicitCallUses, 0, includeCurrentInstr->m_func);
  11818. noImplicitCallUses->SetSrc1(opnd);
  11819. noImplicitCallUses->GetSrc1()->SetIsJITOptimizedReg(true);
  11820. includeCurrentInstr->InsertAfter(noImplicitCallUses);
  11821. return;
  11822. }
  11823. noImplicitCallUsesToInsert->Add(opnd);
  11824. }
  11825. void
  11826. GlobOpt::InsertNoImplicitCallUses(IR::Instr *const instr)
  11827. {
  11828. Assert(noImplicitCallUsesToInsert);
  11829. const int n = noImplicitCallUsesToInsert->Count();
  11830. if(n == 0)
  11831. {
  11832. return;
  11833. }
  11834. IR::Instr *const insertBeforeInstr = instr->GetInsertBeforeByteCodeUsesInstr();
  11835. for(int i = 0; i < n;)
  11836. {
  11837. IR::Instr *const noImplicitCallUses = IR::PragmaInstr::New(Js::OpCode::NoImplicitCallUses, 0, instr->m_func);
  11838. noImplicitCallUses->SetSrc1(noImplicitCallUsesToInsert->Item(i));
  11839. noImplicitCallUses->GetSrc1()->SetIsJITOptimizedReg(true);
  11840. ++i;
  11841. if(i < n)
  11842. {
  11843. noImplicitCallUses->SetSrc2(noImplicitCallUsesToInsert->Item(i));
  11844. noImplicitCallUses->GetSrc2()->SetIsJITOptimizedReg(true);
  11845. ++i;
  11846. }
  11847. noImplicitCallUses->SetByteCodeOffset(instr);
  11848. insertBeforeInstr->InsertBefore(noImplicitCallUses);
  11849. }
  11850. noImplicitCallUsesToInsert->Clear();
  11851. }
  11852. void
  11853. GlobOpt::PrepareLoopArrayCheckHoist()
  11854. {
  11855. if(IsLoopPrePass() || !currentBlock->loop || !currentBlock->isLoopHeader || !currentBlock->loop->parent)
  11856. {
  11857. return;
  11858. }
  11859. if(currentBlock->loop->parent->needImplicitCallBailoutChecksForJsArrayCheckHoist)
  11860. {
  11861. // If the parent loop is an array check elimination candidate, so is the current loop. Even though the current loop may
  11862. // not have array accesses, if the parent loop hoists array checks, the current loop also needs implicit call checks.
  11863. currentBlock->loop->needImplicitCallBailoutChecksForJsArrayCheckHoist = true;
  11864. }
  11865. }
  11866. JsArrayKills
  11867. GlobOpt::CheckJsArrayKills(IR::Instr *const instr)
  11868. {
  11869. Assert(instr);
  11870. JsArrayKills kills;
  11871. if(instr->UsesAllFields())
  11872. {
  11873. // Calls can (but are unlikely to) change a javascript array into an ES5 array, which may have different behavior for
  11874. // index properties.
  11875. kills.SetKillsAllArrays();
  11876. return kills;
  11877. }
  11878. const bool doArrayMissingValueCheckHoist = DoArrayMissingValueCheckHoist();
  11879. const bool doNativeArrayTypeSpec = DoNativeArrayTypeSpec();
  11880. const bool doArraySegmentHoist = DoArraySegmentHoist(ValueType::GetObject(ObjectType::Array));
  11881. Assert(doArraySegmentHoist == DoArraySegmentHoist(ValueType::GetObject(ObjectType::ObjectWithArray)));
  11882. const bool doArrayLengthHoist = DoArrayLengthHoist();
  11883. if(!doArrayMissingValueCheckHoist && !doNativeArrayTypeSpec && !doArraySegmentHoist && !doArrayLengthHoist)
  11884. {
  11885. return kills;
  11886. }
  11887. // The following operations may create missing values in an array in an unlikely circumstance. Even though they don't kill
  11888. // the fact that the 'this' parameter is an array (when implicit calls are disabled), we don't have a way to say the value
  11889. // type is definitely array but it likely has no missing values. So, these will kill the definite value type as well, making
  11890. // it likely array, such that the array checks will have to be redone.
  11891. const bool useValueTypes = !IsLoopPrePass(); // Source value types are not guaranteed to be correct in a loop prepass
  11892. switch(instr->m_opcode)
  11893. {
  11894. case Js::OpCode::StElemC:
  11895. case Js::OpCode::StElemI_A:
  11896. case Js::OpCode::StElemI_A_Strict:
  11897. {
  11898. Assert(instr->GetDst());
  11899. if(!instr->GetDst()->IsIndirOpnd())
  11900. {
  11901. break;
  11902. }
  11903. const ValueType baseValueType =
  11904. useValueTypes ? instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType() : ValueType::Uninitialized;
  11905. if(useValueTypes && baseValueType.IsNotArrayOrObjectWithArray())
  11906. {
  11907. break;
  11908. }
  11909. if(instr->IsProfiledInstr())
  11910. {
  11911. const Js::StElemInfo *const stElemInfo = instr->AsProfiledInstr()->u.stElemInfo;
  11912. if(doArraySegmentHoist && stElemInfo->LikelyStoresOutsideHeadSegmentBounds())
  11913. {
  11914. kills.SetKillsArrayHeadSegments();
  11915. kills.SetKillsArrayHeadSegmentLengths();
  11916. }
  11917. if(doArrayLengthHoist &&
  11918. !(useValueTypes && baseValueType.IsNotArray()) &&
  11919. stElemInfo->LikelyStoresOutsideArrayBounds())
  11920. {
  11921. kills.SetKillsArrayLengths();
  11922. }
  11923. }
  11924. break;
  11925. }
  11926. case Js::OpCode::DeleteElemI_A:
  11927. case Js::OpCode::DeleteElemIStrict_A:
  11928. Assert(instr->GetSrc1());
  11929. if(!instr->GetSrc1()->IsIndirOpnd() ||
  11930. (useValueTypes && instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsNotArrayOrObjectWithArray()))
  11931. {
  11932. break;
  11933. }
  11934. if(doArrayMissingValueCheckHoist)
  11935. {
  11936. kills.SetKillsArraysWithNoMissingValues();
  11937. }
  11938. if(doArraySegmentHoist)
  11939. {
  11940. kills.SetKillsArrayHeadSegmentLengths();
  11941. }
  11942. break;
  11943. case Js::OpCode::ConsoleScopedStFld:
  11944. case Js::OpCode::ConsoleScopedStFldStrict:
  11945. case Js::OpCode::ScopedStFld:
  11946. case Js::OpCode::ScopedStFldStrict:
  11947. case Js::OpCode::StFld:
  11948. case Js::OpCode::StFldStrict:
  11949. case Js::OpCode::StSuperFld:
  11950. {
  11951. Assert(instr->GetDst());
  11952. if(!doArraySegmentHoist && !doArrayLengthHoist)
  11953. {
  11954. break;
  11955. }
  11956. IR::SymOpnd *const symDst = instr->GetDst()->AsSymOpnd();
  11957. if(!symDst->IsPropertySymOpnd())
  11958. {
  11959. break;
  11960. }
  11961. IR::PropertySymOpnd *const dst = symDst->AsPropertySymOpnd();
  11962. if(dst->m_sym->AsPropertySym()->m_propertyId != Js::PropertyIds::length)
  11963. {
  11964. break;
  11965. }
  11966. if(useValueTypes && dst->GetPropertyOwnerValueType().IsNotArray())
  11967. {
  11968. // Setting the 'length' property of an object that is not an array, even if it has an internal array, does
  11969. // not kill the head segment or head segment length of any arrays.
  11970. break;
  11971. }
  11972. if(doArraySegmentHoist)
  11973. {
  11974. kills.SetKillsArrayHeadSegmentLengths();
  11975. }
  11976. if(doArrayLengthHoist)
  11977. {
  11978. kills.SetKillsArrayLengths();
  11979. }
  11980. break;
  11981. }
  11982. case Js::OpCode::InlineArrayPush:
  11983. {
  11984. Assert(instr->GetSrc2());
  11985. IR::Opnd *const arrayOpnd = instr->GetSrc1();
  11986. Assert(arrayOpnd);
  11987. const ValueType arrayValueType(arrayOpnd->GetValueType());
  11988. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  11989. {
  11990. break;
  11991. }
  11992. if(doArrayMissingValueCheckHoist)
  11993. {
  11994. kills.SetKillsArraysWithNoMissingValues();
  11995. }
  11996. if(doArraySegmentHoist)
  11997. {
  11998. kills.SetKillsArrayHeadSegments();
  11999. kills.SetKillsArrayHeadSegmentLengths();
  12000. }
  12001. if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray()))
  12002. {
  12003. kills.SetKillsArrayLengths();
  12004. }
  12005. // Don't kill NativeArray, if there is no mismatch between array's type and element's type.
  12006. if(doNativeArrayTypeSpec &&
  12007. !(useValueTypes && arrayValueType.IsNativeArray() &&
  12008. ((arrayValueType.IsLikelyNativeIntArray() && instr->GetSrc2()->IsInt32()) ||
  12009. (arrayValueType.IsLikelyNativeFloatArray() && instr->GetSrc2()->IsFloat()))
  12010. ) &&
  12011. !(useValueTypes && arrayValueType.IsNotNativeArray()))
  12012. {
  12013. kills.SetKillsNativeArrays();
  12014. }
  12015. break;
  12016. }
  12017. case Js::OpCode::InlineArrayPop:
  12018. {
  12019. IR::Opnd *const arrayOpnd = instr->GetSrc1();
  12020. Assert(arrayOpnd);
  12021. const ValueType arrayValueType(arrayOpnd->GetValueType());
  12022. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  12023. {
  12024. break;
  12025. }
  12026. if(doArraySegmentHoist)
  12027. {
  12028. kills.SetKillsArrayHeadSegmentLengths();
  12029. }
  12030. if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray()))
  12031. {
  12032. kills.SetKillsArrayLengths();
  12033. }
  12034. if(doArrayMissingValueCheckHoist && !(useValueTypes && arrayValueType.IsArray()))
  12035. {
  12036. kills.SetKillsObjectArraysWithNoMissingValues();
  12037. }
  12038. break;
  12039. }
  12040. case Js::OpCode::CallDirect:
  12041. {
  12042. Assert(instr->GetSrc1());
  12043. // Find the 'this' parameter and check if it's possible for it to be an array
  12044. IR::Opnd *const arrayOpnd = instr->FindCallArgumentOpnd(1);
  12045. Assert(arrayOpnd);
  12046. const ValueType arrayValueType(arrayOpnd->GetValueType());
  12047. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  12048. {
  12049. break;
  12050. }
  12051. const IR::JnHelperMethod helperMethod = instr->GetSrc1()->AsHelperCallOpnd()->m_fnHelper;
  12052. if(doArrayMissingValueCheckHoist)
  12053. {
  12054. switch(helperMethod)
  12055. {
  12056. case IR::HelperArray_Reverse:
  12057. case IR::HelperArray_Shift:
  12058. case IR::HelperArray_Splice:
  12059. case IR::HelperArray_Unshift:
  12060. kills.SetKillsArraysWithNoMissingValues();
  12061. break;
  12062. }
  12063. }
  12064. if(doArraySegmentHoist)
  12065. {
  12066. switch(helperMethod)
  12067. {
  12068. case IR::HelperArray_Reverse:
  12069. case IR::HelperArray_Shift:
  12070. case IR::HelperArray_Splice:
  12071. case IR::HelperArray_Unshift:
  12072. case IR::HelperArray_Concat:
  12073. kills.SetKillsArrayHeadSegments();
  12074. kills.SetKillsArrayHeadSegmentLengths();
  12075. break;
  12076. }
  12077. }
  12078. if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray()))
  12079. {
  12080. switch(helperMethod)
  12081. {
  12082. case IR::HelperArray_Shift:
  12083. case IR::HelperArray_Splice:
  12084. case IR::HelperArray_Unshift:
  12085. kills.SetKillsArrayLengths();
  12086. break;
  12087. }
  12088. }
  12089. if(doNativeArrayTypeSpec && !(useValueTypes && arrayValueType.IsNotNativeArray()))
  12090. {
  12091. switch(helperMethod)
  12092. {
  12093. case IR::HelperArray_Reverse:
  12094. case IR::HelperArray_Shift:
  12095. case IR::HelperArray_Slice:
  12096. // Currently not inlined.
  12097. //case IR::HelperArray_Sort:
  12098. case IR::HelperArray_Splice:
  12099. case IR::HelperArray_Unshift:
  12100. case IR::HelperArray_Concat:
  12101. kills.SetKillsNativeArrays();
  12102. break;
  12103. }
  12104. }
  12105. break;
  12106. }
  12107. case Js::OpCode::InitProto:
  12108. {
  12109. // Find the 'this' parameter and check if it's possible for it to be an array
  12110. IR::Opnd *const arrayOpnd = instr->GetSrc1();
  12111. Assert(arrayOpnd);
  12112. const ValueType arrayValueType(arrayOpnd->GetValueType());
  12113. if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray()))
  12114. {
  12115. break;
  12116. }
  12117. if(doNativeArrayTypeSpec && !(useValueTypes && arrayValueType.IsNotNativeArray()))
  12118. {
  12119. kills.SetKillsNativeArrays();
  12120. }
  12121. break;
  12122. }
  12123. case Js::OpCode::InitClass:
  12124. Assert(instr->GetSrc1());
  12125. if (instr->GetSrc2() == nullptr)
  12126. {
  12127. // No extends operand, so the InitClass will not make something into a prototype
  12128. break;
  12129. }
  12130. if(doNativeArrayTypeSpec)
  12131. {
  12132. // Class/object construction can make something a prototype
  12133. kills.SetKillsNativeArrays();
  12134. }
  12135. break;
  12136. case Js::OpCode::NewScObjectNoCtor:
  12137. case Js::OpCode::NewScObjectNoCtorFull:
  12138. if(doNativeArrayTypeSpec)
  12139. {
  12140. // Class/object construction can make something a prototype
  12141. kills.SetKillsNativeArrays();
  12142. }
  12143. break;
  12144. }
  12145. return kills;
  12146. }
  12147. GlobOptBlockData const * GlobOpt::CurrentBlockData() const
  12148. {
  12149. return &this->currentBlock->globOptData;
  12150. }
  12151. GlobOptBlockData * GlobOpt::CurrentBlockData()
  12152. {
  12153. return &this->currentBlock->globOptData;
  12154. }
  12155. void GlobOpt::CommitCapturedValuesCandidate()
  12156. {
  12157. GlobOptBlockData * globOptData = CurrentBlockData();
  12158. globOptData->changedSyms->ClearAll();
  12159. if (!this->changedSymsAfterIncBailoutCandidate->IsEmpty())
  12160. {
  12161. //
  12162. // some symbols are changed after the values for current bailout have been
  12163. // captured (GlobOpt::CapturedValues), need to restore such symbols as changed
  12164. // for following incremental bailout construction, or we will miss capturing
  12165. // values for later bailout
  12166. //
  12167. // swap changedSyms and changedSymsAfterIncBailoutCandidate
  12168. // because both are from this->alloc
  12169. BVSparse<JitArenaAllocator> * tempBvSwap = globOptData->changedSyms;
  12170. globOptData->changedSyms = this->changedSymsAfterIncBailoutCandidate;
  12171. this->changedSymsAfterIncBailoutCandidate = tempBvSwap;
  12172. }
  12173. if (globOptData->capturedValues)
  12174. {
  12175. globOptData->capturedValues->DecrementRefCount();
  12176. }
  12177. globOptData->capturedValues = globOptData->capturedValuesCandidate;
  12178. // null out capturedValuesCandidate to stop tracking symbols change for it
  12179. globOptData->capturedValuesCandidate = nullptr;
  12180. }
  12181. bool
  12182. GlobOpt::IsOperationThatLikelyKillsJsArraysWithNoMissingValues(IR::Instr *const instr)
  12183. {
  12184. // StElem is profiled with information indicating whether it will likely create a missing value in the array. In that case,
  12185. // we prefer to kill the no-missing-values information in the value so that we don't bail out in a likely circumstance.
  12186. return
  12187. (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict) &&
  12188. DoArrayMissingValueCheckHoist() &&
  12189. instr->IsProfiledInstr() &&
  12190. instr->AsProfiledInstr()->u.stElemInfo->LikelyCreatesMissingValue();
  12191. }
  12192. bool
  12193. GlobOpt::NeedBailOnImplicitCallForArrayCheckHoist(BasicBlock const * const block, const bool isForwardPass) const
  12194. {
  12195. Assert(block);
  12196. return isForwardPass && block->loop && block->loop->needImplicitCallBailoutChecksForJsArrayCheckHoist;
  12197. }
  12198. bool
  12199. GlobOpt::PrepareForIgnoringIntOverflow(IR::Instr *const instr)
  12200. {
  12201. Assert(instr);
  12202. const bool isBoundary = instr->m_opcode == Js::OpCode::NoIntOverflowBoundary;
  12203. // Update the instruction's "int overflow matters" flag based on whether we are currently allowing ignoring int overflows.
  12204. // Some operations convert their srcs to int32s, those can still ignore int overflow.
  12205. if(instr->ignoreIntOverflowInRange)
  12206. {
  12207. instr->ignoreIntOverflowInRange = !intOverflowCurrentlyMattersInRange || OpCodeAttr::IsInt32(instr->m_opcode);
  12208. }
  12209. if(!intOverflowDoesNotMatterRange)
  12210. {
  12211. Assert(intOverflowCurrentlyMattersInRange);
  12212. // There are no more ranges of instructions where int overflow does not matter, in this block.
  12213. return isBoundary;
  12214. }
  12215. if(instr == intOverflowDoesNotMatterRange->LastInstr())
  12216. {
  12217. Assert(isBoundary);
  12218. // Reached the last instruction in the range
  12219. intOverflowCurrentlyMattersInRange = true;
  12220. intOverflowDoesNotMatterRange = intOverflowDoesNotMatterRange->Next();
  12221. return isBoundary;
  12222. }
  12223. if(!intOverflowCurrentlyMattersInRange)
  12224. {
  12225. return isBoundary;
  12226. }
  12227. if(instr != intOverflowDoesNotMatterRange->FirstInstr())
  12228. {
  12229. // Have not reached the next range
  12230. return isBoundary;
  12231. }
  12232. Assert(isBoundary);
  12233. // This is the first instruction in a range of instructions where int overflow does not matter. There can be many inputs to
  12234. // instructions in the range, some of which are inputs to the range itself (that is, the values are not defined in the
  12235. // range). Ignoring int overflow is only valid for int operations, so we need to ensure that all inputs to the range are
  12236. // int (not "likely int") before ignoring any overflows in the range. Ensuring that a sym with a "likely int" value is an
  12237. // int requires a bail-out. These bail-out check need to happen before any overflows are ignored, otherwise it's too late.
  12238. // The backward pass tracked all inputs into the range. Iterate over them and verify the values, and insert lossless
  12239. // conversions to int as necessary, before the first instruction in the range. If for any reason all values cannot be
  12240. // guaranteed to be ints, the optimization will be disabled for this range.
  12241. intOverflowCurrentlyMattersInRange = false;
  12242. {
  12243. BVSparse<JitArenaAllocator> tempBv1(tempAlloc);
  12244. BVSparse<JitArenaAllocator> tempBv2(tempAlloc);
  12245. {
  12246. // Just renaming the temp BVs for this section to indicate how they're used so that it makes sense
  12247. BVSparse<JitArenaAllocator> &symsToExclude = tempBv1;
  12248. BVSparse<JitArenaAllocator> &symsToInclude = tempBv2;
  12249. #if DBG_DUMP
  12250. SymID couldNotConvertSymId = 0;
  12251. #endif
  12252. FOREACH_BITSET_IN_SPARSEBV(id, intOverflowDoesNotMatterRange->SymsRequiredToBeInt())
  12253. {
  12254. Sym *const sym = func->m_symTable->Find(id);
  12255. Assert(sym);
  12256. // Some instructions with property syms are also tracked by the backward pass, and may be included in the range
  12257. // (LdSlot for instance). These property syms don't get their values until either copy-prop resolves a value for
  12258. // them, or a new value is created once the use of the property sym is reached. In either case, we're not that
  12259. // far yet, so we need to find the future value of the property sym by evaluating copy-prop in reverse.
  12260. Value *const value = sym->IsStackSym() ? CurrentBlockData()->FindValue(sym) : CurrentBlockData()->FindFuturePropertyValue(sym->AsPropertySym());
  12261. if(!value)
  12262. {
  12263. #if DBG_DUMP
  12264. couldNotConvertSymId = id;
  12265. #endif
  12266. intOverflowCurrentlyMattersInRange = true;
  12267. BREAK_BITSET_IN_SPARSEBV;
  12268. }
  12269. const bool isInt32OrUInt32Float =
  12270. value->GetValueInfo()->IsFloatConstant() &&
  12271. Js::JavascriptNumber::IsInt32OrUInt32(value->GetValueInfo()->AsFloatConstant()->FloatValue());
  12272. if(value->GetValueInfo()->IsInt() || isInt32OrUInt32Float)
  12273. {
  12274. if(!IsLoopPrePass())
  12275. {
  12276. // Input values that are already int can be excluded from int-specialization. We can treat unsigned
  12277. // int32 values as int32 values (ignoring the overflow), since the values will only be used inside the
  12278. // range where overflow does not matter.
  12279. symsToExclude.Set(sym->m_id);
  12280. }
  12281. continue;
  12282. }
  12283. if(!DoAggressiveIntTypeSpec() || !value->GetValueInfo()->IsLikelyInt())
  12284. {
  12285. // When aggressive int specialization is off, syms with "likely int" values cannot be forced to int since
  12286. // int bail-out checks are not allowed in that mode. Similarly, with aggressive int specialization on, it
  12287. // wouldn't make sense to force non-"likely int" values to int since it would almost guarantee a bail-out at
  12288. // runtime. In both cases, just disable ignoring overflow for this range.
  12289. #if DBG_DUMP
  12290. couldNotConvertSymId = id;
  12291. #endif
  12292. intOverflowCurrentlyMattersInRange = true;
  12293. BREAK_BITSET_IN_SPARSEBV;
  12294. }
  12295. if(IsLoopPrePass())
  12296. {
  12297. // The loop prepass does not modify bit-vectors. Since it doesn't add bail-out checks, it also does not need
  12298. // to specialize anything up-front. It only needs to be consistent in how it determines whether to allow
  12299. // ignoring overflow for a range, based on the values of inputs into the range.
  12300. continue;
  12301. }
  12302. // Since input syms are tracked in the backward pass, where there is no value tracking, it will not be aware of
  12303. // copy-prop. If a copy-prop sym is available, it will be used instead, so exclude the original sym and include
  12304. // the copy-prop sym for specialization.
  12305. StackSym *const copyPropSym = CurrentBlockData()->GetCopyPropSym(sym, value);
  12306. if(copyPropSym)
  12307. {
  12308. symsToExclude.Set(sym->m_id);
  12309. Assert(!symsToExclude.Test(copyPropSym->m_id));
  12310. const bool needsToBeLossless =
  12311. !intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Test(sym->m_id);
  12312. if(intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Test(copyPropSym->m_id) ||
  12313. symsToInclude.TestAndSet(copyPropSym->m_id))
  12314. {
  12315. // The copy-prop sym is already included
  12316. if(needsToBeLossless)
  12317. {
  12318. // The original sym needs to be lossless, so make the copy-prop sym lossless as well.
  12319. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Clear(copyPropSym->m_id);
  12320. }
  12321. }
  12322. else if(!needsToBeLossless)
  12323. {
  12324. // The copy-prop sym was not included before, and the original sym can be lossy, so make it lossy.
  12325. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Set(copyPropSym->m_id);
  12326. }
  12327. }
  12328. else if(!sym->IsStackSym())
  12329. {
  12330. // Only stack syms can be converted to int, and copy-prop syms are stack syms. If a copy-prop sym was not
  12331. // found for the property sym, we can't ignore overflows in this range.
  12332. #if DBG_DUMP
  12333. couldNotConvertSymId = id;
  12334. #endif
  12335. intOverflowCurrentlyMattersInRange = true;
  12336. BREAK_BITSET_IN_SPARSEBV;
  12337. }
  12338. } NEXT_BITSET_IN_SPARSEBV;
  12339. if(intOverflowCurrentlyMattersInRange)
  12340. {
  12341. #if DBG_DUMP
  12342. if(PHASE_TRACE(Js::TrackCompoundedIntOverflowPhase, func) && !IsLoopPrePass())
  12343. {
  12344. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  12345. Output::Print(
  12346. _u("TrackCompoundedIntOverflow - Top function: %s (%s), Phase: %s, Block: %u, Disabled ignoring overflows\n"),
  12347. func->GetJITFunctionBody()->GetDisplayName(),
  12348. func->GetDebugNumberSet(debugStringBuffer),
  12349. Js::PhaseNames[Js::ForwardPhase],
  12350. currentBlock->GetBlockNum());
  12351. Output::Print(_u(" Input sym could not be turned into an int: %u\n"), couldNotConvertSymId);
  12352. Output::Print(_u(" First instr: "));
  12353. instr->m_next->Dump();
  12354. Output::Flush();
  12355. }
  12356. #endif
  12357. intOverflowDoesNotMatterRange = intOverflowDoesNotMatterRange->Next();
  12358. return isBoundary;
  12359. }
  12360. if(IsLoopPrePass())
  12361. {
  12362. return isBoundary;
  12363. }
  12364. // Update the syms to specialize after enumeration
  12365. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(&symsToExclude);
  12366. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Minus(&symsToExclude);
  12367. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Or(&symsToInclude);
  12368. }
  12369. {
  12370. // Exclude syms that are already live as lossless int32, and exclude lossy conversions of syms that are already live
  12371. // as lossy int32.
  12372. // symsToExclude = liveInt32Syms - liveLossyInt32Syms // syms live as lossless int
  12373. // lossySymsToExclude = symsRequiredToBeLossyInt & liveLossyInt32Syms; // syms we want as lossy int that are already live as lossy int
  12374. // symsToExclude |= lossySymsToExclude
  12375. // symsRequiredToBeInt -= symsToExclude
  12376. // symsRequiredToBeLossyInt -= symsToExclude
  12377. BVSparse<JitArenaAllocator> &symsToExclude = tempBv1;
  12378. BVSparse<JitArenaAllocator> &lossySymsToExclude = tempBv2;
  12379. symsToExclude.Minus(CurrentBlockData()->liveInt32Syms, CurrentBlockData()->liveLossyInt32Syms);
  12380. lossySymsToExclude.And(
  12381. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt(),
  12382. CurrentBlockData()->liveLossyInt32Syms);
  12383. symsToExclude.Or(&lossySymsToExclude);
  12384. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(&symsToExclude);
  12385. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Minus(&symsToExclude);
  12386. }
  12387. #if DBG
  12388. {
  12389. // Verify that the syms to be converted are live
  12390. // liveSyms = liveInt32Syms | liveFloat64Syms | liveVarSyms
  12391. // deadSymsRequiredToBeInt = symsRequiredToBeInt - liveSyms
  12392. BVSparse<JitArenaAllocator> &liveSyms = tempBv1;
  12393. BVSparse<JitArenaAllocator> &deadSymsRequiredToBeInt = tempBv2;
  12394. liveSyms.Or(CurrentBlockData()->liveInt32Syms, CurrentBlockData()->liveFloat64Syms);
  12395. liveSyms.Or(CurrentBlockData()->liveVarSyms);
  12396. deadSymsRequiredToBeInt.Minus(intOverflowDoesNotMatterRange->SymsRequiredToBeInt(), &liveSyms);
  12397. Assert(deadSymsRequiredToBeInt.IsEmpty());
  12398. }
  12399. #endif
  12400. }
  12401. // Int-specialize the syms before the first instruction of the range (the current instruction)
  12402. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt());
  12403. #if DBG_DUMP
  12404. if(PHASE_TRACE(Js::TrackCompoundedIntOverflowPhase, func))
  12405. {
  12406. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  12407. Output::Print(
  12408. _u("TrackCompoundedIntOverflow - Top function: %s (%s), Phase: %s, Block: %u\n"),
  12409. func->GetJITFunctionBody()->GetDisplayName(),
  12410. func->GetDebugNumberSet(debugStringBuffer),
  12411. Js::PhaseNames[Js::ForwardPhase],
  12412. currentBlock->GetBlockNum());
  12413. Output::Print(_u(" Input syms to be int-specialized (lossless): "));
  12414. intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Dump();
  12415. Output::Print(_u(" Input syms to be converted to int (lossy): "));
  12416. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Dump();
  12417. Output::Print(_u(" First instr: "));
  12418. instr->m_next->Dump();
  12419. Output::Flush();
  12420. }
  12421. #endif
  12422. ToInt32(intOverflowDoesNotMatterRange->SymsRequiredToBeInt(), currentBlock, false /* lossy */, instr);
  12423. ToInt32(intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt(), currentBlock, true /* lossy */, instr);
  12424. return isBoundary;
  12425. }
  12426. void
  12427. GlobOpt::VerifyIntSpecForIgnoringIntOverflow(IR::Instr *const instr)
  12428. {
  12429. if(intOverflowCurrentlyMattersInRange || IsLoopPrePass())
  12430. {
  12431. return;
  12432. }
  12433. Assert(instr->m_opcode != Js::OpCode::Mul_I4 ||
  12434. (instr->m_opcode == Js::OpCode::Mul_I4 && !instr->ShouldCheckFor32BitOverflow() && instr->ShouldCheckForNon32BitOverflow() ));
  12435. // Instructions that are marked as "overflow doesn't matter" in the range must guarantee that they operate on int values and
  12436. // result in int values, for ignoring overflow to be valid. So, int-specialization is required for such instructions in the
  12437. // range. Ld_A is an exception because it only specializes if the src sym is available as a required specialized sym, and it
  12438. // doesn't generate bailouts or cause ignoring int overflow to be invalid.
  12439. // MULs are allowed to start a region and have BailOutInfo since they will bailout on non-32 bit overflow.
  12440. if(instr->m_opcode == Js::OpCode::Ld_A ||
  12441. ((!instr->HasBailOutInfo() || instr->m_opcode == Js::OpCode::Mul_I4) &&
  12442. (!instr->GetDst() || instr->GetDst()->IsInt32()) &&
  12443. (!instr->GetSrc1() || instr->GetSrc1()->IsInt32()) &&
  12444. (!instr->GetSrc2() || instr->GetSrc2()->IsInt32())))
  12445. {
  12446. return;
  12447. }
  12448. if (!instr->HasBailOutInfo() && !instr->HasAnySideEffects())
  12449. {
  12450. return;
  12451. }
  12452. // This can happen for Neg_A if it needs to bail out on negative zero, and perhaps other cases as well. It's too late to fix
  12453. // the problem (overflows may already be ignored), so handle it by bailing out at compile-time and disabling tracking int
  12454. // overflow.
  12455. Assert(!func->IsTrackCompoundedIntOverflowDisabled());
  12456. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  12457. {
  12458. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  12459. Output::Print(
  12460. _u("BailOut (compile-time): function: %s (%s) instr: "),
  12461. func->GetJITFunctionBody()->GetDisplayName(),
  12462. func->GetDebugNumberSet(debugStringBuffer));
  12463. #if DBG_DUMP
  12464. instr->Dump();
  12465. #else
  12466. Output::Print(_u("%s "), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  12467. #endif
  12468. Output::Print(_u("(overflow does not matter but could not int-spec or needed bailout)\n"));
  12469. Output::Flush();
  12470. }
  12471. if(func->IsTrackCompoundedIntOverflowDisabled())
  12472. {
  12473. // Tracking int overflows is already off for some reason. Prevent trying to rejit again because it won't help and the
  12474. // same thing will happen again and cause an infinite loop. Just abort jitting this function.
  12475. if(PHASE_TRACE(Js::BailOutPhase, this->func))
  12476. {
  12477. Output::Print(_u(" Aborting JIT because TrackIntOverflow is already off\n"));
  12478. Output::Flush();
  12479. }
  12480. throw Js::OperationAbortedException();
  12481. }
  12482. throw Js::RejitException(RejitReason::TrackIntOverflowDisabled);
  12483. }
  12484. // It makes lowering easier if it can assume that the first src is never a constant,
  12485. // at least for commutative operators. For non-commutative, just hoist the constant.
  12486. void
  12487. GlobOpt::PreLowerCanonicalize(IR::Instr *instr, Value **pSrc1Val, Value **pSrc2Val)
  12488. {
  12489. IR::Opnd *dst = instr->GetDst();
  12490. IR::Opnd *src1 = instr->GetSrc1();
  12491. IR::Opnd *src2 = instr->GetSrc2();
  12492. if (src1->IsImmediateOpnd())
  12493. {
  12494. // Swap for dst, src
  12495. }
  12496. else if (src2 && dst && src2->IsRegOpnd())
  12497. {
  12498. if (src2->GetIsDead() && !src1->GetIsDead() && !src1->IsEqual(dst))
  12499. {
  12500. // Swap if src2 is dead, as the reg can be reuse for the dst for opEqs like on x86 (ADD r1, r2)
  12501. }
  12502. else if (src2->IsEqual(dst))
  12503. {
  12504. // Helps lowering of opEqs
  12505. }
  12506. else
  12507. {
  12508. return;
  12509. }
  12510. // Make sure we don't swap 2 srcs with valueOf calls.
  12511. if (OpCodeAttr::OpndHasImplicitCall(instr->m_opcode))
  12512. {
  12513. if (instr->IsBranchInstr())
  12514. {
  12515. if (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive())
  12516. {
  12517. return;
  12518. }
  12519. }
  12520. else if (!src1->GetValueType().IsPrimitive() && !src2->GetValueType().IsPrimitive())
  12521. {
  12522. return;
  12523. }
  12524. }
  12525. }
  12526. else
  12527. {
  12528. return;
  12529. }
  12530. Js::OpCode opcode = instr->m_opcode;
  12531. switch (opcode)
  12532. {
  12533. case Js::OpCode::And_A:
  12534. case Js::OpCode::Mul_A:
  12535. case Js::OpCode::Or_A:
  12536. case Js::OpCode::Xor_A:
  12537. case Js::OpCode::And_I4:
  12538. case Js::OpCode::Mul_I4:
  12539. case Js::OpCode::Or_I4:
  12540. case Js::OpCode::Xor_I4:
  12541. case Js::OpCode::Add_I4:
  12542. swap_srcs:
  12543. if (!instr->GetSrc2()->IsImmediateOpnd())
  12544. {
  12545. instr->m_opcode = opcode;
  12546. instr->SwapOpnds();
  12547. Value *tempVal = *pSrc1Val;
  12548. *pSrc1Val = *pSrc2Val;
  12549. *pSrc2Val = tempVal;
  12550. return;
  12551. }
  12552. break;
  12553. case Js::OpCode::BrSrEq_A:
  12554. case Js::OpCode::BrSrNotNeq_A:
  12555. case Js::OpCode::BrEq_I4:
  12556. goto swap_srcs;
  12557. case Js::OpCode::BrSrNeq_A:
  12558. case Js::OpCode::BrNeq_A:
  12559. case Js::OpCode::BrSrNotEq_A:
  12560. case Js::OpCode::BrNotEq_A:
  12561. case Js::OpCode::BrNeq_I4:
  12562. goto swap_srcs;
  12563. case Js::OpCode::BrGe_A:
  12564. opcode = Js::OpCode::BrLe_A;
  12565. goto swap_srcs;
  12566. case Js::OpCode::BrNotGe_A:
  12567. opcode = Js::OpCode::BrNotLe_A;
  12568. goto swap_srcs;
  12569. case Js::OpCode::BrGe_I4:
  12570. opcode = Js::OpCode::BrLe_I4;
  12571. goto swap_srcs;
  12572. case Js::OpCode::BrGt_A:
  12573. opcode = Js::OpCode::BrLt_A;
  12574. goto swap_srcs;
  12575. case Js::OpCode::BrNotGt_A:
  12576. opcode = Js::OpCode::BrNotLt_A;
  12577. goto swap_srcs;
  12578. case Js::OpCode::BrGt_I4:
  12579. opcode = Js::OpCode::BrLt_I4;
  12580. goto swap_srcs;
  12581. case Js::OpCode::BrLe_A:
  12582. opcode = Js::OpCode::BrGe_A;
  12583. goto swap_srcs;
  12584. case Js::OpCode::BrNotLe_A:
  12585. opcode = Js::OpCode::BrNotGe_A;
  12586. goto swap_srcs;
  12587. case Js::OpCode::BrLe_I4:
  12588. opcode = Js::OpCode::BrGe_I4;
  12589. goto swap_srcs;
  12590. case Js::OpCode::BrLt_A:
  12591. opcode = Js::OpCode::BrGt_A;
  12592. goto swap_srcs;
  12593. case Js::OpCode::BrNotLt_A:
  12594. opcode = Js::OpCode::BrNotGt_A;
  12595. goto swap_srcs;
  12596. case Js::OpCode::BrLt_I4:
  12597. opcode = Js::OpCode::BrGt_I4;
  12598. goto swap_srcs;
  12599. case Js::OpCode::BrEq_A:
  12600. case Js::OpCode::BrNotNeq_A:
  12601. case Js::OpCode::CmEq_A:
  12602. case Js::OpCode::CmNeq_A:
  12603. // this == "" not the same as "" == this...
  12604. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12605. {
  12606. return;
  12607. }
  12608. goto swap_srcs;
  12609. case Js::OpCode::CmGe_A:
  12610. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12611. {
  12612. return;
  12613. }
  12614. opcode = Js::OpCode::CmLe_A;
  12615. goto swap_srcs;
  12616. case Js::OpCode::CmGt_A:
  12617. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12618. {
  12619. return;
  12620. }
  12621. opcode = Js::OpCode::CmLt_A;
  12622. goto swap_srcs;
  12623. case Js::OpCode::CmLe_A:
  12624. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12625. {
  12626. return;
  12627. }
  12628. opcode = Js::OpCode::CmGe_A;
  12629. goto swap_srcs;
  12630. case Js::OpCode::CmLt_A:
  12631. if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()))
  12632. {
  12633. return;
  12634. }
  12635. opcode = Js::OpCode::CmGt_A;
  12636. goto swap_srcs;
  12637. case Js::OpCode::CallI:
  12638. case Js::OpCode::CallIFixed:
  12639. case Js::OpCode::NewScObject:
  12640. case Js::OpCode::NewScObjectSpread:
  12641. case Js::OpCode::NewScObjArray:
  12642. case Js::OpCode::NewScObjArraySpread:
  12643. case Js::OpCode::NewScObjectNoCtor:
  12644. // Don't insert load to register if the function operand is a fixed function.
  12645. if (instr->HasFixedFunctionAddressTarget())
  12646. {
  12647. return;
  12648. }
  12649. break;
  12650. // Can't do add because <32 + "Hello"> isn't equal to <"Hello" + 32>
  12651. // Lower can do the swap. Other op-codes listed below don't need immediate source hoisting, as the fast paths handle it,
  12652. // or the lowering handles the hoisting.
  12653. case Js::OpCode::Add_A:
  12654. if (src1->IsFloat())
  12655. {
  12656. goto swap_srcs;
  12657. }
  12658. return;
  12659. case Js::OpCode::Sub_I4:
  12660. case Js::OpCode::Neg_I4:
  12661. case Js::OpCode::Not_I4:
  12662. case Js::OpCode::NewScFunc:
  12663. case Js::OpCode::NewScGenFunc:
  12664. case Js::OpCode::NewScFuncHomeObj:
  12665. case Js::OpCode::NewScGenFuncHomeObj:
  12666. case Js::OpCode::NewScArray:
  12667. case Js::OpCode::NewScIntArray:
  12668. case Js::OpCode::NewScFltArray:
  12669. case Js::OpCode::NewScArrayWithMissingValues:
  12670. case Js::OpCode::NewRegEx:
  12671. case Js::OpCode::Ld_A:
  12672. case Js::OpCode::Ld_I4:
  12673. case Js::OpCode::ThrowRuntimeError:
  12674. case Js::OpCode::TrapIfMinIntOverNegOne:
  12675. case Js::OpCode::TrapIfTruncOverflow:
  12676. case Js::OpCode::TrapIfZero:
  12677. case Js::OpCode::TrapIfUnalignedAccess:
  12678. case Js::OpCode::FromVar:
  12679. case Js::OpCode::Conv_Prim:
  12680. case Js::OpCode::Conv_Prim_Sat:
  12681. case Js::OpCode::LdC_A_I4:
  12682. case Js::OpCode::LdStr:
  12683. case Js::OpCode::InitFld:
  12684. case Js::OpCode::InitRootFld:
  12685. case Js::OpCode::StartCall:
  12686. case Js::OpCode::ArgOut_A:
  12687. case Js::OpCode::ArgOut_A_Inline:
  12688. case Js::OpCode::ArgOut_A_Dynamic:
  12689. case Js::OpCode::ArgOut_A_FromStackArgs:
  12690. case Js::OpCode::ArgOut_A_InlineBuiltIn:
  12691. case Js::OpCode::ArgOut_A_InlineSpecialized:
  12692. case Js::OpCode::ArgOut_A_SpreadArg:
  12693. case Js::OpCode::InlineeEnd:
  12694. case Js::OpCode::EndCallForPolymorphicInlinee:
  12695. case Js::OpCode::InlineeMetaArg:
  12696. case Js::OpCode::InlineBuiltInEnd:
  12697. case Js::OpCode::InlineNonTrackingBuiltInEnd:
  12698. case Js::OpCode::CallHelper:
  12699. case Js::OpCode::LdElemUndef:
  12700. case Js::OpCode::LdElemUndefScoped:
  12701. case Js::OpCode::RuntimeTypeError:
  12702. case Js::OpCode::RuntimeReferenceError:
  12703. case Js::OpCode::Ret:
  12704. case Js::OpCode::NewScObjectSimple:
  12705. case Js::OpCode::NewScObjectLiteral:
  12706. case Js::OpCode::StFld:
  12707. case Js::OpCode::StRootFld:
  12708. case Js::OpCode::StSlot:
  12709. case Js::OpCode::StSlotChkUndecl:
  12710. case Js::OpCode::StElemC:
  12711. case Js::OpCode::StArrSegElemC:
  12712. case Js::OpCode::StElemI_A:
  12713. case Js::OpCode::StElemI_A_Strict:
  12714. case Js::OpCode::CallDirect:
  12715. case Js::OpCode::BrNotHasSideEffects:
  12716. case Js::OpCode::NewConcatStrMulti:
  12717. case Js::OpCode::NewConcatStrMultiBE:
  12718. case Js::OpCode::ExtendArg_A:
  12719. #ifdef ENABLE_DOM_FAST_PATH
  12720. case Js::OpCode::DOMFastPathGetter:
  12721. case Js::OpCode::DOMFastPathSetter:
  12722. #endif
  12723. case Js::OpCode::NewScopeSlots:
  12724. case Js::OpCode::NewScopeSlotsWithoutPropIds:
  12725. case Js::OpCode::NewStackScopeSlots:
  12726. case Js::OpCode::IsInst:
  12727. case Js::OpCode::BailOnEqual:
  12728. case Js::OpCode::BailOnNotEqual:
  12729. case Js::OpCode::StArrViewElem:
  12730. return;
  12731. }
  12732. if (!src1->IsImmediateOpnd())
  12733. {
  12734. return;
  12735. }
  12736. // The fast paths or lowering of the remaining instructions may not support handling immediate opnds for the first src. The
  12737. // immediate src1 is hoisted here into a separate instruction.
  12738. if (src1->IsIntConstOpnd())
  12739. {
  12740. IR::Instr *newInstr = instr->HoistSrc1(Js::OpCode::Ld_I4);
  12741. ToInt32Dst(newInstr, newInstr->GetDst()->AsRegOpnd(), this->currentBlock);
  12742. }
  12743. else if (src1->IsInt64ConstOpnd())
  12744. {
  12745. instr->HoistSrc1(Js::OpCode::Ld_I4);
  12746. }
  12747. else
  12748. {
  12749. instr->HoistSrc1(Js::OpCode::Ld_A);
  12750. }
  12751. src1 = instr->GetSrc1();
  12752. src1->AsRegOpnd()->m_sym->SetIsConst();
  12753. }
  12754. // Clear the ValueMap pf the values invalidated by this instr.
  12755. void
  12756. GlobOpt::ProcessKills(IR::Instr *instr)
  12757. {
  12758. this->ProcessFieldKills(instr);
  12759. this->ProcessValueKills(instr);
  12760. this->ProcessArrayValueKills(instr);
  12761. }
  12762. bool
  12763. GlobOpt::OptIsInvariant(IR::Opnd *src, BasicBlock *block, Loop *loop, Value *srcVal, bool isNotTypeSpecConv, bool allowNonPrimitives)
  12764. {
  12765. if(!loop->CanHoistInvariants())
  12766. {
  12767. return false;
  12768. }
  12769. Sym *sym;
  12770. switch(src->GetKind())
  12771. {
  12772. case IR::OpndKindAddr:
  12773. case IR::OpndKindFloatConst:
  12774. case IR::OpndKindIntConst:
  12775. return true;
  12776. case IR::OpndKindReg:
  12777. sym = src->AsRegOpnd()->m_sym;
  12778. break;
  12779. case IR::OpndKindSym:
  12780. sym = src->AsSymOpnd()->m_sym;
  12781. if (src->AsSymOpnd()->IsPropertySymOpnd())
  12782. {
  12783. if (src->AsSymOpnd()->AsPropertySymOpnd()->IsTypeChecked())
  12784. {
  12785. // We do not handle hoisting these yet. We might be hoisting this across the instr with the type check protecting this one.
  12786. // And somehow, the dead-store pass now removes the type check on that instr later on...
  12787. // For CheckFixedFld, there is no benefit hoisting these if they don't have a type check as they won't generate code.
  12788. return false;
  12789. }
  12790. }
  12791. break;
  12792. case IR::OpndKindHelperCall:
  12793. // Helper calls, like the private slot getter, can be invariant.
  12794. // Consider moving more math builtin to invariant?
  12795. return HelperMethodAttributes::IsInVariant(src->AsHelperCallOpnd()->m_fnHelper);
  12796. default:
  12797. return false;
  12798. }
  12799. return OptIsInvariant(sym, block, loop, srcVal, isNotTypeSpecConv, allowNonPrimitives);
  12800. }
  12801. bool
  12802. GlobOpt::OptIsInvariant(Sym *sym, BasicBlock *block, Loop *loop, Value *srcVal, bool isNotTypeSpecConv, bool allowNonPrimitives, Value **loopHeadValRef)
  12803. {
  12804. Value *localLoopHeadVal;
  12805. if(!loopHeadValRef)
  12806. {
  12807. loopHeadValRef = &localLoopHeadVal;
  12808. }
  12809. Value *&loopHeadVal = *loopHeadValRef;
  12810. loopHeadVal = nullptr;
  12811. if(!loop->CanHoistInvariants())
  12812. {
  12813. return false;
  12814. }
  12815. if (sym->IsStackSym())
  12816. {
  12817. if (sym->AsStackSym()->IsTypeSpec())
  12818. {
  12819. StackSym *varSym = sym->AsStackSym()->GetVarEquivSym(this->func);
  12820. // Make sure the int32/float64 version of this is available.
  12821. // Note: We could handle this by converting the src, but usually the
  12822. // conversion is hoistable if this is hoistable anyway.
  12823. // In some weird cases it may not be however, so we'll bail out.
  12824. if (sym->AsStackSym()->IsInt32())
  12825. {
  12826. Assert(block->globOptData.liveInt32Syms->Test(varSym->m_id));
  12827. if (!loop->landingPad->globOptData.liveInt32Syms->Test(varSym->m_id) ||
  12828. (loop->landingPad->globOptData.liveLossyInt32Syms->Test(varSym->m_id) &&
  12829. !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id)))
  12830. {
  12831. // Either the int32 sym is not live in the landing pad, or it's lossy in the landing pad and the
  12832. // instruction's block is using the lossless version. In either case, the instruction cannot be hoisted
  12833. // without doing a conversion of this operand.
  12834. return false;
  12835. }
  12836. }
  12837. else if (sym->AsStackSym()->IsFloat64())
  12838. {
  12839. if (!loop->landingPad->globOptData.liveFloat64Syms->Test(varSym->m_id))
  12840. {
  12841. return false;
  12842. }
  12843. }
  12844. sym = sym->AsStackSym()->GetVarEquivSym(this->func);
  12845. }
  12846. else
  12847. {
  12848. // Make sure the var version of this is available.
  12849. // Note: We could handle this by converting the src, but usually the
  12850. // conversion is hoistable if this is hoistable anyway.
  12851. // In some weird cases it may not be however, so we'll bail out.
  12852. if (!loop->landingPad->globOptData.liveVarSyms->Test(sym->m_id))
  12853. {
  12854. return false;
  12855. }
  12856. }
  12857. }
  12858. else if (sym->IsPropertySym())
  12859. {
  12860. if (!loop->landingPad->globOptData.liveVarSyms->Test(sym->AsPropertySym()->m_stackSym->m_id))
  12861. {
  12862. return false;
  12863. }
  12864. }
  12865. else
  12866. {
  12867. return false;
  12868. }
  12869. // We rely on having a value.
  12870. if (srcVal == NULL)
  12871. {
  12872. return false;
  12873. }
  12874. // A symbol is invariant if its current value is the same as it was upon entering the loop.
  12875. loopHeadVal = loop->landingPad->globOptData.FindValue(sym);
  12876. if (loopHeadVal == NULL || loopHeadVal->GetValueNumber() != srcVal->GetValueNumber())
  12877. {
  12878. return false;
  12879. }
  12880. // Can't hoist non-primitives, unless we have safeguards against valueof/tostring. Additionally, we need to consider
  12881. // the value annotations on the source *before* the loop: if we hoist this instruction outside the loop, we can't
  12882. // necessarily rely on type annotations added (and enforced) earlier in the loop's body.
  12883. //
  12884. // It might look as though !loopHeadVal->GetValueInfo()->IsPrimitive() implies
  12885. // !loop->landingPad->globOptData.IsTypeSpecialized(sym), but it turns out that this is not always the case. We
  12886. // encountered a test case in which we had previously hoisted a FromVar (to float 64) instruction, but its bailout code was
  12887. // BailoutPrimitiveButString, rather than BailoutNumberOnly, which would have allowed us to conclude that the dest was
  12888. // definitely a float64. Instead, it was only *likely* a float64, causing IsPrimitive to return false.
  12889. if (!allowNonPrimitives && !loopHeadVal->GetValueInfo()->IsPrimitive() && !loop->landingPad->globOptData.IsTypeSpecialized(sym))
  12890. {
  12891. return false;
  12892. }
  12893. if(!isNotTypeSpecConv && loop->symsDefInLoop->Test(sym->m_id))
  12894. {
  12895. // Typically, a sym is considered invariant if it has the same value in the current block and in the loop landing pad.
  12896. // The sym may have had a different value earlier in the loop or on the back-edge, but as long as it's reassigned to its
  12897. // value outside the loop, it would be considered invariant in this block. Consider that case:
  12898. // s1 = s2[invariant]
  12899. // <loop start>
  12900. // s1 = s2[invariant]
  12901. // // s1 now has the same value as in the landing pad, and is considered invariant
  12902. // s1 += s3
  12903. // // s1 is not invariant here, or on the back-edge
  12904. // ++s3 // s3 is not invariant, so the add above cannot be hoisted
  12905. // <loop end>
  12906. //
  12907. // A problem occurs at the point of (s1 += s3) when:
  12908. // - At (s1 = s2) inside the loop, s1 was made to be the sym store of that value. This by itself is legal, because
  12909. // after that transfer, s1 and s2 have the same value.
  12910. // - (s1 += s3) is type-specialized but s1 is not specialized in the loop header. This happens when s1 is not
  12911. // specialized entering the loop, and since s1 is not used before it's defined in the loop, it's not specialized
  12912. // on back-edges.
  12913. //
  12914. // With that, at (s1 += s3), the conversion of s1 to the type-specialized version would be hoisted because s1 is
  12915. // invariant just before that instruction. Since this add is specialized, the specialized version of the sym is modified
  12916. // in the loop without a reassignment at (s1 = s2) inside the loop, and (s1 += s3) would then use an incorrect value of
  12917. // s1 (it would use the value of s1 from the previous loop iteration, instead of using the value of s2).
  12918. //
  12919. // The problem here, is that we cannot hoist the conversion of s1 into its specialized version across the assignment
  12920. // (s1 = s2) inside the loop. So for the purposes of type specialization, don't consider a sym invariant if it has a def
  12921. // inside the loop.
  12922. return false;
  12923. }
  12924. // For values with an int range, require additionally that the range is the same as in the landing pad, as the range may
  12925. // have been changed on this path based on branches, and int specialization and invariant hoisting may rely on the range
  12926. // being the same. For type spec conversions, only require that if the value is an int constant in the current block, that
  12927. // it is also an int constant with the same value in the landing pad. Other range differences don't matter for type spec.
  12928. IntConstantBounds srcIntConstantBounds, loopHeadIntConstantBounds;
  12929. if(srcVal->GetValueInfo()->TryGetIntConstantBounds(&srcIntConstantBounds) &&
  12930. (isNotTypeSpecConv || srcIntConstantBounds.IsConstant()) &&
  12931. (
  12932. !loopHeadVal->GetValueInfo()->TryGetIntConstantBounds(&loopHeadIntConstantBounds) ||
  12933. loopHeadIntConstantBounds.LowerBound() != srcIntConstantBounds.LowerBound() ||
  12934. loopHeadIntConstantBounds.UpperBound() != srcIntConstantBounds.UpperBound()
  12935. ))
  12936. {
  12937. return false;
  12938. }
  12939. // Disabling this assert, because it does not hold true when we force specialize in the loop landing pad
  12940. //Assert((!loopHeadVal->GetValueInfo()->IsPrimitive()) || srcVal->GetValueInfo()->IsLikelyPrimitive());
  12941. return true;
  12942. }
  12943. bool
  12944. GlobOpt::OptIsInvariant(
  12945. IR::Instr *instr,
  12946. BasicBlock *block,
  12947. Loop *loop,
  12948. Value *src1Val,
  12949. Value *src2Val,
  12950. bool isNotTypeSpecConv,
  12951. const bool forceInvariantHoisting)
  12952. {
  12953. if (!loop->CanHoistInvariants())
  12954. {
  12955. return false;
  12956. }
  12957. if (!OpCodeAttr::CanCSE(instr->m_opcode))
  12958. {
  12959. return false;
  12960. }
  12961. bool allowNonPrimitives = !OpCodeAttr::OpndHasImplicitCall(instr->m_opcode);
  12962. switch(instr->m_opcode)
  12963. {
  12964. // Can't legally hoist these
  12965. case Js::OpCode::LdLen_A:
  12966. return false;
  12967. //Can't Hoist BailOnNotStackArgs, as it is necessary as InlineArgsOptimization relies on this opcode
  12968. //to decide whether to throw rejit exception or not.
  12969. case Js::OpCode::BailOnNotStackArgs:
  12970. return false;
  12971. // Usually not worth hoisting these
  12972. case Js::OpCode::Ld_A:
  12973. case Js::OpCode::Ld_I4:
  12974. case Js::OpCode::LdC_A_I4:
  12975. if(!forceInvariantHoisting)
  12976. {
  12977. return false;
  12978. }
  12979. break;
  12980. // Can't hoist these outside the function it's for. The LdArgumentsFromFrame for an inlinee depends on the inlinee meta arg
  12981. // that holds the arguments object, which is only initialized at the start of the inlinee. So, can't hoist this outside the
  12982. // inlinee.
  12983. case Js::OpCode::LdArgumentsFromFrame:
  12984. if(instr->m_func != loop->GetFunc())
  12985. {
  12986. return false;
  12987. }
  12988. break;
  12989. case Js::OpCode::FromVar:
  12990. if (instr->HasBailOutInfo())
  12991. {
  12992. allowNonPrimitives = true;
  12993. }
  12994. break;
  12995. case Js::OpCode::CheckObjType:
  12996. // Bug 11712101: If the operand is a field, ensure that its containing object type is invariant
  12997. // before hoisting -- that is, don't hoist a CheckObjType over a DeleteFld on that object.
  12998. // (CheckObjType only checks the operand and its immediate parent, so we don't need to go
  12999. // any farther up the object graph.)
  13000. Assert(instr->GetSrc1());
  13001. PropertySym *propertySym = instr->GetSrc1()->AsPropertySymOpnd()->GetPropertySym();
  13002. if (propertySym->HasObjectTypeSym()) {
  13003. StackSym *objectTypeSym = propertySym->GetObjectTypeSym();
  13004. if (!this->OptIsInvariant(objectTypeSym, block, loop, this->CurrentBlockData()->FindValue(objectTypeSym), true, true)) {
  13005. return false;
  13006. }
  13007. }
  13008. break;
  13009. }
  13010. IR::Opnd *dst = instr->GetDst();
  13011. if (dst && !dst->IsRegOpnd())
  13012. {
  13013. return false;
  13014. }
  13015. IR::Opnd *src1 = instr->GetSrc1();
  13016. if (src1)
  13017. {
  13018. if (!this->OptIsInvariant(src1, block, loop, src1Val, isNotTypeSpecConv, allowNonPrimitives))
  13019. {
  13020. return false;
  13021. }
  13022. IR::Opnd *src2 = instr->GetSrc2();
  13023. if (src2)
  13024. {
  13025. if (!this->OptIsInvariant(src2, block, loop, src2Val, isNotTypeSpecConv, allowNonPrimitives))
  13026. {
  13027. return false;
  13028. }
  13029. }
  13030. }
  13031. return true;
  13032. }
  13033. bool
  13034. GlobOpt::OptDstIsInvariant(IR::RegOpnd *dst)
  13035. {
  13036. StackSym *dstSym = dst->m_sym;
  13037. if (dstSym->IsTypeSpec())
  13038. {
  13039. // The type-specialized sym may be single def, but not the original...
  13040. dstSym = dstSym->GetVarEquivSym(this->func);
  13041. }
  13042. return (dstSym->m_isSingleDef);
  13043. }
  13044. void
  13045. GlobOpt::OptHoistUpdateValueType(
  13046. Loop* loop,
  13047. IR::Instr* instr,
  13048. IR::Opnd** srcOpndPtr /* All code paths that change src, should update srcOpndPtr*/,
  13049. Value* opndVal)
  13050. {
  13051. if (opndVal == nullptr || instr->m_opcode == Js::OpCode::FromVar || srcOpndPtr == nullptr || *srcOpndPtr == nullptr)
  13052. {
  13053. return;
  13054. }
  13055. IR::Opnd* srcOpnd = *srcOpndPtr;
  13056. Sym* opndSym = srcOpnd->GetSym();;
  13057. if (opndSym)
  13058. {
  13059. BasicBlock* landingPad = loop->landingPad;
  13060. Value* opndValueInLandingPad = landingPad->globOptData.FindValue(opndSym);
  13061. Assert(opndVal->GetValueNumber() == opndValueInLandingPad->GetValueNumber());
  13062. ValueType opndValueTypeInLandingPad = opndValueInLandingPad->GetValueInfo()->Type();
  13063. if (srcOpnd->GetValueType() != opndValueTypeInLandingPad)
  13064. {
  13065. srcOpnd->SetValueType(opndValueTypeInLandingPad);
  13066. if (instr->m_opcode == Js::OpCode::SetConcatStrMultiItemBE)
  13067. {
  13068. Assert(!opndSym->IsPropertySym());
  13069. Assert(!opndValueTypeInLandingPad.IsString());
  13070. Assert(instr->GetDst());
  13071. IR::RegOpnd* strOpnd = IR::RegOpnd::New(TyVar, instr->m_func);
  13072. strOpnd->SetValueType(ValueType::String);
  13073. strOpnd->SetValueTypeFixed();
  13074. IR::Instr* convPrimStrInstr =
  13075. IR::Instr::New(Js::OpCode::Conv_PrimStr, strOpnd, srcOpnd->Use(instr->m_func), instr->m_func);
  13076. instr->ReplaceSrc(srcOpnd, strOpnd);
  13077. // Replace above will free srcOpnd, so reassign it
  13078. *srcOpndPtr = srcOpnd = reinterpret_cast<IR::Opnd *>(strOpnd);
  13079. // We add ConvPrim_Str in the landingpad, and since this instruction doesn't go through the checks in OptInstr, the bailout is never added
  13080. // As we expand hoisting of instructions to new opcode, we need a better framework to handle such cases
  13081. if (IsImplicitCallBailOutCurrentlyNeeded(convPrimStrInstr, opndValueInLandingPad, nullptr, landingPad, landingPad->globOptData.liveFields->IsEmpty(), true, true))
  13082. {
  13083. EnsureBailTarget(loop);
  13084. loop->bailOutInfo->bailOutInstr->InsertBefore(convPrimStrInstr);
  13085. convPrimStrInstr = convPrimStrInstr->ConvertToBailOutInstr(convPrimStrInstr, IR::BailOutOnImplicitCallsPreOp, loop->bailOutInfo->bailOutOffset);
  13086. convPrimStrInstr->ReplaceBailOutInfo(loop->bailOutInfo);
  13087. }
  13088. else
  13089. {
  13090. if (loop->bailOutInfo->bailOutInstr)
  13091. {
  13092. loop->bailOutInfo->bailOutInstr->InsertBefore(convPrimStrInstr);
  13093. }
  13094. else
  13095. {
  13096. landingPad->InsertAfter(convPrimStrInstr);
  13097. }
  13098. }
  13099. // If we came here opndSym can't be PropertySym
  13100. return;
  13101. }
  13102. }
  13103. if (opndSym->IsPropertySym())
  13104. {
  13105. // Also fix valueInfo on objPtr
  13106. StackSym* opndObjPtrSym = opndSym->AsPropertySym()->m_stackSym;
  13107. Value* opndObjPtrSymValInLandingPad = landingPad->globOptData.FindValue(opndObjPtrSym);
  13108. ValueInfo* opndObjPtrSymValueInfoInLandingPad = opndObjPtrSymValInLandingPad->GetValueInfo();
  13109. srcOpnd->AsSymOpnd()->SetPropertyOwnerValueType(opndObjPtrSymValueInfoInLandingPad->Type());
  13110. }
  13111. }
  13112. }
  13113. void
  13114. GlobOpt::OptHoistInvariant(
  13115. IR::Instr *instr,
  13116. BasicBlock *block,
  13117. Loop *loop,
  13118. Value *dstVal,
  13119. Value *const src1Val,
  13120. Value *const src2Val,
  13121. bool isNotTypeSpecConv,
  13122. bool lossy,
  13123. IR::BailOutKind bailoutKind)
  13124. {
  13125. BasicBlock *landingPad = loop->landingPad;
  13126. IR::Opnd* src1 = instr->GetSrc1();
  13127. if (src1)
  13128. {
  13129. // We are hoisting this instruction possibly past other uses, which might invalidate the last use info. Clear it.
  13130. OptHoistUpdateValueType(loop, instr, &src1, src1Val);
  13131. if (src1->IsRegOpnd())
  13132. {
  13133. src1->AsRegOpnd()->m_isTempLastUse = false;
  13134. }
  13135. IR::Opnd* src2 = instr->GetSrc2();
  13136. if (src2)
  13137. {
  13138. OptHoistUpdateValueType(loop, instr, &src2, src2Val);
  13139. if (src2->IsRegOpnd())
  13140. {
  13141. src2->AsRegOpnd()->m_isTempLastUse = false;
  13142. }
  13143. }
  13144. }
  13145. IR::RegOpnd *dst = instr->GetDst() ? instr->GetDst()->AsRegOpnd() : nullptr;
  13146. if(dst)
  13147. {
  13148. switch (instr->m_opcode)
  13149. {
  13150. case Js::OpCode::CmEq_I4:
  13151. case Js::OpCode::CmNeq_I4:
  13152. case Js::OpCode::CmLt_I4:
  13153. case Js::OpCode::CmLe_I4:
  13154. case Js::OpCode::CmGt_I4:
  13155. case Js::OpCode::CmGe_I4:
  13156. case Js::OpCode::CmUnLt_I4:
  13157. case Js::OpCode::CmUnLe_I4:
  13158. case Js::OpCode::CmUnGt_I4:
  13159. case Js::OpCode::CmUnGe_I4:
  13160. // These operations are a special case. They generate a lossy int value, and the var sym is initialized using
  13161. // Conv_Bool. A sym cannot be live only as a lossy int sym, the var needs to be live as well since the lossy int
  13162. // sym cannot be used to convert to var. We don't know however, whether the Conv_Bool will be hoisted. The idea
  13163. // currently is that the sym is only used on the path in which it is initialized inside the loop. So, don't
  13164. // hoist any liveness info for the dst.
  13165. if (!this->GetIsAsmJSFunc())
  13166. {
  13167. lossy = true;
  13168. }
  13169. break;
  13170. case Js::OpCode::FromVar:
  13171. {
  13172. StackSym* src1StackSym = IR::RegOpnd::TryGetStackSym(instr->GetSrc1());
  13173. if (instr->HasBailOutInfo())
  13174. {
  13175. IR::BailOutKind instrBailoutKind = instr->GetBailOutKind();
  13176. Assert(instrBailoutKind == IR::BailOutIntOnly ||
  13177. instrBailoutKind == IR::BailOutExpectingInteger ||
  13178. instrBailoutKind == IR::BailOutOnNotPrimitive ||
  13179. instrBailoutKind == IR::BailOutNumberOnly ||
  13180. instrBailoutKind == IR::BailOutPrimitiveButString);
  13181. }
  13182. else if (src1StackSym && bailoutKind != IR::BailOutInvalid)
  13183. {
  13184. // We may be hoisting FromVar from a region where it didn't need a bailout (src1 had a definite value type) to a region
  13185. // where it would. In such cases, the FromVar needs a bailout based on the value type of src1 in its new position.
  13186. Assert(!src1StackSym->IsTypeSpec());
  13187. Value* landingPadSrc1val = landingPad->globOptData.FindValue(src1StackSym);
  13188. Assert(src1Val->GetValueNumber() == landingPadSrc1val->GetValueNumber());
  13189. ValueInfo *src1ValueInfo = src1Val->GetValueInfo();
  13190. ValueInfo *landingPadSrc1ValueInfo = landingPadSrc1val->GetValueInfo();
  13191. IRType dstType = dst->GetType();
  13192. const auto AddBailOutToFromVar = [&]()
  13193. {
  13194. instr->GetSrc1()->SetValueType(landingPadSrc1val->GetValueInfo()->Type());
  13195. EnsureBailTarget(loop);
  13196. if (block->IsLandingPad())
  13197. {
  13198. instr = instr->ConvertToBailOutInstr(instr, bailoutKind, loop->bailOutInfo->bailOutOffset);
  13199. }
  13200. else
  13201. {
  13202. instr = instr->ConvertToBailOutInstr(instr, bailoutKind);
  13203. }
  13204. };
  13205. // A definite type in the source position and not a definite type in the destination (landing pad)
  13206. // and no bailout on the instruction; we should put a bailout on the hoisted instruction.
  13207. if (dstType == TyInt32)
  13208. {
  13209. if (lossy)
  13210. {
  13211. if ((src1ValueInfo->IsPrimitive() || block->globOptData.IsTypeSpecialized(src1StackSym)) && // didn't need a lossy type spec bailout in the source block
  13212. (!landingPadSrc1ValueInfo->IsPrimitive() && !landingPad->globOptData.IsTypeSpecialized(src1StackSym))) // needs a lossy type spec bailout in the landing pad
  13213. {
  13214. bailoutKind = IR::BailOutOnNotPrimitive;
  13215. AddBailOutToFromVar();
  13216. }
  13217. }
  13218. else if (src1ValueInfo->IsInt() && !landingPadSrc1ValueInfo->IsInt())
  13219. {
  13220. AddBailOutToFromVar();
  13221. }
  13222. }
  13223. else if ((dstType == TyFloat64 && src1ValueInfo->IsNumber() && !landingPadSrc1ValueInfo->IsNumber()))
  13224. {
  13225. AddBailOutToFromVar();
  13226. }
  13227. }
  13228. break;
  13229. }
  13230. }
  13231. if (dstVal == NULL)
  13232. {
  13233. dstVal = this->NewGenericValue(ValueType::Uninitialized, dst);
  13234. }
  13235. // ToVar/FromVar don't need a new dst because it has to be invariant if their src is invariant.
  13236. bool dstDoesntNeedLoad = (!isNotTypeSpecConv && instr->m_opcode != Js::OpCode::LdC_A_I4);
  13237. StackSym *varSym = dst->m_sym;
  13238. if (varSym->IsTypeSpec())
  13239. {
  13240. varSym = varSym->GetVarEquivSym(this->func);
  13241. }
  13242. Value *const landingPadDstVal = loop->landingPad->globOptData.FindValue(varSym);
  13243. if(landingPadDstVal
  13244. ? dstVal->GetValueNumber() != landingPadDstVal->GetValueNumber()
  13245. : loop->symsDefInLoop->Test(varSym->m_id))
  13246. {
  13247. // We need a temp for FromVar/ToVar if dst changes in the loop.
  13248. dstDoesntNeedLoad = false;
  13249. }
  13250. if (!dstDoesntNeedLoad && this->OptDstIsInvariant(dst) == false)
  13251. {
  13252. // Keep dst in place, hoist instr using a new dst.
  13253. instr->UnlinkDst();
  13254. // Set type specialization info correctly for this new sym
  13255. StackSym *copyVarSym;
  13256. IR::RegOpnd *copyReg;
  13257. if (dst->m_sym->IsTypeSpec())
  13258. {
  13259. copyVarSym = StackSym::New(TyVar, instr->m_func);
  13260. StackSym *copySym = copyVarSym;
  13261. if (dst->m_sym->IsInt32())
  13262. {
  13263. if(lossy)
  13264. {
  13265. // The new sym would only be live as a lossy int since we're only hoisting the store to the int version
  13266. // of the sym, and cannot be converted to var. It is not legal to have a sym only live as a lossy int,
  13267. // so don't update liveness info for this sym.
  13268. }
  13269. else
  13270. {
  13271. block->globOptData.liveInt32Syms->Set(copyVarSym->m_id);
  13272. }
  13273. copySym = copySym->GetInt32EquivSym(instr->m_func);
  13274. }
  13275. else if (dst->m_sym->IsFloat64())
  13276. {
  13277. block->globOptData.liveFloat64Syms->Set(copyVarSym->m_id);
  13278. copySym = copySym->GetFloat64EquivSym(instr->m_func);
  13279. }
  13280. copyReg = IR::RegOpnd::New(copySym, copySym->GetType(), instr->m_func);
  13281. }
  13282. else
  13283. {
  13284. copyReg = IR::RegOpnd::New(dst->GetType(), instr->m_func);
  13285. copyVarSym = copyReg->m_sym;
  13286. block->globOptData.liveVarSyms->Set(copyVarSym->m_id);
  13287. }
  13288. copyReg->SetValueType(dst->GetValueType());
  13289. IR::Instr *copyInstr = IR::Instr::New(Js::OpCode::Ld_A, dst, copyReg, instr->m_func);
  13290. copyInstr->SetByteCodeOffset(instr);
  13291. instr->SetDst(copyReg);
  13292. instr->InsertBefore(copyInstr);
  13293. dst->m_sym->m_mayNotBeTempLastUse = true;
  13294. if (instr->GetSrc1() && instr->GetSrc1()->IsImmediateOpnd())
  13295. {
  13296. // Propagate IsIntConst if appropriate
  13297. switch(instr->m_opcode)
  13298. {
  13299. case Js::OpCode::Ld_A:
  13300. case Js::OpCode::Ld_I4:
  13301. case Js::OpCode::LdC_A_I4:
  13302. copyReg->m_sym->SetIsConst();
  13303. break;
  13304. }
  13305. }
  13306. ValueInfo *dstValueInfo = dstVal->GetValueInfo();
  13307. if((!dstValueInfo->GetSymStore() || dstValueInfo->GetSymStore() == varSym) && !lossy)
  13308. {
  13309. // The destination's value may have been transferred from one of the invariant sources, in which case we should
  13310. // keep the sym store intact, as that sym will likely have a better lifetime than this new copy sym. For
  13311. // instance, if we're inside a conditioned block, because we don't make the copy sym live and set its value in
  13312. // all preceding blocks, this sym would not be live after exiting this block, causing this value to not
  13313. // participate in copy-prop after this block.
  13314. this->SetSymStoreDirect(dstValueInfo, copyVarSym);
  13315. }
  13316. block->globOptData.InsertNewValue(dstVal, copyReg);
  13317. dst = copyReg;
  13318. }
  13319. }
  13320. // Move to landing pad
  13321. block->UnlinkInstr(instr);
  13322. if (loop->bailOutInfo->bailOutInstr)
  13323. {
  13324. loop->bailOutInfo->bailOutInstr->InsertBefore(instr);
  13325. }
  13326. else
  13327. {
  13328. landingPad->InsertAfter(instr);
  13329. }
  13330. GlobOpt::MarkNonByteCodeUsed(instr);
  13331. if (instr->HasBailOutInfo() || instr->HasAuxBailOut())
  13332. {
  13333. Assert(loop->bailOutInfo);
  13334. EnsureBailTarget(loop);
  13335. // Copy bailout info of loop top.
  13336. instr->ReplaceBailOutInfo(loop->bailOutInfo);
  13337. }
  13338. if(!dst)
  13339. {
  13340. return;
  13341. }
  13342. // The bailout info's liveness for the dst sym is not updated in loop landing pads because bailout instructions previously
  13343. // hoisted into the loop's landing pad may bail out before the current type of the dst sym became live (perhaps due to this
  13344. // instruction). Since the landing pad will have a shared bailout point, the bailout info cannot assume that the current
  13345. // type of the dst sym was live during every bailout hoisted into the landing pad.
  13346. StackSym *const dstSym = dst->m_sym;
  13347. StackSym *const dstVarSym = dstSym->IsTypeSpec() ? dstSym->GetVarEquivSym(nullptr) : dstSym;
  13348. Assert(dstVarSym);
  13349. if(isNotTypeSpecConv || !loop->landingPad->globOptData.IsLive(dstVarSym))
  13350. {
  13351. // A new dst is being hoisted, or the same single-def dst that would not be live before this block. So, make it live and
  13352. // update the value info with the same value info in this block.
  13353. if(lossy)
  13354. {
  13355. // This is a lossy conversion to int. The instruction was given a new dst specifically for hoisting, so this new dst
  13356. // will not be live as a var before this block. A sym cannot be live only as a lossy int sym, the var needs to be
  13357. // live as well since the lossy int sym cannot be used to convert to var. Since the var version of the sym is not
  13358. // going to be initialized, don't hoist any liveness info for the dst. The sym is only going to be used on the path
  13359. // in which it is initialized inside the loop.
  13360. Assert(dstSym->IsTypeSpec());
  13361. Assert(dstSym->IsInt32());
  13362. return;
  13363. }
  13364. // Check if the dst value was transferred from the src. If so, the value transfer needs to be replicated.
  13365. bool isTransfer = dstVal == src1Val;
  13366. StackSym *transferValueOfSym = nullptr;
  13367. if(isTransfer)
  13368. {
  13369. Assert(instr->GetSrc1());
  13370. if(instr->GetSrc1()->IsRegOpnd())
  13371. {
  13372. StackSym *src1Sym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  13373. if(src1Sym->IsTypeSpec())
  13374. {
  13375. src1Sym = src1Sym->GetVarEquivSym(nullptr);
  13376. Assert(src1Sym);
  13377. }
  13378. if(dstVal == block->globOptData.FindValue(src1Sym))
  13379. {
  13380. transferValueOfSym = src1Sym;
  13381. }
  13382. }
  13383. }
  13384. // SIMD_JS
  13385. if (instr->m_opcode == Js::OpCode::ExtendArg_A)
  13386. {
  13387. // Check if we should have CSE'ed this EA
  13388. Assert(instr->GetSrc1());
  13389. // If the dstVal symstore is not the dst itself, then we copied the Value from another expression.
  13390. if (dstVal->GetValueInfo()->GetSymStore() != instr->GetDst()->GetStackSym())
  13391. {
  13392. isTransfer = true;
  13393. transferValueOfSym = dstVal->GetValueInfo()->GetSymStore()->AsStackSym();
  13394. }
  13395. }
  13396. const ValueNumber dstValueNumber = dstVal->GetValueNumber();
  13397. ValueNumber dstNewValueNumber = InvalidValueNumber;
  13398. for(InvariantBlockBackwardIterator it(this, block, loop->landingPad, nullptr); it.IsValid(); it.MoveNext())
  13399. {
  13400. BasicBlock *const hoistBlock = it.Block();
  13401. GlobOptBlockData &hoistBlockData = hoistBlock->globOptData;
  13402. Assert(!hoistBlockData.IsLive(dstVarSym));
  13403. hoistBlockData.MakeLive(dstSym, lossy);
  13404. Value *newDstValue;
  13405. do
  13406. {
  13407. if(isTransfer)
  13408. {
  13409. if(transferValueOfSym)
  13410. {
  13411. newDstValue = hoistBlockData.FindValue(transferValueOfSym);
  13412. if(newDstValue && newDstValue->GetValueNumber() == dstValueNumber)
  13413. {
  13414. break;
  13415. }
  13416. }
  13417. // It's a transfer, but we don't have a sym whose value number matches in the target block. Use a new value
  13418. // number since we don't know if there is already a value with the current number for the target block.
  13419. if(dstNewValueNumber == InvalidValueNumber)
  13420. {
  13421. dstNewValueNumber = NewValueNumber();
  13422. }
  13423. newDstValue = CopyValue(dstVal, dstNewValueNumber);
  13424. break;
  13425. }
  13426. newDstValue = CopyValue(dstVal, dstValueNumber);
  13427. } while(false);
  13428. hoistBlockData.SetValue(newDstValue, dstVarSym);
  13429. }
  13430. return;
  13431. }
  13432. #if DBG
  13433. if(instr->GetSrc1()->IsRegOpnd()) // Type spec conversion may load a constant into a dst sym
  13434. {
  13435. StackSym *const srcSym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  13436. Assert(srcSym != dstSym); // Type spec conversion must be changing the type, so the syms must be different
  13437. StackSym *const srcVarSym = srcSym->IsTypeSpec() ? srcSym->GetVarEquivSym(nullptr) : srcSym;
  13438. Assert(srcVarSym == dstVarSym); // Type spec conversion must be between variants of the same var sym
  13439. }
  13440. #endif
  13441. bool changeValueType = false, changeValueTypeToInt = false;
  13442. if(dstSym->IsTypeSpec())
  13443. {
  13444. if(dst->IsInt32())
  13445. {
  13446. if(!lossy)
  13447. {
  13448. Assert(
  13449. !instr->HasBailOutInfo() ||
  13450. instr->GetBailOutKind() == IR::BailOutIntOnly ||
  13451. instr->GetBailOutKind() == IR::BailOutExpectingInteger);
  13452. changeValueType = changeValueTypeToInt = true;
  13453. }
  13454. }
  13455. else if (dst->IsFloat64())
  13456. {
  13457. if(instr->HasBailOutInfo() && instr->GetBailOutKind() == IR::BailOutNumberOnly)
  13458. {
  13459. changeValueType = true;
  13460. }
  13461. }
  13462. }
  13463. ValueInfo *previousValueInfoBeforeUpdate = nullptr, *previousValueInfoAfterUpdate = nullptr;
  13464. for(InvariantBlockBackwardIterator it(
  13465. this,
  13466. block,
  13467. loop->landingPad,
  13468. dstVarSym,
  13469. dstVal->GetValueNumber());
  13470. it.IsValid();
  13471. it.MoveNext())
  13472. {
  13473. BasicBlock *const hoistBlock = it.Block();
  13474. GlobOptBlockData &hoistBlockData = hoistBlock->globOptData;
  13475. #if DBG
  13476. // TODO: There are some odd cases with field hoisting where the sym is invariant in only part of the loop and the info
  13477. // does not flow through all blocks. Un-comment the verification below after PRE replaces field hoisting.
  13478. //// Verify that the src sym is live as the required type, and that the conversion is valid
  13479. //Assert(IsLive(dstVarSym, &hoistBlockData));
  13480. //if(instr->GetSrc1()->IsRegOpnd())
  13481. //{
  13482. // IR::RegOpnd *const src = instr->GetSrc1()->AsRegOpnd();
  13483. // StackSym *const srcSym = instr->GetSrc1()->AsRegOpnd()->m_sym;
  13484. // if(srcSym->IsTypeSpec())
  13485. // {
  13486. // if(src->IsInt32())
  13487. // {
  13488. // Assert(hoistBlockData.liveInt32Syms->Test(dstVarSym->m_id));
  13489. // Assert(!hoistBlockData.liveLossyInt32Syms->Test(dstVarSym->m_id)); // shouldn't try to convert a lossy int32 to anything
  13490. // }
  13491. // else
  13492. // {
  13493. // Assert(src->IsFloat64());
  13494. // Assert(hoistBlockData.liveFloat64Syms->Test(dstVarSym->m_id));
  13495. // if(dstSym->IsTypeSpec() && dst->IsInt32())
  13496. // {
  13497. // Assert(lossy); // shouldn't try to do a lossless conversion from float64 to int32
  13498. // }
  13499. // }
  13500. // }
  13501. // else
  13502. // {
  13503. // Assert(hoistBlockData.liveVarSyms->Test(dstVarSym->m_id));
  13504. // }
  13505. //}
  13506. //if(dstSym->IsTypeSpec() && dst->IsInt32())
  13507. //{
  13508. // // If the sym is already specialized as required in the block to which we are attempting to hoist the conversion,
  13509. // // that info should have flowed into this block
  13510. // if(lossy)
  13511. // {
  13512. // Assert(!hoistBlockData.liveInt32Syms->Test(dstVarSym->m_id));
  13513. // }
  13514. // else
  13515. // {
  13516. // Assert(!IsInt32TypeSpecialized(dstVarSym, hoistBlock));
  13517. // }
  13518. //}
  13519. #endif
  13520. hoistBlockData.MakeLive(dstSym, lossy);
  13521. if(!changeValueType)
  13522. {
  13523. continue;
  13524. }
  13525. Value *const hoistBlockValue = it.InvariantSymValue();
  13526. ValueInfo *const hoistBlockValueInfo = hoistBlockValue->GetValueInfo();
  13527. if(hoistBlockValueInfo == previousValueInfoBeforeUpdate)
  13528. {
  13529. if(hoistBlockValueInfo != previousValueInfoAfterUpdate)
  13530. {
  13531. HoistInvariantValueInfo(previousValueInfoAfterUpdate, hoistBlockValue, hoistBlock);
  13532. }
  13533. }
  13534. else
  13535. {
  13536. previousValueInfoBeforeUpdate = hoistBlockValueInfo;
  13537. ValueInfo *const newValueInfo =
  13538. changeValueTypeToInt
  13539. ? hoistBlockValueInfo->SpecializeToInt32(alloc)
  13540. : hoistBlockValueInfo->SpecializeToFloat64(alloc);
  13541. previousValueInfoAfterUpdate = newValueInfo;
  13542. ChangeValueInfo(changeValueTypeToInt ? nullptr : hoistBlock, hoistBlockValue, newValueInfo);
  13543. }
  13544. }
  13545. }
  13546. bool
  13547. GlobOpt::TryHoistInvariant(
  13548. IR::Instr *instr,
  13549. BasicBlock *block,
  13550. Value *dstVal,
  13551. Value *src1Val,
  13552. Value *src2Val,
  13553. bool isNotTypeSpecConv,
  13554. const bool lossy,
  13555. const bool forceInvariantHoisting,
  13556. IR::BailOutKind bailoutKind)
  13557. {
  13558. Assert(!this->IsLoopPrePass());
  13559. if (OptIsInvariant(instr, block, block->loop, src1Val, src2Val, isNotTypeSpecConv, forceInvariantHoisting))
  13560. {
  13561. #if DBG
  13562. if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::InvariantsPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()))
  13563. {
  13564. Output::Print(_u(" **** INVARIANT *** "));
  13565. instr->Dump();
  13566. }
  13567. #endif
  13568. #if ENABLE_DEBUG_CONFIG_OPTIONS
  13569. if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::InvariantsPhase))
  13570. {
  13571. Output::Print(_u(" **** INVARIANT *** "));
  13572. Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
  13573. }
  13574. #endif
  13575. Loop *loop = block->loop;
  13576. // Try hoisting from to outer most loop
  13577. while (loop->parent && OptIsInvariant(instr, block, loop->parent, src1Val, src2Val, isNotTypeSpecConv, forceInvariantHoisting))
  13578. {
  13579. loop = loop->parent;
  13580. }
  13581. // Record the byte code use here since we are going to move this instruction up
  13582. if (isNotTypeSpecConv)
  13583. {
  13584. InsertNoImplicitCallUses(instr);
  13585. this->CaptureByteCodeSymUses(instr);
  13586. this->InsertByteCodeUses(instr, true);
  13587. }
  13588. #if DBG
  13589. else
  13590. {
  13591. PropertySym *propertySymUse = NULL;
  13592. NoRecoverMemoryJitArenaAllocator tempAllocator(_u("BE-GlobOpt-Temp"), this->alloc->GetPageAllocator(), Js::Throw::OutOfMemory);
  13593. BVSparse<JitArenaAllocator> * tempByteCodeUse = JitAnew(&tempAllocator, BVSparse<JitArenaAllocator>, &tempAllocator);
  13594. GlobOpt::TrackByteCodeSymUsed(instr, tempByteCodeUse, &propertySymUse);
  13595. Assert(tempByteCodeUse->Count() == 0 && propertySymUse == NULL);
  13596. }
  13597. #endif
  13598. OptHoistInvariant(instr, block, loop, dstVal, src1Val, src2Val, isNotTypeSpecConv, lossy, bailoutKind);
  13599. return true;
  13600. }
  13601. return false;
  13602. }
  13603. InvariantBlockBackwardIterator::InvariantBlockBackwardIterator(
  13604. GlobOpt *const globOpt,
  13605. BasicBlock *const exclusiveBeginBlock,
  13606. BasicBlock *const inclusiveEndBlock,
  13607. StackSym *const invariantSym,
  13608. const ValueNumber invariantSymValueNumber,
  13609. bool followFlow)
  13610. : globOpt(globOpt),
  13611. exclusiveEndBlock(inclusiveEndBlock->prev),
  13612. invariantSym(invariantSym),
  13613. invariantSymValueNumber(invariantSymValueNumber),
  13614. block(exclusiveBeginBlock),
  13615. blockBV(globOpt->tempAlloc),
  13616. followFlow(followFlow)
  13617. #if DBG
  13618. ,
  13619. inclusiveEndBlock(inclusiveEndBlock)
  13620. #endif
  13621. {
  13622. Assert(exclusiveBeginBlock);
  13623. Assert(inclusiveEndBlock);
  13624. Assert(!inclusiveEndBlock->isDeleted);
  13625. Assert(exclusiveBeginBlock != inclusiveEndBlock);
  13626. Assert(!invariantSym == (invariantSymValueNumber == InvalidValueNumber));
  13627. MoveNext();
  13628. }
  13629. bool
  13630. InvariantBlockBackwardIterator::IsValid() const
  13631. {
  13632. return block != exclusiveEndBlock;
  13633. }
  13634. void
  13635. InvariantBlockBackwardIterator::MoveNext()
  13636. {
  13637. Assert(IsValid());
  13638. while(true)
  13639. {
  13640. #if DBG
  13641. BasicBlock *const previouslyIteratedBlock = block;
  13642. #endif
  13643. block = block->prev;
  13644. if(!IsValid())
  13645. {
  13646. Assert(previouslyIteratedBlock == inclusiveEndBlock);
  13647. break;
  13648. }
  13649. if (!this->UpdatePredBlockBV())
  13650. {
  13651. continue;
  13652. }
  13653. if (!this->UpdatePredBlockBV())
  13654. {
  13655. continue;
  13656. }
  13657. if(block->isDeleted)
  13658. {
  13659. continue;
  13660. }
  13661. if(!block->globOptData.HasData())
  13662. {
  13663. // This block's info has already been merged with all of its successors
  13664. continue;
  13665. }
  13666. if(!invariantSym)
  13667. {
  13668. break;
  13669. }
  13670. invariantSymValue = block->globOptData.FindValue(invariantSym);
  13671. if(!invariantSymValue || invariantSymValue->GetValueNumber() != invariantSymValueNumber)
  13672. {
  13673. // BailOnNoProfile and throw blocks are not moved outside loops. A sym table cleanup on these paths may delete the
  13674. // values. Field hoisting also has some odd cases where the hoisted stack sym is invariant in only part of the loop.
  13675. continue;
  13676. }
  13677. break;
  13678. }
  13679. }
  13680. bool
  13681. InvariantBlockBackwardIterator::UpdatePredBlockBV()
  13682. {
  13683. if (!this->followFlow)
  13684. {
  13685. return true;
  13686. }
  13687. // Track blocks we've visited to ensure that we only iterate over predecessor blocks
  13688. if (!this->blockBV.IsEmpty() && !this->blockBV.Test(this->block->GetBlockNum()))
  13689. {
  13690. return false;
  13691. }
  13692. FOREACH_SLISTBASECOUNTED_ENTRY(FlowEdge*, edge, this->block->GetPredList())
  13693. {
  13694. this->blockBV.Set(edge->GetPred()->GetBlockNum());
  13695. } NEXT_SLISTBASECOUNTED_ENTRY;
  13696. return true;
  13697. }
  13698. BasicBlock *
  13699. InvariantBlockBackwardIterator::Block() const
  13700. {
  13701. Assert(IsValid());
  13702. return block;
  13703. }
  13704. Value *
  13705. InvariantBlockBackwardIterator::InvariantSymValue() const
  13706. {
  13707. Assert(IsValid());
  13708. Assert(invariantSym);
  13709. return invariantSymValue;
  13710. }
  13711. void
  13712. GlobOpt::HoistInvariantValueInfo(
  13713. ValueInfo *const invariantValueInfoToHoist,
  13714. Value *const valueToUpdate,
  13715. BasicBlock *const targetBlock)
  13716. {
  13717. Assert(invariantValueInfoToHoist);
  13718. Assert(valueToUpdate);
  13719. Assert(targetBlock);
  13720. // Why are we trying to change the value type of the type sym value? Asserting here to make sure we don't deep copy the type sym's value info.
  13721. Assert(!invariantValueInfoToHoist->IsJsType());
  13722. Sym *const symStore = valueToUpdate->GetValueInfo()->GetSymStore();
  13723. ValueInfo *newValueInfo;
  13724. if(invariantValueInfoToHoist->GetSymStore() == symStore)
  13725. {
  13726. newValueInfo = invariantValueInfoToHoist;
  13727. }
  13728. else
  13729. {
  13730. newValueInfo = invariantValueInfoToHoist->Copy(alloc);
  13731. this->SetSymStoreDirect(newValueInfo, symStore);
  13732. }
  13733. ChangeValueInfo(targetBlock, valueToUpdate, newValueInfo, true);
  13734. }
  13735. // static
  13736. bool
  13737. GlobOpt::DoInlineArgsOpt(Func const * func)
  13738. {
  13739. Func const * topFunc = func->GetTopFunc();
  13740. Assert(topFunc != func);
  13741. bool doInlineArgsOpt =
  13742. !PHASE_OFF(Js::InlineArgsOptPhase, topFunc) &&
  13743. !func->GetHasCalls() &&
  13744. !func->GetHasUnoptimizedArgumentsAccess() &&
  13745. func->m_canDoInlineArgsOpt;
  13746. return doInlineArgsOpt;
  13747. }
  13748. bool
  13749. GlobOpt::IsSwitchOptEnabled(Func const * func)
  13750. {
  13751. Assert(func->IsTopFunc());
  13752. return !PHASE_OFF(Js::SwitchOptPhase, func) && !func->IsSwitchOptDisabled() && func->DoGlobOpt();
  13753. }
  13754. bool
  13755. GlobOpt::IsSwitchOptEnabledForIntTypeSpec(Func const * func)
  13756. {
  13757. return IsSwitchOptEnabled(func) && !IsTypeSpecPhaseOff(func) && DoAggressiveIntTypeSpec(func);
  13758. }
  13759. bool
  13760. GlobOpt::DoConstFold() const
  13761. {
  13762. return !PHASE_OFF(Js::ConstFoldPhase, func);
  13763. }
  13764. bool
  13765. GlobOpt::IsTypeSpecPhaseOff(Func const *func)
  13766. {
  13767. return PHASE_OFF(Js::TypeSpecPhase, func) || func->IsJitInDebugMode() || !func->DoGlobOptsForGeneratorFunc();
  13768. }
  13769. bool
  13770. GlobOpt::DoTypeSpec() const
  13771. {
  13772. return doTypeSpec;
  13773. }
  13774. bool
  13775. GlobOpt::DoAggressiveIntTypeSpec(Func const * func)
  13776. {
  13777. return
  13778. !PHASE_OFF(Js::AggressiveIntTypeSpecPhase, func) &&
  13779. !IsTypeSpecPhaseOff(func) &&
  13780. !func->IsAggressiveIntTypeSpecDisabled();
  13781. }
  13782. bool
  13783. GlobOpt::DoAggressiveIntTypeSpec() const
  13784. {
  13785. return doAggressiveIntTypeSpec;
  13786. }
  13787. bool
  13788. GlobOpt::DoAggressiveMulIntTypeSpec() const
  13789. {
  13790. return doAggressiveMulIntTypeSpec;
  13791. }
  13792. bool
  13793. GlobOpt::DoDivIntTypeSpec() const
  13794. {
  13795. return doDivIntTypeSpec;
  13796. }
  13797. // static
  13798. bool
  13799. GlobOpt::DoLossyIntTypeSpec(Func const * func)
  13800. {
  13801. return
  13802. !PHASE_OFF(Js::LossyIntTypeSpecPhase, func) &&
  13803. !IsTypeSpecPhaseOff(func) &&
  13804. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsLossyIntTypeSpecDisabled());
  13805. }
  13806. bool
  13807. GlobOpt::DoLossyIntTypeSpec() const
  13808. {
  13809. return doLossyIntTypeSpec;
  13810. }
  13811. // static
  13812. bool
  13813. GlobOpt::DoFloatTypeSpec(Func const * func)
  13814. {
  13815. return
  13816. !PHASE_OFF(Js::FloatTypeSpecPhase, func) &&
  13817. !IsTypeSpecPhaseOff(func) &&
  13818. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsFloatTypeSpecDisabled()) &&
  13819. AutoSystemInfo::Data.SSE2Available();
  13820. }
  13821. bool
  13822. GlobOpt::DoFloatTypeSpec() const
  13823. {
  13824. return doFloatTypeSpec;
  13825. }
  13826. bool
  13827. GlobOpt::DoStringTypeSpec(Func const * func)
  13828. {
  13829. return !PHASE_OFF(Js::StringTypeSpecPhase, func) && !IsTypeSpecPhaseOff(func);
  13830. }
  13831. // static
  13832. bool
  13833. GlobOpt::DoTypedArrayTypeSpec(Func const * func)
  13834. {
  13835. return !PHASE_OFF(Js::TypedArrayTypeSpecPhase, func) &&
  13836. !IsTypeSpecPhaseOff(func) &&
  13837. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsTypedArrayTypeSpecDisabled(func->IsLoopBody()))
  13838. #if defined(_M_IX86)
  13839. && AutoSystemInfo::Data.SSE2Available()
  13840. #endif
  13841. ;
  13842. }
  13843. // static
  13844. bool
  13845. GlobOpt::DoNativeArrayTypeSpec(Func const * func)
  13846. {
  13847. return !PHASE_OFF(Js::NativeArrayPhase, func) &&
  13848. !IsTypeSpecPhaseOff(func)
  13849. #if defined(_M_IX86)
  13850. && AutoSystemInfo::Data.SSE2Available()
  13851. #endif
  13852. ;
  13853. }
  13854. bool
  13855. GlobOpt::DoArrayCheckHoist(Func const * const func)
  13856. {
  13857. Assert(func->IsTopFunc());
  13858. return
  13859. !PHASE_OFF(Js::ArrayCheckHoistPhase, func) &&
  13860. !func->IsArrayCheckHoistDisabled() &&
  13861. !func->IsJitInDebugMode() && // StElemI fast path is not allowed when in debug mode, so it cannot have bailout
  13862. func->DoGlobOptsForGeneratorFunc();
  13863. }
  13864. bool
  13865. GlobOpt::DoArrayCheckHoist() const
  13866. {
  13867. return doArrayCheckHoist;
  13868. }
  13869. bool
  13870. GlobOpt::DoArrayCheckHoist(const ValueType baseValueType, Loop* loop, IR::Instr const * const instr) const
  13871. {
  13872. if(!DoArrayCheckHoist() || (instr && !IsLoopPrePass() && instr->DoStackArgsOpt(func)))
  13873. {
  13874. return false;
  13875. }
  13876. // This includes typed arrays, but not virtual typed arrays, whose vtable can change if the buffer goes away.
  13877. // Note that in the virtual case the vtable check is the only way to catch this, since there's no bound check.
  13878. if(!(baseValueType.IsLikelyArrayOrObjectWithArray() || baseValueType.IsLikelyOptimizedVirtualTypedArray()) ||
  13879. (loop ? ImplicitCallFlagsAllowOpts(loop) : ImplicitCallFlagsAllowOpts(func)))
  13880. {
  13881. return true;
  13882. }
  13883. // The function or loop does not allow disabling implicit calls, which is required to eliminate redundant JS array checks
  13884. #if DBG_DUMP
  13885. if((((loop ? loop->GetImplicitCallFlags() : func->m_fg->implicitCallFlags) & ~Js::ImplicitCall_External) == 0) &&
  13886. Js::Configuration::Global.flags.Trace.IsEnabled(Js::HostOptPhase))
  13887. {
  13888. Output::Print(_u("DoArrayCheckHoist disabled for JS arrays because of external: "));
  13889. func->DumpFullFunctionName();
  13890. Output::Print(_u("\n"));
  13891. Output::Flush();
  13892. }
  13893. #endif
  13894. return false;
  13895. }
  13896. bool
  13897. GlobOpt::DoArrayMissingValueCheckHoist(Func const * const func)
  13898. {
  13899. return
  13900. DoArrayCheckHoist(func) &&
  13901. !PHASE_OFF(Js::ArrayMissingValueCheckHoistPhase, func) &&
  13902. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsArrayMissingValueCheckHoistDisabled(func->IsLoopBody()));
  13903. }
  13904. bool
  13905. GlobOpt::DoArrayMissingValueCheckHoist() const
  13906. {
  13907. return doArrayMissingValueCheckHoist;
  13908. }
  13909. bool
  13910. GlobOpt::DoArraySegmentHoist(const ValueType baseValueType, Func const * const func)
  13911. {
  13912. Assert(baseValueType.IsLikelyAnyOptimizedArray());
  13913. if(!DoArrayCheckHoist(func) || PHASE_OFF(Js::ArraySegmentHoistPhase, func))
  13914. {
  13915. return false;
  13916. }
  13917. if(!baseValueType.IsLikelyArrayOrObjectWithArray())
  13918. {
  13919. return true;
  13920. }
  13921. return
  13922. !PHASE_OFF(Js::JsArraySegmentHoistPhase, func) &&
  13923. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsJsArraySegmentHoistDisabled(func->IsLoopBody()));
  13924. }
  13925. bool
  13926. GlobOpt::DoArraySegmentHoist(const ValueType baseValueType) const
  13927. {
  13928. Assert(baseValueType.IsLikelyAnyOptimizedArray());
  13929. return baseValueType.IsLikelyArrayOrObjectWithArray() ? doJsArraySegmentHoist : doArraySegmentHoist;
  13930. }
  13931. bool
  13932. GlobOpt::DoTypedArraySegmentLengthHoist(Loop *const loop) const
  13933. {
  13934. if(!DoArraySegmentHoist(ValueType::GetObject(ObjectType::Int32Array)))
  13935. {
  13936. return false;
  13937. }
  13938. if(loop ? ImplicitCallFlagsAllowOpts(loop) : ImplicitCallFlagsAllowOpts(func))
  13939. {
  13940. return true;
  13941. }
  13942. // The function or loop does not allow disabling implicit calls, which is required to eliminate redundant typed array
  13943. // segment length loads.
  13944. #if DBG_DUMP
  13945. if((((loop ? loop->GetImplicitCallFlags() : func->m_fg->implicitCallFlags) & ~Js::ImplicitCall_External) == 0) &&
  13946. Js::Configuration::Global.flags.Trace.IsEnabled(Js::HostOptPhase))
  13947. {
  13948. Output::Print(_u("DoArraySegmentLengthHoist disabled for typed arrays because of external: "));
  13949. func->DumpFullFunctionName();
  13950. Output::Print(_u("\n"));
  13951. Output::Flush();
  13952. }
  13953. #endif
  13954. return false;
  13955. }
  13956. bool
  13957. GlobOpt::DoArrayLengthHoist(Func const * const func)
  13958. {
  13959. return
  13960. DoArrayCheckHoist(func) &&
  13961. !PHASE_OFF(Js::Phase::ArrayLengthHoistPhase, func) &&
  13962. (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsArrayLengthHoistDisabled(func->IsLoopBody()));
  13963. }
  13964. bool
  13965. GlobOpt::DoArrayLengthHoist() const
  13966. {
  13967. return doArrayLengthHoist;
  13968. }
  13969. bool
  13970. GlobOpt::DoEliminateArrayAccessHelperCall(Func *const func)
  13971. {
  13972. return DoArrayCheckHoist(func);
  13973. }
  13974. bool
  13975. GlobOpt::DoEliminateArrayAccessHelperCall() const
  13976. {
  13977. return doEliminateArrayAccessHelperCall;
  13978. }
  13979. bool
  13980. GlobOpt::DoLdLenIntSpec(IR::Instr * const instr, const ValueType baseValueType)
  13981. {
  13982. Assert(!instr || instr->m_opcode == Js::OpCode::LdLen_A);
  13983. Assert(!instr || instr->GetDst());
  13984. Assert(!instr || instr->GetSrc1());
  13985. if(PHASE_OFF(Js::LdLenIntSpecPhase, func) ||
  13986. IsTypeSpecPhaseOff(func) ||
  13987. (func->HasProfileInfo() && func->GetReadOnlyProfileInfo()->IsLdLenIntSpecDisabled()) ||
  13988. (instr && !IsLoopPrePass() && instr->DoStackArgsOpt(func)))
  13989. {
  13990. return false;
  13991. }
  13992. if(instr &&
  13993. instr->IsProfiledInstr() &&
  13994. (
  13995. !instr->AsProfiledInstr()->u.FldInfo().valueType.IsLikelyInt() ||
  13996. instr->GetDst()->AsRegOpnd()->m_sym->m_isNotNumber
  13997. ))
  13998. {
  13999. return false;
  14000. }
  14001. Assert(!instr || baseValueType == instr->GetSrc1()->GetValueType());
  14002. return
  14003. baseValueType.HasBeenString() ||
  14004. (baseValueType.IsLikelyAnyOptimizedArray() && baseValueType.GetObjectType() != ObjectType::ObjectWithArray);
  14005. }
  14006. bool
  14007. GlobOpt::DoPathDependentValues() const
  14008. {
  14009. return !PHASE_OFF(Js::Phase::PathDependentValuesPhase, func);
  14010. }
  14011. bool
  14012. GlobOpt::DoTrackRelativeIntBounds() const
  14013. {
  14014. return doTrackRelativeIntBounds;
  14015. }
  14016. bool
  14017. GlobOpt::DoBoundCheckElimination() const
  14018. {
  14019. return doBoundCheckElimination;
  14020. }
  14021. bool
  14022. GlobOpt::DoBoundCheckHoist() const
  14023. {
  14024. return doBoundCheckHoist;
  14025. }
  14026. bool
  14027. GlobOpt::DoLoopCountBasedBoundCheckHoist() const
  14028. {
  14029. return doLoopCountBasedBoundCheckHoist;
  14030. }
  14031. bool
  14032. GlobOpt::DoPowIntIntTypeSpec() const
  14033. {
  14034. return doPowIntIntTypeSpec;
  14035. }
  14036. bool
  14037. GlobOpt::DoTagChecks() const
  14038. {
  14039. return doTagChecks;
  14040. }
  14041. bool
  14042. GlobOpt::TrackArgumentsObject()
  14043. {
  14044. if (PHASE_OFF(Js::StackArgOptPhase, this->func))
  14045. {
  14046. this->CannotAllocateArgumentsObjectOnStack();
  14047. return false;
  14048. }
  14049. return func->GetHasStackArgs();
  14050. }
  14051. void
  14052. GlobOpt::CannotAllocateArgumentsObjectOnStack()
  14053. {
  14054. func->SetHasStackArgs(false);
  14055. #ifdef ENABLE_DEBUG_CONFIG_OPTIONS
  14056. if (PHASE_TESTTRACE(Js::StackArgOptPhase, this->func))
  14057. {
  14058. char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
  14059. Output::Print(_u("Stack args disabled for function %s(%s)\n"), func->GetJITFunctionBody()->GetDisplayName(), func->GetDebugNumberSet(debugStringBuffer));
  14060. Output::Flush();
  14061. }
  14062. #endif
  14063. }
  14064. IR::Instr *
  14065. GlobOpt::PreOptPeep(IR::Instr *instr)
  14066. {
  14067. if (OpCodeAttr::HasDeadFallThrough(instr->m_opcode))
  14068. {
  14069. switch (instr->m_opcode)
  14070. {
  14071. case Js::OpCode::BailOnNoProfile:
  14072. {
  14073. // Handle BailOnNoProfile
  14074. if (instr->HasBailOutInfo())
  14075. {
  14076. if (!this->prePassLoop)
  14077. {
  14078. FillBailOutInfo(this->currentBlock, instr);
  14079. }
  14080. // Already processed.
  14081. return instr;
  14082. }
  14083. // Convert to bailout instr
  14084. IR::Instr *nextBytecodeOffsetInstr = instr->GetNextRealInstrOrLabel();
  14085. while(nextBytecodeOffsetInstr->GetByteCodeOffset() == Js::Constants::NoByteCodeOffset)
  14086. {
  14087. nextBytecodeOffsetInstr = nextBytecodeOffsetInstr->GetNextRealInstrOrLabel();
  14088. Assert(!nextBytecodeOffsetInstr->IsLabelInstr());
  14089. }
  14090. instr = instr->ConvertToBailOutInstr(nextBytecodeOffsetInstr, IR::BailOutOnNoProfile);
  14091. instr->ClearByteCodeOffset();
  14092. instr->SetByteCodeOffset(nextBytecodeOffsetInstr);
  14093. if (!this->currentBlock->loop)
  14094. {
  14095. FillBailOutInfo(this->currentBlock, instr);
  14096. }
  14097. else
  14098. {
  14099. Assert(this->prePassLoop);
  14100. }
  14101. break;
  14102. }
  14103. case Js::OpCode::BailOnException:
  14104. {
  14105. Assert(
  14106. (
  14107. this->func->HasTry() && this->func->DoOptimizeTry() &&
  14108. instr->m_prev->m_opcode == Js::OpCode::Catch &&
  14109. instr->m_prev->m_prev->IsLabelInstr() &&
  14110. instr->m_prev->m_prev->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeCatch
  14111. )
  14112. ||
  14113. (
  14114. this->func->HasFinally() && this->func->DoOptimizeTry() &&
  14115. instr->m_prev->AsLabelInstr() &&
  14116. instr->m_prev->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeFinally
  14117. )
  14118. );
  14119. break;
  14120. }
  14121. case Js::OpCode::BailOnEarlyExit:
  14122. {
  14123. Assert(this->func->HasFinally() && this->func->DoOptimizeTry());
  14124. break;
  14125. }
  14126. default:
  14127. {
  14128. if(this->currentBlock->loop && !this->IsLoopPrePass())
  14129. {
  14130. return instr;
  14131. }
  14132. break;
  14133. }
  14134. }
  14135. RemoveCodeAfterNoFallthroughInstr(instr);
  14136. }
  14137. return instr;
  14138. }
  14139. void
  14140. GlobOpt::RemoveCodeAfterNoFallthroughInstr(IR::Instr *instr)
  14141. {
  14142. if (instr != this->currentBlock->GetLastInstr())
  14143. {
  14144. // Remove dead code after bailout
  14145. IR::Instr *instrDead = instr->m_next;
  14146. IR::Instr *instrNext;
  14147. for (; instrDead != this->currentBlock->GetLastInstr(); instrDead = instrNext)
  14148. {
  14149. instrNext = instrDead->m_next;
  14150. if (instrNext->m_opcode == Js::OpCode::FunctionExit)
  14151. {
  14152. break;
  14153. }
  14154. this->func->m_fg->RemoveInstr(instrDead, this);
  14155. }
  14156. IR::Instr *instrNextBlock = instrDead->m_next;
  14157. this->func->m_fg->RemoveInstr(instrDead, this);
  14158. this->currentBlock->SetLastInstr(instrNextBlock->m_prev);
  14159. }
  14160. // Cleanup dead successors
  14161. FOREACH_SUCCESSOR_BLOCK_EDITING(deadBlock, this->currentBlock, iter)
  14162. {
  14163. this->currentBlock->RemoveDeadSucc(deadBlock, this->func->m_fg);
  14164. if (this->currentBlock->GetDataUseCount() > 0)
  14165. {
  14166. this->currentBlock->DecrementDataUseCount();
  14167. }
  14168. } NEXT_SUCCESSOR_BLOCK_EDITING;
  14169. }
  14170. void
  14171. GlobOpt::ProcessTryHandler(IR::Instr* instr)
  14172. {
  14173. Assert(instr->m_next->IsLabelInstr() && instr->m_next->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeTry);
  14174. Region* tryRegion = instr->m_next->AsLabelInstr()->GetRegion();
  14175. BVSparse<JitArenaAllocator> * writeThroughSymbolsSet = tryRegion->writeThroughSymbolsSet;
  14176. ToVar(writeThroughSymbolsSet, this->currentBlock);
  14177. }
  14178. bool
  14179. GlobOpt::ProcessExceptionHandlingEdges(IR::Instr* instr)
  14180. {
  14181. Assert(instr->m_opcode == Js::OpCode::BrOnException || instr->m_opcode == Js::OpCode::BrOnNoException);
  14182. if (instr->m_opcode == Js::OpCode::BrOnException)
  14183. {
  14184. if (instr->AsBranchInstr()->GetTarget()->GetRegion()->GetType() == RegionType::RegionTypeCatch)
  14185. {
  14186. // BrOnException was added to model flow from try region to the catch region to assist
  14187. // the backward pass in propagating bytecode upward exposed info from the catch block
  14188. // to the try, and to handle break blocks. Removing it here as it has served its purpose
  14189. // and keeping it around might also have unintended effects while merging block data for
  14190. // the catch block's predecessors.
  14191. // Note that the Deadstore pass will still be able to propagate bytecode upward exposed info
  14192. // because it doesn't skip dead blocks for that.
  14193. this->RemoveFlowEdgeToCatchBlock(instr);
  14194. this->currentBlock->RemoveInstr(instr);
  14195. return true;
  14196. }
  14197. else
  14198. {
  14199. // We add BrOnException from a finally region to early exit, remove that since it has served its purpose
  14200. return this->RemoveFlowEdgeToFinallyOnExceptionBlock(instr);
  14201. }
  14202. }
  14203. else if (instr->m_opcode == Js::OpCode::BrOnNoException)
  14204. {
  14205. if (instr->AsBranchInstr()->GetTarget()->GetRegion()->GetType() == RegionType::RegionTypeCatch)
  14206. {
  14207. this->RemoveFlowEdgeToCatchBlock(instr);
  14208. }
  14209. else
  14210. {
  14211. this->RemoveFlowEdgeToFinallyOnExceptionBlock(instr);
  14212. }
  14213. }
  14214. return false;
  14215. }
  14216. void
  14217. GlobOpt::InsertToVarAtDefInTryRegion(IR::Instr * instr, IR::Opnd * dstOpnd)
  14218. {
  14219. if ((this->currentRegion->GetType() == RegionTypeTry || this->currentRegion->GetType() == RegionTypeFinally) &&
  14220. dstOpnd->IsRegOpnd() && dstOpnd->AsRegOpnd()->m_sym->HasByteCodeRegSlot())
  14221. {
  14222. StackSym * sym = dstOpnd->AsRegOpnd()->m_sym;
  14223. if (sym->IsVar())
  14224. {
  14225. return;
  14226. }
  14227. StackSym * varSym = sym->GetVarEquivSym(nullptr);
  14228. if ((this->currentRegion->GetType() == RegionTypeTry && this->currentRegion->writeThroughSymbolsSet->Test(varSym->m_id)) ||
  14229. ((this->currentRegion->GetType() == RegionTypeFinally && this->currentRegion->GetMatchingTryRegion()->writeThroughSymbolsSet->Test(varSym->m_id))))
  14230. {
  14231. IR::RegOpnd * regOpnd = IR::RegOpnd::New(varSym, IRType::TyVar, instr->m_func);
  14232. this->ToVar(instr->m_next, regOpnd, this->currentBlock, NULL, false);
  14233. }
  14234. }
  14235. }
  14236. void
  14237. GlobOpt::RemoveFlowEdgeToCatchBlock(IR::Instr * instr)
  14238. {
  14239. Assert(instr->IsBranchInstr());
  14240. BasicBlock * catchBlock = nullptr;
  14241. BasicBlock * predBlock = nullptr;
  14242. if (instr->m_opcode == Js::OpCode::BrOnException)
  14243. {
  14244. catchBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock();
  14245. predBlock = this->currentBlock;
  14246. }
  14247. else
  14248. {
  14249. Assert(instr->m_opcode == Js::OpCode::BrOnNoException);
  14250. IR::Instr * nextInstr = instr->GetNextRealInstrOrLabel();
  14251. Assert(nextInstr->IsLabelInstr());
  14252. IR::LabelInstr * nextLabel = nextInstr->AsLabelInstr();
  14253. if (nextLabel->GetRegion() && nextLabel->GetRegion()->GetType() == RegionTypeCatch)
  14254. {
  14255. catchBlock = nextLabel->GetBasicBlock();
  14256. predBlock = this->currentBlock;
  14257. }
  14258. else
  14259. {
  14260. Assert(nextLabel->m_next->IsBranchInstr() && nextLabel->m_next->AsBranchInstr()->IsUnconditional());
  14261. BasicBlock * nextBlock = nextLabel->GetBasicBlock();
  14262. IR::BranchInstr * branchToCatchBlock = nextLabel->m_next->AsBranchInstr();
  14263. IR::LabelInstr * catchBlockLabel = branchToCatchBlock->GetTarget();
  14264. Assert(catchBlockLabel->GetRegion()->GetType() == RegionTypeCatch);
  14265. catchBlock = catchBlockLabel->GetBasicBlock();
  14266. predBlock = nextBlock;
  14267. }
  14268. }
  14269. Assert(catchBlock);
  14270. Assert(predBlock);
  14271. if (this->func->m_fg->FindEdge(predBlock, catchBlock))
  14272. {
  14273. predBlock->RemoveDeadSucc(catchBlock, this->func->m_fg);
  14274. if (predBlock == this->currentBlock)
  14275. {
  14276. predBlock->DecrementDataUseCount();
  14277. }
  14278. }
  14279. }
  14280. bool
  14281. GlobOpt::RemoveFlowEdgeToFinallyOnExceptionBlock(IR::Instr * instr)
  14282. {
  14283. Assert(instr->IsBranchInstr());
  14284. if (instr->m_opcode == Js::OpCode::BrOnNoException && instr->AsBranchInstr()->m_brFinallyToEarlyExit)
  14285. {
  14286. // We add edge from finally to early exit block
  14287. // We should not remove this edge
  14288. // If a loop has continue, and we add edge in finally to continue
  14289. // Break block removal can move all continues inside the loop to branch to the continue added within finally
  14290. // If we get rid of this edge, then loop may loose all backedges
  14291. // Ideally, doing tail duplication before globopt would enable us to remove these edges, but since we do it after globopt, keep it this way for now
  14292. // See test1() in core/test/tryfinallytests.js
  14293. return false;
  14294. }
  14295. BasicBlock * finallyBlock = nullptr;
  14296. BasicBlock * predBlock = nullptr;
  14297. if (instr->m_opcode == Js::OpCode::BrOnException)
  14298. {
  14299. finallyBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock();
  14300. predBlock = this->currentBlock;
  14301. }
  14302. else
  14303. {
  14304. Assert(instr->m_opcode == Js::OpCode::BrOnNoException);
  14305. IR::Instr * nextInstr = instr->GetNextRealInstrOrLabel();
  14306. Assert(nextInstr->IsLabelInstr());
  14307. IR::LabelInstr * nextLabel = nextInstr->AsLabelInstr();
  14308. if (nextLabel->GetRegion() && nextLabel->GetRegion()->GetType() == RegionTypeFinally)
  14309. {
  14310. finallyBlock = nextLabel->GetBasicBlock();
  14311. predBlock = this->currentBlock;
  14312. }
  14313. else
  14314. {
  14315. if (!(nextLabel->m_next->IsBranchInstr() && nextLabel->m_next->AsBranchInstr()->IsUnconditional()))
  14316. {
  14317. return false;
  14318. }
  14319. BasicBlock * nextBlock = nextLabel->GetBasicBlock();
  14320. IR::BranchInstr * branchTofinallyBlockOrEarlyExit = nextLabel->m_next->AsBranchInstr();
  14321. IR::LabelInstr * finallyBlockLabelOrEarlyExitLabel = branchTofinallyBlockOrEarlyExit->GetTarget();
  14322. finallyBlock = finallyBlockLabelOrEarlyExitLabel->GetBasicBlock();
  14323. predBlock = nextBlock;
  14324. }
  14325. }
  14326. Assert(finallyBlock && predBlock);
  14327. if (this->func->m_fg->FindEdge(predBlock, finallyBlock))
  14328. {
  14329. predBlock->RemoveDeadSucc(finallyBlock, this->func->m_fg);
  14330. if (instr->m_opcode == Js::OpCode::BrOnException)
  14331. {
  14332. this->currentBlock->RemoveInstr(instr);
  14333. }
  14334. if (finallyBlock->GetFirstInstr()->AsLabelInstr()->IsUnreferenced())
  14335. {
  14336. // Traverse predBlocks of finallyBlock, if any of the preds have a different region, set m_hasNonBranchRef to true
  14337. // If not, this label can get eliminated and an incorrect region from the predecessor can get propagated in lowered code
  14338. // See test3() in tryfinallytests.js
  14339. Region * finallyRegion = finallyBlock->GetFirstInstr()->AsLabelInstr()->GetRegion();
  14340. FOREACH_PREDECESSOR_BLOCK(pred, finallyBlock)
  14341. {
  14342. Region * predRegion = pred->GetFirstInstr()->AsLabelInstr()->GetRegion();
  14343. if (predRegion != finallyRegion)
  14344. {
  14345. finallyBlock->GetFirstInstr()->AsLabelInstr()->m_hasNonBranchRef = true;
  14346. }
  14347. } NEXT_PREDECESSOR_BLOCK;
  14348. }
  14349. if (predBlock == this->currentBlock)
  14350. {
  14351. predBlock->DecrementDataUseCount();
  14352. }
  14353. }
  14354. return true;
  14355. }
  14356. IR::Instr *
  14357. GlobOpt::OptPeep(IR::Instr *instr, Value *src1Val, Value *src2Val)
  14358. {
  14359. IR::Opnd *dst, *src1, *src2;
  14360. if (this->IsLoopPrePass())
  14361. {
  14362. return instr;
  14363. }
  14364. switch (instr->m_opcode)
  14365. {
  14366. case Js::OpCode::DeadBrEqual:
  14367. case Js::OpCode::DeadBrRelational:
  14368. case Js::OpCode::DeadBrSrEqual:
  14369. src1 = instr->GetSrc1();
  14370. src2 = instr->GetSrc2();
  14371. // These branches were turned into dead branches because they were unnecessary (branch to next, ...).
  14372. // The DeadBr are necessary in case the evaluation of the sources have side-effects.
  14373. // If we know for sure the srcs are primitive or have been type specialized, we don't need these instructions
  14374. if (((src1Val && src1Val->GetValueInfo()->IsPrimitive()) || (src1->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src1->AsRegOpnd()->m_sym))) &&
  14375. ((src2Val && src2Val->GetValueInfo()->IsPrimitive()) || (src2->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src2->AsRegOpnd()->m_sym))))
  14376. {
  14377. this->CaptureByteCodeSymUses(instr);
  14378. instr->m_opcode = Js::OpCode::Nop;
  14379. }
  14380. break;
  14381. case Js::OpCode::DeadBrOnHasProperty:
  14382. src1 = instr->GetSrc1();
  14383. if (((src1Val && src1Val->GetValueInfo()->IsPrimitive()) || (src1->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src1->AsRegOpnd()->m_sym))))
  14384. {
  14385. this->CaptureByteCodeSymUses(instr);
  14386. instr->m_opcode = Js::OpCode::Nop;
  14387. }
  14388. break;
  14389. case Js::OpCode::Ld_A:
  14390. case Js::OpCode::Ld_I4:
  14391. src1 = instr->GetSrc1();
  14392. dst = instr->GetDst();
  14393. if (dst->IsRegOpnd() && dst->IsEqual(src1))
  14394. {
  14395. dst = instr->UnlinkDst();
  14396. if (!dst->GetIsJITOptimizedReg())
  14397. {
  14398. IR::ByteCodeUsesInstr *bytecodeUse = IR::ByteCodeUsesInstr::New(instr);
  14399. bytecodeUse->SetDst(dst);
  14400. instr->InsertAfter(bytecodeUse);
  14401. }
  14402. instr->FreeSrc1();
  14403. instr->m_opcode = Js::OpCode::Nop;
  14404. }
  14405. break;
  14406. }
  14407. return instr;
  14408. }
  14409. void
  14410. GlobOpt::OptimizeIndirUses(IR::IndirOpnd *indirOpnd, IR::Instr * *pInstr, Value **indirIndexValRef)
  14411. {
  14412. IR::Instr * &instr = *pInstr;
  14413. Assert(!indirIndexValRef || !*indirIndexValRef);
  14414. // Update value types and copy-prop the base
  14415. OptSrc(indirOpnd->GetBaseOpnd(), &instr, nullptr, indirOpnd);
  14416. IR::RegOpnd *indexOpnd = indirOpnd->GetIndexOpnd();
  14417. if (!indexOpnd)
  14418. {
  14419. return;
  14420. }
  14421. // Update value types and copy-prop the index
  14422. Value *indexVal = OptSrc(indexOpnd, &instr, nullptr, indirOpnd);
  14423. if(indirIndexValRef)
  14424. {
  14425. *indirIndexValRef = indexVal;
  14426. }
  14427. }
  14428. bool
  14429. GlobOpt::IsPREInstrCandidateLoad(Js::OpCode opcode)
  14430. {
  14431. switch (opcode)
  14432. {
  14433. case Js::OpCode::LdFld:
  14434. case Js::OpCode::LdFldForTypeOf:
  14435. case Js::OpCode::LdRootFld:
  14436. case Js::OpCode::LdRootFldForTypeOf:
  14437. case Js::OpCode::LdMethodFld:
  14438. case Js::OpCode::LdRootMethodFld:
  14439. case Js::OpCode::LdSlot:
  14440. case Js::OpCode::LdSlotArr:
  14441. return true;
  14442. }
  14443. return false;
  14444. }
  14445. bool
  14446. GlobOpt::IsPREInstrSequenceCandidateLoad(Js::OpCode opcode)
  14447. {
  14448. switch (opcode)
  14449. {
  14450. default:
  14451. return IsPREInstrCandidateLoad(opcode);
  14452. case Js::OpCode::Ld_A:
  14453. case Js::OpCode::BytecodeArgOutCapture:
  14454. return true;
  14455. }
  14456. }
  14457. bool
  14458. GlobOpt::IsPREInstrCandidateStore(Js::OpCode opcode)
  14459. {
  14460. switch (opcode)
  14461. {
  14462. case Js::OpCode::StFld:
  14463. case Js::OpCode::StRootFld:
  14464. case Js::OpCode::StSlot:
  14465. return true;
  14466. }
  14467. return false;
  14468. }
  14469. bool
  14470. GlobOpt::ImplicitCallFlagsAllowOpts(Loop *loop)
  14471. {
  14472. return loop->GetImplicitCallFlags() != Js::ImplicitCall_HasNoInfo &&
  14473. (((loop->GetImplicitCallFlags() & ~Js::ImplicitCall_Accessor) | Js::ImplicitCall_None) == Js::ImplicitCall_None);
  14474. }
  14475. bool
  14476. GlobOpt::ImplicitCallFlagsAllowOpts(Func const *func)
  14477. {
  14478. return func->m_fg->implicitCallFlags != Js::ImplicitCall_HasNoInfo &&
  14479. (((func->m_fg->implicitCallFlags & ~Js::ImplicitCall_Accessor) | Js::ImplicitCall_None) == Js::ImplicitCall_None);
  14480. }
  14481. #if DBG_DUMP
  14482. void
  14483. GlobOpt::Dump() const
  14484. {
  14485. this->DumpSymToValueMap();
  14486. }
  14487. void
  14488. GlobOpt::DumpSymToValueMap(BasicBlock const * block) const
  14489. {
  14490. Output::Print(_u("\n*** SymToValueMap ***\n"));
  14491. block->globOptData.DumpSymToValueMap();
  14492. }
  14493. void
  14494. GlobOpt::DumpSymToValueMap() const
  14495. {
  14496. DumpSymToValueMap(this->currentBlock);
  14497. }
  14498. void
  14499. GlobOpt::DumpSymVal(int index)
  14500. {
  14501. SymID id = index;
  14502. extern Func *CurrentFunc;
  14503. Sym *sym = this->func->m_symTable->Find(id);
  14504. AssertMsg(sym, "Sym not found!!!");
  14505. Output::Print(_u("Sym: "));
  14506. sym->Dump();
  14507. Output::Print(_u("\t\tValueNumber: "));
  14508. Value * pValue = CurrentBlockData()->FindValueFromMapDirect(sym->m_id);
  14509. pValue->Dump();
  14510. Output::Print(_u("\n"));
  14511. }
  14512. void
  14513. GlobOpt::Trace(BasicBlock * block, bool before) const
  14514. {
  14515. bool globOptTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::GlobOptPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14516. bool typeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::TypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14517. bool floatTypeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FloatTypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14518. bool fieldCopyPropTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14519. bool objTypeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::ObjTypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14520. bool valueTableTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::ValueTablePhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14521. bool fieldPRETrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldPREPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId());
  14522. bool anyTrace = globOptTrace || typeSpecTrace || floatTypeSpecTrace || fieldCopyPropTrace || objTypeSpecTrace || valueTableTrace || fieldPRETrace;
  14523. if (!anyTrace)
  14524. {
  14525. return;
  14526. }
  14527. if (fieldPRETrace && this->IsLoopPrePass())
  14528. {
  14529. if (block->isLoopHeader && before)
  14530. {
  14531. Output::Print(_u("==== Loop Prepass block header #%-3d, Visiting Loop block head #%-3d\n"),
  14532. this->prePassLoop->GetHeadBlock()->GetBlockNum(), block->GetBlockNum());
  14533. }
  14534. }
  14535. if (!typeSpecTrace && !floatTypeSpecTrace && !valueTableTrace && !Js::Configuration::Global.flags.Verbose)
  14536. {
  14537. return;
  14538. }
  14539. if (before)
  14540. {
  14541. Output::Print(_u("========================================================================\n"));
  14542. Output::Print(_u("Begin OptBlock: Block #%-3d"), block->GetBlockNum());
  14543. if (block->loop)
  14544. {
  14545. Output::Print(_u(" Loop block header:%-3d currentLoop block head:%-3d %s"),
  14546. block->loop->GetHeadBlock()->GetBlockNum(),
  14547. this->prePassLoop ? this->prePassLoop->GetHeadBlock()->GetBlockNum() : 0,
  14548. this->IsLoopPrePass() ? _u("PrePass") : _u(""));
  14549. }
  14550. Output::Print(_u("\n"));
  14551. }
  14552. else
  14553. {
  14554. Output::Print(_u("-----------------------------------------------------------------------\n"));
  14555. Output::Print(_u("After OptBlock: Block #%-3d\n"), block->GetBlockNum());
  14556. }
  14557. if ((typeSpecTrace || floatTypeSpecTrace) && !block->globOptData.liveVarSyms->IsEmpty())
  14558. {
  14559. Output::Print(_u(" Live var syms: "));
  14560. block->globOptData.liveVarSyms->Dump();
  14561. }
  14562. if (typeSpecTrace && !block->globOptData.liveInt32Syms->IsEmpty())
  14563. {
  14564. Assert(this->tempBv->IsEmpty());
  14565. this->tempBv->Minus(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms);
  14566. if(!this->tempBv->IsEmpty())
  14567. {
  14568. Output::Print(_u(" Int32 type specialized (lossless) syms: "));
  14569. this->tempBv->Dump();
  14570. }
  14571. this->tempBv->ClearAll();
  14572. if(!block->globOptData.liveLossyInt32Syms->IsEmpty())
  14573. {
  14574. Output::Print(_u(" Int32 converted (lossy) syms: "));
  14575. block->globOptData.liveLossyInt32Syms->Dump();
  14576. }
  14577. }
  14578. if (floatTypeSpecTrace && !block->globOptData.liveFloat64Syms->IsEmpty())
  14579. {
  14580. Output::Print(_u(" Float64 type specialized syms: "));
  14581. block->globOptData.liveFloat64Syms->Dump();
  14582. }
  14583. if ((fieldCopyPropTrace || objTypeSpecTrace) && this->DoFieldCopyProp(block->loop) && !block->globOptData.liveFields->IsEmpty())
  14584. {
  14585. Output::Print(_u(" Live field syms: "));
  14586. block->globOptData.liveFields->Dump();
  14587. }
  14588. if (objTypeSpecTrace || valueTableTrace)
  14589. {
  14590. Output::Print(_u(" Value table:\n"));
  14591. block->globOptData.DumpSymToValueMap();
  14592. }
  14593. if (before)
  14594. {
  14595. Output::Print(_u("-----------------------------------------------------------------------\n")); \
  14596. }
  14597. Output::Flush();
  14598. }
  14599. void
  14600. GlobOpt::TraceSettings() const
  14601. {
  14602. Output::Print(_u("GlobOpt Settings:\r\n"));
  14603. Output::Print(_u(" FloatTypeSpec: %s\r\n"), this->DoFloatTypeSpec() ? _u("enabled") : _u("disabled"));
  14604. Output::Print(_u(" AggressiveIntTypeSpec: %s\r\n"), this->DoAggressiveIntTypeSpec() ? _u("enabled") : _u("disabled"));
  14605. Output::Print(_u(" LossyIntTypeSpec: %s\r\n"), this->DoLossyIntTypeSpec() ? _u("enabled") : _u("disabled"));
  14606. Output::Print(_u(" ArrayCheckHoist: %s\r\n"), this->func->IsArrayCheckHoistDisabled() ? _u("disabled") : _u("enabled"));
  14607. Output::Print(_u(" ImplicitCallFlags: %s\r\n"), Js::DynamicProfileInfo::GetImplicitCallFlagsString(this->func->m_fg->implicitCallFlags));
  14608. for (Loop * loop = this->func->m_fg->loopList; loop != NULL; loop = loop->next)
  14609. {
  14610. Output::Print(_u(" loop: %d, ImplicitCallFlags: %s\r\n"), loop->GetLoopNumber(),
  14611. Js::DynamicProfileInfo::GetImplicitCallFlagsString(loop->GetImplicitCallFlags()));
  14612. }
  14613. Output::Flush();
  14614. }
  14615. #endif // DBG_DUMP
  14616. IR::Instr *
  14617. GlobOpt::TrackMarkTempObject(IR::Instr * instrStart, IR::Instr * instrLast)
  14618. {
  14619. if (!this->func->GetHasMarkTempObjects())
  14620. {
  14621. return instrLast;
  14622. }
  14623. IR::Instr * instr = instrStart;
  14624. IR::Instr * instrEnd = instrLast->m_next;
  14625. IR::Instr * lastInstr = nullptr;
  14626. GlobOptBlockData& globOptData = *CurrentBlockData();
  14627. do
  14628. {
  14629. bool mayNeedBailOnImplicitCallsPreOp = !this->IsLoopPrePass()
  14630. && instr->HasAnyImplicitCalls()
  14631. && globOptData.maybeTempObjectSyms != nullptr;
  14632. if (mayNeedBailOnImplicitCallsPreOp)
  14633. {
  14634. IR::Opnd * src1 = instr->GetSrc1();
  14635. if (src1)
  14636. {
  14637. instr = GenerateBailOutMarkTempObjectIfNeeded(instr, src1, false);
  14638. IR::Opnd * src2 = instr->GetSrc2();
  14639. if (src2)
  14640. {
  14641. instr = GenerateBailOutMarkTempObjectIfNeeded(instr, src2, false);
  14642. }
  14643. }
  14644. }
  14645. IR::Opnd *dst = instr->GetDst();
  14646. if (dst)
  14647. {
  14648. if (dst->IsRegOpnd())
  14649. {
  14650. TrackTempObjectSyms(instr, dst->AsRegOpnd());
  14651. }
  14652. else if (mayNeedBailOnImplicitCallsPreOp)
  14653. {
  14654. instr = GenerateBailOutMarkTempObjectIfNeeded(instr, dst, true);
  14655. }
  14656. }
  14657. lastInstr = instr;
  14658. instr = instr->m_next;
  14659. }
  14660. while (instr != instrEnd);
  14661. return lastInstr;
  14662. }
  14663. void
  14664. GlobOpt::TrackTempObjectSyms(IR::Instr * instr, IR::RegOpnd * opnd)
  14665. {
  14666. // If it is marked as dstIsTempObject, we should have mark temped it, or type specialized it to Ld_I4.
  14667. Assert(!instr->dstIsTempObject || ObjectTempVerify::CanMarkTemp(instr, nullptr));
  14668. GlobOptBlockData& globOptData = *CurrentBlockData();
  14669. bool canStoreTemp = false;
  14670. bool maybeTemp = false;
  14671. if (OpCodeAttr::TempObjectProducing(instr->m_opcode))
  14672. {
  14673. maybeTemp = instr->dstIsTempObject;
  14674. // We have to make sure that lower will always generate code to do stack allocation
  14675. // before we can store any other stack instance onto it. Otherwise, we would not
  14676. // walk object to box the stack property.
  14677. canStoreTemp = instr->dstIsTempObject && ObjectTemp::CanStoreTemp(instr);
  14678. }
  14679. else if (OpCodeAttr::TempObjectTransfer(instr->m_opcode))
  14680. {
  14681. // Need to check both sources, GetNewScObject has two srcs for transfer.
  14682. // No need to get var equiv sym here as transfer of type spec value does not transfer a mark temp object.
  14683. maybeTemp = globOptData.maybeTempObjectSyms && (
  14684. (instr->GetSrc1()->IsRegOpnd() && globOptData.maybeTempObjectSyms->Test(instr->GetSrc1()->AsRegOpnd()->m_sym->m_id))
  14685. || (instr->GetSrc2() && instr->GetSrc2()->IsRegOpnd() && globOptData.maybeTempObjectSyms->Test(instr->GetSrc2()->AsRegOpnd()->m_sym->m_id)));
  14686. canStoreTemp = globOptData.canStoreTempObjectSyms && (
  14687. (instr->GetSrc1()->IsRegOpnd() && globOptData.canStoreTempObjectSyms->Test(instr->GetSrc1()->AsRegOpnd()->m_sym->m_id))
  14688. && (!instr->GetSrc2() || (instr->GetSrc2()->IsRegOpnd() && globOptData.canStoreTempObjectSyms->Test(instr->GetSrc2()->AsRegOpnd()->m_sym->m_id))));
  14689. AssertOrFailFast(!canStoreTemp || instr->dstIsTempObject);
  14690. AssertOrFailFast(!maybeTemp || instr->dstIsTempObject);
  14691. }
  14692. // Need to get the var equiv sym as assignment of type specialized sym kill the var sym value anyway.
  14693. StackSym * sym = opnd->m_sym;
  14694. if (!sym->IsVar())
  14695. {
  14696. sym = sym->GetVarEquivSym(nullptr);
  14697. if (sym == nullptr)
  14698. {
  14699. return;
  14700. }
  14701. }
  14702. SymID symId = sym->m_id;
  14703. if (maybeTemp)
  14704. {
  14705. // Only var sym should be temp objects
  14706. Assert(opnd->m_sym == sym);
  14707. if (globOptData.maybeTempObjectSyms == nullptr)
  14708. {
  14709. globOptData.maybeTempObjectSyms = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  14710. }
  14711. globOptData.maybeTempObjectSyms->Set(symId);
  14712. if (canStoreTemp)
  14713. {
  14714. if (instr->m_opcode == Js::OpCode::NewScObjectLiteral && !this->IsLoopPrePass())
  14715. {
  14716. // For object literal, we install the final type up front.
  14717. // If there are bailout before we finish initializing all the fields, we need to
  14718. // zero out the rest if we stack allocate the literal, so that the boxing would not
  14719. // try to box trash pointer in the properties.
  14720. // Although object Literal initialization can be done lexically, BailOnNoProfile may cause some path
  14721. // to disappear. Do it is flow base make it easier to stop propagate those entries.
  14722. IR::IntConstOpnd * propertyArrayIdOpnd = instr->GetSrc1()->AsIntConstOpnd();
  14723. const Js::PropertyIdArray * propIds = instr->m_func->GetJITFunctionBody()->ReadPropertyIdArrayFromAuxData(propertyArrayIdOpnd->AsUint32());
  14724. // Duplicates are removed by parser
  14725. Assert(!propIds->hadDuplicates);
  14726. if (globOptData.stackLiteralInitFldDataMap == nullptr)
  14727. {
  14728. globOptData.stackLiteralInitFldDataMap = JitAnew(alloc, StackLiteralInitFldDataMap, alloc);
  14729. }
  14730. else
  14731. {
  14732. Assert(!globOptData.stackLiteralInitFldDataMap->ContainsKey(sym));
  14733. }
  14734. StackLiteralInitFldData data = { propIds, 0};
  14735. globOptData.stackLiteralInitFldDataMap->AddNew(sym, data);
  14736. }
  14737. if (globOptData.canStoreTempObjectSyms == nullptr)
  14738. {
  14739. globOptData.canStoreTempObjectSyms = JitAnew(this->alloc, BVSparse<JitArenaAllocator>, this->alloc);
  14740. }
  14741. globOptData.canStoreTempObjectSyms->Set(symId);
  14742. }
  14743. else if (globOptData.canStoreTempObjectSyms)
  14744. {
  14745. globOptData.canStoreTempObjectSyms->Clear(symId);
  14746. }
  14747. }
  14748. else
  14749. {
  14750. Assert(!canStoreTemp);
  14751. if (globOptData.maybeTempObjectSyms)
  14752. {
  14753. if (globOptData.canStoreTempObjectSyms)
  14754. {
  14755. globOptData.canStoreTempObjectSyms->Clear(symId);
  14756. }
  14757. globOptData.maybeTempObjectSyms->Clear(symId);
  14758. }
  14759. else
  14760. {
  14761. Assert(!globOptData.canStoreTempObjectSyms);
  14762. }
  14763. // The symbol is being assigned to, the sym shouldn't still be in the stackLiteralInitFldDataMap
  14764. Assert(this->IsLoopPrePass() ||
  14765. globOptData.stackLiteralInitFldDataMap == nullptr
  14766. || globOptData.stackLiteralInitFldDataMap->Count() == 0
  14767. || !globOptData.stackLiteralInitFldDataMap->ContainsKey(sym));
  14768. }
  14769. }
  14770. IR::Instr *
  14771. GlobOpt::GenerateBailOutMarkTempObjectIfNeeded(IR::Instr * instr, IR::Opnd * opnd, bool isDst)
  14772. {
  14773. Assert(opnd);
  14774. Assert(isDst == (opnd == instr->GetDst()));
  14775. Assert(opnd != instr->GetDst() || !opnd->IsRegOpnd());
  14776. Assert(!this->IsLoopPrePass());
  14777. Assert(instr->HasAnyImplicitCalls());
  14778. // Only dst reg opnd opcode or ArgOut_A should have dstIsTempObject marked
  14779. Assert(!isDst || !instr->dstIsTempObject || instr->m_opcode == Js::OpCode::ArgOut_A);
  14780. // Post-op implicit call shouldn't have installed yet
  14781. Assert(!instr->HasBailOutInfo() || (instr->GetBailOutKind() & IR::BailOutKindBits) != IR::BailOutOnImplicitCalls);
  14782. GlobOptBlockData& globOptData = *CurrentBlockData();
  14783. Assert(globOptData.maybeTempObjectSyms != nullptr);
  14784. IR::PropertySymOpnd * propertySymOpnd = nullptr;
  14785. StackSym * stackSym = ObjectTemp::GetStackSym(opnd, &propertySymOpnd);
  14786. // It is okay to not get the var equiv sym here, as use of a type specialized sym is not use of the temp object
  14787. // so no need to add mark temp bailout.
  14788. // TempObjectSysm doesn't contain any type spec sym, so we will get false here for all type spec sym.
  14789. if (stackSym && globOptData.maybeTempObjectSyms->Test(stackSym->m_id))
  14790. {
  14791. if (instr->HasBailOutInfo())
  14792. {
  14793. instr->SetBailOutKind(instr->GetBailOutKind() | IR::BailOutMarkTempObject);
  14794. }
  14795. else
  14796. {
  14797. // On insert the pre op bailout if it is not Direct field access do nothing, don't check the dst yet.
  14798. // SetTypeCheckBailout will clear this out if it is direct field access.
  14799. if (isDst
  14800. || (instr->m_opcode == Js::OpCode::FromVar && !opnd->GetValueType().IsPrimitive())
  14801. || propertySymOpnd == nullptr
  14802. || !propertySymOpnd->IsTypeCheckProtected())
  14803. {
  14804. this->GenerateBailAtOperation(&instr, IR::BailOutMarkTempObject);
  14805. }
  14806. }
  14807. if (!opnd->IsRegOpnd() && (!isDst || (globOptData.canStoreTempObjectSyms && globOptData.canStoreTempObjectSyms->Test(stackSym->m_id))))
  14808. {
  14809. // If this opnd is a dst, that means that the object pointer is a stack object,
  14810. // and we can store temp object/number on it.
  14811. // If the opnd is a src, that means that the object pointer may be a stack object
  14812. // so the load may be a temp object/number and we need to track its use.
  14813. // Don't mark start of indir as can store temp, because we don't actually know
  14814. // what it is assigning to.
  14815. if (!isDst || !opnd->IsIndirOpnd())
  14816. {
  14817. opnd->SetCanStoreTemp();
  14818. }
  14819. if (propertySymOpnd)
  14820. {
  14821. // Track initfld of stack literals
  14822. if (isDst && instr->m_opcode == Js::OpCode::InitFld)
  14823. {
  14824. const Js::PropertyId propertyId = propertySymOpnd->m_sym->AsPropertySym()->m_propertyId;
  14825. // We don't need to track numeric properties init
  14826. if (!this->func->GetThreadContextInfo()->IsNumericProperty(propertyId))
  14827. {
  14828. DebugOnly(bool found = false);
  14829. globOptData.stackLiteralInitFldDataMap->RemoveIf(stackSym,
  14830. [&](StackSym * key, StackLiteralInitFldData & data)
  14831. {
  14832. DebugOnly(found = true);
  14833. Assert(key == stackSym);
  14834. Assert(data.currentInitFldCount < data.propIds->count);
  14835. if (data.propIds->elements[data.currentInitFldCount] != propertyId)
  14836. {
  14837. #if DBG
  14838. bool duplicate = false;
  14839. for (uint i = 0; i < data.currentInitFldCount; i++)
  14840. {
  14841. if (data.propIds->elements[i] == propertyId)
  14842. {
  14843. duplicate = true;
  14844. break;
  14845. }
  14846. }
  14847. Assert(duplicate);
  14848. #endif
  14849. // duplicate initialization
  14850. return false;
  14851. }
  14852. bool finished = (++data.currentInitFldCount == data.propIds->count);
  14853. #if DBG
  14854. if (finished)
  14855. {
  14856. // We can still track the finished stack literal InitFld lexically.
  14857. this->finishedStackLiteralInitFld->Set(stackSym->m_id);
  14858. }
  14859. #endif
  14860. return finished;
  14861. });
  14862. // We might still see InitFld even we have finished with all the property Id because
  14863. // of duplicate entries at the end
  14864. Assert(found || finishedStackLiteralInitFld->Test(stackSym->m_id));
  14865. }
  14866. }
  14867. }
  14868. }
  14869. }
  14870. return instr;
  14871. }
  14872. LoopCount *
  14873. GlobOpt::GetOrGenerateLoopCountForMemOp(Loop *loop)
  14874. {
  14875. LoopCount *loopCount = loop->loopCount;
  14876. if (loopCount && !loopCount->HasGeneratedLoopCountSym())
  14877. {
  14878. Assert(loop->bailOutInfo);
  14879. EnsureBailTarget(loop);
  14880. GenerateLoopCountPlusOne(loop, loopCount);
  14881. }
  14882. return loopCount;
  14883. }
  14884. IR::Opnd *
  14885. GlobOpt::GenerateInductionVariableChangeForMemOp(Loop *loop, byte unroll, IR::Instr *insertBeforeInstr)
  14886. {
  14887. LoopCount *loopCount = loop->loopCount;
  14888. IR::Opnd *sizeOpnd = nullptr;
  14889. Assert(loopCount);
  14890. Assert(loop->memOpInfo->inductionVariableOpndPerUnrollMap);
  14891. if (loop->memOpInfo->inductionVariableOpndPerUnrollMap->TryGetValue(unroll, &sizeOpnd))
  14892. {
  14893. return sizeOpnd;
  14894. }
  14895. Func *localFunc = loop->GetFunc();
  14896. const auto InsertInstr = [&](IR::Instr *instr)
  14897. {
  14898. if (insertBeforeInstr == nullptr)
  14899. {
  14900. loop->landingPad->InsertAfter(instr);
  14901. }
  14902. else
  14903. {
  14904. insertBeforeInstr->InsertBefore(instr);
  14905. }
  14906. };
  14907. if (loopCount->LoopCountMinusOneSym())
  14908. {
  14909. IRType type = loopCount->LoopCountSym()->GetType();
  14910. // Loop count is off by one, so add one
  14911. IR::RegOpnd *loopCountOpnd = IR::RegOpnd::New(loopCount->LoopCountSym(), type, localFunc);
  14912. sizeOpnd = loopCountOpnd;
  14913. if (unroll != 1)
  14914. {
  14915. sizeOpnd = IR::RegOpnd::New(TyUint32, this->func);
  14916. IR::Opnd *unrollOpnd = IR::IntConstOpnd::New(unroll, type, localFunc);
  14917. InsertInstr(IR::Instr::New(Js::OpCode::Mul_I4,
  14918. sizeOpnd,
  14919. loopCountOpnd,
  14920. unrollOpnd,
  14921. localFunc));
  14922. }
  14923. }
  14924. else
  14925. {
  14926. uint size = (loopCount->LoopCountMinusOneConstantValue() + 1) * unroll;
  14927. sizeOpnd = IR::IntConstOpnd::New(size, IRType::TyUint32, localFunc);
  14928. }
  14929. loop->memOpInfo->inductionVariableOpndPerUnrollMap->Add(unroll, sizeOpnd);
  14930. return sizeOpnd;
  14931. }
  14932. IR::RegOpnd*
  14933. GlobOpt::GenerateStartIndexOpndForMemop(Loop *loop, IR::Opnd *indexOpnd, IR::Opnd *sizeOpnd, bool isInductionVariableChangeIncremental, bool bIndexAlreadyChanged, IR::Instr *insertBeforeInstr)
  14934. {
  14935. IR::RegOpnd *startIndexOpnd = nullptr;
  14936. Func *localFunc = loop->GetFunc();
  14937. IRType type = indexOpnd->GetType();
  14938. const int cacheIndex = ((int)isInductionVariableChangeIncremental << 1) | (int)bIndexAlreadyChanged;
  14939. if (loop->memOpInfo->startIndexOpndCache[cacheIndex])
  14940. {
  14941. return loop->memOpInfo->startIndexOpndCache[cacheIndex];
  14942. }
  14943. const auto InsertInstr = [&](IR::Instr *instr)
  14944. {
  14945. if (insertBeforeInstr == nullptr)
  14946. {
  14947. loop->landingPad->InsertAfter(instr);
  14948. }
  14949. else
  14950. {
  14951. insertBeforeInstr->InsertBefore(instr);
  14952. }
  14953. };
  14954. startIndexOpnd = IR::RegOpnd::New(type, localFunc);
  14955. // If the 2 are different we can simply use indexOpnd
  14956. if (isInductionVariableChangeIncremental != bIndexAlreadyChanged)
  14957. {
  14958. InsertInstr(IR::Instr::New(Js::OpCode::Ld_A,
  14959. startIndexOpnd,
  14960. indexOpnd,
  14961. localFunc));
  14962. }
  14963. else
  14964. {
  14965. // Otherwise add 1 to it
  14966. InsertInstr(IR::Instr::New(Js::OpCode::Add_I4,
  14967. startIndexOpnd,
  14968. indexOpnd,
  14969. IR::IntConstOpnd::New(1, type, localFunc, true),
  14970. localFunc));
  14971. }
  14972. if (!isInductionVariableChangeIncremental)
  14973. {
  14974. InsertInstr(IR::Instr::New(Js::OpCode::Sub_I4,
  14975. startIndexOpnd,
  14976. startIndexOpnd,
  14977. sizeOpnd,
  14978. localFunc));
  14979. }
  14980. loop->memOpInfo->startIndexOpndCache[cacheIndex] = startIndexOpnd;
  14981. return startIndexOpnd;
  14982. }
  14983. IR::Instr*
  14984. GlobOpt::FindUpperBoundsCheckInstr(IR::Instr* fromInstr)
  14985. {
  14986. IR::Instr *upperBoundCheck = fromInstr;
  14987. do
  14988. {
  14989. upperBoundCheck = upperBoundCheck->m_prev;
  14990. Assert(upperBoundCheck);
  14991. Assert(!upperBoundCheck->IsLabelInstr());
  14992. } while (upperBoundCheck->m_opcode != Js::OpCode::BoundCheck);
  14993. return upperBoundCheck;
  14994. }
  14995. IR::Instr*
  14996. GlobOpt::FindArraySegmentLoadInstr(IR::Instr* fromInstr)
  14997. {
  14998. IR::Instr *headSegmentLengthLoad = fromInstr;
  14999. do
  15000. {
  15001. headSegmentLengthLoad = headSegmentLengthLoad->m_prev;
  15002. Assert(headSegmentLengthLoad);
  15003. Assert(!headSegmentLengthLoad->IsLabelInstr());
  15004. } while (headSegmentLengthLoad->m_opcode != Js::OpCode::LdIndir);
  15005. return headSegmentLengthLoad;
  15006. }
  15007. void
  15008. GlobOpt::RemoveMemOpSrcInstr(IR::Instr* memopInstr, IR::Instr* srcInstr, BasicBlock* block)
  15009. {
  15010. Assert(srcInstr && (srcInstr->m_opcode == Js::OpCode::LdElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A_Strict));
  15011. Assert(memopInstr && (memopInstr->m_opcode == Js::OpCode::Memcopy || memopInstr->m_opcode == Js::OpCode::Memset));
  15012. Assert(block);
  15013. const bool isDst = srcInstr->m_opcode == Js::OpCode::StElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A_Strict;
  15014. IR::RegOpnd* opnd = (isDst ? memopInstr->GetDst() : memopInstr->GetSrc1())->AsIndirOpnd()->GetBaseOpnd();
  15015. IR::ArrayRegOpnd* arrayOpnd = opnd->IsArrayRegOpnd() ? opnd->AsArrayRegOpnd() : nullptr;
  15016. IR::Instr* topInstr = srcInstr;
  15017. if (srcInstr->extractedUpperBoundCheckWithoutHoisting)
  15018. {
  15019. IR::Instr *upperBoundCheck = FindUpperBoundsCheckInstr(srcInstr);
  15020. Assert(upperBoundCheck && upperBoundCheck != srcInstr);
  15021. topInstr = upperBoundCheck;
  15022. }
  15023. if (srcInstr->loadedArrayHeadSegmentLength && arrayOpnd && arrayOpnd->HeadSegmentLengthSym())
  15024. {
  15025. IR::Instr *arrayLoadSegmentHeadLength = FindArraySegmentLoadInstr(topInstr);
  15026. Assert(arrayLoadSegmentHeadLength);
  15027. topInstr = arrayLoadSegmentHeadLength;
  15028. arrayOpnd->RemoveHeadSegmentLengthSym();
  15029. }
  15030. if (srcInstr->loadedArrayHeadSegment && arrayOpnd && arrayOpnd->HeadSegmentSym())
  15031. {
  15032. IR::Instr *arrayLoadSegmentHead = FindArraySegmentLoadInstr(topInstr);
  15033. Assert(arrayLoadSegmentHead);
  15034. topInstr = arrayLoadSegmentHead;
  15035. arrayOpnd->RemoveHeadSegmentSym();
  15036. }
  15037. // If no bounds check are present, simply look up for instruction added for instrumentation
  15038. if(topInstr == srcInstr)
  15039. {
  15040. bool checkPrev = true;
  15041. while (checkPrev)
  15042. {
  15043. switch (topInstr->m_prev->m_opcode)
  15044. {
  15045. case Js::OpCode::BailOnNotArray:
  15046. case Js::OpCode::NoImplicitCallUses:
  15047. case Js::OpCode::ByteCodeUses:
  15048. topInstr = topInstr->m_prev;
  15049. checkPrev = !!topInstr->m_prev;
  15050. break;
  15051. default:
  15052. checkPrev = false;
  15053. break;
  15054. }
  15055. }
  15056. }
  15057. while (topInstr != srcInstr)
  15058. {
  15059. IR::Instr* removeInstr = topInstr;
  15060. topInstr = topInstr->m_next;
  15061. Assert(
  15062. removeInstr->m_opcode == Js::OpCode::BailOnNotArray ||
  15063. removeInstr->m_opcode == Js::OpCode::NoImplicitCallUses ||
  15064. removeInstr->m_opcode == Js::OpCode::ByteCodeUses ||
  15065. removeInstr->m_opcode == Js::OpCode::LdIndir ||
  15066. removeInstr->m_opcode == Js::OpCode::BoundCheck
  15067. );
  15068. if (removeInstr->m_opcode != Js::OpCode::ByteCodeUses)
  15069. {
  15070. block->RemoveInstr(removeInstr);
  15071. }
  15072. }
  15073. this->ConvertToByteCodeUses(srcInstr);
  15074. }
  15075. void
  15076. GlobOpt::GetMemOpSrcInfo(Loop* loop, IR::Instr* instr, IR::RegOpnd*& base, IR::RegOpnd*& index, IRType& arrayType)
  15077. {
  15078. Assert(instr && (instr->m_opcode == Js::OpCode::LdElemI_A || instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict));
  15079. IR::Opnd* arrayOpnd = instr->m_opcode == Js::OpCode::LdElemI_A ? instr->GetSrc1() : instr->GetDst();
  15080. Assert(arrayOpnd->IsIndirOpnd());
  15081. IR::IndirOpnd* indirArrayOpnd = arrayOpnd->AsIndirOpnd();
  15082. IR::RegOpnd* baseOpnd = (IR::RegOpnd*)indirArrayOpnd->GetBaseOpnd();
  15083. IR::RegOpnd* indexOpnd = (IR::RegOpnd*)indirArrayOpnd->GetIndexOpnd();
  15084. Assert(baseOpnd);
  15085. Assert(indexOpnd);
  15086. // Process Out Params
  15087. base = baseOpnd;
  15088. index = indexOpnd;
  15089. arrayType = indirArrayOpnd->GetType();
  15090. }
  15091. void
  15092. GlobOpt::EmitMemop(Loop * loop, LoopCount *loopCount, const MemOpEmitData* emitData)
  15093. {
  15094. Assert(emitData);
  15095. Assert(emitData->candidate);
  15096. Assert(emitData->stElemInstr);
  15097. Assert(emitData->stElemInstr->m_opcode == Js::OpCode::StElemI_A || emitData->stElemInstr->m_opcode == Js::OpCode::StElemI_A_Strict);
  15098. IR::BailOutKind bailOutKind = emitData->bailOutKind;
  15099. const byte unroll = emitData->inductionVar.unroll;
  15100. Assert(unroll == 1);
  15101. const bool isInductionVariableChangeIncremental = emitData->inductionVar.isIncremental;
  15102. const bool bIndexAlreadyChanged = emitData->candidate->bIndexAlreadyChanged;
  15103. IR::RegOpnd *baseOpnd = nullptr;
  15104. IR::RegOpnd *indexOpnd = nullptr;
  15105. IRType dstType;
  15106. GetMemOpSrcInfo(loop, emitData->stElemInstr, baseOpnd, indexOpnd, dstType);
  15107. Func *localFunc = loop->GetFunc();
  15108. // Handle bailout info
  15109. EnsureBailTarget(loop);
  15110. Assert(bailOutKind != IR::BailOutInvalid);
  15111. // Keep only Array bits bailOuts. Consider handling these bailouts instead of simply ignoring them
  15112. bailOutKind &= IR::BailOutForArrayBits;
  15113. // Add our custom bailout to handle Op_MemCopy return value.
  15114. bailOutKind |= IR::BailOutOnMemOpError;
  15115. BailOutInfo *const bailOutInfo = loop->bailOutInfo;
  15116. Assert(bailOutInfo);
  15117. IR::Instr *insertBeforeInstr = bailOutInfo->bailOutInstr;
  15118. Assert(insertBeforeInstr);
  15119. IR::Opnd *sizeOpnd = GenerateInductionVariableChangeForMemOp(loop, unroll, insertBeforeInstr);
  15120. IR::RegOpnd *startIndexOpnd = GenerateStartIndexOpndForMemop(loop, indexOpnd, sizeOpnd, isInductionVariableChangeIncremental, bIndexAlreadyChanged, insertBeforeInstr);
  15121. IR::IndirOpnd* dstOpnd = IR::IndirOpnd::New(baseOpnd, startIndexOpnd, dstType, localFunc);
  15122. IR::Opnd *src1;
  15123. const bool isMemset = emitData->candidate->IsMemSet();
  15124. // Get the source according to the memop type
  15125. if (isMemset)
  15126. {
  15127. MemSetEmitData* data = (MemSetEmitData*)emitData;
  15128. const Loop::MemSetCandidate* candidate = data->candidate->AsMemSet();
  15129. if (candidate->srcSym)
  15130. {
  15131. IR::RegOpnd* regSrc = IR::RegOpnd::New(candidate->srcSym, candidate->srcSym->GetType(), func);
  15132. regSrc->SetIsJITOptimizedReg(true);
  15133. src1 = regSrc;
  15134. }
  15135. else
  15136. {
  15137. src1 = IR::AddrOpnd::New(candidate->constant.ToVar(localFunc), IR::AddrOpndKindConstantAddress, localFunc);
  15138. }
  15139. }
  15140. else
  15141. {
  15142. Assert(emitData->candidate->IsMemCopy());
  15143. MemCopyEmitData* data = (MemCopyEmitData*)emitData;
  15144. Assert(data->ldElemInstr);
  15145. Assert(data->ldElemInstr->m_opcode == Js::OpCode::LdElemI_A);
  15146. IR::RegOpnd *srcBaseOpnd = nullptr;
  15147. IR::RegOpnd *srcIndexOpnd = nullptr;
  15148. IRType srcType;
  15149. GetMemOpSrcInfo(loop, data->ldElemInstr, srcBaseOpnd, srcIndexOpnd, srcType);
  15150. Assert(GetVarSymID(srcIndexOpnd->GetStackSym()) == GetVarSymID(indexOpnd->GetStackSym()));
  15151. src1 = IR::IndirOpnd::New(srcBaseOpnd, startIndexOpnd, srcType, localFunc);
  15152. }
  15153. // Generate memcopy
  15154. IR::Instr* memopInstr = IR::BailOutInstr::New(isMemset ? Js::OpCode::Memset : Js::OpCode::Memcopy, bailOutKind, bailOutInfo, localFunc);
  15155. memopInstr->SetDst(dstOpnd);
  15156. memopInstr->SetSrc1(src1);
  15157. memopInstr->SetSrc2(sizeOpnd);
  15158. insertBeforeInstr->InsertBefore(memopInstr);
  15159. loop->memOpInfo->instr = memopInstr;
  15160. #if DBG_DUMP
  15161. if (DO_MEMOP_TRACE())
  15162. {
  15163. char valueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  15164. baseOpnd->GetValueType().ToString(valueTypeStr);
  15165. const int loopCountBufSize = 16;
  15166. char16 loopCountBuf[loopCountBufSize];
  15167. if (loopCount->LoopCountMinusOneSym())
  15168. {
  15169. swprintf_s(loopCountBuf, _u("s%u"), loopCount->LoopCountMinusOneSym()->m_id);
  15170. }
  15171. else
  15172. {
  15173. swprintf_s(loopCountBuf, _u("%u"), loopCount->LoopCountMinusOneConstantValue() + 1);
  15174. }
  15175. if (isMemset)
  15176. {
  15177. const Loop::MemSetCandidate* candidate = emitData->candidate->AsMemSet();
  15178. const int constBufSize = 32;
  15179. char16 constBuf[constBufSize];
  15180. if (candidate->srcSym)
  15181. {
  15182. swprintf_s(constBuf, _u("s%u"), candidate->srcSym->m_id);
  15183. }
  15184. else
  15185. {
  15186. switch (candidate->constant.type)
  15187. {
  15188. case TyInt8:
  15189. case TyInt16:
  15190. case TyInt32:
  15191. case TyInt64:
  15192. swprintf_s(constBuf, sizeof(IntConstType) == 8 ? _u("%lld") : _u("%d"), candidate->constant.u.intConst.value);
  15193. break;
  15194. case TyFloat32:
  15195. case TyFloat64:
  15196. swprintf_s(constBuf, _u("%.4f"), candidate->constant.u.floatConst.value);
  15197. break;
  15198. case TyVar:
  15199. swprintf_s(constBuf, sizeof(Js::Var) == 8 ? _u("0x%.16llX") : _u("0x%.8X"), candidate->constant.u.varConst.value);
  15200. break;
  15201. default:
  15202. AssertMsg(false, "Unsupported constant type");
  15203. swprintf_s(constBuf, _u("Unknown"));
  15204. break;
  15205. }
  15206. }
  15207. TRACE_MEMOP_PHASE(MemSet, loop, emitData->stElemInstr,
  15208. _u("ValueType: %S, Base: s%u, Index: s%u, Constant: %s, LoopCount: %s, IsIndexChangedBeforeUse: %d"),
  15209. valueTypeStr,
  15210. candidate->base,
  15211. candidate->index,
  15212. constBuf,
  15213. loopCountBuf,
  15214. bIndexAlreadyChanged);
  15215. }
  15216. else
  15217. {
  15218. const Loop::MemCopyCandidate* candidate = emitData->candidate->AsMemCopy();
  15219. TRACE_MEMOP_PHASE(MemCopy, loop, emitData->stElemInstr,
  15220. _u("ValueType: %S, StBase: s%u, Index: s%u, LdBase: s%u, LoopCount: %s, IsIndexChangedBeforeUse: %d"),
  15221. valueTypeStr,
  15222. candidate->base,
  15223. candidate->index,
  15224. candidate->ldBase,
  15225. loopCountBuf,
  15226. bIndexAlreadyChanged);
  15227. }
  15228. }
  15229. #endif
  15230. RemoveMemOpSrcInstr(memopInstr, emitData->stElemInstr, emitData->block);
  15231. if (!isMemset)
  15232. {
  15233. RemoveMemOpSrcInstr(memopInstr, ((MemCopyEmitData*)emitData)->ldElemInstr, emitData->block);
  15234. }
  15235. }
  15236. bool
  15237. GlobOpt::InspectInstrForMemSetCandidate(Loop* loop, IR::Instr* instr, MemSetEmitData* emitData, bool& errorInInstr)
  15238. {
  15239. Assert(emitData && emitData->candidate && emitData->candidate->IsMemSet());
  15240. Loop::MemSetCandidate* candidate = (Loop::MemSetCandidate*)emitData->candidate;
  15241. if (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict)
  15242. {
  15243. if (instr->GetDst()->IsIndirOpnd()
  15244. && (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->base)
  15245. && (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index)
  15246. )
  15247. {
  15248. Assert(instr->IsProfiledInstr());
  15249. emitData->stElemInstr = instr;
  15250. emitData->bailOutKind = instr->GetBailOutKind();
  15251. return true;
  15252. }
  15253. TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Orphan StElemI_A detected"));
  15254. errorInInstr = true;
  15255. }
  15256. else if (instr->m_opcode == Js::OpCode::LdElemI_A)
  15257. {
  15258. TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Orphan LdElemI_A detected"));
  15259. errorInInstr = true;
  15260. }
  15261. return false;
  15262. }
  15263. bool
  15264. GlobOpt::InspectInstrForMemCopyCandidate(Loop* loop, IR::Instr* instr, MemCopyEmitData* emitData, bool& errorInInstr)
  15265. {
  15266. Assert(emitData && emitData->candidate && emitData->candidate->IsMemCopy());
  15267. Loop::MemCopyCandidate* candidate = (Loop::MemCopyCandidate*)emitData->candidate;
  15268. if (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict)
  15269. {
  15270. if (
  15271. instr->GetDst()->IsIndirOpnd() &&
  15272. (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->base) &&
  15273. (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index)
  15274. )
  15275. {
  15276. Assert(instr->IsProfiledInstr());
  15277. emitData->stElemInstr = instr;
  15278. emitData->bailOutKind = instr->GetBailOutKind();
  15279. // Still need to find the LdElem
  15280. return false;
  15281. }
  15282. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Orphan StElemI_A detected"));
  15283. errorInInstr = true;
  15284. }
  15285. else if (instr->m_opcode == Js::OpCode::LdElemI_A)
  15286. {
  15287. if (
  15288. emitData->stElemInstr &&
  15289. instr->GetSrc1()->IsIndirOpnd() &&
  15290. (GetVarSymID(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->ldBase) &&
  15291. (GetVarSymID(instr->GetSrc1()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index)
  15292. )
  15293. {
  15294. Assert(instr->IsProfiledInstr());
  15295. emitData->ldElemInstr = instr;
  15296. ValueType stValueType = emitData->stElemInstr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType();
  15297. ValueType ldValueType = emitData->ldElemInstr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType();
  15298. if (stValueType != ldValueType)
  15299. {
  15300. #if DBG_DUMP
  15301. char16 stValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  15302. stValueType.ToString(stValueTypeStr);
  15303. char16 ldValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE];
  15304. ldValueType.ToString(ldValueTypeStr);
  15305. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("for mismatch in Load(%s) and Store(%s) value type"), ldValueTypeStr, stValueTypeStr);
  15306. #endif
  15307. errorInInstr = true;
  15308. return false;
  15309. }
  15310. // We found both instruction for this candidate
  15311. return true;
  15312. }
  15313. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Orphan LdElemI_A detected"));
  15314. errorInInstr = true;
  15315. }
  15316. return false;
  15317. }
  15318. // The caller is responsible to free the memory allocated between inOrderEmitData[iEmitData -> end]
  15319. bool
  15320. GlobOpt::ValidateMemOpCandidates(Loop * loop, _Out_writes_(iEmitData) MemOpEmitData** inOrderEmitData, int& iEmitData)
  15321. {
  15322. AnalysisAssert(iEmitData == (int)loop->memOpInfo->candidates->Count());
  15323. // We iterate over the second block of the loop only. MemOp Works only if the loop has exactly 2 blocks
  15324. Assert(loop->blockList.HasTwo());
  15325. Loop::MemOpList::Iterator iter(loop->memOpInfo->candidates);
  15326. BasicBlock* bblock = loop->blockList.Head()->next;
  15327. Loop::MemOpCandidate* candidate = nullptr;
  15328. MemOpEmitData* emitData = nullptr;
  15329. // Iterate backward because the list of candidate is reversed
  15330. FOREACH_INSTR_BACKWARD_IN_BLOCK(instr, bblock)
  15331. {
  15332. if (!candidate)
  15333. {
  15334. // Time to check next candidate
  15335. if (!iter.Next())
  15336. {
  15337. // We have been through the whole list of candidates, finish
  15338. break;
  15339. }
  15340. candidate = iter.Data();
  15341. if (!candidate)
  15342. {
  15343. continue;
  15344. }
  15345. // Common check for memset and memcopy
  15346. Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 };
  15347. // Get the inductionVariable changeInfo
  15348. if (!loop->memOpInfo->inductionVariableChangeInfoMap->TryGetValue(candidate->index, &inductionVariableChangeInfo))
  15349. {
  15350. TRACE_MEMOP_VERBOSE(loop, nullptr, _u("MemOp skipped (s%d): no induction variable"), candidate->base);
  15351. return false;
  15352. }
  15353. if (inductionVariableChangeInfo.unroll != candidate->count)
  15354. {
  15355. TRACE_MEMOP_VERBOSE(loop, nullptr, _u("MemOp skipped (s%d): not matching unroll count"), candidate->base);
  15356. return false;
  15357. }
  15358. if (candidate->IsMemSet())
  15359. {
  15360. Assert(!PHASE_OFF(Js::MemSetPhase, this->func));
  15361. emitData = JitAnew(this->alloc, MemSetEmitData);
  15362. }
  15363. else
  15364. {
  15365. Assert(!PHASE_OFF(Js::MemCopyPhase, this->func));
  15366. // Specific check for memcopy
  15367. Assert(candidate->IsMemCopy());
  15368. Loop::MemCopyCandidate* memcopyCandidate = candidate->AsMemCopy();
  15369. if (memcopyCandidate->base == Js::Constants::InvalidSymID
  15370. || memcopyCandidate->ldBase == Js::Constants::InvalidSymID
  15371. || (memcopyCandidate->ldCount != memcopyCandidate->count))
  15372. {
  15373. TRACE_MEMOP_PHASE(MemCopy, loop, nullptr, _u("(s%d): not matching ldElem and stElem"), candidate->base);
  15374. return false;
  15375. }
  15376. emitData = JitAnew(this->alloc, MemCopyEmitData);
  15377. }
  15378. Assert(emitData);
  15379. emitData->block = bblock;
  15380. emitData->inductionVar = inductionVariableChangeInfo;
  15381. emitData->candidate = candidate;
  15382. }
  15383. bool errorInInstr = false;
  15384. bool candidateFound = candidate->IsMemSet() ?
  15385. InspectInstrForMemSetCandidate(loop, instr, (MemSetEmitData*)emitData, errorInInstr)
  15386. : InspectInstrForMemCopyCandidate(loop, instr, (MemCopyEmitData*)emitData, errorInInstr);
  15387. if (errorInInstr)
  15388. {
  15389. JitAdelete(this->alloc, emitData);
  15390. return false;
  15391. }
  15392. if (candidateFound)
  15393. {
  15394. AnalysisAssert(iEmitData > 0);
  15395. if (iEmitData == 0)
  15396. {
  15397. // Explicit for OACR
  15398. break;
  15399. }
  15400. inOrderEmitData[--iEmitData] = emitData;
  15401. candidate = nullptr;
  15402. emitData = nullptr;
  15403. }
  15404. } NEXT_INSTR_BACKWARD_IN_BLOCK;
  15405. if (iter.IsValid())
  15406. {
  15407. TRACE_MEMOP(loop, nullptr, _u("Candidates not found in loop while validating"));
  15408. return false;
  15409. }
  15410. return true;
  15411. }
  15412. void
  15413. GlobOpt::ProcessMemOp()
  15414. {
  15415. FOREACH_LOOP_IN_FUNC_EDITING(loop, this->func)
  15416. {
  15417. if (HasMemOp(loop))
  15418. {
  15419. const int candidateCount = loop->memOpInfo->candidates->Count();
  15420. Assert(candidateCount > 0);
  15421. LoopCount * loopCount = GetOrGenerateLoopCountForMemOp(loop);
  15422. // If loopCount is not available we can not continue with memop
  15423. if (!loopCount || !(loopCount->LoopCountMinusOneSym() || loopCount->LoopCountMinusOneConstantValue()))
  15424. {
  15425. TRACE_MEMOP(loop, nullptr, _u("MemOp skipped for no loop count"));
  15426. loop->doMemOp = false;
  15427. loop->memOpInfo->candidates->Clear();
  15428. continue;
  15429. }
  15430. // The list is reversed, check them and place them in order in the following array
  15431. MemOpEmitData** inOrderCandidates = JitAnewArray(this->alloc, MemOpEmitData*, candidateCount);
  15432. int i = candidateCount;
  15433. if (ValidateMemOpCandidates(loop, inOrderCandidates, i))
  15434. {
  15435. Assert(i == 0);
  15436. // Process the valid MemOp candidate in order.
  15437. for (; i < candidateCount; ++i)
  15438. {
  15439. // Emit
  15440. EmitMemop(loop, loopCount, inOrderCandidates[i]);
  15441. JitAdelete(this->alloc, inOrderCandidates[i]);
  15442. }
  15443. }
  15444. else
  15445. {
  15446. Assert(i != 0);
  15447. for (; i < candidateCount; ++i)
  15448. {
  15449. JitAdelete(this->alloc, inOrderCandidates[i]);
  15450. }
  15451. // One of the memop candidates did not validate. Do not emit for this loop.
  15452. loop->doMemOp = false;
  15453. loop->memOpInfo->candidates->Clear();
  15454. }
  15455. // Free memory
  15456. JitAdeleteArray(this->alloc, candidateCount, inOrderCandidates);
  15457. }
  15458. } NEXT_LOOP_EDITING;
  15459. }
  15460. void GlobOpt::PRE::FieldPRE(Loop *loop)
  15461. {
  15462. JitArenaAllocator *alloc = this->globOpt->tempAlloc;
  15463. this->FindPossiblePRECandidates(loop, alloc);
  15464. this->PreloadPRECandidates(loop);
  15465. this->RemoveOverlyOptimisticInitialValues(loop);
  15466. }
  15467. bool
  15468. GlobOpt::PRE::InsertSymDefinitionInLandingPad(StackSym * sym, Loop * loop, Sym ** objPtrCopyPropSym)
  15469. {
  15470. Assert(sym->IsSingleDef());
  15471. IR::Instr * symDefInstr = sym->GetInstrDef();
  15472. if (!GlobOpt::IsPREInstrSequenceCandidateLoad(symDefInstr->m_opcode))
  15473. {
  15474. return false;
  15475. }
  15476. IR::Opnd * symDefInstrSrc1 = symDefInstr->GetSrc1();
  15477. if (symDefInstrSrc1->IsSymOpnd())
  15478. {
  15479. Assert(symDefInstrSrc1->AsSymOpnd()->m_sym->IsPropertySym());
  15480. // $L1
  15481. // T1 = o.x (v1|T3)
  15482. // T2 = T1.y (v2|T4) <-- T1 is not live in the loop landing pad
  15483. // jmp $L1
  15484. // Trying to make T1 live in the landing pad
  15485. // o.x
  15486. PropertySym* propSym = symDefInstrSrc1->AsSymOpnd()->m_sym->AsPropertySym();
  15487. if (candidates->candidatesBv->Test(propSym->m_id))
  15488. {
  15489. // If propsym is a PRE candidate, then it must have had the same value on all back edges.
  15490. // So, just look up the value on one of the back edges.
  15491. BasicBlock* loopTail = loop->GetAnyTailBlock();
  15492. Value * valueOnBackEdge = loopTail->globOptData.FindValue(propSym);
  15493. // If o.x is not invariant in the loop, we can't use the preloaded value of o.x.y in the landing pad
  15494. Value * valueInLandingPad = loop->landingPad->globOptData.FindValue(propSym);
  15495. if (valueOnBackEdge->GetValueNumber() != valueInLandingPad->GetValueNumber())
  15496. {
  15497. return false;
  15498. }
  15499. *objPtrCopyPropSym = valueOnBackEdge->GetValueInfo()->GetSymStore();
  15500. if (candidates->candidatesToProcess->Test(propSym->m_id))
  15501. {
  15502. GlobHashBucket bucket;
  15503. bucket.element = valueOnBackEdge;
  15504. bucket.value = propSym;
  15505. if (!PreloadPRECandidate(loop, &bucket))
  15506. {
  15507. return false;
  15508. }
  15509. Assert(!candidates->candidatesToProcess->Test(propSym->m_id));
  15510. Assert(loop->landingPad->globOptData.IsLive(valueOnBackEdge->GetValueInfo()->GetSymStore()));
  15511. // Inserted T3 = o.x
  15512. // Now, we want to
  15513. // 1. Insert T1 = o.x
  15514. // 2. Insert T4 = T1.y
  15515. // 3. Indentify T3 as the objptr copy prop sym for T1, and make T3.y live on the back-edges
  15516. // #1 is done next. #2 and #3 are done as part of preloading T1.y
  15517. // Insert T1 = o.x
  15518. if (!InsertPropertySymPreloadInLandingPad(symDefInstr->Copy(), loop, propSym))
  15519. {
  15520. return false;
  15521. }
  15522. return true;
  15523. }
  15524. else
  15525. {
  15526. // o.x was already processed as a PRE candidate. If we were successful in preloading o.x,
  15527. // we can now insert T1 = o.x
  15528. if (loop->landingPad->globOptData.IsLive(*objPtrCopyPropSym))
  15529. {
  15530. // insert T1 = o.x
  15531. if (!InsertPropertySymPreloadInLandingPad(symDefInstr->Copy(), loop, propSym))
  15532. {
  15533. return false;
  15534. }
  15535. return true;
  15536. }
  15537. else
  15538. {
  15539. return false;
  15540. }
  15541. }
  15542. }
  15543. else
  15544. {
  15545. return false;
  15546. }
  15547. }
  15548. else if (symDefInstrSrc1->IsRegOpnd())
  15549. {
  15550. // T2 = T1
  15551. // T3 = T2.y
  15552. // trying to insert def of T2
  15553. // T1
  15554. StackSym * symDefInstrSrc1Sym = symDefInstrSrc1->AsRegOpnd()->GetStackSym();
  15555. if (!loop->landingPad->globOptData.IsLive(symDefInstrSrc1Sym))
  15556. {
  15557. if (symDefInstrSrc1Sym->IsSingleDef())
  15558. {
  15559. if (!InsertSymDefinitionInLandingPad(symDefInstrSrc1Sym, loop, objPtrCopyPropSym))
  15560. {
  15561. return false;
  15562. }
  15563. }
  15564. }
  15565. else
  15566. {
  15567. *objPtrCopyPropSym = symDefInstrSrc1Sym;
  15568. }
  15569. if (!(OpCodeAttr::TempNumberTransfer(symDefInstr->m_opcode) && OpCodeAttr::TempObjectTransfer(symDefInstr->m_opcode)))
  15570. {
  15571. *objPtrCopyPropSym = sym;
  15572. }
  15573. IR::Instr * instr = symDefInstr->Copy();
  15574. if (instr->m_opcode == Js::OpCode::BytecodeArgOutCapture)
  15575. {
  15576. instr->m_opcode = Js::OpCode::Ld_A;
  15577. }
  15578. InsertInstrInLandingPad(instr, loop);
  15579. return true;
  15580. }
  15581. else
  15582. {
  15583. return false;
  15584. }
  15585. }
  15586. void
  15587. GlobOpt::PRE::InsertInstrInLandingPad(IR::Instr * instr, Loop * loop)
  15588. {
  15589. instr->GetSrc1()->SetIsJITOptimizedReg(true);
  15590. if (instr->GetDst())
  15591. {
  15592. instr->GetDst()->SetIsJITOptimizedReg(true);
  15593. loop->landingPad->globOptData.liveVarSyms->Set(instr->GetDst()->GetStackSym()->m_id);
  15594. }
  15595. if (instr->HasAnyImplicitCalls())
  15596. {
  15597. IR::Instr * bailInstr = globOpt->EnsureDisableImplicitCallRegion(loop);
  15598. bailInstr->InsertBefore(instr);
  15599. }
  15600. else if (loop->endDisableImplicitCall)
  15601. {
  15602. loop->endDisableImplicitCall->InsertBefore(instr);
  15603. }
  15604. else
  15605. {
  15606. loop->landingPad->InsertAfter(instr);
  15607. }
  15608. instr->ClearByteCodeOffset();
  15609. instr->SetByteCodeOffset(loop->landingPad->GetFirstInstr());
  15610. }
  15611. IR::Instr *
  15612. GlobOpt::PRE::InsertPropertySymPreloadInLandingPad(IR::Instr * ldInstr, Loop * loop, PropertySym * propertySym)
  15613. {
  15614. IR::SymOpnd *ldSrc = ldInstr->GetSrc1()->AsSymOpnd();
  15615. if (ldSrc->m_sym != propertySym)
  15616. {
  15617. // It's possible that the property syms are different but have equivalent objPtrs. Verify their values.
  15618. Value *val1 = globOpt->CurrentBlockData()->FindValue(ldSrc->m_sym->AsPropertySym()->m_stackSym);
  15619. Value *val2 = globOpt->CurrentBlockData()->FindValue(propertySym->m_stackSym);
  15620. if (!val1 || !val2 || val1->GetValueNumber() != val2->GetValueNumber())
  15621. {
  15622. return nullptr;
  15623. }
  15624. }
  15625. // Consider: Shouldn't be necessary once we have copy-prop in prepass...
  15626. ldInstr->GetSrc1()->AsSymOpnd()->m_sym = propertySym;
  15627. ldSrc = ldInstr->GetSrc1()->AsSymOpnd();
  15628. if (ldSrc->IsPropertySymOpnd())
  15629. {
  15630. IR::PropertySymOpnd *propSymOpnd = ldSrc->AsPropertySymOpnd();
  15631. IR::PropertySymOpnd *newPropSymOpnd;
  15632. newPropSymOpnd = propSymOpnd->AsPropertySymOpnd()->CopyWithoutFlowSensitiveInfo(this->globOpt->func);
  15633. ldInstr->ReplaceSrc1(newPropSymOpnd);
  15634. }
  15635. if (ldInstr->GetDst())
  15636. {
  15637. loop->landingPad->globOptData.liveVarSyms->Set(ldInstr->GetDst()->GetStackSym()->m_id);
  15638. }
  15639. InsertInstrInLandingPad(ldInstr, loop);
  15640. return ldInstr;
  15641. }
  15642. void
  15643. GlobOpt::PRE::MakePropertySymLiveOnBackEdges(PropertySym * propertySym, Loop * loop, Value * valueToAdd)
  15644. {
  15645. BasicBlock * loopHeader = loop->GetHeadBlock();
  15646. FOREACH_PREDECESSOR_BLOCK(blockPred, loopHeader)
  15647. {
  15648. if (!loop->IsDescendentOrSelf(blockPred->loop))
  15649. {
  15650. // Not a loop back-edge
  15651. continue;
  15652. }
  15653. // Insert it in the value table
  15654. blockPred->globOptData.SetValue(valueToAdd, propertySym);
  15655. // Make it a live field
  15656. blockPred->globOptData.liveFields->Set(propertySym->m_id);
  15657. } NEXT_PREDECESSOR_BLOCK;
  15658. }
  15659. void GlobOpt::PRE::RemoveOverlyOptimisticInitialValues(Loop * loop)
  15660. {
  15661. BasicBlock * landingPad = loop->landingPad;
  15662. // For a property sym whose obj ptr sym wasn't live in the landing pad, we can optmistically (if the obj ptr sym was
  15663. // single def) insert an initial value in the landing pad, with the hope that PRE could make the obj ptr sym live.
  15664. // But, if PRE couldn't make the obj ptr sym live, we need to clear the value for the property sym from the landing pad
  15665. for (auto it = loop->initialValueFieldMap.GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext())
  15666. {
  15667. PropertySym * propertySym = it.CurrentKey();
  15668. StackSym * objPtrSym = propertySym->m_stackSym;
  15669. if (!landingPad->globOptData.IsLive(objPtrSym))
  15670. {
  15671. Value * landingPadPropSymValue = landingPad->globOptData.FindValue(propertySym);
  15672. Assert(landingPadPropSymValue);
  15673. Assert(landingPadPropSymValue->GetValueNumber() == it.CurrentValue()->GetValueNumber());
  15674. Assert(landingPadPropSymValue->GetValueInfo()->GetSymStore() == propertySym);
  15675. landingPad->globOptData.ClearSymValue(propertySym);
  15676. it.RemoveCurrent();
  15677. }
  15678. }
  15679. }
  15680. #if DBG_DUMP
  15681. void GlobOpt::PRE::TraceFailedPreloadInLandingPad(const Loop *const loop, PropertySym * propertySym, const char16* reason) const
  15682. {
  15683. if (PHASE_TRACE(Js::FieldPREPhase, this->globOpt->func))
  15684. {
  15685. int32 propertyId = propertySym->m_propertyId;
  15686. SymID objectSymId = propertySym->m_stackSym->m_id;
  15687. char16 propSymStr[32];
  15688. switch (propertySym->m_fieldKind)
  15689. {
  15690. case PropertyKindData:
  15691. if (JITManager::GetJITManager()->IsOOPJITEnabled())
  15692. {
  15693. swprintf_s(propSymStr, _u("s%d->#%d"), objectSymId, propertyId);
  15694. }
  15695. else
  15696. {
  15697. Js::PropertyRecord const* fieldName = propertySym->m_func->GetInProcThreadContext()->GetPropertyRecord(propertyId);
  15698. swprintf_s(propSymStr, _u("s%d->%s"), objectSymId, fieldName->GetBuffer());
  15699. }
  15700. break;
  15701. case PropertyKindSlots:
  15702. case PropertyKindSlotArray:
  15703. swprintf_s(propSymStr, _u("s%d[%d]"), objectSymId, propertyId);
  15704. break;
  15705. case PropertyKindLocalSlots:
  15706. swprintf_s(propSymStr, _u("s%dl[%d]"), objectSymId, propertyId);
  15707. break;
  15708. default:
  15709. AssertMsg(0, "Unknown field kind");
  15710. break;
  15711. }
  15712. Output::Print(_u("** TRACE: Field PRE: "));
  15713. this->globOpt->func->DumpFullFunctionName();
  15714. Output::Print(_u(": Failed to pre-load (%s) in landing pad of loop #%d. Reason: %s "), propSymStr, loop->GetLoopNumber(), reason);
  15715. Output::Print(_u("\n"));
  15716. }
  15717. }
  15718. #endif