//------------------------------------------------------------------------------------------------------- // Copyright (C) Microsoft Corporation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. //------------------------------------------------------------------------------------------------------- #include "Backend.h" #if DBG_DUMP #define DO_MEMOP_TRACE() (PHASE_TRACE(Js::MemOpPhase, this->func) ||\ PHASE_TRACE(Js::MemSetPhase, this->func) ||\ PHASE_TRACE(Js::MemCopyPhase, this->func)) #define DO_MEMOP_TRACE_PHASE(phase) (PHASE_TRACE(Js::MemOpPhase, this->func) || PHASE_TRACE(Js::phase ## Phase, this->func)) #define OUTPUT_MEMOP_TRACE(loop, instr, ...) {\ char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];\ Output::Print(15, _u("Function: %s%s, Loop: %u: "), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), loop->GetLoopNumber());\ Output::Print(__VA_ARGS__);\ IR::Instr* __instr__ = instr;\ if(__instr__) __instr__->DumpByteCodeOffset();\ if(__instr__) Output::Print(_u(" (%s)"), Js::OpCodeUtil::GetOpCodeName(__instr__->m_opcode));\ Output::Print(_u("\n"));\ Output::Flush(); \ } #define TRACE_MEMOP(loop, instr, ...) \ if (DO_MEMOP_TRACE()) {\ Output::Print(_u("TRACE MemOp:"));\ OUTPUT_MEMOP_TRACE(loop, instr, __VA_ARGS__)\ } #define TRACE_MEMOP_VERBOSE(loop, instr, ...) if(CONFIG_FLAG(Verbose)) {TRACE_MEMOP(loop, instr, __VA_ARGS__)} #define TRACE_MEMOP_PHASE(phase, loop, instr, ...) \ if (DO_MEMOP_TRACE_PHASE(phase))\ {\ Output::Print(_u("TRACE ") _u(#phase) _u(":"));\ OUTPUT_MEMOP_TRACE(loop, instr, __VA_ARGS__)\ } #define TRACE_MEMOP_PHASE_VERBOSE(phase, loop, instr, ...) if(CONFIG_FLAG(Verbose)) {TRACE_MEMOP_PHASE(phase, loop, instr, __VA_ARGS__)} #else #define DO_MEMOP_TRACE() #define DO_MEMOP_TRACE_PHASE(phase) #define OUTPUT_MEMOP_TRACE(loop, instr, ...) #define TRACE_MEMOP(loop, instr, ...) #define TRACE_MEMOP_VERBOSE(loop, instr, ...) #define TRACE_MEMOP_PHASE(phase, loop, instr, ...) #define TRACE_MEMOP_PHASE_VERBOSE(phase, loop, instr, ...) #endif class AutoRestoreVal { private: Value *const originalValue; Value *const tempValue; Value * *const valueRef; public: AutoRestoreVal(Value *const originalValue, Value * *const tempValueRef) : originalValue(originalValue), tempValue(*tempValueRef), valueRef(tempValueRef) { } ~AutoRestoreVal() { if(*valueRef == tempValue) { *valueRef = originalValue; } } PREVENT_COPY(AutoRestoreVal); }; GlobOpt::GlobOpt(Func * func) : func(func), intConstantToStackSymMap(nullptr), intConstantToValueMap(nullptr), currentValue(FirstNewValueNumber), prePassLoop(nullptr), alloc(nullptr), isCallHelper(false), inInlinedBuiltIn(false), rootLoopPrePass(nullptr), noImplicitCallUsesToInsert(nullptr), valuesCreatedForClone(nullptr), valuesCreatedForMerge(nullptr), instrCountSinceLastCleanUp(0), isRecursiveCallOnLandingPad(false), updateInductionVariableValueNumber(false), isPerformingLoopBackEdgeCompensation(false), currentRegion(nullptr), auxSlotPtrSyms(nullptr), changedSymsAfterIncBailoutCandidate(nullptr), doTypeSpec( !IsTypeSpecPhaseOff(func)), doAggressiveIntTypeSpec( doTypeSpec && DoAggressiveIntTypeSpec(func)), doAggressiveMulIntTypeSpec( doTypeSpec && !PHASE_OFF(Js::AggressiveMulIntTypeSpecPhase, func) && (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsAggressiveMulIntTypeSpecDisabled(func->IsLoopBody()))), doDivIntTypeSpec( doAggressiveIntTypeSpec && (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsDivIntTypeSpecDisabled(func->IsLoopBody()))), doLossyIntTypeSpec( doTypeSpec && DoLossyIntTypeSpec(func)), doFloatTypeSpec( doTypeSpec && DoFloatTypeSpec(func)), doArrayCheckHoist( DoArrayCheckHoist(func)), doArrayMissingValueCheckHoist( doArrayCheckHoist && DoArrayMissingValueCheckHoist(func)), doArraySegmentHoist( doArrayCheckHoist && DoArraySegmentHoist(ValueType::GetObject(ObjectType::Int32Array), func)), doJsArraySegmentHoist( doArraySegmentHoist && DoArraySegmentHoist(ValueType::GetObject(ObjectType::Array), func)), doArrayLengthHoist( doArrayCheckHoist && DoArrayLengthHoist(func)), doEliminateArrayAccessHelperCall( doArrayCheckHoist && !PHASE_OFF(Js::EliminateArrayAccessHelperCallPhase, func)), doTrackRelativeIntBounds( doAggressiveIntTypeSpec && DoPathDependentValues() && !PHASE_OFF(Js::Phase::TrackRelativeIntBoundsPhase, func)), doBoundCheckElimination( doTrackRelativeIntBounds && !PHASE_OFF(Js::Phase::BoundCheckEliminationPhase, func)), doBoundCheckHoist( doEliminateArrayAccessHelperCall && doBoundCheckElimination && DoConstFold() && !PHASE_OFF(Js::Phase::BoundCheckHoistPhase, func) && (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsBoundCheckHoistDisabled(func->IsLoopBody()))), doLoopCountBasedBoundCheckHoist( doBoundCheckHoist && !PHASE_OFF(Js::Phase::LoopCountBasedBoundCheckHoistPhase, func) && (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsLoopCountBasedBoundCheckHoistDisabled(func->IsLoopBody()))), doPowIntIntTypeSpec( doAggressiveIntTypeSpec && (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsPowIntIntTypeSpecDisabled())), doTagChecks( (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsTagCheckDisabled())), isAsmJSFunc(func->GetJITFunctionBody()->IsAsmJsMode()) { } void GlobOpt::BackwardPass(Js::Phase tag) { BEGIN_CODEGEN_PHASE(this->func, tag); ::BackwardPass backwardPass(this->func, this, tag); backwardPass.Optimize(); END_CODEGEN_PHASE(this->func, tag); } void GlobOpt::Optimize() { this->objectTypeSyms = nullptr; this->func->argInsCount = this->func->GetInParamsCount(); if (!func->GetJITFunctionBody()->IsAsmJsMode()) { // Don't include "this" pointer in the count when not in AsmJs mode (AsmJS does not have "this"). this->func->argInsCount--; } if (!func->DoGlobOpt()) { this->lengthEquivBv = nullptr; this->argumentsEquivBv = nullptr; this->callerEquivBv = nullptr; // Still need to run the dead store phase to calculate the live reg on back edge this->BackwardPass(Js::DeadStorePhase); CannotAllocateArgumentsObjectOnStack(nullptr); return; } { this->lengthEquivBv = this->func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::length, nullptr); // Used to kill live "length" properties this->argumentsEquivBv = func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::arguments, nullptr); // Used to kill live "arguments" properties this->callerEquivBv = func->m_symTable->m_propertyEquivBvMap->Lookup(Js::PropertyIds::caller, nullptr); // Used to kill live "caller" properties // The backward phase needs the glob opt's allocator to allocate the propertyTypeValueMap // in GlobOpt::EnsurePropertyTypeValue and ranges of instructions where int overflow may be ignored. // (see BackwardPass::TrackIntUsage) PageAllocator * pageAllocator = this->func->m_alloc->GetPageAllocator(); NoRecoverMemoryJitArenaAllocator localAlloc(_u("BE-GlobOpt"), pageAllocator, Js::Throw::OutOfMemory); this->alloc = &localAlloc; NoRecoverMemoryJitArenaAllocator localTempAlloc(_u("BE-GlobOpt temp"), pageAllocator, Js::Throw::OutOfMemory); this->tempAlloc = &localTempAlloc; // The forward passes use info (upwardExposedUses) from the backward pass. This info // isn't available for some of the symbols created during the backward pass, or the forward pass. // Keep track of the last symbol for which we're guaranteed to have data. this->maxInitialSymID = this->func->m_symTable->GetMaxSymID(); #if DBG this->BackwardPass(Js::CaptureByteCodeRegUsePhase); #endif this->BackwardPass(Js::BackwardPhase); this->ForwardPass(); this->BackwardPass(Js::DeadStorePhase); } this->TailDupPass(); } bool GlobOpt::ShouldExpectConventionalArrayIndexValue(IR::IndirOpnd *const indirOpnd) { Assert(indirOpnd); if(!indirOpnd->GetIndexOpnd()) { return indirOpnd->GetOffset() >= 0; } IR::RegOpnd *const indexOpnd = indirOpnd->GetIndexOpnd(); if(indexOpnd->m_sym->m_isNotNumber) { // Typically, single-def or any sym-specific information for type-specialized syms should not be used because all of // their defs will not have been accounted for until after the forward pass. But m_isNotNumber is only ever changed from // false to true, so it's okay in this case. return false; } StackSym *indexVarSym = indexOpnd->m_sym; if(indexVarSym->IsTypeSpec()) { indexVarSym = indexVarSym->GetVarEquivSym(nullptr); Assert(indexVarSym); } else if(!IsLoopPrePass()) { // Don't use single-def info or const flags for type-specialized syms, as all of their defs will not have been accounted // for until after the forward pass. Also, don't use the const flags in a loop prepass because the const flags may not // be up-to-date. if (indexOpnd->IsNotInt()) { return false; } StackSym *const indexSym = indexOpnd->m_sym; if(indexSym->IsIntConst()) { return indexSym->GetIntConstValue() >= 0; } } Value *const indexValue = CurrentBlockData()->FindValue(indexVarSym); if(!indexValue) { // Treat it as Uninitialized, assume it's going to be valid return true; } ValueInfo *const indexValueInfo = indexValue->GetValueInfo(); int32 indexConstantValue; if(indexValueInfo->TryGetIntConstantValue(&indexConstantValue)) { return indexConstantValue >= 0; } if(indexValueInfo->IsUninitialized()) { // Assume it's going to be valid return true; } return indexValueInfo->HasBeenNumber() && !indexValueInfo->HasBeenFloat(); } // // Either result is float or 1/x or cst1/cst2 where cst1%cst2 != 0 // ValueType GlobOpt::GetDivValueType(IR::Instr* instr, Value* src1Val, Value* src2Val, bool specialize) { ValueInfo *src1ValueInfo = (src1Val ? src1Val->GetValueInfo() : nullptr); ValueInfo *src2ValueInfo = (src2Val ? src2Val->GetValueInfo() : nullptr); if (instr->IsProfiledInstr() && instr->m_func->HasProfileInfo()) { ValueType resultType = instr->m_func->GetReadOnlyProfileInfo()->GetDivProfileInfo(static_cast(instr->AsProfiledInstr()->u.profileId)); if (resultType.IsLikelyInt()) { if (specialize && src1ValueInfo && src2ValueInfo && ((src1ValueInfo->IsInt() && src2ValueInfo->IsInt()) || (this->DoDivIntTypeSpec() && src1ValueInfo->IsLikelyInt() && src2ValueInfo->IsLikelyInt()))) { return ValueType::GetInt(true); } return resultType; } // Consider: Checking that the sources are numbers. if (resultType.IsLikelyFloat()) { return ValueType::Float; } return resultType; } int32 src1IntConstantValue; if(!src1ValueInfo || !src1ValueInfo->TryGetIntConstantValue(&src1IntConstantValue)) { return ValueType::Number; } if (src1IntConstantValue == 1) { return ValueType::Float; } int32 src2IntConstantValue; if(!src2Val || !src2ValueInfo->TryGetIntConstantValue(&src2IntConstantValue)) { return ValueType::Number; } if (src2IntConstantValue // Avoid divide by zero && !(src1IntConstantValue == 0x80000000 && src2IntConstantValue == -1) // Avoid integer overflow && (src1IntConstantValue % src2IntConstantValue) != 0) { return ValueType::Float; } return ValueType::Number; } void GlobOpt::ForwardPass() { BEGIN_CODEGEN_PHASE(this->func, Js::ForwardPhase); #if DBG_DUMP if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::GlobOptPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId())) { this->func->DumpHeader(); } if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::GlobOptPhase)) { this->TraceSettings(); } #endif // GetConstantCount() gives us the right size to pick for the SparseArray, but we may need more if we've inlined // functions with constants. There will be a gap in the symbol numbering between the main constants and // the inlined ones, so we'll most likely need a new array chunk. Make the min size of the array chunks be 64 // in case we have a main function with very few constants and a bunch of constants from inlined functions. this->byteCodeConstantValueArray = SparseArray::New(this->alloc, max(this->func->GetJITFunctionBody()->GetConstCount(), 64U)); this->byteCodeConstantValueNumbersBv = JitAnew(this->alloc, BVSparse, this->alloc); this->tempBv = JitAnew(this->alloc, BVSparse, this->alloc); this->prePassCopyPropSym = JitAnew(this->alloc, BVSparse, this->alloc); this->slotSyms = JitAnew(this->alloc, BVSparse, this->alloc); this->byteCodeUses = nullptr; this->propertySymUse = nullptr; // changedSymsAfterIncBailoutCandidate helps track building incremental bailout in ForwardPass this->changedSymsAfterIncBailoutCandidate = JitAnew(alloc, BVSparse, alloc); this->auxSlotPtrSyms = JitAnew(alloc, BVSparse, alloc); #if DBG this->byteCodeUsesBeforeOpt = JitAnew(this->alloc, BVSparse, this->alloc); if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase) && this->DoFunctionFieldCopyProp()) { Output::Print(_u("TRACE: CanDoFieldCopyProp Func: ")); this->func->DumpFullFunctionName(); Output::Print(_u("\n")); } #endif OpndList localNoImplicitCallUsesToInsert(alloc); this->noImplicitCallUsesToInsert = &localNoImplicitCallUsesToInsert; IntConstantToStackSymMap localIntConstantToStackSymMap(alloc); this->intConstantToStackSymMap = &localIntConstantToStackSymMap; IntConstantToValueMap localIntConstantToValueMap(alloc); this->intConstantToValueMap = &localIntConstantToValueMap; Int64ConstantToValueMap localInt64ConstantToValueMap(alloc); this->int64ConstantToValueMap = &localInt64ConstantToValueMap; AddrConstantToValueMap localAddrConstantToValueMap(alloc); this->addrConstantToValueMap = &localAddrConstantToValueMap; StringConstantToValueMap localStringConstantToValueMap(alloc); this->stringConstantToValueMap = &localStringConstantToValueMap; SymIdToInstrMap localPrePassInstrMap(alloc); this->prePassInstrMap = &localPrePassInstrMap; ValueSetByValueNumber localValuesCreatedForClone(alloc, 64); this->valuesCreatedForClone = &localValuesCreatedForClone; ValueNumberPairToValueMap localValuesCreatedForMerge(alloc, 64); this->valuesCreatedForMerge = &localValuesCreatedForMerge; #if DBG BVSparse localFinishedStackLiteralInitFld(alloc); this->finishedStackLiteralInitFld = &localFinishedStackLiteralInitFld; #endif FOREACH_BLOCK_IN_FUNC_EDITING(block, this->func) { this->OptBlock(block); } NEXT_BLOCK_IN_FUNC_EDITING; if (!PHASE_OFF(Js::MemOpPhase, this->func)) { ProcessMemOp(); } this->noImplicitCallUsesToInsert = nullptr; this->intConstantToStackSymMap = nullptr; this->intConstantToValueMap = nullptr; this->int64ConstantToValueMap = nullptr; this->addrConstantToValueMap = nullptr; this->stringConstantToValueMap = nullptr; #if DBG this->finishedStackLiteralInitFld = nullptr; uint freedCount = 0; uint spilledCount = 0; #endif FOREACH_BLOCK_IN_FUNC(block, this->func) { #if DBG if (block->GetDataUseCount() == 0) { freedCount++; } else { spilledCount++; } #endif block->SetDataUseCount(0); if (block->cloneStrCandidates) { JitAdelete(this->alloc, block->cloneStrCandidates); block->cloneStrCandidates = nullptr; } } NEXT_BLOCK_IN_FUNC; // Make sure we free most of them. Assert(freedCount >= spilledCount); // this->alloc will be freed right after return, no need to free it here this->changedSymsAfterIncBailoutCandidate = nullptr; this->auxSlotPtrSyms = nullptr; END_CODEGEN_PHASE(this->func, Js::ForwardPhase); } void GlobOpt::OptBlock(BasicBlock *block) { if (this->func->m_fg->RemoveUnreachableBlock(block, this)) { GOPT_TRACE(_u("Removing unreachable block #%d\n"), block->GetBlockNum()); return; } Loop * loop = block->loop; if (loop && block->isLoopHeader) { if (loop != this->prePassLoop) { OptLoops(loop); if (!IsLoopPrePass() && loop->parent) { loop->fieldPRESymStores->Or(loop->parent->fieldPRESymStores); } if (!this->IsLoopPrePass() && DoFieldPRE(loop)) { // Note: !IsLoopPrePass means this was a root loop pre-pass. FieldPre() is called once per loop. this->FieldPRE(loop); // Re-optimize the landing pad BasicBlock *landingPad = loop->landingPad; this->isRecursiveCallOnLandingPad = true; this->OptBlock(landingPad); this->isRecursiveCallOnLandingPad = false; } } } this->currentBlock = block; PrepareLoopArrayCheckHoist(); block->MergePredBlocksValueMaps(this); this->intOverflowCurrentlyMattersInRange = true; this->intOverflowDoesNotMatterRange = this->currentBlock->intOverflowDoesNotMatterRange; if (!DoFieldCopyProp() && !DoFieldRefOpts()) { this->KillAllFields(CurrentBlockData()->liveFields); } this->tempAlloc->Reset(); if(loop && block->isLoopHeader) { loop->firstValueNumberInLoop = this->currentValue; } GOPT_TRACE_BLOCK(block, true); FOREACH_INSTR_IN_BLOCK_EDITING(instr, instrNext, block) { GOPT_TRACE_INSTRTRACE(instr); BailOutInfo* oldBailOutInfo = nullptr; bool isCheckAuxBailoutNeeded = this->func->IsJitInDebugMode() && !this->IsLoopPrePass(); if (isCheckAuxBailoutNeeded && instr->HasAuxBailOut() && !instr->HasBailOutInfo()) { oldBailOutInfo = instr->GetBailOutInfo(); Assert(oldBailOutInfo); } bool isInstrRemoved = false; instrNext = this->OptInstr(instr, &isInstrRemoved); // If we still have instrs with only aux bail out, convert aux bail out back to regular bail out and fill it. // During OptInstr some instr can be moved out to a different block, in this case bailout info is going to be replaced // with e.g. loop bailout info which is filled as part of processing that block, thus we don't need to fill it here. if (isCheckAuxBailoutNeeded && !isInstrRemoved && instr->HasAuxBailOut() && !instr->HasBailOutInfo()) { if (instr->GetBailOutInfo() == oldBailOutInfo) { instr->PromoteAuxBailOut(); FillBailOutInfo(block, instr); } else { AssertMsg(instr->GetBailOutInfo(), "With aux bailout, the bailout info should not be removed by OptInstr."); } } } NEXT_INSTR_IN_BLOCK_EDITING; GOPT_TRACE_BLOCK(block, false); if (block->loop) { if (IsLoopPrePass()) { if (DoBoundCheckHoist()) { DetectUnknownChangesToInductionVariables(&block->globOptData); } } else { isPerformingLoopBackEdgeCompensation = true; Assert(this->tempBv->IsEmpty()); BVSparse tempBv2(this->tempAlloc); // On loop back-edges, we need to restore the state of the type specialized // symbols to that of the loop header. FOREACH_SUCCESSOR_BLOCK(succ, block) { if (succ->isLoopHeader && succ->loop->IsDescendentOrSelf(block->loop)) { BVSparse *liveOnBackEdge = block->loop->regAlloc.liveOnBackEdgeSyms; liveOnBackEdge->Or(block->loop->fieldPRESymStores); this->tempBv->Minus(block->loop->varSymsOnEntry, block->globOptData.liveVarSyms); this->tempBv->And(liveOnBackEdge); this->ToVar(this->tempBv, block); // Lossy int in the loop header, and no int on the back-edge - need a lossy conversion to int this->tempBv->Minus(block->loop->lossyInt32SymsOnEntry, block->globOptData.liveInt32Syms); this->tempBv->And(liveOnBackEdge); this->ToInt32(this->tempBv, block, true /* lossy */); // Lossless int in the loop header, and no lossless int on the back-edge - need a lossless conversion to int this->tempBv->Minus(block->loop->int32SymsOnEntry, block->loop->lossyInt32SymsOnEntry); tempBv2.Minus(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms); this->tempBv->Minus(&tempBv2); this->tempBv->And(liveOnBackEdge); this->ToInt32(this->tempBv, block, false /* lossy */); this->tempBv->Minus(block->loop->float64SymsOnEntry, block->globOptData.liveFloat64Syms); this->tempBv->And(liveOnBackEdge); this->ToFloat64(this->tempBv, block); // For ints and floats, go aggressive and type specialize in the landing pad any symbol which was specialized on // entry to the loop body (in the loop header), and is still specialized on this tail, but wasn't specialized in // the landing pad. // Lossy int in the loop header and no int in the landing pad - need a lossy conversion to int // (entry.lossyInt32 - landingPad.int32) this->tempBv->Minus(block->loop->lossyInt32SymsOnEntry, block->loop->landingPad->globOptData.liveInt32Syms); this->tempBv->And(liveOnBackEdge); this->ToInt32(this->tempBv, block->loop->landingPad, true /* lossy */); // Lossless int in the loop header, and no lossless int in the landing pad - need a lossless conversion to int // ((entry.int32 - entry.lossyInt32) - (landingPad.int32 - landingPad.lossyInt32)) this->tempBv->Minus(block->loop->int32SymsOnEntry, block->loop->lossyInt32SymsOnEntry); tempBv2.Minus( block->loop->landingPad->globOptData.liveInt32Syms, block->loop->landingPad->globOptData.liveLossyInt32Syms); this->tempBv->Minus(&tempBv2); this->tempBv->And(liveOnBackEdge); this->ToInt32(this->tempBv, block->loop->landingPad, false /* lossy */); // ((entry.float64 - landingPad.float64) & block.float64) this->tempBv->Minus(block->loop->float64SymsOnEntry, block->loop->landingPad->globOptData.liveFloat64Syms); this->tempBv->And(block->globOptData.liveFloat64Syms); this->tempBv->And(liveOnBackEdge); this->ToFloat64(this->tempBv, block->loop->landingPad); if (block->loop->symsRequiringCompensationToMergedValueInfoMap) { InsertValueCompensation(block, succ, block->loop->symsRequiringCompensationToMergedValueInfoMap); } // Now that we're done with the liveFields within this loop, trim the set to those syms // that the backward pass told us were live out of the loop. // This assumes we have no further need of the liveFields within the loop. if (block->loop->liveOutFields) { block->globOptData.liveFields->And(block->loop->liveOutFields); } } } NEXT_SUCCESSOR_BLOCK; this->tempBv->ClearAll(); isPerformingLoopBackEdgeCompensation = false; } } block->PathDepBranchFolding(this); #if DBG // The set of live lossy int32 syms should be a subset of all live int32 syms this->tempBv->And(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms); Assert(this->tempBv->Count() == block->globOptData.liveLossyInt32Syms->Count()); // The set of live lossy int32 syms should be a subset of live var or float syms (var or float sym containing the lossless // value of the sym should be live) this->tempBv->Or(block->globOptData.liveVarSyms, block->globOptData.liveFloat64Syms); this->tempBv->And(block->globOptData.liveLossyInt32Syms); Assert(this->tempBv->Count() == block->globOptData.liveLossyInt32Syms->Count()); this->tempBv->ClearAll(); Assert(this->currentBlock == block); #endif } void GlobOpt::OptLoops(Loop *loop) { Assert(loop != nullptr); #if DBG if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase) && !DoFunctionFieldCopyProp() && DoFieldCopyProp(loop)) { Output::Print(_u("TRACE: CanDoFieldCopyProp Loop: ")); this->func->DumpFullFunctionName(); uint loopNumber = loop->GetLoopNumber(); Assert(loopNumber != Js::LoopHeader::NoLoop); Output::Print(_u(" Loop: %d\n"), loopNumber); } #endif Loop *previousLoop = this->prePassLoop; this->prePassLoop = loop; if (previousLoop == nullptr) { Assert(this->rootLoopPrePass == nullptr); this->rootLoopPrePass = loop; this->prePassInstrMap->Clear(); if (loop->parent == nullptr) { // Outer most loop... this->prePassCopyPropSym->ClearAll(); } } Assert(loop->symsAssignedToInLoop != nullptr); if (loop->symsUsedBeforeDefined == nullptr) { loop->symsUsedBeforeDefined = JitAnew(alloc, BVSparse, this->alloc); loop->likelyIntSymsUsedBeforeDefined = JitAnew(alloc, BVSparse, this->alloc); loop->likelyNumberSymsUsedBeforeDefined = JitAnew(alloc, BVSparse, this->alloc); loop->forceFloat64SymsOnEntry = JitAnew(this->alloc, BVSparse, this->alloc); loop->symsDefInLoop = JitAnew(this->alloc, BVSparse, this->alloc); loop->fieldKilled = JitAnew(alloc, BVSparse, this->alloc); loop->fieldPRESymStores = JitAnew(alloc, BVSparse, this->alloc); loop->allFieldsKilled = false; } else { loop->symsUsedBeforeDefined->ClearAll(); loop->likelyIntSymsUsedBeforeDefined->ClearAll(); loop->likelyNumberSymsUsedBeforeDefined->ClearAll(); loop->forceFloat64SymsOnEntry->ClearAll(); loop->symsDefInLoop->ClearAll(); loop->fieldKilled->ClearAll(); loop->allFieldsKilled = false; loop->initialValueFieldMap.Reset(); } FOREACH_BLOCK_IN_LOOP(block, loop) { block->SetDataUseCount(block->GetSuccList()->Count()); OptBlock(block); } NEXT_BLOCK_IN_LOOP; if (previousLoop == nullptr) { Assert(this->rootLoopPrePass == loop); this->rootLoopPrePass = nullptr; } this->prePassLoop = previousLoop; } void GlobOpt::TailDupPass() { FOREACH_LOOP_IN_FUNC_EDITING(loop, this->func) { BasicBlock* header = loop->GetHeadBlock(); BasicBlock* loopTail = nullptr; FOREACH_PREDECESSOR_BLOCK(pred, header) { if (loop->IsDescendentOrSelf(pred->loop)) { loopTail = pred; break; } } NEXT_PREDECESSOR_BLOCK; if (loopTail) { AssertMsg(loopTail->GetLastInstr()->IsBranchInstr(), "LastInstr of loop should always be a branch no?"); if (!loopTail->GetPredList()->HasOne()) { TryTailDup(loopTail->GetLastInstr()->AsBranchInstr()); } } } NEXT_LOOP_IN_FUNC_EDITING; } bool GlobOpt::TryTailDup(IR::BranchInstr *tailBranch) { if (PHASE_OFF(Js::TailDupPhase, tailBranch->m_func->GetTopFunc())) { return false; } if (tailBranch->IsConditional()) { return false; } IR::Instr *instr; uint instrCount = 0; for (instr = tailBranch->GetPrevRealInstrOrLabel(); !instr->IsLabelInstr(); instr = instr->GetPrevRealInstrOrLabel()) { if (instr->HasBailOutInfo()) { break; } if (!OpCodeAttr::CanCSE(instr->m_opcode)) { // Consider: We could be more aggressive here break; } instrCount++; if (instrCount > 1) { // Consider: If copy handled single-def tmps renaming, we could do more instrs break; } } if (!instr->IsLabelInstr()) { return false; } IR::LabelInstr *mergeLabel = instr->AsLabelInstr(); IR::Instr *mergeLabelPrev = mergeLabel->m_prev; // Skip unreferenced labels while (mergeLabelPrev->IsLabelInstr() && mergeLabelPrev->AsLabelInstr()->labelRefs.Empty()) { mergeLabelPrev = mergeLabelPrev->m_prev; } BasicBlock* labelBlock = mergeLabel->GetBasicBlock(); uint origPredCount = labelBlock->GetPredList()->Count(); uint dupCount = 0; // We are good to go. Let's do the tail duplication. FOREACH_SLISTCOUNTED_ENTRY_EDITING(IR::BranchInstr*, branchEntry, &mergeLabel->labelRefs, iter) { if (branchEntry->IsUnconditional() && !branchEntry->IsMultiBranch() && branchEntry != mergeLabelPrev && branchEntry != tailBranch) { for (instr = mergeLabel->m_next; instr != tailBranch; instr = instr->m_next) { branchEntry->InsertBefore(instr->Copy()); } instr = branchEntry; branchEntry->ReplaceTarget(mergeLabel, tailBranch->GetTarget()); while(!instr->IsLabelInstr()) { instr = instr->m_prev; } BasicBlock* branchBlock = instr->AsLabelInstr()->GetBasicBlock(); labelBlock->RemovePred(branchBlock, func->m_fg); func->m_fg->AddEdge(branchBlock, tailBranch->GetTarget()->GetBasicBlock()); dupCount++; } } NEXT_SLISTCOUNTED_ENTRY_EDITING; // If we've duplicated everywhere, tail block is dead and should be removed. if (dupCount == origPredCount) { AssertMsg(mergeLabel->labelRefs.Empty(), "Should not remove block with referenced label."); func->m_fg->RemoveBlock(labelBlock, nullptr, true); } return true; } void GlobOpt::ToVar(BVSparse *bv, BasicBlock *block, IR::Instr* insertBeforeInstr /* = nullptr */) { FOREACH_BITSET_IN_SPARSEBV(id, bv) { StackSym *stackSym = this->func->m_symTable->FindStackSym(id); IR::RegOpnd *newOpnd = IR::RegOpnd::New(stackSym, TyVar, this->func); IR::Instr* lastInstr = block->GetLastInstr(); if (insertBeforeInstr != nullptr) { this->ToVar(insertBeforeInstr, newOpnd, block, nullptr, false); } else if (lastInstr->IsBranchInstr() || lastInstr->m_opcode == Js::OpCode::BailTarget) { // If branch is using this symbol, hoist the operand as the ToVar load will get // inserted right before the branch. IR::Opnd *src1 = lastInstr->GetSrc1(); if (src1) { if (src1->IsRegOpnd() && src1->AsRegOpnd()->m_sym == stackSym) { lastInstr->HoistSrc1(Js::OpCode::Ld_A); } IR::Opnd *src2 = lastInstr->GetSrc2(); if (src2) { if (src2->IsRegOpnd() && src2->AsRegOpnd()->m_sym == stackSym) { lastInstr->HoistSrc2(Js::OpCode::Ld_A); } } } this->ToVar(lastInstr, newOpnd, block, nullptr, false); } else { IR::Instr *lastNextInstr = lastInstr->m_next; this->ToVar(lastNextInstr, newOpnd, block, nullptr, false); } } NEXT_BITSET_IN_SPARSEBV; } void GlobOpt::ToInt32(BVSparse *bv, BasicBlock *block, bool lossy, IR::Instr *insertBeforeInstr) { return this->ToTypeSpec(bv, block, TyInt32, IR::BailOutIntOnly, lossy, insertBeforeInstr); } void GlobOpt::ToFloat64(BVSparse *bv, BasicBlock *block) { return this->ToTypeSpec(bv, block, TyFloat64, IR::BailOutNumberOnly); } void GlobOpt::ToTypeSpec(BVSparse *bv, BasicBlock *block, IRType toType, IR::BailOutKind bailOutKind, bool lossy, IR::Instr *insertBeforeInstr) { FOREACH_BITSET_IN_SPARSEBV(id, bv) { StackSym *stackSym = this->func->m_symTable->FindStackSym(id); IRType fromType = TyIllegal; // Win8 bug: 757126. If we are trying to type specialize the arguments object, // let's make sure stack args optimization is not enabled. This is a problem, particularly, // if the instruction comes from an unreachable block. In other cases, the pass on the // instruction itself should disable arguments object optimization. if(block->globOptData.argObjSyms && block->globOptData.IsArgumentsSymID(id)) { CannotAllocateArgumentsObjectOnStack(nullptr); } if (block->globOptData.liveVarSyms->Test(id)) { fromType = TyVar; } else if (block->globOptData.liveInt32Syms->Test(id) && !block->globOptData.liveLossyInt32Syms->Test(id)) { fromType = TyInt32; stackSym = stackSym->GetInt32EquivSym(this->func); } else if (block->globOptData.liveFloat64Syms->Test(id)) { fromType = TyFloat64; stackSym = stackSym->GetFloat64EquivSym(this->func); } else { Assert(UNREACHED); } IR::RegOpnd *newOpnd = IR::RegOpnd::New(stackSym, fromType, this->func); this->ToTypeSpecUse(nullptr, newOpnd, block, nullptr, nullptr, toType, bailOutKind, lossy, insertBeforeInstr); } NEXT_BITSET_IN_SPARSEBV; } void GlobOpt::PRE::FindPossiblePRECandidates(Loop *loop, JitArenaAllocator *alloc) { // Find the set of PRE candidates BasicBlock *loopHeader = loop->GetHeadBlock(); PRECandidates *candidates = nullptr; bool firstBackEdge = true; FOREACH_PREDECESSOR_BLOCK(blockPred, loopHeader) { if (!loop->IsDescendentOrSelf(blockPred->loop)) { // Not a loop back-edge continue; } if (firstBackEdge) { candidates = this->globOpt->FindBackEdgePRECandidates(blockPred, alloc); } else { blockPred->globOptData.RemoveUnavailableCandidates(candidates); } } NEXT_PREDECESSOR_BLOCK; this->candidates = candidates; } BOOL GlobOpt::PRE::PreloadPRECandidate(Loop *loop, GlobHashBucket* candidate) { // Insert a load for each field PRE candidate. PropertySym *propertySym = candidate->value->AsPropertySym(); if (!candidates->candidatesToProcess->TestAndClear(propertySym->m_id)) { return false; } Value * propSymValueOnBackEdge = candidate->element; StackSym *objPtrSym = propertySym->m_stackSym; Sym * objPtrCopyPropSym = nullptr; if (!loop->landingPad->globOptData.IsLive(objPtrSym)) { if (PHASE_OFF(Js::MakeObjSymLiveInLandingPadPhase, this->globOpt->func)) { return false; } if (objPtrSym->IsSingleDef()) { // We can still try to do PRE if the object sym is single def, even if its not live in the landing pad. // We'll have to add a def instruction for the object sym in the landing pad, and then we can continue // pre-loading the current PRE candidate. // Case in point: // $L1 // value|symStore // t1 = o.x (v1|t3) // t2 = t1.y (v2|t4) <-- t1 is not live in the loop landing pad // jmp $L1 if (!InsertSymDefinitionInLandingPad(objPtrSym, loop, &objPtrCopyPropSym)) { #if DBG_DUMP TraceFailedPreloadInLandingPad(loop, propertySym, _u("Failed to insert load of object sym in landing pad")); #endif return false; } } else { #if DBG_DUMP TraceFailedPreloadInLandingPad(loop, propertySym, _u("Object sym not live in landing pad and not single-def")); #endif return false; } } Assert(loop->landingPad->globOptData.IsLive(objPtrSym)); BasicBlock *landingPad = loop->landingPad; Sym *symStore = propSymValueOnBackEdge->GetValueInfo()->GetSymStore(); // The symStore can't be live into the loop // The symStore needs to still have the same value Assert(symStore && symStore->IsStackSym()); if (loop->landingPad->globOptData.IsLive(symStore)) { // May have already been hoisted: // o.x = t1; // o.y = t1; return false; } Value *landingPadValue = landingPad->globOptData.FindValue(propertySym); // Value should be added as initial value or already be there. Assert(landingPadValue); IR::Instr * ldInstrInLoop = this->globOpt->prePassInstrMap->Lookup(propertySym->m_id, nullptr); Assert(ldInstrInLoop); Assert(ldInstrInLoop->GetDst() == nullptr); // Create instr to put in landing pad for compensation Assert(IsPREInstrCandidateLoad(ldInstrInLoop->m_opcode)); IR::Instr * ldInstr = InsertPropertySymPreloadInLandingPad(ldInstrInLoop, loop, propertySym); if (!ldInstr) { return false; } Assert(ldInstr->GetDst() == nullptr); ldInstr->SetDst(IR::RegOpnd::New(symStore->AsStackSym(), TyVar, this->globOpt->func)); loop->fieldPRESymStores->Set(symStore->m_id); landingPad->globOptData.liveVarSyms->Set(symStore->m_id); Value * objPtrValue = landingPad->globOptData.FindValue(objPtrSym); objPtrCopyPropSym = objPtrCopyPropSym ? objPtrCopyPropSym : objPtrValue ? landingPad->globOptData.GetCopyPropSym(objPtrSym, objPtrValue) : nullptr; if (objPtrCopyPropSym) { // If we inserted T4 = T1.y, and T3 is the copy prop sym for T1 in the landing pad, we need T3.y // to be live on back edges to have the merge produce a value for T3.y. Having a value for T1.y // produced from the merge is not enough as the T1.y in the loop will get obj-ptr-copy-propped to // T3.y // T3.y PropertySym *newPropSym = PropertySym::FindOrCreate( objPtrCopyPropSym->m_id, propertySym->m_propertyId, propertySym->GetPropertyIdIndex(), propertySym->GetInlineCacheIndex(), propertySym->m_fieldKind, this->globOpt->func); if (!landingPad->globOptData.FindValue(newPropSym)) { landingPad->globOptData.SetValue(landingPadValue, newPropSym); landingPad->globOptData.liveFields->Set(newPropSym->m_id); MakePropertySymLiveOnBackEdges(newPropSym, loop, propSymValueOnBackEdge); } } ValueType valueType(ValueType::Uninitialized); Value *initialValue = nullptr; if (loop->initialValueFieldMap.TryGetValue(propertySym, &initialValue)) { if (ldInstr->IsProfiledInstr()) { if (initialValue->GetValueNumber() == propSymValueOnBackEdge->GetValueNumber()) { if (propSymValueOnBackEdge->GetValueInfo()->IsUninitialized()) { valueType = ldInstr->AsProfiledInstr()->u.FldInfo().valueType; } else { valueType = propSymValueOnBackEdge->GetValueInfo()->Type(); } } else { valueType = ValueType::Uninitialized; } ldInstr->AsProfiledInstr()->u.FldInfo().valueType = valueType; } } else { valueType = landingPadValue->GetValueInfo()->Type(); } loop->symsUsedBeforeDefined->Set(symStore->m_id); if (valueType.IsLikelyNumber()) { loop->likelyNumberSymsUsedBeforeDefined->Set(symStore->m_id); if (globOpt->DoAggressiveIntTypeSpec() ? valueType.IsLikelyInt() : valueType.IsInt()) { // Can only force int conversions in the landing pad based on likely-int values if aggressive int type // specialization is enabled loop->likelyIntSymsUsedBeforeDefined->Set(symStore->m_id); } } #if DBG_DUMP if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldPREPhase, this->globOpt->func->GetSourceContextId(), this->globOpt->func->GetLocalFunctionId())) { Output::Print(_u("** TRACE: Field PRE: field pre-loaded in landing pad of loop head #%-3d: "), loop->GetHeadBlock()->GetBlockNum()); ldInstr->Dump(); Output::Print(_u("\n")); Output::Flush(); } #endif return true; } void GlobOpt::PRE::PreloadPRECandidates(Loop *loop) { // Insert loads in landing pad for field PRE candidates. Iterate while(changed) // for the o.x.y cases. BOOL changed = true; if (!candidates || !candidates->candidatesList) { return; } Assert(loop->landingPad->GetFirstInstr() == loop->landingPad->GetLastInstr()); while (changed) { changed = false; FOREACH_SLIST_ENTRY_EDITING(GlobHashBucket*, candidate, (SList*)candidates->candidatesList, iter) { if (this->PreloadPRECandidate(loop, candidate)) { changed = true; iter.RemoveCurrent(); } if (PHASE_TRACE(Js::FieldPREPhase, this->globOpt->func)) { Output::Print(_u("============================\n")); Output::Flush(); } } NEXT_SLIST_ENTRY_EDITING; } } void GlobOpt::FieldPRE(Loop *loop) { if (!DoFieldPRE(loop)) { return; } GlobOpt::PRE pre(this); pre.FieldPRE(loop); } void GlobOpt::InsertValueCompensation( BasicBlock *const predecessor, BasicBlock *const successor, const SymToValueInfoMap *symsRequiringCompensationToMergedValueInfoMap) { Assert(predecessor); Assert(successor); AssertOrFailFast(predecessor != successor); Assert(symsRequiringCompensationToMergedValueInfoMap->Count() != 0); IR::Instr *insertBeforeInstr = predecessor->GetLastInstr(); Func *const func = insertBeforeInstr->m_func; bool setLastInstrInPredecessor; // If this is a loop back edge, and the successor has been completed, don't attempt to update its block data. // The update is unnecessary, and the data has likely been freed. bool updateSuccessorBlockData = !this->isPerformingLoopBackEdgeCompensation || successor->GetDataUseCount() > 0; if(insertBeforeInstr->IsBranchInstr() || insertBeforeInstr->m_opcode == Js::OpCode::BailTarget) { // Don't insert code between the branch and the corresponding ByteCodeUses instructions while(insertBeforeInstr->m_prev->m_opcode == Js::OpCode::ByteCodeUses) { insertBeforeInstr = insertBeforeInstr->m_prev; } setLastInstrInPredecessor = false; } else { // Insert at the end of the block and set the last instruction Assert(insertBeforeInstr->m_next); insertBeforeInstr = insertBeforeInstr->m_next; // Instruction after the last instruction in the predecessor setLastInstrInPredecessor = true; } GlobOptBlockData &predecessorBlockData = predecessor->globOptData; GlobOptBlockData &successorBlockData = successor->globOptData; struct DelayChangeValueInfo { Value* predecessorValue; ArrayValueInfo* valueInfo; void ChangeValueInfo(BasicBlock* predecessor, GlobOpt* g) { g->ChangeValueInfo( predecessor, predecessorValue, valueInfo, false /*allowIncompatibleType*/, true /*compensated*/); } }; JsUtil::List delayChangeValueInfo(alloc); for(auto it = symsRequiringCompensationToMergedValueInfoMap->GetIterator(); it.IsValid(); it.MoveNext()) { const auto &entry = it.Current(); Sym *const sym = entry.Key(); Value *const predecessorValue = predecessorBlockData.FindValue(sym); Assert(predecessorValue); ValueInfo *const predecessorValueInfo = predecessorValue->GetValueInfo(); // Currently, array value infos are the only ones that require compensation based on values Assert(predecessorValueInfo->IsAnyOptimizedArray()); const ArrayValueInfo *const predecessorArrayValueInfo = predecessorValueInfo->AsArrayValueInfo(); StackSym *const predecessorHeadSegmentSym = predecessorArrayValueInfo->HeadSegmentSym(); StackSym *const predecessorHeadSegmentLengthSym = predecessorArrayValueInfo->HeadSegmentLengthSym(); StackSym *const predecessorLengthSym = predecessorArrayValueInfo->LengthSym(); ValueInfo *const mergedValueInfo = entry.Value(); const ArrayValueInfo *const mergedArrayValueInfo = mergedValueInfo->AsArrayValueInfo(); StackSym *const mergedHeadSegmentSym = mergedArrayValueInfo->HeadSegmentSym(); StackSym *const mergedHeadSegmentLengthSym = mergedArrayValueInfo->HeadSegmentLengthSym(); StackSym *const mergedLengthSym = mergedArrayValueInfo->LengthSym(); Assert(!mergedHeadSegmentSym || predecessorHeadSegmentSym); Assert(!mergedHeadSegmentLengthSym || predecessorHeadSegmentLengthSym); Assert(!mergedLengthSym || predecessorLengthSym); bool compensated = false; if(mergedHeadSegmentSym && predecessorHeadSegmentSym != mergedHeadSegmentSym) { IR::Instr *const newInstr = IR::Instr::New( Js::OpCode::Ld_A, IR::RegOpnd::New(mergedHeadSegmentSym, mergedHeadSegmentSym->GetType(), func), IR::RegOpnd::New(predecessorHeadSegmentSym, predecessorHeadSegmentSym->GetType(), func), func); newInstr->GetDst()->SetIsJITOptimizedReg(true); newInstr->GetSrc1()->SetIsJITOptimizedReg(true); newInstr->SetByteCodeOffset(insertBeforeInstr); insertBeforeInstr->InsertBefore(newInstr); compensated = true; } if(mergedHeadSegmentLengthSym && predecessorHeadSegmentLengthSym != mergedHeadSegmentLengthSym) { IR::Instr *const newInstr = IR::Instr::New( Js::OpCode::Ld_A, IR::RegOpnd::New(mergedHeadSegmentLengthSym, mergedHeadSegmentLengthSym->GetType(), func), IR::RegOpnd::New(predecessorHeadSegmentLengthSym, predecessorHeadSegmentLengthSym->GetType(), func), func); newInstr->GetDst()->SetIsJITOptimizedReg(true); newInstr->GetSrc1()->SetIsJITOptimizedReg(true); newInstr->SetByteCodeOffset(insertBeforeInstr); insertBeforeInstr->InsertBefore(newInstr); compensated = true; // Merge the head segment length value Assert(predecessorBlockData.liveVarSyms->Test(predecessorHeadSegmentLengthSym->m_id)); predecessorBlockData.liveVarSyms->Set(mergedHeadSegmentLengthSym->m_id); Value *const predecessorHeadSegmentLengthValue = predecessorBlockData.FindValue(predecessorHeadSegmentLengthSym); Assert(predecessorHeadSegmentLengthValue); predecessorBlockData.SetValue(predecessorHeadSegmentLengthValue, mergedHeadSegmentLengthSym); if (updateSuccessorBlockData) { successorBlockData.liveVarSyms->Set(mergedHeadSegmentLengthSym->m_id); Value *const mergedHeadSegmentLengthValue = successorBlockData.FindValue(mergedHeadSegmentLengthSym); if(mergedHeadSegmentLengthValue) { Assert(mergedHeadSegmentLengthValue->GetValueNumber() != predecessorHeadSegmentLengthValue->GetValueNumber()); if(predecessorHeadSegmentLengthValue->GetValueInfo() != mergedHeadSegmentLengthValue->GetValueInfo()) { mergedHeadSegmentLengthValue->SetValueInfo( ValueInfo::MergeLikelyIntValueInfo( this->alloc, mergedHeadSegmentLengthValue, predecessorHeadSegmentLengthValue, mergedHeadSegmentLengthValue->GetValueInfo()->Type() .Merge(predecessorHeadSegmentLengthValue->GetValueInfo()->Type()))); } } else { successorBlockData.SetValue(CopyValue(predecessorHeadSegmentLengthValue), mergedHeadSegmentLengthSym); } } } if(mergedLengthSym && predecessorLengthSym != mergedLengthSym) { IR::Instr *const newInstr = IR::Instr::New( Js::OpCode::Ld_I4, IR::RegOpnd::New(mergedLengthSym, mergedLengthSym->GetType(), func), IR::RegOpnd::New(predecessorLengthSym, predecessorLengthSym->GetType(), func), func); newInstr->GetDst()->SetIsJITOptimizedReg(true); newInstr->GetSrc1()->SetIsJITOptimizedReg(true); newInstr->SetByteCodeOffset(insertBeforeInstr); insertBeforeInstr->InsertBefore(newInstr); compensated = true; // Merge the length value Assert(predecessorBlockData.liveVarSyms->Test(predecessorLengthSym->m_id)); predecessorBlockData.liveVarSyms->Set(mergedLengthSym->m_id); Value *const predecessorLengthValue = predecessorBlockData.FindValue(predecessorLengthSym); Assert(predecessorLengthValue); predecessorBlockData.SetValue(predecessorLengthValue, mergedLengthSym); if (updateSuccessorBlockData) { successorBlockData.liveVarSyms->Set(mergedLengthSym->m_id); Value *const mergedLengthValue = successorBlockData.FindValue(mergedLengthSym); if(mergedLengthValue) { Assert(mergedLengthValue->GetValueNumber() != predecessorLengthValue->GetValueNumber()); if(predecessorLengthValue->GetValueInfo() != mergedLengthValue->GetValueInfo()) { mergedLengthValue->SetValueInfo( ValueInfo::MergeLikelyIntValueInfo( this->alloc, mergedLengthValue, predecessorLengthValue, mergedLengthValue->GetValueInfo()->Type().Merge(predecessorLengthValue->GetValueInfo()->Type()))); } } else { successorBlockData.SetValue(CopyValue(predecessorLengthValue), mergedLengthSym); } } } if(compensated) { // Save the new ValueInfo for later. // We don't want other symbols needing compensation to see this new one delayChangeValueInfo.Add({ predecessorValue, ArrayValueInfo::New( alloc, predecessorValueInfo->Type(), mergedHeadSegmentSym ? mergedHeadSegmentSym : predecessorHeadSegmentSym, mergedHeadSegmentLengthSym ? mergedHeadSegmentLengthSym : predecessorHeadSegmentLengthSym, mergedLengthSym ? mergedLengthSym : predecessorLengthSym, predecessorValueInfo->GetSymStore()) }); } } // Once we've compensated all the symbols, update the new ValueInfo. delayChangeValueInfo.Map([predecessor, this](int, DelayChangeValueInfo d) { d.ChangeValueInfo(predecessor, this); }); if(setLastInstrInPredecessor) { predecessor->SetLastInstr(insertBeforeInstr->m_prev); } } bool GlobOpt::AreFromSameBytecodeFunc(IR::RegOpnd const* src1, IR::RegOpnd const* dst) const { Assert(this->func->m_symTable->FindStackSym(src1->m_sym->m_id) == src1->m_sym); Assert(this->func->m_symTable->FindStackSym(dst->m_sym->m_id) == dst->m_sym); if (dst->m_sym->HasByteCodeRegSlot() && src1->m_sym->HasByteCodeRegSlot()) { return src1->m_sym->GetByteCodeFunc() == dst->m_sym->GetByteCodeFunc(); } return false; } /* * This is for scope object removal along with Heap Arguments optimization. * We track several instructions to facilitate the removal of scope object. * - LdSlotArr - This instr is tracked to keep track of the formals array (the dest) * - InlineeStart - To keep track of the stack syms for the formals of the inlinee. */ void GlobOpt::TrackInstrsForScopeObjectRemoval(IR::Instr * instr) { IR::Opnd* dst = instr->GetDst(); IR::Opnd* src1 = instr->GetSrc1(); if (instr->m_opcode == Js::OpCode::Ld_A && src1->IsRegOpnd()) { AssertMsg(!instr->m_func->IsStackArgsEnabled() || !src1->IsScopeObjOpnd(instr->m_func), "There can be no aliasing for scope object."); } // The following is to track formals array for Stack Arguments optimization with Formals if (instr->m_func->IsStackArgsEnabled() && !this->IsLoopPrePass()) { if (instr->m_opcode == Js::OpCode::LdSlotArr) { if (instr->GetSrc1()->IsScopeObjOpnd(instr->m_func)) { AssertMsg(!instr->m_func->GetJITFunctionBody()->HasImplicitArgIns(), "No mapping is required in this case. So it should already be generating ArgIns."); instr->m_func->TrackFormalsArraySym(dst->GetStackSym()->m_id); } } else if (instr->m_opcode == Js::OpCode::InlineeStart) { Assert(instr->m_func->IsInlined()); Js::ArgSlot actualsCount = instr->m_func->actualCount - 1; Js::ArgSlot formalsCount = instr->m_func->GetJITFunctionBody()->GetInParamsCount() - 1; Func * func = instr->m_func; Func * inlinerFunc = func->GetParentFunc(); //Inliner's func IR::Instr * argOutInstr = instr->GetSrc2()->GetStackSym()->GetInstrDef(); //The argout immediately before the InlineeStart will be the ArgOut for NewScObject //So we don't want to track the stack sym for this argout.- Skipping it here. if (instr->m_func->IsInlinedConstructor()) { //PRE might introduce a second defintion for the Src1. So assert for the opcode only when it has single definition. Assert(argOutInstr->GetSrc1()->GetStackSym()->GetInstrDef() == nullptr || argOutInstr->GetSrc1()->GetStackSym()->GetInstrDef()->m_opcode == Js::OpCode::NewScObjectNoCtor); argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef(); } if (formalsCount < actualsCount) { Js::ArgSlot extraActuals = actualsCount - formalsCount; //Skipping extra actuals passed for (Js::ArgSlot i = 0; i < extraActuals; i++) { argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef(); } } StackSym * undefinedSym = nullptr; for (Js::ArgSlot param = formalsCount; param > 0; param--) { StackSym * argOutSym = nullptr; if (argOutInstr->GetSrc1()) { if (argOutInstr->GetSrc1()->IsRegOpnd()) { argOutSym = argOutInstr->GetSrc1()->GetStackSym(); } else { // We will always have ArgOut instr - so the source operand will not be removed. argOutSym = StackSym::New(inlinerFunc); IR::Opnd * srcOpnd = argOutInstr->GetSrc1(); IR::Opnd * dstOpnd = IR::RegOpnd::New(argOutSym, TyVar, inlinerFunc); IR::Instr * assignInstr = IR::Instr::New(Js::OpCode::Ld_A, dstOpnd, srcOpnd, inlinerFunc); instr->InsertBefore(assignInstr); } } Assert(!func->HasStackSymForFormal(param - 1)); if (param <= actualsCount) { Assert(argOutSym); func->TrackStackSymForFormalIndex(param - 1, argOutSym); argOutInstr = argOutInstr->GetSrc2()->GetStackSym()->GetInstrDef(); } else { /*When param is out of range of actuals count, load undefined*/ // TODO: saravind: This will insert undefined for each of the param not having an actual. - Clean up this by having a sym for undefined on func ? Assert(formalsCount > actualsCount); if (undefinedSym == nullptr) { undefinedSym = StackSym::New(inlinerFunc); IR::Opnd * srcOpnd = IR::AddrOpnd::New(inlinerFunc->GetScriptContextInfo()->GetUndefinedAddr(), IR::AddrOpndKindDynamicMisc, inlinerFunc); IR::Opnd * dstOpnd = IR::RegOpnd::New(undefinedSym, TyVar, inlinerFunc); IR::Instr * assignUndefined = IR::Instr::New(Js::OpCode::Ld_A, dstOpnd, srcOpnd, inlinerFunc); instr->InsertBefore(assignUndefined); } func->TrackStackSymForFormalIndex(param - 1, undefinedSym); } } } } } void GlobOpt::OptArguments(IR::Instr *instr) { IR::Opnd* dst = instr->GetDst(); IR::Opnd* src1 = instr->GetSrc1(); IR::Opnd* src2 = instr->GetSrc2(); TrackInstrsForScopeObjectRemoval(instr); if (!TrackArgumentsObject()) { return; } if (instr->HasAnyLoadHeapArgsOpCode()) { #ifdef ENABLE_DEBUG_CONFIG_OPTIONS if (instr->m_func->IsStackArgsEnabled()) { if (instr->GetSrc1()->IsRegOpnd() && instr->m_func->GetJITFunctionBody()->GetInParamsCount() > 1) { StackSym * scopeObjSym = instr->GetSrc1()->GetStackSym(); Assert(scopeObjSym); Assert(scopeObjSym->GetInstrDef()->m_opcode == Js::OpCode::InitCachedScope || scopeObjSym->GetInstrDef()->m_opcode == Js::OpCode::NewScopeObject); Assert(instr->m_func->GetScopeObjSym() == scopeObjSym); if (PHASE_VERBOSE_TRACE1(Js::StackArgFormalsOptPhase)) { Output::Print(_u("StackArgFormals : %s (%d) :Setting scopeObjSym in forward pass. \n"), instr->m_func->GetJITFunctionBody()->GetDisplayName(), instr->m_func->GetJITFunctionBody()->GetFunctionNumber()); Output::Flush(); } } } #endif if (instr->m_func->GetJITFunctionBody()->GetInParamsCount() != 1 && !instr->m_func->IsStackArgsEnabled()) { CannotAllocateArgumentsObjectOnStack(instr->m_func); } else { CurrentBlockData()->TrackArgumentsSym(dst->AsRegOpnd()); } return; } // Keep track of arguments objects and its aliases // LdHeapArguments loads the arguments object and Ld_A tracks the aliases. if ((instr->m_opcode == Js::OpCode::Ld_A || instr->m_opcode == Js::OpCode::BytecodeArgOutCapture) && (src1->IsRegOpnd() && CurrentBlockData()->IsArgumentsOpnd(src1))) { // In the debug mode, we don't want to optimize away the aliases. Since we may have to show them on the inspection. if (((!AreFromSameBytecodeFunc(src1->AsRegOpnd(), dst->AsRegOpnd()) || this->currentBlock->loop) && instr->m_opcode != Js::OpCode::BytecodeArgOutCapture) || this->func->IsJitInDebugMode()) { CannotAllocateArgumentsObjectOnStack(instr->m_func); return; } // Disable stack args if we are aliasing arguments inside try block to a writethrough symbol. // We don't have precise tracking of these symbols, so bailout couldn't know if it needs to restore arguments object or not after exception Region* tryRegion = this->currentRegion ? this->currentRegion->GetSelfOrFirstTryAncestor() : nullptr; if (tryRegion && tryRegion->GetType() == RegionTypeTry && tryRegion->writeThroughSymbolsSet && tryRegion->writeThroughSymbolsSet->Test(dst->AsRegOpnd()->m_sym->m_id)) { CannotAllocateArgumentsObjectOnStack(instr->m_func); return; } if(!dst->AsRegOpnd()->GetStackSym()->m_nonEscapingArgObjAlias) { CurrentBlockData()->TrackArgumentsSym(dst->AsRegOpnd()); } return; } if (!CurrentBlockData()->TestAnyArgumentsSym()) { // There are no syms to track yet, don't start tracking arguments sym. return; } // Avoid loop prepass if (this->currentBlock->loop && this->IsLoopPrePass()) { return; } SymID id = 0; switch(instr->m_opcode) { case Js::OpCode::LdElemI_A: case Js::OpCode::TypeofElem: { Assert(src1->IsIndirOpnd()); IR::RegOpnd *indexOpnd = src1->AsIndirOpnd()->GetIndexOpnd(); if (indexOpnd && CurrentBlockData()->IsArgumentsSymID(indexOpnd->m_sym->m_id)) { // Pathological test cases such as a[arguments] CannotAllocateArgumentsObjectOnStack(instr->m_func); return; } IR::RegOpnd *baseOpnd = src1->AsIndirOpnd()->GetBaseOpnd(); id = baseOpnd->m_sym->m_id; if (CurrentBlockData()->IsArgumentsSymID(id)) { instr->usesStackArgumentsObject = true; } break; } case Js::OpCode::LdLen_A: { Assert(src1->IsRegOpnd()); if(CurrentBlockData()->IsArgumentsOpnd(src1)) { instr->usesStackArgumentsObject = true; } break; } case Js::OpCode::ArgOut_A_InlineBuiltIn: { if (CurrentBlockData()->IsArgumentsOpnd(src1)) { instr->usesStackArgumentsObject = true; instr->m_func->unoptimizableArgumentsObjReference++; } if (CurrentBlockData()->IsArgumentsOpnd(src1) && src1->AsRegOpnd()->m_sym->GetInstrDef()->m_opcode == Js::OpCode::BytecodeArgOutCapture) { // Apply inlining results in such usage - this is to ignore this sym that is def'd by ByteCodeArgOutCapture // It's needed because we do not have block level merging of arguments object and this def due to inlining can turn off stack args opt. IR::Instr* builtinStart = instr->GetNextRealInstr(); if (builtinStart->m_opcode == Js::OpCode::InlineBuiltInStart) { IR::Opnd* builtinOpnd = builtinStart->GetSrc1(); if (builtinStart->GetSrc1()->IsAddrOpnd()) { Assert(builtinOpnd->AsAddrOpnd()->m_isFunction); Js::BuiltinFunction builtinFunction = Js::JavascriptLibrary::GetBuiltInForFuncInfo(((FixedFieldInfo*)builtinOpnd->AsAddrOpnd()->m_metadata)->GetLocalFuncId()); if (builtinFunction == Js::BuiltinFunction::JavascriptFunction_Apply) { CurrentBlockData()->ClearArgumentsSym(src1->AsRegOpnd()); instr->m_func->unoptimizableArgumentsObjReference--; } } else if (builtinOpnd->IsRegOpnd()) { if (builtinOpnd->AsRegOpnd()->m_sym->m_builtInIndex == Js::BuiltinFunction::JavascriptFunction_Apply) { CurrentBlockData()->ClearArgumentsSym(src1->AsRegOpnd()); instr->m_func->unoptimizableArgumentsObjReference--; } } } } break; } case Js::OpCode::BailOnNotStackArgs: case Js::OpCode::ArgOut_A_FromStackArgs: case Js::OpCode::BytecodeArgOutUse: { if (src1 && CurrentBlockData()->IsArgumentsOpnd(src1)) { instr->usesStackArgumentsObject = true; } break; } default: { // Super conservative here, if we see the arguments or any of its alias being used in any // other opcode just don't do this optimization. Revisit this to optimize further if we see any common // case is missed. if (src1) { if (src1->IsRegOpnd() || src1->IsSymOpnd() || src1->IsIndirOpnd()) { if (CurrentBlockData()->IsArgumentsOpnd(src1)) { #ifdef PERF_HINT if (PHASE_TRACE1(Js::PerfHintPhase)) { WritePerfHint(PerfHints::HeapArgumentsCreated, instr->m_func, instr->GetByteCodeOffset()); } #endif CannotAllocateArgumentsObjectOnStack(instr->m_func); return; } } } if (src2) { if (src2->IsRegOpnd() || src2->IsSymOpnd() || src2->IsIndirOpnd()) { if (CurrentBlockData()->IsArgumentsOpnd(src2)) { #ifdef PERF_HINT if (PHASE_TRACE1(Js::PerfHintPhase)) { WritePerfHint(PerfHints::HeapArgumentsCreated, instr->m_func, instr->GetByteCodeOffset()); } #endif CannotAllocateArgumentsObjectOnStack(instr->m_func); return; } } } // We should look at dst last to correctly handle cases where it's the same as one of the src operands. if (dst) { if (dst->IsIndirOpnd() || dst->IsSymOpnd()) { if (CurrentBlockData()->IsArgumentsOpnd(dst)) { #ifdef PERF_HINT if (PHASE_TRACE1(Js::PerfHintPhase)) { WritePerfHint(PerfHints::HeapArgumentsModification, instr->m_func, instr->GetByteCodeOffset()); } #endif CannotAllocateArgumentsObjectOnStack(instr->m_func); return; } } else if (dst->IsRegOpnd()) { if (this->currentBlock->loop && CurrentBlockData()->IsArgumentsOpnd(dst)) { #ifdef PERF_HINT if (PHASE_TRACE1(Js::PerfHintPhase)) { WritePerfHint(PerfHints::HeapArgumentsModification, instr->m_func, instr->GetByteCodeOffset()); } #endif CannotAllocateArgumentsObjectOnStack(instr->m_func); return; } CurrentBlockData()->ClearArgumentsSym(dst->AsRegOpnd()); } } } break; } return; } void GlobOpt::MarkArgumentsUsedForBranch(IR::Instr * instr) { // If it's a conditional branch instruction and the operand used for branching is one of the arguments // to the function, tag the m_argUsedForBranch of the functionBody so that it can be used later for inlining decisions. if (instr->IsBranchInstr() && !instr->AsBranchInstr()->IsUnconditional()) { IR::BranchInstr * bInstr = instr->AsBranchInstr(); IR::Opnd *src1 = bInstr->GetSrc1(); IR::Opnd *src2 = bInstr->GetSrc2(); // These are used because we don't want to rely on src1 or src2 to always be the register/constant IR::RegOpnd *regOpnd = nullptr; if (!src2 && (instr->m_opcode == Js::OpCode::BrFalse_A || instr->m_opcode == Js::OpCode::BrTrue_A) && src1->IsRegOpnd()) { regOpnd = src1->AsRegOpnd(); } // We need to check for (0===arg) and (arg===0); this is especially important since some minifiers // change all instances of one to the other. else if (src2 && src2->IsConstOpnd() && src1->IsRegOpnd()) { regOpnd = src1->AsRegOpnd(); } else if (src2 && src2->IsRegOpnd() && src1->IsConstOpnd()) { regOpnd = src2->AsRegOpnd(); } if (regOpnd != nullptr) { if (regOpnd->m_sym->IsSingleDef()) { IR::Instr * defInst = regOpnd->m_sym->GetInstrDef(); IR::Opnd *defSym = defInst->GetSrc1(); if (defSym && defSym->IsSymOpnd() && defSym->AsSymOpnd()->m_sym->IsStackSym() && defSym->AsSymOpnd()->m_sym->AsStackSym()->IsParamSlotSym()) { uint16 param = defSym->AsSymOpnd()->m_sym->AsStackSym()->GetParamSlotNum(); // We only support functions with 13 arguments to ensure optimal size of callSiteInfo if (param < Js::Constants::MaximumArgumentCountForConstantArgumentInlining) { this->func->GetJITOutput()->SetArgUsedForBranch((uint8)param); } } } } } } const InductionVariable* GlobOpt::GetInductionVariable(SymID sym, Loop *loop) { if (loop->inductionVariables) { for (auto it = loop->inductionVariables->GetIterator(); it.IsValid(); it.MoveNext()) { InductionVariable* iv = &it.CurrentValueReference(); if (!iv->IsChangeDeterminate() || !iv->IsChangeUnidirectional()) { continue; } if (iv->Sym()->m_id == sym) { return iv; } } } return nullptr; } bool GlobOpt::IsSymIDInductionVariable(SymID sym, Loop *loop) { return GetInductionVariable(sym, loop) != nullptr; } SymID GlobOpt::GetVarSymID(StackSym *sym) { if (sym && sym->m_type != TyVar) { sym = sym->GetVarEquivSym(nullptr); } if (!sym) { return Js::Constants::InvalidSymID; } return sym->m_id; } bool GlobOpt::IsAllowedForMemOpt(IR::Instr* instr, bool isMemset, IR::RegOpnd *baseOpnd, IR::Opnd *indexOpnd) { Assert(instr); if (!baseOpnd || !indexOpnd) { return false; } Loop* loop = this->currentBlock->loop; const ValueType baseValueType(baseOpnd->GetValueType()); const ValueType indexValueType(indexOpnd->GetValueType()); // Validate the array and index types if ( !indexValueType.IsInt() || !( baseValueType.IsTypedIntOrFloatArray() || baseValueType.IsArray() ) ) { #if DBG_DUMP wchar indexValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; indexValueType.ToString(indexValueTypeStr); wchar baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; baseValueType.ToString(baseValueTypeStr); TRACE_MEMOP_VERBOSE(loop, instr, _u("Index[%s] or Array[%s] value type is invalid"), indexValueTypeStr, baseValueTypeStr); #endif return false; } // The following is conservative and works around a bug in induction variable analysis. if (baseOpnd->IsArrayRegOpnd()) { IR::ArrayRegOpnd *baseArrayOp = baseOpnd->AsArrayRegOpnd(); bool hasBoundChecksRemoved = ( baseArrayOp->EliminatedLowerBoundCheck() && baseArrayOp->EliminatedUpperBoundCheck() && !instr->extractedUpperBoundCheckWithoutHoisting && !instr->loadedArrayHeadSegment && !instr->loadedArrayHeadSegmentLength ); if (!hasBoundChecksRemoved) { TRACE_MEMOP_VERBOSE(loop, instr, _u("Missing bounds check optimization")); return false; } } else { return false; } if (!baseValueType.IsTypedArray()) { // Check if the instr can kill the value type of the array JsArrayKills arrayKills = CheckJsArrayKills(instr); if (arrayKills.KillsValueType(baseValueType)) { TRACE_MEMOP_VERBOSE(loop, instr, _u("The array (s%d) can lose its value type"), GetVarSymID(baseOpnd->GetStackSym())); return false; } } // Process the Index Operand if (!this->OptIsInvariant(baseOpnd, this->currentBlock, loop, CurrentBlockData()->FindValue(baseOpnd->m_sym), false, true)) { TRACE_MEMOP_VERBOSE(loop, instr, _u("Base (s%d) is not invariant"), GetVarSymID(baseOpnd->GetStackSym())); return false; } // Validate the index Assert(indexOpnd->GetStackSym()); SymID indexSymID = GetVarSymID(indexOpnd->GetStackSym()); const InductionVariable* iv = GetInductionVariable(indexSymID, loop); if (!iv) { // If the index is not an induction variable return TRACE_MEMOP_VERBOSE(loop, instr, _u("Index (s%d) is not an induction variable"), indexSymID); return false; } Assert(iv->IsChangeDeterminate() && iv->IsChangeUnidirectional()); const IntConstantBounds & bounds = iv->ChangeBounds(); if (loop->memOpInfo) { // Only accept induction variables that increments by 1 Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 }; inductionVariableChangeInfo = loop->memOpInfo->inductionVariableChangeInfoMap->Lookup(indexSymID, inductionVariableChangeInfo); if ( (bounds.LowerBound() != 1 && bounds.LowerBound() != -1) || (bounds.UpperBound() != bounds.LowerBound()) || inductionVariableChangeInfo.unroll > 1 // Must be 0 (not seen yet) or 1 (already seen) ) { TRACE_MEMOP_VERBOSE(loop, instr, _u("The index does not change by 1: %d><%d, unroll=%d"), bounds.LowerBound(), bounds.UpperBound(), inductionVariableChangeInfo.unroll); return false; } // Check if the index is the same in all MemOp optimization in this loop if (!loop->memOpInfo->candidates->Empty()) { Loop::MemOpCandidate* previousCandidate = loop->memOpInfo->candidates->Head(); // All MemOp operations within the same loop must use the same index if (previousCandidate->index != indexSymID) { TRACE_MEMOP_VERBOSE(loop, instr, _u("The index is not the same as other MemOp in the loop")); return false; } } } return true; } bool GlobOpt::CollectMemcopyLdElementI(IR::Instr *instr, Loop *loop) { Assert(instr->GetSrc1()->IsIndirOpnd()); IR::IndirOpnd *src1 = instr->GetSrc1()->AsIndirOpnd(); IR::Opnd *indexOpnd = src1->GetIndexOpnd(); IR::RegOpnd *baseOpnd = src1->GetBaseOpnd()->AsRegOpnd(); SymID baseSymID = GetVarSymID(baseOpnd->GetStackSym()); if (!IsAllowedForMemOpt(instr, false, baseOpnd, indexOpnd)) { return false; } SymID inductionSymID = GetVarSymID(indexOpnd->GetStackSym()); Assert(IsSymIDInductionVariable(inductionSymID, loop)); loop->EnsureMemOpVariablesInitialized(); bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID); IR::Opnd * dst = instr->GetDst(); if (!dst->IsRegOpnd() || !dst->AsRegOpnd()->GetStackSym()->IsSingleDef()) { return false; } Loop::MemCopyCandidate* memcopyInfo = memcopyInfo = JitAnewStruct(this->func->GetTopFunc()->m_fg->alloc, Loop::MemCopyCandidate); memcopyInfo->ldBase = baseSymID; memcopyInfo->ldCount = 1; memcopyInfo->count = 0; memcopyInfo->bIndexAlreadyChanged = isIndexPreIncr; memcopyInfo->base = Js::Constants::InvalidSymID; //need to find the stElem first memcopyInfo->index = inductionSymID; memcopyInfo->transferSym = dst->AsRegOpnd()->GetStackSym(); loop->memOpInfo->candidates->Prepend(memcopyInfo); return true; } bool GlobOpt::CollectMemsetStElementI(IR::Instr *instr, Loop *loop) { Assert(instr->GetDst()->IsIndirOpnd()); IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd(); IR::Opnd *indexOp = dst->GetIndexOpnd(); IR::RegOpnd *baseOp = dst->GetBaseOpnd()->AsRegOpnd(); if (!IsAllowedForMemOpt(instr, true, baseOp, indexOp)) { return false; } SymID baseSymID = GetVarSymID(baseOp->GetStackSym()); IR::Opnd *srcDef = instr->GetSrc1(); StackSym *srcSym = nullptr; if (srcDef->IsRegOpnd()) { IR::RegOpnd* opnd = srcDef->AsRegOpnd(); if (this->OptIsInvariant(opnd, this->currentBlock, loop, CurrentBlockData()->FindValue(opnd->m_sym), true, true)) { srcSym = opnd->GetStackSym(); } } BailoutConstantValue constant = {TyIllegal, 0}; if (srcDef->IsFloatConstOpnd()) { constant.InitFloatConstValue(srcDef->AsFloatConstOpnd()->m_value); } else if (srcDef->IsIntConstOpnd()) { constant.InitIntConstValue(srcDef->AsIntConstOpnd()->GetValue(), srcDef->AsIntConstOpnd()->GetType()); } else if (srcDef->IsAddrOpnd()) { constant.InitVarConstValue(srcDef->AsAddrOpnd()->m_address); } else if(!srcSym) { TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Source is not an invariant")); return false; } // Process the Index Operand Assert(indexOp->GetStackSym()); SymID inductionSymID = GetVarSymID(indexOp->GetStackSym()); Assert(IsSymIDInductionVariable(inductionSymID, loop)); loop->EnsureMemOpVariablesInitialized(); bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID); Loop::MemSetCandidate* memsetInfo = JitAnewStruct(this->func->GetTopFunc()->m_fg->alloc, Loop::MemSetCandidate); memsetInfo->base = baseSymID; memsetInfo->index = inductionSymID; memsetInfo->constant = constant; memsetInfo->srcSym = srcSym; memsetInfo->count = 1; memsetInfo->bIndexAlreadyChanged = isIndexPreIncr; loop->memOpInfo->candidates->Prepend(memsetInfo); return true; } bool GlobOpt::CollectMemcopyStElementI(IR::Instr *instr, Loop *loop) { if (!loop->memOpInfo || loop->memOpInfo->candidates->Empty()) { // There is no ldElem matching this stElem return false; } Assert(instr->GetDst()->IsIndirOpnd()); IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd(); IR::Opnd *indexOp = dst->GetIndexOpnd(); IR::RegOpnd *baseOp = dst->GetBaseOpnd()->AsRegOpnd(); SymID baseSymID = GetVarSymID(baseOp->GetStackSym()); if (!instr->GetSrc1()->IsRegOpnd()) { return false; } IR::RegOpnd* src1 = instr->GetSrc1()->AsRegOpnd(); if (!src1->GetIsDead()) { // This must be the last use of the register. // It will invalidate `var m = a[i]; b[i] = m;` but this is not a very interesting case. TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Source (s%d) is still alive after StElemI"), baseSymID); return false; } if (!IsAllowedForMemOpt(instr, false, baseOp, indexOp)) { return false; } SymID srcSymID = GetVarSymID(src1->GetStackSym()); // Prepare the memcopyCandidate entry Loop::MemOpCandidate* previousCandidate = loop->memOpInfo->candidates->Head(); if (!previousCandidate->IsMemCopy()) { return false; } Loop::MemCopyCandidate* memcopyInfo = previousCandidate->AsMemCopy(); // The previous candidate has to have been created by the matching ldElem if ( memcopyInfo->base != Js::Constants::InvalidSymID || GetVarSymID(memcopyInfo->transferSym) != srcSymID ) { TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("No matching LdElem found (s%d)"), baseSymID); return false; } Assert(indexOp->GetStackSym()); SymID inductionSymID = GetVarSymID(indexOp->GetStackSym()); Assert(IsSymIDInductionVariable(inductionSymID, loop)); bool isIndexPreIncr = loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID); if (isIndexPreIncr != memcopyInfo->bIndexAlreadyChanged) { // The index changed between the load and the store TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Index value changed between ldElem and stElem")); return false; } // Consider: Can we remove the count field? memcopyInfo->count++; AssertOrFailFast(memcopyInfo->count <= 1); memcopyInfo->base = baseSymID; return true; } bool GlobOpt::CollectMemOpLdElementI(IR::Instr *instr, Loop *loop) { Assert(instr->m_opcode == Js::OpCode::LdElemI_A); return (!PHASE_OFF(Js::MemCopyPhase, this->func) && CollectMemcopyLdElementI(instr, loop)); } bool GlobOpt::CollectMemOpStElementI(IR::Instr *instr, Loop *loop) { Assert(instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict); Assert(instr->GetSrc1()); return (!PHASE_OFF(Js::MemSetPhase, this->func) && CollectMemsetStElementI(instr, loop)) || (!PHASE_OFF(Js::MemCopyPhase, this->func) && CollectMemcopyStElementI(instr, loop)); } bool GlobOpt::CollectMemOpInfo(IR::Instr *instrBegin, IR::Instr *instr, Value *src1Val, Value *src2Val) { Assert(this->currentBlock->loop); Loop *loop = this->currentBlock->loop; if (!loop->blockList.HasTwo()) { // We support memcopy and memset for loops which have only two blocks. return false; } if (loop->GetLoopFlags().isInterpreted && !loop->GetLoopFlags().memopMinCountReached) { TRACE_MEMOP_VERBOSE(loop, instr, _u("minimum loop count not reached")) loop->doMemOp = false; return false; } Assert(loop->doMemOp); bool isIncr = true, isChangedByOne = false; switch (instr->m_opcode) { case Js::OpCode::StElemI_A: case Js::OpCode::StElemI_A_Strict: if (!CollectMemOpStElementI(instr, loop)) { loop->doMemOp = false; return false; } break; case Js::OpCode::LdElemI_A: if (!CollectMemOpLdElementI(instr, loop)) { loop->doMemOp = false; return false; } break; case Js::OpCode::Sub_I4: isIncr = false; case Js::OpCode::Add_I4: { // The only case in which these OpCodes can contribute to an inductionVariableChangeInfo // is when the induction variable is being modified and overwritten aswell (ex: j = j + 1) // and not when the induction variable is modified but not overwritten (ex: k = j + 1). // This can either be detected in IR as // s1 = Add_I4 s1 1 // Case #1, can be seen with "j++". // or as // s4(s2) = Add_I4 s3(s1) 1 // Case #2, can be see with "j = j + 1". // s1 = Ld_A s2 bool isInductionVar = false; IR::Instr* nextInstr = instr->m_next; if ( // Checks for Case #1 and Case #2 instr->GetDst()->GetStackSym() != nullptr && instr->GetDst()->IsRegOpnd() && ( // Checks for Case #1 (instr->GetDst()->GetStackSym() == instr->GetSrc1()->GetStackSym()) || // Checks for Case #2 (nextInstr&& nextInstr->m_opcode == Js::OpCode::Ld_A && nextInstr->GetSrc1()->IsRegOpnd() && nextInstr->GetDst()->IsRegOpnd() && GetVarSymID(instr->GetDst()->GetStackSym()) == nextInstr->GetSrc1()->GetStackSym()->m_id && GetVarSymID(instr->GetSrc1()->GetStackSym()) == nextInstr->GetDst()->GetStackSym()->m_id) ) ) { isInductionVar = true; } // Even if dstIsInductionVar then dst == src1 so it's safe to use src1 as the induction sym always. StackSym* sym = instr->GetSrc1()->GetStackSym(); SymID inductionSymID = GetVarSymID(sym); if (isInductionVar && IsSymIDInductionVariable(inductionSymID, this->currentBlock->loop)) { if (!isChangedByOne) { IR::Opnd *src1, *src2; src1 = instr->GetSrc1(); src2 = instr->GetSrc2(); if (src2->IsRegOpnd()) { Value *val = CurrentBlockData()->FindValue(src2->AsRegOpnd()->m_sym); if (val) { ValueInfo *vi = val->GetValueInfo(); int constValue; if (vi && vi->TryGetIntConstantValue(&constValue)) { if (constValue == 1) { isChangedByOne = true; } } } } else if (src2->IsIntConstOpnd()) { if (src2->AsIntConstOpnd()->GetValue() == 1) { isChangedByOne = true; } } } loop->EnsureMemOpVariablesInitialized(); if (!isChangedByOne) { Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { Js::Constants::InvalidLoopUnrollFactor, 0 }; if (!loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID)) { loop->memOpInfo->inductionVariableChangeInfoMap->Add(inductionSymID, inductionVariableChangeInfo); if (sym->m_id != inductionSymID) { // Backwards pass uses this bit-vector to lookup upwardExposedUsed/bytecodeUpwardExposedUsed symbols, which are not necessarily vars. Just add both. loop->memOpInfo->inductionVariableChangeInfoMap->Add(sym->m_id, inductionVariableChangeInfo); } } else { loop->memOpInfo->inductionVariableChangeInfoMap->Item(inductionSymID, inductionVariableChangeInfo); if (sym->m_id != inductionSymID) { // Backwards pass uses this bit-vector to lookup upwardExposedUsed/bytecodeUpwardExposedUsed symbols, which are not necessarily vars. Just add both. loop->memOpInfo->inductionVariableChangeInfoMap->Item(sym->m_id, inductionVariableChangeInfo); } } } else { if (!loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(inductionSymID)) { Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 1, isIncr }; loop->memOpInfo->inductionVariableChangeInfoMap->Add(inductionSymID, inductionVariableChangeInfo); if (sym->m_id != inductionSymID) { // Backwards pass uses this bit-vector to lookup upwardExposedUsed/bytecodeUpwardExposedUsed symbols, which are not necessarily vars. Just add both. loop->memOpInfo->inductionVariableChangeInfoMap->Add(sym->m_id, inductionVariableChangeInfo); } } else { Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 }; inductionVariableChangeInfo = loop->memOpInfo->inductionVariableChangeInfoMap->Lookup(inductionSymID, inductionVariableChangeInfo); // If inductionVariableChangeInfo.unroll has been invalidated, do // not modify the Js::Constants::InvalidLoopUnrollFactor value if (inductionVariableChangeInfo.unroll != Js::Constants::InvalidLoopUnrollFactor) { inductionVariableChangeInfo.unroll++; } inductionVariableChangeInfo.isIncremental = isIncr; loop->memOpInfo->inductionVariableChangeInfoMap->Item(inductionSymID, inductionVariableChangeInfo); if (sym->m_id != inductionSymID) { // Backwards pass uses this bit-vector to lookup upwardExposedUsed/bytecodeUpwardExposedUsed symbols, which are not necessarily vars. Just add both. loop->memOpInfo->inductionVariableChangeInfoMap->Item(sym->m_id, inductionVariableChangeInfo); } } } break; } // Fallthrough if not an induction variable } default: FOREACH_INSTR_IN_RANGE(chkInstr, instrBegin->m_next, instr) { if (IsInstrInvalidForMemOp(chkInstr, loop, src1Val, src2Val)) { loop->doMemOp = false; return false; } // Make sure this instruction doesn't use the memcopy transfer sym before it is checked by StElemI if (loop->memOpInfo && !loop->memOpInfo->candidates->Empty()) { Loop::MemOpCandidate* prevCandidate = loop->memOpInfo->candidates->Head(); if (prevCandidate->IsMemCopy()) { Loop::MemCopyCandidate* memcopyCandidate = prevCandidate->AsMemCopy(); if (memcopyCandidate->base == Js::Constants::InvalidSymID) { if (chkInstr->HasSymUse(memcopyCandidate->transferSym)) { loop->doMemOp = false; TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, chkInstr, _u("Found illegal use of LdElemI value(s%d)"), GetVarSymID(memcopyCandidate->transferSym)); return false; } } } } } NEXT_INSTR_IN_RANGE; IR::Instr* prevInstr = instr->m_prev; // If an instr where the dst is an induction variable (and thus is being written to) is not caught by a case in the above // switch statement (which implies that this instr does not contributes to a inductionVariableChangeInfo) and in the default // case does not set doMemOp to false (which implies that this instr does not invalidate this MemOp), then FailFast as we // should not be performing a MemOp under these conditions. AssertOrFailFast(!instr->GetDst() || instr->m_opcode == Js::OpCode::IncrLoopBodyCount || !loop->memOpInfo || // Refer to "Case #2" described above in this function. For the following IR: // Line #1: s4(s2) = Add_I4 s3(s1) 1 // Line #2: s3(s1) = Ld_A s4(s2) // do not consider line #2 as a violating instr (instr->m_opcode == Js::OpCode::Ld_I4 && prevInstr && (prevInstr->m_opcode == Js::OpCode::Add_I4 || prevInstr->m_opcode == Js::OpCode::Sub_I4) && instr->GetSrc1()->IsRegOpnd() && instr->GetDst()->IsRegOpnd() && prevInstr->GetDst()->IsRegOpnd() && instr->GetDst()->GetStackSym() == prevInstr->GetSrc1()->GetStackSym() && instr->GetSrc1()->GetStackSym() == prevInstr->GetDst()->GetStackSym()) || !loop->memOpInfo->inductionVariableChangeInfoMap->ContainsKey(GetVarSymID(instr->GetDst()->GetStackSym()))); } return true; } bool GlobOpt::IsInstrInvalidForMemOp(IR::Instr *instr, Loop *loop, Value *src1Val, Value *src2Val) { // List of instruction that are valid with memop (ie: instr that gets removed if memop is emitted) if ( this->currentBlock != loop->GetHeadBlock() && !instr->IsLabelInstr() && instr->IsRealInstr() && instr->m_opcode != Js::OpCode::IncrLoopBodyCount && instr->m_opcode != Js::OpCode::StLoopBodyCount && instr->m_opcode != Js::OpCode::Ld_A && instr->m_opcode != Js::OpCode::Ld_I4 && !(instr->IsBranchInstr() && instr->AsBranchInstr()->IsUnconditional()) ) { TRACE_MEMOP_VERBOSE(loop, instr, _u("Instruction not accepted for memop")); return true; } // Check prev instr because it could have been added by an optimization and we won't see it here. if (OpCodeAttr::FastFldInstr(instr->m_opcode) || (instr->m_prev && OpCodeAttr::FastFldInstr(instr->m_prev->m_opcode))) { // Refuse any operations interacting with Fields TRACE_MEMOP_VERBOSE(loop, instr, _u("Field interaction detected")); return true; } if (Js::OpCodeUtil::GetOpCodeLayout(instr->m_opcode) == Js::OpLayoutType::ElementSlot) { // Refuse any operations interacting with slots TRACE_MEMOP_VERBOSE(loop, instr, _u("Slot interaction detected")); return true; } if (this->MayNeedBailOnImplicitCall(instr, src1Val, src2Val)) { TRACE_MEMOP_VERBOSE(loop, instr, _u("Implicit call bailout detected")); return true; } return false; } void GlobOpt::TryReplaceLdLen(IR::Instr *& instr) { // Change LdLen on objects other than arrays, strings, and 'arguments' to LdFld. Otherwise, convert the SymOpnd to a RegOpnd here. if (instr->m_opcode == Js::OpCode::LdLen_A && instr->GetSrc1() && instr->GetSrc1()->IsSymOpnd()) { IR::SymOpnd * opnd = instr->GetSrc1()->AsSymOpnd(); Sym *sym = opnd->m_sym; Assert(sym->IsPropertySym()); PropertySym *originalPropertySym = sym->AsPropertySym(); IR::RegOpnd* newopnd = IR::RegOpnd::New(originalPropertySym->m_stackSym, IRType::TyVar, instr->m_func); ValueInfo *const objectValueInfo = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym)->GetValueInfo(); // things we'd emit a fast path for if ( objectValueInfo->IsLikelyAnyArray() || objectValueInfo->HasHadStringTag() || objectValueInfo->IsLikelyString() || newopnd->IsArgumentsObject() || (CurrentBlockData()->argObjSyms && CurrentBlockData()->IsArgumentsOpnd(newopnd)) ) { // We need to properly transfer over the information from the old operand, which is // a SymOpnd, to the new one, which is a RegOpnd. Unfortunately, the types mean the // normal copy methods won't work here, so we're going to directly copy data. newopnd->SetIsJITOptimizedReg(opnd->GetIsJITOptimizedReg()); newopnd->SetValueType(objectValueInfo->Type()); newopnd->SetIsDead(opnd->GetIsDead()); instr->ReplaceSrc1(newopnd); } else { // otherwise, change the instruction to an LdFld here. instr->m_opcode = Js::OpCode::LdFld; } } } IR::Instr * GlobOpt::OptInstr(IR::Instr *&instr, bool* isInstrRemoved) { Assert(instr->m_func->IsTopFunc() || instr->m_func->isGetterSetter || instr->m_func->callSiteIdInParentFunc != UINT16_MAX); IR::Opnd *src1, *src2; Value *src1Val = nullptr, *src2Val = nullptr, *dstVal = nullptr; Value *src1IndirIndexVal = nullptr, *dstIndirIndexVal = nullptr; IR::Instr *instrPrev = instr->m_prev; IR::Instr *instrNext = instr->m_next; if (instr->IsLabelInstr() && this->func->HasTry() && this->func->DoOptimizeTry()) { this->currentRegion = instr->AsLabelInstr()->GetRegion(); Assert(this->currentRegion); } if(PrepareForIgnoringIntOverflow(instr)) { if(!IsLoopPrePass()) { *isInstrRemoved = true; currentBlock->RemoveInstr(instr); } return instrNext; } if (instr->m_opcode == Js::OpCode::Yield) { // TODO[generators][ianhall]: Can this and the FillBailOutInfo call below be moved to after Src1 and Src2 so that Yield can be optimized right up to the actual yield? this->ProcessKills(instr); } if (!instr->IsRealInstr() || instr->IsByteCodeUsesInstr() || instr->m_opcode == Js::OpCode::Conv_Bool) { return instrNext; } if (!IsLoopPrePass()) { // Change LdLen on objects other than arrays, strings, and 'arguments' to LdFld. this->TryReplaceLdLen(instr); } // Consider: Do we ever get post-op bailout here, and if so is the FillBailOutInfo call in the right place? if (instr->HasBailOutInfo() && !this->IsLoopPrePass()) { this->FillBailOutInfo(this->currentBlock, instr); } this->instrCountSinceLastCleanUp++; instr = this->PreOptPeep(instr); this->OptArguments(instr); //StackArguments Optimization - We bail out if the index is out of range of actuals. if ((instr->m_opcode == Js::OpCode::LdElemI_A || instr->m_opcode == Js::OpCode::TypeofElem) && instr->DoStackArgsOpt() && !this->IsLoopPrePass()) { GenerateBailAtOperation(&instr, IR::BailOnStackArgsOutOfActualsRange); } #if DBG PropertySym *propertySymUseBefore = nullptr; Assert(this->byteCodeUses == nullptr); this->byteCodeUsesBeforeOpt->ClearAll(); GlobOpt::TrackByteCodeSymUsed(instr, this->byteCodeUsesBeforeOpt, &propertySymUseBefore); Assert(noImplicitCallUsesToInsert->Count() == 0); #endif this->ignoredIntOverflowForCurrentInstr = false; this->ignoredNegativeZeroForCurrentInstr = false; src1 = instr->GetSrc1(); src2 = instr->GetSrc2(); if (src1) { src1Val = this->OptSrc(src1, &instr, &src1IndirIndexVal); GOPT_TRACE_VALUENUMBER(_u("[src1] "), instr->GetSrc1(), _u("%d"), src1Val ? src1Val->GetValueNumber() : -1); instr = this->SetTypeCheckBailOut(instr->GetSrc1(), instr, nullptr); if (src2) { src2Val = this->OptSrc(src2, &instr); GOPT_TRACE_VALUENUMBER(_u("[src2] "), instr->GetSrc2(), _u("%d"), src2Val ? src2Val->GetValueNumber() : -1); } } if(instr->GetDst() && instr->GetDst()->IsIndirOpnd()) { this->OptSrc(instr->GetDst(), &instr, &dstIndirIndexVal); } MarkArgumentsUsedForBranch(instr); CSEOptimize(this->currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal); OptimizeChecks(instr); OptArraySrc(&instr, &src1Val, &src2Val); OptNewScObject(&instr, src1Val); OptStackArgLenAndConst(instr, &src1Val); instr = this->OptPeep(instr, src1Val, src2Val); if (instr->m_opcode == Js::OpCode::Nop || (instr->m_opcode == Js::OpCode::CheckThis && instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isSafeThis)) { instrNext = instr->m_next; InsertNoImplicitCallUses(instr); if (this->byteCodeUses) { this->InsertByteCodeUses(instr); } *isInstrRemoved = true; this->currentBlock->RemoveInstr(instr); return instrNext; } else if (instr->m_opcode == Js::OpCode::GetNewScObject && !this->IsLoopPrePass() && src1Val->GetValueInfo()->IsPrimitive()) { // Constructor returned (src1) a primitive value, so fold this into "dst = Ld_A src2", where src2 is the new object that // was passed into the constructor as its 'this' parameter instr->FreeSrc1(); instr->SetSrc1(instr->UnlinkSrc2()); instr->m_opcode = Js::OpCode::Ld_A; src1Val = src2Val; src2Val = nullptr; } else if ((instr->m_opcode == Js::OpCode::TryCatch && this->func->DoOptimizeTry()) || (instr->m_opcode == Js::OpCode::TryFinally && this->func->DoOptimizeTry())) { ProcessTryHandler(instr); } else if (instr->m_opcode == Js::OpCode::BrOnException || instr->m_opcode == Js::OpCode::BrOnNoException) { if (this->ProcessExceptionHandlingEdges(instr)) { *isInstrRemoved = true; return instrNext; } } bool isAlreadyTypeSpecialized = false; if (!IsLoopPrePass() && instr->HasBailOutInfo()) { if (instr->GetBailOutKind() == IR::BailOutExpectingInteger) { isAlreadyTypeSpecialized = TypeSpecializeBailoutExpectedInteger(instr, src1Val, &dstVal); } else if (instr->GetBailOutKind() == IR::BailOutExpectingString) { if (instr->GetSrc1()->IsRegOpnd()) { if (!src1Val || !src1Val->GetValueInfo()->IsLikelyString()) { // Disable SwitchOpt if the source is definitely not a string - This may be realized only in Globopt Assert(IsSwitchOptEnabled()); throw Js::RejitException(RejitReason::DisableSwitchOptExpectingString); } } } } bool forceInvariantHoisting = false; const bool ignoreIntOverflowInRangeForInstr = instr->ignoreIntOverflowInRange; // Save it since the instr can change if (!isAlreadyTypeSpecialized) { bool redoTypeSpec; instr = this->TypeSpecialization(instr, &src1Val, &src2Val, &dstVal, &redoTypeSpec, &forceInvariantHoisting); if(redoTypeSpec && instr->m_opcode != Js::OpCode::Nop) { forceInvariantHoisting = false; instr = this->TypeSpecialization(instr, &src1Val, &src2Val, &dstVal, &redoTypeSpec, &forceInvariantHoisting); Assert(!redoTypeSpec); } if (instr->m_opcode == Js::OpCode::Nop) { InsertNoImplicitCallUses(instr); if (this->byteCodeUses) { this->InsertByteCodeUses(instr); } instrNext = instr->m_next; *isInstrRemoved = true; this->currentBlock->RemoveInstr(instr); return instrNext; } } if (ignoreIntOverflowInRangeForInstr) { VerifyIntSpecForIgnoringIntOverflow(instr); } // Track calls after any pre-op bailouts have been inserted before the call, because they will need to restore out params. this->TrackCalls(instr); if (instr->GetSrc1()) { this->UpdateObjPtrValueType(instr->GetSrc1(), instr); } IR::Opnd *dst = instr->GetDst(); if (dst) { // Copy prop dst uses and mark live/available type syms before tracking kills. CopyPropDstUses(dst, instr, src1Val); } // Track mark temp object before we process the dst so we can generate pre-op bailout instr = this->TrackMarkTempObject(instrPrev->m_next, instr); bool removed = OptTagChecks(instr); if (removed) { *isInstrRemoved = true; return instrNext; } dstVal = this->OptDst(&instr, dstVal, src1Val, src2Val, dstIndirIndexVal, src1IndirIndexVal); if (dst) { GOPT_TRACE_VALUENUMBER(_u("[dst] "), instr->GetDst(), _u("%d\n"), dstVal ? dstVal->GetValueNumber() : -1); } dst = instr->GetDst(); instrNext = instr->m_next; if (dst) { if (this->func->HasTry() && this->func->DoOptimizeTry()) { this->InsertToVarAtDefInTryRegion(instr, dst); } instr = this->SetTypeCheckBailOut(dst, instr, nullptr); this->UpdateObjPtrValueType(dst, instr); } BVSparse instrByteCodeStackSymUsedAfter(this->alloc); PropertySym *propertySymUseAfter = nullptr; if (this->byteCodeUses != nullptr) { GlobOpt::TrackByteCodeSymUsed(instr, &instrByteCodeStackSymUsedAfter, &propertySymUseAfter); } #if DBG else { GlobOpt::TrackByteCodeSymUsed(instr, &instrByteCodeStackSymUsedAfter, &propertySymUseAfter); instrByteCodeStackSymUsedAfter.Equal(this->byteCodeUsesBeforeOpt); Assert(propertySymUseAfter == propertySymUseBefore); } #endif bool isHoisted = false; if (this->currentBlock->loop && !this->IsLoopPrePass()) { isHoisted = this->TryHoistInvariant(instr, this->currentBlock, dstVal, src1Val, src2Val, true, false, forceInvariantHoisting); } src1 = instr->GetSrc1(); if (!this->IsLoopPrePass() && src1) { // instr const, nonConst => canonicalize by swapping operands // This simplifies lowering. (somewhat machine dependent) // Note that because of Var overflows, src1 may not have been constant prop'd to an IntConst this->PreLowerCanonicalize(instr, &src1Val, &src2Val); } if (!PHASE_OFF(Js::MemOpPhase, this->func) && !isHoisted && !(instr->IsJitProfilingInstr()) && this->currentBlock->loop && !IsLoopPrePass() && !func->IsJitInDebugMode() && !func->IsMemOpDisabled() && this->currentBlock->loop->doMemOp) { CollectMemOpInfo(instrPrev, instr, src1Val, src2Val); } InsertNoImplicitCallUses(instr); if (this->byteCodeUses != nullptr) { // Optimization removed some uses from the instruction. // Need to insert fake uses so we can get the correct live register to restore in bailout. this->byteCodeUses->Minus(&instrByteCodeStackSymUsedAfter); if (this->propertySymUse == propertySymUseAfter) { this->propertySymUse = nullptr; } this->InsertByteCodeUses(instr); } if (!this->IsLoopPrePass() && !isHoisted && this->IsImplicitCallBailOutCurrentlyNeeded(instr, src1Val, src2Val)) { IR::BailOutKind kind = IR::BailOutOnImplicitCalls; if(instr->HasBailOutInfo()) { Assert(instr->GetBailOutInfo()->bailOutOffset == instr->GetByteCodeOffset()); const IR::BailOutKind bailOutKind = instr->GetBailOutKind(); if((bailOutKind & ~IR::BailOutKindBits) != IR::BailOutOnImplicitCallsPreOp) { Assert(!(bailOutKind & ~IR::BailOutKindBits)); instr->SetBailOutKind(bailOutKind + IR::BailOutOnImplicitCallsPreOp); } } else if (instr->forcePreOpBailOutIfNeeded || this->isRecursiveCallOnLandingPad) { // We can't have a byte code reg slot as dst to generate a // pre-op implicit call after we have processed the dst. // Consider: This might miss an opportunity to use a copy prop sym to restore // some other byte code reg if the dst is that copy prop that we already killed. Assert(!instr->GetDst() || !instr->GetDst()->IsRegOpnd() || instr->GetDst()->AsRegOpnd()->GetIsJITOptimizedReg() || !instr->GetDst()->AsRegOpnd()->m_sym->HasByteCodeRegSlot()); this->GenerateBailAtOperation(&instr, IR::BailOutOnImplicitCallsPreOp); } else { // Capture value of the bailout after the operation is done. this->GenerateBailAfterOperation(&instr, kind); } } if (this->IsLazyBailOutCurrentlyNeeded(instr, src1Val, src2Val, isHoisted)) { this->GenerateLazyBailOut(instr); } if (CurrentBlockData()->capturedValuesCandidate && !this->IsLoopPrePass()) { this->CommitCapturedValuesCandidate(); } #if DBG if (CONFIG_FLAG(ValidateIntRanges) && !IsLoopPrePass()) { if (instr->ShouldEmitIntRangeCheck()) { this->EmitIntRangeChecks(instr); } } #endif return instrNext; } bool GlobOpt::IsNonNumericRegOpnd(IR::RegOpnd* opnd, bool inGlobOpt, bool* isSafeToTransferInPrepass /*=nullptr*/) const { if (opnd == nullptr) { return false; } if (opnd->m_sym->m_isNotNumber) { return true; } if (!inGlobOpt) { return false; } if (opnd->GetValueType().IsNumber() || currentBlock->globOptData.IsTypeSpecialized(opnd->m_sym)) { if (!this->IsLoopPrePass()) { return false; } Value * opndValue = this->currentBlock->globOptData.FindValue(opnd->m_sym); ValueInfo * opndValueInfo = opndValue ? opndValue->GetValueInfo() : nullptr; if (!opndValueInfo) { return true; } bool isSafeToTransfer = this->IsSafeToTransferInPrepass(opnd->m_sym, opndValueInfo); if (isSafeToTransferInPrepass != nullptr) { *isSafeToTransferInPrepass = isSafeToTransfer; } if (this->prePassLoop->preservesNumberValue->Test(opnd->m_sym->m_id)) { return false; } return !isSafeToTransfer; } return true; } bool GlobOpt::OptTagChecks(IR::Instr *instr) { if (PHASE_OFF(Js::OptTagChecksPhase, this->func) || !this->DoTagChecks()) { return false; } StackSym *stackSym = nullptr; IR::SymOpnd *symOpnd = nullptr; IR::RegOpnd *regOpnd = nullptr; switch(instr->m_opcode) { case Js::OpCode::LdFld: case Js::OpCode::LdMethodFld: case Js::OpCode::CheckFixedFld: case Js::OpCode::CheckPropertyGuardAndLoadType: symOpnd = instr->GetSrc1()->AsSymOpnd(); stackSym = symOpnd->m_sym->AsPropertySym()->m_stackSym; break; case Js::OpCode::BailOnNotObject: case Js::OpCode::BailOnNotArray: if (instr->GetSrc1()->IsRegOpnd()) { regOpnd = instr->GetSrc1()->AsRegOpnd(); stackSym = regOpnd->m_sym; } break; case Js::OpCode::StFld: symOpnd = instr->GetDst()->AsSymOpnd(); stackSym = symOpnd->m_sym->AsPropertySym()->m_stackSym; break; } if (stackSym) { Value *value = CurrentBlockData()->FindValue(stackSym); if (value) { ValueInfo *valInfo = value->GetValueInfo(); if (valInfo->GetSymStore() && valInfo->GetSymStore()->IsStackSym() && valInfo->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable()) { return false; } ValueType valueType = value->GetValueInfo()->Type(); if (instr->m_opcode == Js::OpCode::BailOnNotObject) { if (valueType.CanBeTaggedValue()) { // We're not adding new information to the value other than changing the value type. Preserve any existing // information and just change the value type. ChangeValueType(nullptr, value, valueType.SetCanBeTaggedValue(false), true /*preserveSubClassInfo*/); return false; } if (!this->IsLoopPrePass()) { if (this->byteCodeUses) { this->InsertByteCodeUses(instr); } this->currentBlock->RemoveInstr(instr); } return true; } if (valueType.CanBeTaggedValue() && !valueType.HasBeenNumber() && !this->IsLoopPrePass()) { ValueType newValueType = valueType.SetCanBeTaggedValue(false); // Split out the tag check as a separate instruction. IR::Instr *bailOutInstr; bailOutInstr = IR::BailOutInstr::New(Js::OpCode::BailOnNotObject, IR::BailOutOnTaggedValue, instr, instr->m_func); if (!this->IsLoopPrePass()) { FillBailOutInfo(this->currentBlock, bailOutInstr); } IR::RegOpnd *srcOpnd = regOpnd; if (!srcOpnd) { srcOpnd = IR::RegOpnd::New(stackSym, stackSym->GetType(), instr->m_func); AnalysisAssert(symOpnd); if (symOpnd->GetIsJITOptimizedReg()) { srcOpnd->SetIsJITOptimizedReg(true); } } bailOutInstr->SetSrc1(srcOpnd); bailOutInstr->GetSrc1()->SetValueType(valueType); bailOutInstr->SetByteCodeOffset(instr); instr->InsertBefore(bailOutInstr); if (this->currentBlock->loop) { // Try hoisting the BailOnNotObject instr. // But since this isn't the current instr being optimized, we need to play tricks with // the byteCodeUse fields... TrackByteCodeUsesForInstrAddedInOptInstr(bailOutInstr, [&]() { if (TryHoistInvariant(bailOutInstr, this->currentBlock, nullptr, value, nullptr, true, false, false, IR::BailOutOnTaggedValue)) { Value* landingPadValue = this->currentBlock->loop->landingPad->globOptData.FindValue(stackSym); ValueType newLandingPadValueType = landingPadValue->GetValueInfo()->Type().SetCanBeTaggedValue(false); ChangeValueType(nullptr, landingPadValue, newLandingPadValueType, false); } }); } if (symOpnd) { symOpnd->SetPropertyOwnerValueType(newValueType); } else { regOpnd->SetValueType(newValueType); } ChangeValueType(nullptr, value, newValueType, false); } } } return false; } bool GlobOpt::TypeSpecializeBailoutExpectedInteger(IR::Instr* instr, Value* src1Val, Value** dstVal) { bool isAlreadyTypeSpecialized = false; if(instr->GetSrc1()->IsRegOpnd()) { if (!src1Val || !src1Val->GetValueInfo()->IsLikelyInt() || instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber) { Assert(IsSwitchOptEnabledForIntTypeSpec()); throw Js::RejitException(RejitReason::DisableSwitchOptExpectingInteger); } // Attach the BailOutExpectingInteger to FromVar and Remove the bail out info on the Ld_A (Begin Switch) instr. this->ToTypeSpecUse(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, TyInt32, IR::BailOutExpectingInteger, false, instr); //TypeSpecialize the dst of Ld_A TypeSpecializeIntDst(instr, instr->m_opcode, src1Val, src1Val, nullptr, IR::BailOutInvalid, INT32_MIN, INT32_MAX, dstVal); isAlreadyTypeSpecialized = true; } instr->ClearBailOutInfo(); return isAlreadyTypeSpecialized; } Value* GlobOpt::OptDst( IR::Instr ** pInstr, Value *dstVal, Value *src1Val, Value *src2Val, Value *dstIndirIndexVal, Value *src1IndirIndexVal) { IR::Instr *&instr = *pInstr; IR::Opnd *opnd = instr->GetDst(); if (opnd) { if (opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd()) { this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd()); } if (opnd->IsIndirOpnd() && !this->IsLoopPrePass()) { IR::RegOpnd *baseOpnd = opnd->AsIndirOpnd()->GetBaseOpnd(); const ValueType baseValueType(baseOpnd->GetValueType()); if (( baseValueType.IsLikelyNativeArray() || #ifdef _M_IX86 ( !AutoSystemInfo::Data.SSE2Available() && baseValueType.IsLikelyObject() && ( baseValueType.GetObjectType() == ObjectType::Float32Array || baseValueType.GetObjectType() == ObjectType::Float64Array ) ) #else false #endif ) && instr->GetSrc1()->IsVar()) { if(instr->m_opcode == Js::OpCode::StElemC) { // StElemC has different code that handles native array conversion or missing value stores. Add a bailout // for those cases. Assert(baseValueType.IsLikelyNativeArray()); Assert(!instr->HasBailOutInfo()); GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly); } else if(instr->HasBailOutInfo()) { // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast // path. Note that the removed bailouts should not be necessary for correctness. Bailout on native array // conversion will be handled automatically as normal. IR::BailOutKind bailOutKind = instr->GetBailOutKind(); if(bailOutKind & IR::BailOutOnArrayAccessHelperCall) { bailOutKind -= IR::BailOutOnArrayAccessHelperCall; } if(bailOutKind == IR::BailOutOnImplicitCallsPreOp) { bailOutKind -= IR::BailOutOnImplicitCallsPreOp; } if(bailOutKind) { instr->SetBailOutKind(bailOutKind); } else { instr->ClearBailOutInfo(); } } } } } this->ProcessKills(instr); if (opnd) { if (dstVal == nullptr) { dstVal = ValueNumberDst(pInstr, src1Val, src2Val); } if (this->IsLoopPrePass()) { // Keep track of symbols defined in the loop. if (opnd->IsRegOpnd()) { StackSym *symDst = opnd->AsRegOpnd()->m_sym; rootLoopPrePass->symsDefInLoop->Set(symDst->m_id); } } else if (dstVal) { opnd->SetValueType(dstVal->GetValueInfo()->Type()); if (currentBlock->loop && !IsLoopPrePass() && (instr->m_opcode == Js::OpCode::Ld_A || instr->m_opcode == Js::OpCode::Ld_I4) && instr->GetSrc1()->IsRegOpnd() && !func->IsJitInDebugMode()) { // Look for the following patterns: // // Pattern 1: // s1[liveOnBackEdge] = s3[dead] // // Pattern 2: // s3 = operation(s1[liveOnBackEdge], s2) // s1[liveOnBackEdge] = s3 // // In both patterns, s1 and s3 have the same value by the end. Prefer to use s1 as the sym store instead of s3 // since s1 is live on back-edge, as otherwise, their lifetimes overlap, requiring two registers to hold the // value instead of one. do { IR::RegOpnd *const src = instr->GetSrc1()->AsRegOpnd(); StackSym *srcVarSym = src->m_sym; if(srcVarSym->IsTypeSpec()) { srcVarSym = srcVarSym->GetVarEquivSym(nullptr); Assert(srcVarSym); } if(dstVal->GetValueInfo()->GetSymStore() != srcVarSym) { break; } IR::RegOpnd *const dst = opnd->AsRegOpnd(); StackSym *dstVarSym = dst->m_sym; if(dstVarSym->IsTypeSpec()) { dstVarSym = dstVarSym->GetVarEquivSym(nullptr); Assert(dstVarSym); } if(!currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(dstVarSym->m_id)) { break; } Value *const srcValue = CurrentBlockData()->FindValue(srcVarSym); if(srcValue->GetValueNumber() != dstVal->GetValueNumber()) { break; } if(!src->GetIsDead()) { IR::Instr *const prevInstr = instr->GetPrevRealInstrOrLabel(); IR::Opnd *const prevDst = prevInstr->GetDst(); if(!prevDst || !src->IsEqualInternal(prevDst) || !( (prevInstr->GetSrc1() && dst->IsEqual(prevInstr->GetSrc1())) || (prevInstr->GetSrc2() && dst->IsEqual(prevInstr->GetSrc2())) )) { break; } } this->SetSymStoreDirect(dstVal->GetValueInfo(), dstVarSym); } while(false); } } this->ValueNumberObjectType(opnd, instr); } this->CSEAddInstr(this->currentBlock, *pInstr, dstVal, src1Val, src2Val, dstIndirIndexVal, src1IndirIndexVal); return dstVal; } void GlobOpt::CopyPropDstUses(IR::Opnd *opnd, IR::Instr *instr, Value *src1Val) { if (opnd->IsSymOpnd()) { IR::SymOpnd *symOpnd = opnd->AsSymOpnd(); if (symOpnd->m_sym->IsPropertySym()) { PropertySym * originalPropertySym = symOpnd->m_sym->AsPropertySym(); Value *const objectValue = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym); symOpnd->SetPropertyOwnerValueType(objectValue ? objectValue->GetValueInfo()->Type() : ValueType::Uninitialized); this->CopyPropPropertySymObj(symOpnd, instr); } } } void GlobOpt::SetLoopFieldInitialValue(Loop *loop, IR::Instr *instr, PropertySym *propertySym, PropertySym *originalPropertySym) { Value *initialValue = nullptr; StackSym *symStore; if (loop->allFieldsKilled || loop->fieldKilled->Test(originalPropertySym->m_id) || loop->fieldKilled->Test(propertySym->m_id)) { return; } // Value already exists if (CurrentBlockData()->FindValue(propertySym)) { return; } // If this initial value was already added, we would find in the current value table. Assert(!loop->initialValueFieldMap.TryGetValue(propertySym, &initialValue)); // If propertySym is live in landingPad, we don't need an initial value. if (loop->landingPad->globOptData.liveFields->Test(propertySym->m_id)) { return; } StackSym * objectSym = propertySym->m_stackSym; Value *landingPadObjPtrVal, *currentObjPtrVal; landingPadObjPtrVal = loop->landingPad->globOptData.FindValue(objectSym); currentObjPtrVal = CurrentBlockData()->FindValue(objectSym); auto CanSetInitialValue = [&]() -> bool { if (!currentObjPtrVal) { return false; } if (landingPadObjPtrVal) { return currentObjPtrVal->GetValueNumber() == landingPadObjPtrVal->GetValueNumber(); } else { if (!objectSym->IsSingleDef()) { return false; } IR::Instr * defInstr = objectSym->GetInstrDef(); IR::Opnd * src1 = defInstr->GetSrc1(); while (!(src1 && src1->IsSymOpnd() && src1->AsSymOpnd()->m_sym->IsPropertySym())) { if (src1 && src1->IsRegOpnd() && src1->AsRegOpnd()->GetStackSym()->IsSingleDef()) { defInstr = src1->AsRegOpnd()->GetStackSym()->GetInstrDef(); src1 = defInstr->GetSrc1(); } else { return false; } } return true; // Todo: allow other kinds of operands as src1 of instr def of the object sym of the current propertySym // SymOpnd, but not PropertySymOpnd - LdSlotArr, some LdSlots (?) // nullptr - NewScObject } }; if (!CanSetInitialValue()) { // objPtr has a different value in the landing pad. return; } // The opnd's value type has not yet been initialized. Since the property sym doesn't have a value, it effectively has an // Uninitialized value type. Use the profiled value type from the instruction. const ValueType profiledValueType = instr->IsProfiledInstr() ? instr->AsProfiledInstr()->u.FldInfo().valueType : ValueType::Uninitialized; Assert(!profiledValueType.IsDefinite()); // Hence the values created here don't need to be tracked for kills initialValue = this->NewGenericValue(profiledValueType, propertySym); symStore = StackSym::New(this->func); initialValue->GetValueInfo()->SetSymStore(symStore); loop->initialValueFieldMap.Add(propertySym, initialValue->Copy(this->alloc, initialValue->GetValueNumber())); // Copy the initial value into the landing pad, but without a symStore Value *landingPadInitialValue = Value::New(this->alloc, initialValue->GetValueNumber(), ValueInfo::New(this->alloc, initialValue->GetValueInfo()->Type())); loop->landingPad->globOptData.SetValue(landingPadInitialValue, propertySym); loop->landingPad->globOptData.liveFields->Set(propertySym->m_id); #if DBG_DUMP if (PHASE_TRACE(Js::FieldPREPhase, this->func)) { Output::Print(_u("** TRACE: Field PRE initial value for loop head #%d. Val:%d symStore:"), loop->GetHeadBlock()->GetBlockNum(), initialValue->GetValueNumber()); symStore->Dump(); Output::Print(_u("\n Instr: ")); instr->Dump(); Output::Flush(); } #endif // Add initial value to all the previous blocks in the loop. FOREACH_BLOCK_BACKWARD_IN_RANGE(block, this->currentBlock->GetPrev(), loop->GetHeadBlock()) { if (block->GetDataUseCount() == 0) { // All successor blocks have been processed, no point in adding the value. continue; } Value *newValue = initialValue->Copy(this->alloc, initialValue->GetValueNumber()); block->globOptData.SetValue(newValue, propertySym); block->globOptData.liveFields->Set(propertySym->m_id); block->globOptData.SetValue(newValue, symStore); block->globOptData.liveVarSyms->Set(symStore->m_id); } NEXT_BLOCK_BACKWARD_IN_RANGE; CurrentBlockData()->SetValue(initialValue, symStore); CurrentBlockData()->liveVarSyms->Set(symStore->m_id); CurrentBlockData()->liveFields->Set(propertySym->m_id); } // Examine src, apply copy prop and value number it Value* GlobOpt::OptSrc(IR::Opnd *opnd, IR::Instr * *pInstr, Value **indirIndexValRef, IR::IndirOpnd *parentIndirOpnd) { IR::Instr * &instr = *pInstr; Assert(!indirIndexValRef || !*indirIndexValRef); Assert( parentIndirOpnd ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd() : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd()); Sym *sym; Value *val; PropertySym *originalPropertySym = nullptr; switch(opnd->GetKind()) { case IR::OpndKindIntConst: val = this->GetIntConstantValue(opnd->AsIntConstOpnd()->AsInt32(), instr); opnd->SetValueType(val->GetValueInfo()->Type()); return val; case IR::OpndKindInt64Const: val = this->GetIntConstantValue(opnd->AsInt64ConstOpnd()->GetValue(), instr); opnd->SetValueType(val->GetValueInfo()->Type()); return val; case IR::OpndKindFloatConst: { const FloatConstType floatValue = opnd->AsFloatConstOpnd()->m_value; int32 int32Value; if(Js::JavascriptNumber::TryGetInt32Value(floatValue, &int32Value)) { val = GetIntConstantValue(int32Value, instr); } else { val = NewFloatConstantValue(floatValue); } opnd->SetValueType(val->GetValueInfo()->Type()); return val; } case IR::OpndKindAddr: { IR::AddrOpnd *addrOpnd = opnd->AsAddrOpnd(); if (addrOpnd->m_isFunction) { AssertMsg(!PHASE_OFF(Js::FixedMethodsPhase, instr->m_func), "Fixed function address operand with fixed method calls phase disabled?"); val = NewFixedFunctionValue((Js::JavascriptFunction *)addrOpnd->m_address, addrOpnd); opnd->SetValueType(val->GetValueInfo()->Type()); return val; } else if (addrOpnd->IsVar() && Js::TaggedInt::Is(addrOpnd->m_address)) { val = this->GetIntConstantValue(Js::TaggedInt::ToInt32(addrOpnd->m_address), instr); opnd->SetValueType(val->GetValueInfo()->Type()); return val; } val = this->GetVarConstantValue(addrOpnd); return val; } case IR::OpndKindSym: { // Clear the opnd's value type up-front, so that this code cannot accidentally use the value type set from a previous // OptSrc on the same instruction (for instance, from an earlier loop prepass). The value type will be set from the // value if available, before returning from this function. opnd->SetValueType(ValueType::Uninitialized); sym = opnd->AsSymOpnd()->m_sym; // Don't create a new value for ArgSlots and don't copy prop them away. if (sym->IsStackSym() && sym->AsStackSym()->IsArgSlotSym()) { return nullptr; } // Unless we have profile info, don't create a new value for ArgSlots and don't copy prop them away. if (sym->IsStackSym() && sym->AsStackSym()->IsParamSlotSym()) { if (!instr->m_func->IsLoopBody() && instr->m_func->HasProfileInfo()) { // Skip "this" pointer. int paramSlotNum = sym->AsStackSym()->GetParamSlotNum() - 2; if (paramSlotNum >= 0) { const auto parameterType = instr->m_func->GetReadOnlyProfileInfo()->GetParameterInfo(static_cast(paramSlotNum)); val = NewGenericValue(parameterType); opnd->SetValueType(val->GetValueInfo()->Type()); return val; } } return nullptr; } if (!sym->IsPropertySym()) { break; } originalPropertySym = sym->AsPropertySym(); // Don't give a value to 'arguments' property sym to prevent field copy prop of 'arguments' if (originalPropertySym->AsPropertySym()->m_propertyId == Js::PropertyIds::arguments && originalPropertySym->AsPropertySym()->m_fieldKind == PropertyKindData) { if (opnd->AsSymOpnd()->IsPropertySymOpnd()) { this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd()); } return nullptr; } Value *const objectValue = CurrentBlockData()->FindValue(originalPropertySym->m_stackSym); opnd->AsSymOpnd()->SetPropertyOwnerValueType( objectValue ? objectValue->GetValueInfo()->Type() : ValueType::Uninitialized); sym = this->CopyPropPropertySymObj(opnd->AsSymOpnd(), instr); if (!DoFieldCopyProp()) { if (opnd->AsSymOpnd()->IsPropertySymOpnd()) { this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd()); } return nullptr; } switch (instr->m_opcode) { // These need the symbolic reference to the field, don't copy prop the value of the field case Js::OpCode::DeleteFld: case Js::OpCode::DeleteRootFld: case Js::OpCode::DeleteFldStrict: case Js::OpCode::DeleteRootFldStrict: case Js::OpCode::ScopedDeleteFld: case Js::OpCode::ScopedDeleteFldStrict: case Js::OpCode::LdMethodFromFlags: case Js::OpCode::BrOnNoProperty: case Js::OpCode::BrOnNoLocalProperty: case Js::OpCode::BrOnHasProperty: case Js::OpCode::BrOnHasLocalProperty: case Js::OpCode::LdMethodFldPolyInlineMiss: case Js::OpCode::StSlotChkUndecl: case Js::OpCode::ScopedLdInst: return nullptr; }; if (instr->CallsGetter()) { return nullptr; } if (this->IsLoopPrePass() && this->DoFieldPRE(this->rootLoopPrePass)) { if (!this->prePassLoop->allFieldsKilled && !this->prePassLoop->fieldKilled->Test(sym->m_id)) { this->SetLoopFieldInitialValue(this->rootLoopPrePass, instr, sym->AsPropertySym(), originalPropertySym); } if (this->IsPREInstrCandidateLoad(instr->m_opcode)) { // Foreach property sym, remember the first instruction that loads it. // Can this be done in one call? if (!this->prePassInstrMap->ContainsKey(sym->m_id)) { this->prePassInstrMap->AddNew(sym->m_id, instr->CopyWithoutDst()); } } } break; } case IR::OpndKindReg: // Clear the opnd's value type up-front, so that this code cannot accidentally use the value type set from a previous // OptSrc on the same instruction (for instance, from an earlier loop prepass). The value type will be set from the // value if available, before returning from this function. opnd->SetValueType(ValueType::Uninitialized); sym = opnd->AsRegOpnd()->m_sym; CurrentBlockData()->MarkTempLastUse(instr, opnd->AsRegOpnd()); if (sym->AsStackSym()->IsTypeSpec()) { sym = sym->AsStackSym()->GetVarEquivSym(this->func); } break; case IR::OpndKindIndir: this->OptimizeIndirUses(opnd->AsIndirOpnd(), &instr, indirIndexValRef); return nullptr; default: return nullptr; } val = CurrentBlockData()->FindValue(sym); if (val) { Assert(CurrentBlockData()->IsLive(sym) || (sym->IsPropertySym())); if (instr) { opnd = this->CopyProp(opnd, instr, val, parentIndirOpnd); } // Check if we freed the operand. if (opnd == nullptr) { return nullptr; } // In a loop prepass, determine stack syms that are used before they are defined in the root loop for which the prepass // is being done. This information is used to do type specialization conversions in the landing pad where appropriate. if(IsLoopPrePass() && sym->IsStackSym() && !rootLoopPrePass->symsUsedBeforeDefined->Test(sym->m_id) && rootLoopPrePass->landingPad->globOptData.IsLive(sym) && !isAsmJSFunc) // no typespec in asmjs and hence skipping this { Value *const landingPadValue = rootLoopPrePass->landingPad->globOptData.FindValue(sym); if(landingPadValue && val->GetValueNumber() == landingPadValue->GetValueNumber()) { rootLoopPrePass->symsUsedBeforeDefined->Set(sym->m_id); ValueInfo *landingPadValueInfo = landingPadValue->GetValueInfo(); if(landingPadValueInfo->IsLikelyNumber()) { rootLoopPrePass->likelyNumberSymsUsedBeforeDefined->Set(sym->m_id); if(DoAggressiveIntTypeSpec() ? landingPadValueInfo->IsLikelyInt() : landingPadValueInfo->IsInt()) { // Can only force int conversions in the landing pad based on likely-int values if aggressive int type // specialization is enabled. rootLoopPrePass->likelyIntSymsUsedBeforeDefined->Set(sym->m_id); } } } } } else if ((instr->TransfersSrcValue() || OpCodeAttr::CanCSE(instr->m_opcode)) && (opnd == instr->GetSrc1() || opnd == instr->GetSrc2())) { if (sym->IsPropertySym()) { val = this->CreateFieldSrcValue(sym->AsPropertySym(), originalPropertySym, &opnd, instr); } else { val = this->NewGenericValue(ValueType::Uninitialized, opnd); } } if (opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd()) { TryOptimizeInstrWithFixedDataProperty(&instr); this->FinishOptPropOp(instr, opnd->AsPropertySymOpnd()); } if (val) { ValueType valueType(val->GetValueInfo()->Type()); // This block uses per-instruction profile information on array types to optimize using the best available profile // information and to prevent infinite bailouts by ensuring array type information is updated on bailouts. if (valueType.IsLikelyArray() && !valueType.IsDefinite() && !valueType.IsObject() && instr->IsProfiledInstr()) { // See if we have profile data for the array type IR::ProfiledInstr *const profiledInstr = instr->AsProfiledInstr(); ValueType profiledArrayType; bool useAggressiveSpecialization = true; switch(instr->m_opcode) { case Js::OpCode::LdElemI_A: if(instr->GetSrc1()->IsIndirOpnd() && opnd == instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()) { profiledArrayType = profiledInstr->u.ldElemInfo->GetArrayType(); useAggressiveSpecialization = !profiledInstr->u.ldElemInfo->IsAggressiveSpecializationDisabled(); } break; case Js::OpCode::StElemI_A: case Js::OpCode::StElemI_A_Strict: case Js::OpCode::StElemC: if(instr->GetDst()->IsIndirOpnd() && opnd == instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()) { profiledArrayType = profiledInstr->u.stElemInfo->GetArrayType(); useAggressiveSpecialization = !profiledInstr->u.stElemInfo->IsAggressiveSpecializationDisabled(); } break; case Js::OpCode::LdLen_A: if(instr->GetSrc1()->IsRegOpnd() && opnd == instr->GetSrc1()) { profiledArrayType = profiledInstr->u.LdLenInfo().GetArrayType(); useAggressiveSpecialization = !profiledInstr->u.LdLenInfo().IsAggressiveSpecializationDisabled(); } break; case Js::OpCode::IsIn: if (instr->GetSrc2()->IsRegOpnd() && opnd == instr->GetSrc2()) { profiledArrayType = profiledInstr->u.ldElemInfo->GetArrayType(); useAggressiveSpecialization = !profiledInstr->u.ldElemInfo->IsAggressiveSpecializationDisabled(); } break; } if (profiledArrayType.IsLikelyObject()) { // Ideally we want to use the most specialized type seen by this path, but when that causes bailouts use the least specialized type instead. if (useAggressiveSpecialization && profiledArrayType.GetObjectType() == valueType.GetObjectType() && !valueType.IsLikelyNativeIntArray() && ( profiledArrayType.HasIntElements() || (valueType.HasVarElements() && profiledArrayType.HasFloatElements()) )) { // use the more specialized type profiled by the instruction. valueType = profiledArrayType.SetHasNoMissingValues(valueType.HasNoMissingValues()); ChangeValueType(this->currentBlock, CurrentBlockData()->FindValue(opnd->AsRegOpnd()->m_sym), valueType, false); } else if (!useAggressiveSpecialization && (profiledArrayType.GetObjectType() != valueType.GetObjectType() || ( valueType.IsLikelyNativeArray() && ( profiledArrayType.HasVarElements() || (valueType.HasIntElements() && profiledArrayType.HasFloatElements()) ) ) )) { // Merge array type we pulled from profile with type propagated by dataflow. if (profiledArrayType.IsLikelyArray()) { valueType = valueType.Merge(profiledArrayType).SetHasNoMissingValues(valueType.HasNoMissingValues()); } else { valueType = valueType.Merge(profiledArrayType); } ChangeValueType(this->currentBlock, CurrentBlockData()->FindValue(opnd->AsRegOpnd()->m_sym), valueType, false, true); } } } opnd->SetValueType(valueType); if(!IsLoopPrePass() && opnd->IsSymOpnd() && (valueType.IsDefinite() || valueType.IsNotTaggedValue())) { if (opnd->AsSymOpnd()->m_sym->IsPropertySym()) { // A property sym can only be guaranteed to have a definite value type when implicit calls are disabled from the // point where the sym was defined with the definite value type. Insert an instruction to indicate to the // dead-store pass that implicit calls need to be kept disabled until after this instruction. Assert(DoFieldCopyProp()); CaptureNoImplicitCallUses(opnd, false, instr); } } } else { opnd->SetValueType(ValueType::Uninitialized); } return val; } /* * GlobOpt::TryOptimizeInstrWithFixedDataProperty * Converts Ld[Root]Fld instr to * * CheckFixedFld * * Dst = Ld_A * This API assumes that the source operand is a Sym/PropertySym kind. */ void GlobOpt::TryOptimizeInstrWithFixedDataProperty(IR::Instr ** const pInstr) { Assert(pInstr); IR::Instr * &instr = *pInstr; IR::Opnd * src1 = instr->GetSrc1(); Assert(src1 && src1->IsSymOpnd() && src1->AsSymOpnd()->IsPropertySymOpnd()); if(PHASE_OFF(Js::UseFixedDataPropsPhase, instr->m_func)) { return; } if (!this->IsLoopPrePass() && !this->isRecursiveCallOnLandingPad && OpCodeAttr::CanLoadFixedFields(instr->m_opcode)) { instr->TryOptimizeInstrWithFixedDataProperty(&instr, this); } } // Constant prop if possible, otherwise if this value already resides in another // symbol, reuse this previous symbol. This should help register allocation. IR::Opnd * GlobOpt::CopyProp(IR::Opnd *opnd, IR::Instr *instr, Value *val, IR::IndirOpnd *parentIndirOpnd) { Assert( parentIndirOpnd ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd() : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd()); if (this->IsLoopPrePass()) { // Transformations are not legal in prepass... return opnd; } if (instr->m_opcode == Js::OpCode::CheckFixedFld || instr->m_opcode == Js::OpCode::CheckPropertyGuardAndLoadType) { // Don't copy prop into CheckFixedFld or CheckPropertyGuardAndLoadType return opnd; } // Don't copy-prop link operands of ExtendedArgs if (instr->m_opcode == Js::OpCode::ExtendArg_A && opnd == instr->GetSrc2()) { return opnd; } // Don't copy-prop operand of SIMD instr with ExtendedArg operands. Each instr should have its exclusive EA sequence. if ( Js::IsSimd128Opcode(instr->m_opcode) && instr->GetSrc1() != nullptr && instr->GetSrc1()->IsRegOpnd() && instr->GetSrc2() == nullptr ) { StackSym *sym = instr->GetSrc1()->GetStackSym(); if (sym && sym->IsSingleDef() && sym->GetInstrDef()->m_opcode == Js::OpCode::ExtendArg_A) { return opnd; } } ValueInfo *valueInfo = val->GetValueInfo(); if (this->func->HasFinally()) { // s0 = undefined was added on functions with early exit in try-finally functions, that can get copy-proped and case incorrect results if (instr->m_opcode == Js::OpCode::ArgOut_A_Inline && valueInfo->GetSymStore() && valueInfo->GetSymStore()->m_id == 0) { // We don't want to copy-prop s0 (return symbol) into inlinee code return opnd; } } // Constant prop? int32 intConstantValue; int64 int64ConstantValue; if (valueInfo->TryGetIntConstantValue(&intConstantValue)) { if (PHASE_OFF(Js::ConstPropPhase, this->func)) { return opnd; } if (( instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict || instr->m_opcode == Js::OpCode::StElemC ) && instr->GetSrc1() == opnd) { // Disabling prop to src of native array store, because we were losing the chance to type specialize. // Is it possible to type specialize this src if we allow constants, etc., to be prop'd here? if (instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyNativeArray()) { return opnd; } } if(opnd != instr->GetSrc1() && opnd != instr->GetSrc2()) { if(PHASE_OFF(Js::IndirCopyPropPhase, instr->m_func)) { return opnd; } // Const-prop an indir opnd's constant index into its offset IR::Opnd *srcs[] = { instr->GetSrc1(), instr->GetSrc2(), instr->GetDst() }; for(int i = 0; i < sizeof(srcs) / sizeof(srcs[0]); ++i) { const auto src = srcs[i]; if(!src || !src->IsIndirOpnd()) { continue; } const auto indir = src->AsIndirOpnd(); if ((int64)indir->GetOffset() + intConstantValue > INT32_MAX) { continue; } if(opnd == indir->GetIndexOpnd()) { Assert(indir->GetScale() == 0); GOPT_TRACE_OPND(opnd, _u("Constant prop indir index into offset (value: %d)\n"), intConstantValue); this->CaptureByteCodeSymUses(instr); indir->SetOffset(indir->GetOffset() + intConstantValue); indir->SetIndexOpnd(nullptr); } } return opnd; } if (Js::TaggedInt::IsOverflow(intConstantValue)) { return opnd; } IR::Opnd *constOpnd; if (opnd->IsVar()) { IR::AddrOpnd *addrOpnd = IR::AddrOpnd::New(Js::TaggedInt::ToVarUnchecked((int)intConstantValue), IR::AddrOpndKindConstantVar, instr->m_func); GOPT_TRACE_OPND(opnd, _u("Constant prop %d (value:%d)\n"), addrOpnd->m_address, intConstantValue); constOpnd = addrOpnd; } else { // Note: Jit loop body generates some i32 operands... Assert(opnd->IsInt32() || opnd->IsInt64() || opnd->IsUInt32()); IRType opndType; IntConstType constVal; if (opnd->IsUInt32()) { // avoid sign extension constVal = (uint32)intConstantValue; opndType = TyUint32; } else { constVal = intConstantValue; opndType = TyInt32; } IR::IntConstOpnd *intOpnd = IR::IntConstOpnd::New(constVal, opndType, instr->m_func); GOPT_TRACE_OPND(opnd, _u("Constant prop %d (value:%d)\n"), intOpnd->GetImmediateValue(instr->m_func), intConstantValue); constOpnd = intOpnd; } #if ENABLE_DEBUG_CONFIG_OPTIONS //Need to update DumpFieldCopyPropTestTrace for every new opcode that is added for fieldcopyprop if(Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FieldCopyPropPhase)) { instr->DumpFieldCopyPropTestTrace(this->isRecursiveCallOnLandingPad); } #endif this->CaptureByteCodeSymUses(instr); opnd = instr->ReplaceSrc(opnd, constOpnd); switch (instr->m_opcode) { case Js::OpCode::LdSlot: case Js::OpCode::LdSlotArr: case Js::OpCode::LdFld: case Js::OpCode::LdFldForTypeOf: case Js::OpCode::LdRootFldForTypeOf: case Js::OpCode::LdFldForCallApplyTarget: case Js::OpCode::LdRootFld: case Js::OpCode::LdMethodFld: case Js::OpCode::LdRootMethodFld: case Js::OpCode::LdMethodFromFlags: case Js::OpCode::ScopedLdMethodFld: case Js::OpCode::ScopedLdFld: case Js::OpCode::ScopedLdFldForTypeOf: instr->m_opcode = Js::OpCode::Ld_A; case Js::OpCode::Ld_A: { IR::Opnd * dst = instr->GetDst(); if (dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->IsSingleDef()) { dst->AsRegOpnd()->m_sym->SetIsIntConst((int)intConstantValue); } break; } case Js::OpCode::ArgOut_A: case Js::OpCode::ArgOut_A_Inline: case Js::OpCode::ArgOut_A_FixupForStackArgs: case Js::OpCode::ArgOut_A_InlineBuiltIn: if (instr->GetDst()->IsRegOpnd()) { Assert(instr->GetDst()->AsRegOpnd()->m_sym->m_isSingleDef); instr->GetDst()->AsRegOpnd()->m_sym->AsStackSym()->SetIsIntConst((int)intConstantValue); } else { instr->GetDst()->AsSymOpnd()->m_sym->AsStackSym()->SetIsIntConst((int)intConstantValue); } break; case Js::OpCode::TypeofElem: instr->m_opcode = Js::OpCode::Typeof; break; case Js::OpCode::StSlotChkUndecl: if (instr->GetSrc2() == opnd) { // Src2 here should refer to the same location as the Dst operand, which we need to keep live // due to the implicit read for ChkUndecl. instr->m_opcode = Js::OpCode::StSlot; instr->FreeSrc2(); opnd = nullptr; } break; } return opnd; } else if (valueInfo->TryGetIntConstantValue(&int64ConstantValue, false)) { if (PHASE_OFF(Js::ConstPropPhase, this->func) || !PHASE_ON(Js::Int64ConstPropPhase, this->func)) { return opnd; } Assert(this->func->GetJITFunctionBody()->IsWasmFunction()); if (this->func->GetJITFunctionBody()->IsWasmFunction() && opnd->IsInt64()) { IR::Int64ConstOpnd *intOpnd = IR::Int64ConstOpnd::New(int64ConstantValue, opnd->GetType(), instr->m_func); GOPT_TRACE_OPND(opnd, _u("Constant prop %lld (value:%lld)\n"), intOpnd->GetImmediateValue(instr->m_func), int64ConstantValue); this->CaptureByteCodeSymUses(instr); opnd = instr->ReplaceSrc(opnd, intOpnd); } return opnd; } Sym *opndSym = nullptr; if (opnd->IsRegOpnd()) { IR::RegOpnd *regOpnd = opnd->AsRegOpnd(); opndSym = regOpnd->m_sym; } else if (opnd->IsSymOpnd()) { IR::SymOpnd *symOpnd = opnd->AsSymOpnd(); opndSym = symOpnd->m_sym; } if (!opndSym) { return opnd; } if (PHASE_OFF(Js::CopyPropPhase, this->func)) { this->SetSymStoreDirect(valueInfo, opndSym); return opnd; } StackSym *copySym = CurrentBlockData()->GetCopyPropSym(opndSym, val); if (copySym != nullptr) { Assert(!opndSym->IsStackSym() || copySym->GetSymSize() == opndSym->AsStackSym()->GetSymSize()); // Copy prop. return CopyPropReplaceOpnd(instr, opnd, copySym, parentIndirOpnd); } else { if (valueInfo->GetSymStore() && instr->m_opcode == Js::OpCode::Ld_A && instr->GetDst()->IsRegOpnd() && valueInfo->GetSymStore() == instr->GetDst()->AsRegOpnd()->m_sym) { // Avoid resetting symStore after fieldHoisting: // t1 = LdFld field <- set symStore to fieldHoistSym // fieldHoistSym = Ld_A t1 <- we're looking at t1 now, but want to copy-prop fieldHoistSym forward return opnd; } this->SetSymStoreDirect(valueInfo, opndSym); } return opnd; } IR::Opnd * GlobOpt::CopyPropReplaceOpnd(IR::Instr * instr, IR::Opnd * opnd, StackSym * copySym, IR::IndirOpnd *parentIndirOpnd) { Assert( parentIndirOpnd ? opnd == parentIndirOpnd->GetBaseOpnd() || opnd == parentIndirOpnd->GetIndexOpnd() : opnd == instr->GetSrc1() || opnd == instr->GetSrc2() || opnd == instr->GetDst() && opnd->IsIndirOpnd()); Assert(CurrentBlockData()->IsLive(copySym)); IR::RegOpnd *regOpnd; StackSym *newSym = copySym; GOPT_TRACE_OPND(opnd, _u("Copy prop s%d\n"), newSym->m_id); #if ENABLE_DEBUG_CONFIG_OPTIONS //Need to update DumpFieldCopyPropTestTrace for every new opcode that is added for fieldcopyprop if(Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FieldCopyPropPhase)) { instr->DumpFieldCopyPropTestTrace(this->isRecursiveCallOnLandingPad); } #endif this->CaptureByteCodeSymUses(instr); if (opnd->IsRegOpnd()) { regOpnd = opnd->AsRegOpnd(); regOpnd->m_sym = newSym; regOpnd->SetIsJITOptimizedReg(true); // The dead bit on the opnd is specific to the sym it is referencing. Since we replaced the sym, the bit is reset. regOpnd->SetIsDead(false); if(parentIndirOpnd) { return regOpnd; } } else { // If this is an object type specialized field load inside a loop, and it produces a type value which wasn't live // before, make sure the type check is left in the loop, because it may be the last type check in the loop protecting // other fields which are not hoistable and are lexically upstream in the loop. If the check is not ultimately // needed, the dead store pass will remove it. if (this->currentBlock->loop != nullptr && opnd->IsSymOpnd() && opnd->AsSymOpnd()->IsPropertySymOpnd()) { IR::PropertySymOpnd* propertySymOpnd = opnd->AsPropertySymOpnd(); if (CheckIfPropOpEmitsTypeCheck(instr, propertySymOpnd)) { // We only set guarded properties in the dead store pass, so they shouldn't be set here yet. If they were // we would need to move them from this operand to the operand which is being copy propagated. Assert(propertySymOpnd->GetGuardedPropOps() == nullptr); // We're creating a copy of this operand to be reused in the same spot in the flow, so we can copy all // flow sensitive fields. However, we will do only a type check here (no property access) and only for // the sake of downstream instructions, so the flags pertaining to this property access are irrelevant. IR::PropertySymOpnd* checkObjTypeOpnd = CreateOpndForTypeCheckOnly(propertySymOpnd, instr->m_func); IR::Instr* checkObjTypeInstr = IR::Instr::New(Js::OpCode::CheckObjType, instr->m_func); checkObjTypeInstr->SetSrc1(checkObjTypeOpnd); checkObjTypeInstr->SetByteCodeOffset(instr); instr->InsertBefore(checkObjTypeInstr); // Since we inserted this instruction before the one that is being processed in natural flow, we must process // it for object type spec explicitly here. FinishOptPropOp(checkObjTypeInstr, checkObjTypeOpnd); Assert(!propertySymOpnd->IsTypeChecked()); checkObjTypeInstr = this->SetTypeCheckBailOut(checkObjTypeOpnd, checkObjTypeInstr, nullptr); Assert(checkObjTypeInstr->HasBailOutInfo()); if (this->currentBlock->loop && !this->IsLoopPrePass()) { // Try hoisting this checkObjType. // But since this isn't the current instr being optimized, we need to play tricks with // the byteCodeUse fields... TrackByteCodeUsesForInstrAddedInOptInstr(checkObjTypeInstr, [&]() { TryHoistInvariant(checkObjTypeInstr, this->currentBlock, NULL, CurrentBlockData()->FindValue(copySym), NULL, true); }); } } } if (opnd->IsSymOpnd() && opnd->GetIsDead()) { // Take the property sym out of the live fields set this->EndFieldLifetime(opnd->AsSymOpnd()); } regOpnd = IR::RegOpnd::New(newSym, opnd->GetType(), instr->m_func); regOpnd->SetIsJITOptimizedReg(true); instr->ReplaceSrc(opnd, regOpnd); } switch (instr->m_opcode) { case Js::OpCode::Ld_A: if (instr->GetDst()->IsRegOpnd() && instr->GetSrc1()->IsRegOpnd() && instr->GetDst()->AsRegOpnd()->GetStackSym() == instr->GetSrc1()->AsRegOpnd()->GetStackSym()) { this->InsertByteCodeUses(instr, true); instr->m_opcode = Js::OpCode::Nop; } break; case Js::OpCode::LdSlot: case Js::OpCode::LdSlotArr: if (instr->GetDst()->IsRegOpnd() && instr->GetSrc1()->IsRegOpnd() && instr->GetDst()->AsRegOpnd()->GetStackSym() == instr->GetSrc1()->AsRegOpnd()->GetStackSym()) { this->InsertByteCodeUses(instr, true); instr->m_opcode = Js::OpCode::Nop; } else { instr->m_opcode = Js::OpCode::Ld_A; } break; case Js::OpCode::StSlotChkUndecl: if (instr->GetSrc2()->IsRegOpnd()) { // Src2 here should refer to the same location as the Dst operand, which we need to keep live // due to the implicit read for ChkUndecl. instr->m_opcode = Js::OpCode::StSlot; instr->FreeSrc2(); return nullptr; } break; case Js::OpCode::LdFld: case Js::OpCode::LdFldForTypeOf: case Js::OpCode::LdRootFldForTypeOf: case Js::OpCode::LdFldForCallApplyTarget: case Js::OpCode::LdRootFld: case Js::OpCode::LdMethodFld: case Js::OpCode::LdRootMethodFld: case Js::OpCode::ScopedLdMethodFld: case Js::OpCode::ScopedLdFld: case Js::OpCode::ScopedLdFldForTypeOf: instr->m_opcode = Js::OpCode::Ld_A; break; case Js::OpCode::LdMethodFromFlags: // The bailout is checked on the loop top and we don't need to check bailout again in loop. instr->m_opcode = Js::OpCode::Ld_A; instr->ClearBailOutInfo(); break; case Js::OpCode::TypeofElem: instr->m_opcode = Js::OpCode::Typeof; break; } CurrentBlockData()->MarkTempLastUse(instr, regOpnd); return regOpnd; } ValueNumber GlobOpt::NewValueNumber() { ValueNumber valueNumber = this->currentValue++; if (valueNumber == 0) { Js::Throw::OutOfMemory(); } return valueNumber; } Value *GlobOpt::NewValue(ValueInfo *const valueInfo) { return NewValue(NewValueNumber(), valueInfo); } Value *GlobOpt::NewValue(const ValueNumber valueNumber, ValueInfo *const valueInfo) { Assert(valueInfo); return Value::New(alloc, valueNumber, valueInfo); } Value *GlobOpt::CopyValue(Value const *const value) { return CopyValue(value, NewValueNumber()); } Value *GlobOpt::CopyValue(Value const *const value, const ValueNumber valueNumber) { Assert(value); return value->Copy(alloc, valueNumber); } Value * GlobOpt::NewGenericValue(const ValueType valueType) { return NewGenericValue(valueType, static_cast(nullptr)); } Value * GlobOpt::NewGenericValue(const ValueType valueType, IR::Opnd *const opnd) { // Shouldn't assign a likely-int value to something that is definitely not an int Assert(!(valueType.IsLikelyInt() && opnd && opnd->IsNotInt())); ValueInfo *valueInfo = ValueInfo::New(this->alloc, valueType); Value *val = NewValue(valueInfo); TrackNewValueForKills(val); CurrentBlockData()->InsertNewValue(val, opnd); return val; } Value * GlobOpt::NewGenericValue(const ValueType valueType, Sym *const sym) { ValueInfo *valueInfo = ValueInfo::New(this->alloc, valueType); Value *val = NewValue(valueInfo); TrackNewValueForKills(val); CurrentBlockData()->SetValue(val, sym); return val; } Value * GlobOpt::GetIntConstantValue(const int32 intConst, IR::Instr * instr, IR::Opnd *const opnd) { Value *value = nullptr; Value *const cachedValue = this->intConstantToValueMap->Lookup(intConst, nullptr); if(cachedValue) { // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since // values are cloned for each block, we can't use the same value object. We also can't have two values with the same // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast // way to determine if a value with the same value number exists for this block. So the best we can do with a global // cache is to check the sym-store's value in the current block to see if it has a value with the same number. // Otherwise, we have to create a new value with a new value number. Sym *const symStore = cachedValue->GetValueInfo()->GetSymStore(); if (symStore && CurrentBlockData()->IsLive(symStore)) { Value *const symStoreValue = CurrentBlockData()->FindValue(symStore); int32 symStoreIntConstantValue; if (symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber() && symStoreValue->GetValueInfo()->TryGetIntConstantValue(&symStoreIntConstantValue) && symStoreIntConstantValue == intConst) { value = symStoreValue; } } } if (!value) { value = NewIntConstantValue(intConst, instr, !Js::TaggedInt::IsOverflow(intConst)); } return CurrentBlockData()->InsertNewValue(value, opnd); } Value * GlobOpt::GetIntConstantValue(const int64 intConst, IR::Instr * instr, IR::Opnd *const opnd) { Assert(instr->m_func->GetJITFunctionBody()->IsWasmFunction()); Value *value = nullptr; Value *const cachedValue = this->int64ConstantToValueMap->Lookup(intConst, nullptr); if (cachedValue) { // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since // values are cloned for each block, we can't use the same value object. We also can't have two values with the same // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast // way to determine if a value with the same value number exists for this block. So the best we can do with a global // cache is to check the sym-store's value in the current block to see if it has a value with the same number. // Otherwise, we have to create a new value with a new value number. Sym *const symStore = cachedValue->GetValueInfo()->GetSymStore(); if (symStore && this->currentBlock->globOptData.IsLive(symStore)) { Value *const symStoreValue = this->currentBlock->globOptData.FindValue(symStore); int64 symStoreIntConstantValue; if (symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber() && symStoreValue->GetValueInfo()->TryGetInt64ConstantValue(&symStoreIntConstantValue, false) && symStoreIntConstantValue == intConst) { value = symStoreValue; } } } if (!value) { value = NewInt64ConstantValue(intConst, instr); } return this->currentBlock->globOptData.InsertNewValue(value, opnd); } Value * GlobOpt::NewInt64ConstantValue(const int64 intConst, IR::Instr* instr) { Value * value = NewValue(Int64ConstantValueInfo::New(this->alloc, intConst)); this->int64ConstantToValueMap->Item(intConst, value); if (!value->GetValueInfo()->GetSymStore() && (instr->m_opcode == Js::OpCode::LdC_A_I4 || instr->m_opcode == Js::OpCode::Ld_I4)) { StackSym * sym = instr->GetDst()->GetStackSym(); Assert(sym && !sym->IsTypeSpec()); this->currentBlock->globOptData.SetValue(value, sym); this->currentBlock->globOptData.liveVarSyms->Set(sym->m_id); } return value; } Value * GlobOpt::NewIntConstantValue(const int32 intConst, IR::Instr * instr, bool isTaggable) { Value * value = NewValue(IntConstantValueInfo::New(this->alloc, intConst)); this->intConstantToValueMap->Item(intConst, value); if (isTaggable && !PHASE_OFF(Js::HoistConstIntPhase, this->func)) { // When creating a new int constant value, make sure it gets a symstore. If the int const doesn't have a symstore, // any downstream instruction using the same int will have to create a new value (object) for the int. // This gets in the way of CSE. value = HoistConstantLoadAndPropagateValueBackward(Js::TaggedInt::ToVarUnchecked(intConst), instr, value); if (!value->GetValueInfo()->GetSymStore() && (instr->m_opcode == Js::OpCode::LdC_A_I4 || instr->m_opcode == Js::OpCode::Ld_I4)) { StackSym * sym = instr->GetDst()->GetStackSym(); Assert(sym); if (sym->IsTypeSpec()) { Assert(sym->IsInt32()); StackSym * varSym = sym->GetVarEquivSym(instr->m_func); CurrentBlockData()->SetValue(value, varSym); CurrentBlockData()->liveInt32Syms->Set(varSym->m_id); } else { CurrentBlockData()->SetValue(value, sym); CurrentBlockData()->liveVarSyms->Set(sym->m_id); } } } return value; } ValueInfo * GlobOpt::NewIntRangeValueInfo(const int32 min, const int32 max, const bool wasNegativeZeroPreventedByBailout) { return ValueInfo::NewIntRangeValueInfo(this->alloc, min, max, wasNegativeZeroPreventedByBailout); } ValueInfo *GlobOpt::NewIntRangeValueInfo( const ValueInfo *const originalValueInfo, const int32 min, const int32 max) const { Assert(originalValueInfo); ValueInfo *valueInfo; if(min == max) { // Since int constant values are const-propped, negative zero tracking does not track them, and so it's okay to ignore // 'wasNegativeZeroPreventedByBailout' valueInfo = IntConstantValueInfo::New(alloc, min); } else { valueInfo = IntRangeValueInfo::New( alloc, min, max, min <= 0 && max >= 0 && originalValueInfo->WasNegativeZeroPreventedByBailout()); } valueInfo->SetSymStore(originalValueInfo->GetSymStore()); return valueInfo; } Value * GlobOpt::NewIntRangeValue( const int32 min, const int32 max, const bool wasNegativeZeroPreventedByBailout, IR::Opnd *const opnd) { ValueInfo *valueInfo = this->NewIntRangeValueInfo(min, max, wasNegativeZeroPreventedByBailout); Value *val = NewValue(valueInfo); if (opnd) { GOPT_TRACE_OPND(opnd, _u("Range %d (0x%X) to %d (0x%X)\n"), min, min, max, max); } CurrentBlockData()->InsertNewValue(val, opnd); return val; } IntBoundedValueInfo *GlobOpt::NewIntBoundedValueInfo( const ValueInfo *const originalValueInfo, const IntBounds *const bounds) const { Assert(originalValueInfo); bounds->Verify(); IntBoundedValueInfo *const valueInfo = IntBoundedValueInfo::New( originalValueInfo->Type(), bounds, ( bounds->ConstantLowerBound() <= 0 && bounds->ConstantUpperBound() >= 0 && originalValueInfo->WasNegativeZeroPreventedByBailout() ), alloc); valueInfo->SetSymStore(originalValueInfo->GetSymStore()); return valueInfo; } Value *GlobOpt::NewIntBoundedValue( const ValueType valueType, const IntBounds *const bounds, const bool wasNegativeZeroPreventedByBailout, IR::Opnd *const opnd) { Value *const value = NewValue(IntBoundedValueInfo::New(valueType, bounds, wasNegativeZeroPreventedByBailout, alloc)); CurrentBlockData()->InsertNewValue(value, opnd); return value; } Value * GlobOpt::NewFloatConstantValue(const FloatConstType floatValue, IR::Opnd *const opnd) { FloatConstantValueInfo *valueInfo = FloatConstantValueInfo::New(this->alloc, floatValue); Value *val = NewValue(valueInfo); CurrentBlockData()->InsertNewValue(val, opnd); return val; } Value * GlobOpt::GetVarConstantValue(IR::AddrOpnd *addrOpnd) { bool isVar = addrOpnd->IsVar(); bool isString = isVar && addrOpnd->m_localAddress && JITJavascriptString::Is(addrOpnd->m_localAddress); Value *val = nullptr; Value *cachedValue = nullptr; if(this->addrConstantToValueMap->TryGetValue(addrOpnd->m_address, &cachedValue)) { // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since // values are cloned for each block, we can't use the same value object. We also can't have two values with the same // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast // way to determine if a value with the same value number exists for this block. So the best we can do with a global // cache is to check the sym-store's value in the current block to see if it has a value with the same number. // Otherwise, we have to create a new value with a new value number. Sym *symStore = cachedValue->GetValueInfo()->GetSymStore(); if(symStore && CurrentBlockData()->IsLive(symStore)) { Value *const symStoreValue = CurrentBlockData()->FindValue(symStore); if(symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber()) { ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo(); if(symStoreValueInfo->IsVarConstant() && symStoreValueInfo->AsVarConstant()->VarValue() == addrOpnd->m_address) { val = symStoreValue; } } } } else if (isString) { JITJavascriptString* jsString = JITJavascriptString::FromVar(addrOpnd->m_localAddress); Js::InternalString internalString(jsString->GetString(), jsString->GetLength()); if (this->stringConstantToValueMap->TryGetValue(internalString, &cachedValue)) { Sym *symStore = cachedValue->GetValueInfo()->GetSymStore(); if (symStore && CurrentBlockData()->IsLive(symStore)) { Value *const symStoreValue = CurrentBlockData()->FindValue(symStore); if (symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber()) { ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo(); if (symStoreValueInfo->IsVarConstant()) { JITJavascriptString * cachedString = JITJavascriptString::FromVar(symStoreValue->GetValueInfo()->AsVarConstant()->VarValue(true)); Js::InternalString cachedInternalString(cachedString->GetString(), cachedString->GetLength()); if (Js::InternalStringComparer::Equals(internalString, cachedInternalString)) { val = symStoreValue; } } } } } } if(!val) { val = NewVarConstantValue(addrOpnd, isString); } addrOpnd->SetValueType(val->GetValueInfo()->Type()); return val; } Value * GlobOpt::NewVarConstantValue(IR::AddrOpnd *addrOpnd, bool isString) { VarConstantValueInfo *valueInfo = VarConstantValueInfo::New(this->alloc, addrOpnd->m_address, addrOpnd->GetValueType(), false, addrOpnd->m_localAddress); Value * value = NewValue(valueInfo); this->addrConstantToValueMap->Item(addrOpnd->m_address, value); if (isString) { JITJavascriptString* jsString = JITJavascriptString::FromVar(addrOpnd->m_localAddress); Js::InternalString internalString(jsString->GetString(), jsString->GetLength()); this->stringConstantToValueMap->Item(internalString, value); } return value; } Value * GlobOpt::HoistConstantLoadAndPropagateValueBackward(Js::Var varConst, IR::Instr * origInstr, Value * value) { if (this->IsLoopPrePass() || ((this->currentBlock == this->func->m_fg->blockList) && origInstr->TransfersSrcValue())) { return value; } // Only hoisting taggable int const loads for now. Could be extended to other constants (floats, strings, addr opnds) if we see some benefit. Assert(Js::TaggedInt::Is(varConst)); // Insert a load of the constant at the top of the function StackSym * dstSym = StackSym::New(this->func); IR::RegOpnd * constRegOpnd = IR::RegOpnd::New(dstSym, TyVar, this->func); IR::Instr * loadInstr = IR::Instr::NewConstantLoad(constRegOpnd, (intptr_t)varConst, ValueType::GetInt(true), this->func); this->func->m_fg->blockList->GetFirstInstr()->InsertAfter(loadInstr); // Type-spec the load (Support for floats needs to be added when we start hoisting float constants). bool typeSpecedToInt = false; if (Js::TaggedInt::Is(varConst) && !IsTypeSpecPhaseOff(this->func)) { typeSpecedToInt = true; loadInstr->m_opcode = Js::OpCode::Ld_I4; ToInt32Dst(loadInstr, loadInstr->GetDst()->AsRegOpnd(), this->currentBlock); loadInstr->GetDst()->GetStackSym()->SetIsConst(); } else { CurrentBlockData()->liveVarSyms->Set(dstSym->m_id); } // Add the value (object) to the current block's symToValueMap and propagate the value backward to all relevant blocks so it is available on merges. value = CurrentBlockData()->InsertNewValue(value, constRegOpnd); BVSparse* GlobOptBlockData::*bv; bv = typeSpecedToInt ? &GlobOptBlockData::liveInt32Syms : &GlobOptBlockData::liveVarSyms; // Will need to be expanded when we start hoisting float constants. if (this->currentBlock != this->func->m_fg->blockList) { for (InvariantBlockBackwardIterator it(this, this->currentBlock, this->func->m_fg->blockList, nullptr); it.IsValid(); it.MoveNext()) { BasicBlock * block = it.Block(); (block->globOptData.*bv)->Set(dstSym->m_id); if (!block->globOptData.FindValue(dstSym)) { Value *const valueCopy = CopyValue(value, value->GetValueNumber()); block->globOptData.SetValue(valueCopy, dstSym); } } } return value; } Value * GlobOpt::NewFixedFunctionValue(Js::JavascriptFunction *function, IR::AddrOpnd *addrOpnd) { Assert(function != nullptr); Value *val = nullptr; Value *cachedValue = nullptr; if(this->addrConstantToValueMap->TryGetValue(addrOpnd->m_address, &cachedValue)) { // The cached value could be from a different block since this is a global (as opposed to a per-block) cache. Since // values are cloned for each block, we can't use the same value object. We also can't have two values with the same // number in one block, so we can't simply copy the cached value either. And finally, there is no deterministic and fast // way to determine if a value with the same value number exists for this block. So the best we can do with a global // cache is to check the sym-store's value in the current block to see if it has a value with the same number. // Otherwise, we have to create a new value with a new value number. Sym *symStore = cachedValue->GetValueInfo()->GetSymStore(); if(symStore && CurrentBlockData()->IsLive(symStore)) { Value *const symStoreValue = CurrentBlockData()->FindValue(symStore); if(symStoreValue && symStoreValue->GetValueNumber() == cachedValue->GetValueNumber()) { ValueInfo *const symStoreValueInfo = symStoreValue->GetValueInfo(); if(symStoreValueInfo->IsVarConstant()) { VarConstantValueInfo *const symStoreVarConstantValueInfo = symStoreValueInfo->AsVarConstant(); if(symStoreVarConstantValueInfo->VarValue() == addrOpnd->m_address && symStoreVarConstantValueInfo->IsFunction()) { val = symStoreValue; } } } } } if(!val) { VarConstantValueInfo *valueInfo = VarConstantValueInfo::New(this->alloc, function, addrOpnd->GetValueType(), true, addrOpnd->m_localAddress); val = NewValue(valueInfo); this->addrConstantToValueMap->AddNew(addrOpnd->m_address, val); } CurrentBlockData()->InsertNewValue(val, addrOpnd); return val; } StackSym *GlobOpt::GetTaggedIntConstantStackSym(const int32 intConstantValue) const { Assert(!Js::TaggedInt::IsOverflow(intConstantValue)); return intConstantToStackSymMap->Lookup(intConstantValue, nullptr); } StackSym *GlobOpt::GetOrCreateTaggedIntConstantStackSym(const int32 intConstantValue) const { StackSym *stackSym = GetTaggedIntConstantStackSym(intConstantValue); if(stackSym) { return stackSym; } stackSym = StackSym::New(TyVar,func); intConstantToStackSymMap->Add(intConstantValue, stackSym); return stackSym; } Sym * GlobOpt::SetSymStore(ValueInfo *valueInfo, Sym *sym) { if (sym->IsStackSym()) { StackSym *stackSym = sym->AsStackSym(); if (stackSym->IsTypeSpec()) { stackSym = stackSym->GetVarEquivSym(this->func); sym = stackSym; } } if (valueInfo->GetSymStore() == nullptr || valueInfo->GetSymStore()->IsPropertySym()) { SetSymStoreDirect(valueInfo, sym); } return sym; } void GlobOpt::SetSymStoreDirect(ValueInfo * valueInfo, Sym * sym) { Sym * prevSymStore = valueInfo->GetSymStore(); CurrentBlockData()->SetChangedSym(prevSymStore); valueInfo->SetSymStore(sym); } // Figure out the Value of this dst. Value * GlobOpt::ValueNumberDst(IR::Instr **pInstr, Value *src1Val, Value *src2Val) { IR::Instr *&instr = *pInstr; IR::Opnd *dst = instr->GetDst(); Value *dstVal = nullptr; Sym *sym; if (instr->CallsSetter()) { return nullptr; } if (dst == nullptr) { return nullptr; } switch (dst->GetKind()) { case IR::OpndKindSym: sym = dst->AsSymOpnd()->m_sym; break; case IR::OpndKindReg: sym = dst->AsRegOpnd()->m_sym; if (OpCodeAttr::TempNumberProducing(instr->m_opcode)) { CurrentBlockData()->isTempSrc->Set(sym->m_id); } else if (OpCodeAttr::TempNumberTransfer(instr->m_opcode)) { IR::Opnd *src1 = instr->GetSrc1(); if (src1->IsRegOpnd() && CurrentBlockData()->isTempSrc->Test(src1->AsRegOpnd()->m_sym->m_id)) { StackSym *src1Sym = src1->AsRegOpnd()->m_sym; // isTempSrc is used for marking isTempLastUse, which is used to generate AddLeftDead() // calls instead of the normal Add helpers. It tells the runtime that concats can use string // builders. // We need to be careful in the case where src1 points to a string builder and is getting aliased. // Clear the bit on src and dst of the transfer instr in this case, unless we can prove src1 // isn't pointing at a string builder, like if it is single def and the def instr is not an Add, // but TempProducing. if (src1Sym->IsSingleDef() && src1Sym->m_instrDef->m_opcode != Js::OpCode::Add_A && OpCodeAttr::TempNumberProducing(src1Sym->m_instrDef->m_opcode)) { CurrentBlockData()->isTempSrc->Set(sym->m_id); } else { CurrentBlockData()->isTempSrc->Clear(src1->AsRegOpnd()->m_sym->m_id); CurrentBlockData()->isTempSrc->Clear(sym->m_id); } } else { CurrentBlockData()->isTempSrc->Clear(sym->m_id); } } else { CurrentBlockData()->isTempSrc->Clear(sym->m_id); } break; case IR::OpndKindIndir: return nullptr; default: return nullptr; } int32 min1, max1, min2, max2, newMin, newMax; ValueInfo *src1ValueInfo = (src1Val ? src1Val->GetValueInfo() : nullptr); ValueInfo *src2ValueInfo = (src2Val ? src2Val->GetValueInfo() : nullptr); switch (instr->m_opcode) { case Js::OpCode::Conv_PrimStr: AssertMsg(instr->GetDst()->GetValueType().IsString(), "Creator of this instruction should have set the type"); if (this->IsLoopPrePass() || src1ValueInfo == nullptr || !src1ValueInfo->IsPrimitive()) { break; } instr->m_opcode = Js::OpCode::Conv_Str; // fall-through case Js::OpCode::Conv_Str: // This opcode is commented out since we don't track regex information in GlobOpt now. //case Js::OpCode::Coerce_Regex: case Js::OpCode::Coerce_Str: AssertMsg(instr->GetDst()->GetValueType().IsString(), "Creator of this instruction should have set the type"); // Due to fall through and the fact that Ld_A only takes one source, // free the other source here. if (instr->GetSrc2() && !(this->IsLoopPrePass() || src1ValueInfo == nullptr || !src1ValueInfo->IsString())) { instr->FreeSrc2(); } // fall-through case Js::OpCode::Coerce_StrOrRegex: // We don't set the ValueType of src1 for Coerce_StrOrRegex, hence skip the ASSERT if (this->IsLoopPrePass() || src1ValueInfo == nullptr || !src1ValueInfo->IsString()) { break; } instr->m_opcode = Js::OpCode::Ld_A; // fall-through case Js::OpCode::BytecodeArgOutCapture: case Js::OpCode::LdAsmJsFunc: case Js::OpCode::Ld_A: case Js::OpCode::Ld_I4: // Propagate sym attributes across the reg copy. if (!this->IsLoopPrePass() && instr->GetSrc1()->IsRegOpnd()) { if (dst->AsRegOpnd()->m_sym->IsSingleDef()) { dst->AsRegOpnd()->m_sym->CopySymAttrs(instr->GetSrc1()->AsRegOpnd()->m_sym); } } if (instr->IsProfiledInstr()) { const ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType); if(!( profiledValueType.IsLikelyInt() && ( (dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber) || (instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber) ) )) { if(!src1ValueInfo) { dstVal = this->NewGenericValue(profiledValueType, dst); } else if(src1ValueInfo->IsUninitialized()) { if(IsLoopPrePass()) { dstVal = this->NewGenericValue(profiledValueType, dst); } else { // Assuming the profile data gives more precise value types based on the path it took at runtime, we // can improve the original value type. src1ValueInfo->Type() = profiledValueType; instr->GetSrc1()->SetValueType(profiledValueType); } } } } if (dstVal == nullptr) { // Ld_A is just transferring the value dstVal = this->ValueNumberTransferDst(instr, src1Val); } break; case Js::OpCode::ExtendArg_A: { // SIMD_JS // We avoid transforming EAs to Lds to keep the IR shape consistent and avoid CSEing of EAs. // CSEOptimize only assigns a Value to the EA dst, and doesn't turn it to a Ld. If this happened, we shouldn't assign a new Value here. if (DoCSE()) { IR::Opnd * currDst = instr->GetDst(); Value * currDstVal = CurrentBlockData()->FindValue(currDst->GetStackSym()); if (currDstVal != nullptr) { return currDstVal; } } break; } case Js::OpCode::CheckFixedFld: AssertMsg(false, "CheckFixedFld doesn't have a dst, so we should never get here"); break; case Js::OpCode::LdSlot: case Js::OpCode::LdSlotArr: case Js::OpCode::LdFld: case Js::OpCode::LdFldForTypeOf: case Js::OpCode::LdFldForCallApplyTarget: // Do not transfer value type on LdRootFldForTypeOf to prevent copy-prop to LdRootFld in case the field doesn't exist since LdRootFldForTypeOf does not throw. // Same goes for ScopedLdFldForTypeOf as we'll end up loading the property from the root object if the property is not in the scope chain. //case Js::OpCode::LdRootFldForTypeOf: //case Js::OpCode::ScopedLdFldForTypeOf: case Js::OpCode::LdRootFld: case Js::OpCode::LdMethodFld: case Js::OpCode::LdRootMethodFld: case Js::OpCode::ScopedLdMethodFld: case Js::OpCode::LdMethodFromFlags: case Js::OpCode::ScopedLdFld: if (instr->IsProfiledInstr()) { ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType); if(!(profiledValueType.IsLikelyInt() && dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber)) { if(!src1ValueInfo) { dstVal = this->NewGenericValue(profiledValueType, dst); } else if(src1ValueInfo->IsUninitialized()) { if(IsLoopPrePass() && (!dst->IsRegOpnd() || !dst->AsRegOpnd()->m_sym->IsSingleDef())) { dstVal = this->NewGenericValue(profiledValueType, dst); } else { // Assuming the profile data gives more precise value types based on the path it took at runtime, we // can improve the original value type. src1ValueInfo->Type() = profiledValueType; instr->GetSrc1()->SetValueType(profiledValueType); } } } } if (dstVal == nullptr) { dstVal = this->ValueNumberTransferDst(instr, src1Val); } if(!this->IsLoopPrePass()) { // We cannot transfer value if the field hasn't been copy prop'd because we don't generate // an implicit call bailout between those values if we don't have "live fields" unless, we are hoisting the field. ValueInfo *dstValueInfo = (dstVal ? dstVal->GetValueInfo() : nullptr); // Update symStore if it isn't a stackSym if (dstVal && (!dstValueInfo->GetSymStore() || !dstValueInfo->GetSymStore()->IsStackSym())) { Assert(dst->IsRegOpnd()); this->SetSymStoreDirect(dstValueInfo, dst->AsRegOpnd()->m_sym); } if (src1Val != dstVal) { CurrentBlockData()->SetValue(dstVal, instr->GetSrc1()); } } break; case Js::OpCode::LdC_A_R8: case Js::OpCode::LdC_A_I4: case Js::OpCode::ArgIn_A: dstVal = src1Val; break; case Js::OpCode::LdStr: if (src1Val == nullptr) { src1Val = NewGenericValue(ValueType::String, dst); } dstVal = src1Val; break; // LdElemUndef only assign undef if the field doesn't exist. // So we don't actually know what the value is, so we can't really copy prop it. //case Js::OpCode::LdElemUndef: case Js::OpCode::StSlot: case Js::OpCode::StSlotChkUndecl: case Js::OpCode::StFld: case Js::OpCode::StRootFld: case Js::OpCode::StFldStrict: case Js::OpCode::StRootFldStrict: case Js::OpCode::InitFld: case Js::OpCode::InitComputedProperty: if (DoFieldCopyProp()) { if (src1Val == nullptr) { // src1 may have no value if it's not a valid var, e.g., NULL for let/const initialization. // Consider creating generic values for such things. return nullptr; } AssertMsg(!src2Val, "Bad src Values..."); Assert(sym->IsPropertySym()); SymID symId = sym->m_id; Assert(instr->m_opcode == Js::OpCode::StSlot || instr->m_opcode == Js::OpCode::StSlotChkUndecl || !CurrentBlockData()->liveFields->Test(symId)); CurrentBlockData()->liveFields->Set(symId); if (!this->IsLoopPrePass() && dst->GetIsDead()) { // Take the property sym out of the live fields set (with special handling for loops). this->EndFieldLifetime(dst->AsSymOpnd()); } dstVal = this->ValueNumberTransferDst(instr, src1Val); } else { return nullptr; } break; case Js::OpCode::Conv_Num: if(src1ValueInfo->IsNumber()) { dstVal = ValueNumberTransferDst(instr, src1Val); } else { return NewGenericValue(src1ValueInfo->Type().ToDefiniteAnyNumber().SetCanBeTaggedValue(true), dst); } break; case Js::OpCode::Not_A: { if (!src1Val || !src1ValueInfo->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec())) { min1 = INT32_MIN; max1 = INT32_MAX; } this->PropagateIntRangeForNot(min1, max1, &newMin, &newMax); return CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val); } case Js::OpCode::Xor_A: case Js::OpCode::Or_A: case Js::OpCode::And_A: case Js::OpCode::Shl_A: case Js::OpCode::Shr_A: case Js::OpCode::ShrU_A: { if (!src1Val || !src1ValueInfo->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec())) { min1 = INT32_MIN; max1 = INT32_MAX; } if (!src2Val || !src2ValueInfo->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec())) { min2 = INT32_MIN; max2 = INT32_MAX; } if (instr->m_opcode == Js::OpCode::ShrU_A && min1 < 0 && IntConstantBounds(min2, max2).And_0x1f().Contains(0)) { // Src1 may be too large to represent as a signed int32, and src2 may be zero. // Since the result can therefore be too large to represent as a signed int32, // include Number in the value type. return CreateDstUntransferredValue( ValueType::AnyNumber.SetCanBeTaggedValue(true), instr, src1Val, src2Val); } this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax); return CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val); } case Js::OpCode::Incr_A: case Js::OpCode::Decr_A: { ValueType valueType; if(src1Val) { valueType = src1Val->GetValueInfo()->Type().ToDefiniteAnyNumber(); } else { valueType = ValueType::Number; } return CreateDstUntransferredValue(valueType.SetCanBeTaggedValue(true), instr, src1Val, src2Val); } case Js::OpCode::Add_A: { ValueType valueType; if (src1Val && src1ValueInfo->IsLikelyNumber() && src2Val && src2ValueInfo->IsLikelyNumber()) { if(src1ValueInfo->IsLikelyInt() && src2ValueInfo->IsLikelyInt()) { // When doing aggressiveIntType, just assume the result is likely going to be int // if both input is int. const bool isLikelyTagged = src1ValueInfo->IsLikelyTaggedInt() && src2ValueInfo->IsLikelyTaggedInt(); if(src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber()) { // If both of them are numbers then we can definitely say that the result is a number. valueType = ValueType::GetNumberAndLikelyInt(isLikelyTagged); } else { // This is only likely going to be int but can be a string as well. valueType = ValueType::GetInt(isLikelyTagged).ToLikely(); } } else { // We can only be certain of any thing if both of them are numbers. // Otherwise, the result could be string. if (src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber()) { if (src1ValueInfo->IsFloat() || src2ValueInfo->IsFloat()) { // If one of them is a float, the result probably is a float instead of just int // but should always be a number. valueType = ValueType::Float.SetCanBeTaggedValue(true); } else { // Could be int, could be number valueType = ValueType::Number.SetCanBeTaggedValue(true); } } else if (src1ValueInfo->IsLikelyFloat() || src2ValueInfo->IsLikelyFloat()) { // Result is likely a float (but can be anything) valueType = ValueType::Float.ToLikely(); } else { // Otherwise it is a likely int or float (but can be anything) valueType = ValueType::Number.ToLikely(); } } } else if((src1Val && src1ValueInfo->IsString()) || (src2Val && src2ValueInfo->IsString())) { // String + anything should always result in a string valueType = ValueType::String; } else if((src1Val && src1ValueInfo->IsNotString() && src1ValueInfo->IsPrimitive()) && (src2Val && src2ValueInfo->IsNotString() && src2ValueInfo->IsPrimitive())) { // If src1 and src2 are not strings and primitive, add should yield a number. valueType = ValueType::Number.SetCanBeTaggedValue(true); } else if((src1Val && src1ValueInfo->IsLikelyString()) || (src2Val && src2ValueInfo->IsLikelyString())) { // likelystring + anything should always result in a likelystring valueType = ValueType::String.ToLikely(); } else { // Number or string. Could make the value a merge of Number and String, but Uninitialized is more useful at the moment. Assert(valueType.IsUninitialized()); } return CreateDstUntransferredValue(valueType, instr, src1Val, src2Val); } case Js::OpCode::Div_A: { ValueType divValueType = GetDivValueType(instr, src1Val, src2Val, false); if (divValueType.IsLikelyInt() || divValueType.IsFloat()) { return CreateDstUntransferredValue(divValueType.SetCanBeTaggedValue(true), instr, src1Val, src2Val); } } // fall-through case Js::OpCode::Sub_A: case Js::OpCode::Mul_A: case Js::OpCode::Rem_A: { ValueType valueType; if( src1Val && src1ValueInfo->IsLikelyInt() && src2Val && src2ValueInfo->IsLikelyInt() && instr->m_opcode != Js::OpCode::Div_A) { const bool isLikelyTagged = src1ValueInfo->IsLikelyTaggedInt() && (src2ValueInfo->IsLikelyTaggedInt() || instr->m_opcode == Js::OpCode::Rem_A); if(src1ValueInfo->IsNumber() && src2ValueInfo->IsNumber()) { valueType = ValueType::GetNumberAndLikelyInt(isLikelyTagged); } else { valueType = ValueType::GetInt(isLikelyTagged).ToLikely(); } } else if ((src1Val && src1ValueInfo->IsLikelyFloat()) || (src2Val && src2ValueInfo->IsLikelyFloat())) { // This should ideally be NewNumberAndLikelyFloatValue since we know the result is a number but not sure if it will // be a float value. However, that Number/LikelyFloat value type doesn't exist currently and all the necessary // checks are done for float values (tagged int checks, etc.) so it's sufficient to just create a float value here. valueType = ValueType::Float.SetCanBeTaggedValue(true); } else { valueType = ValueType::Number.SetCanBeTaggedValue(true); } return CreateDstUntransferredValue(valueType, instr, src1Val, src2Val); } case Js::OpCode::CallI: Assert(dst->IsRegOpnd()); return NewGenericValue(dst->AsRegOpnd()->GetValueType(), dst); case Js::OpCode::LdElemI_A: { dstVal = ValueNumberLdElemDst(pInstr, src1Val); const ValueType baseValueType(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType()); if( ( baseValueType.IsLikelyNativeArray() || #ifdef _M_IX86 ( !AutoSystemInfo::Data.SSE2Available() && baseValueType.IsLikelyObject() && ( baseValueType.GetObjectType() == ObjectType::Float32Array || baseValueType.GetObjectType() == ObjectType::Float64Array ) ) #else false #endif ) && instr->GetDst()->IsVar() && instr->HasBailOutInfo()) { // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast // path. Note that the removed bailouts should not be necessary for correctness. IR::BailOutKind bailOutKind = instr->GetBailOutKind(); if(bailOutKind & IR::BailOutOnArrayAccessHelperCall) { bailOutKind -= IR::BailOutOnArrayAccessHelperCall; } if(bailOutKind == IR::BailOutOnImplicitCallsPreOp) { bailOutKind -= IR::BailOutOnImplicitCallsPreOp; } if(bailOutKind) { instr->SetBailOutKind(bailOutKind); } else { instr->ClearBailOutInfo(); } } return dstVal; } case Js::OpCode::LdMethodElem: // Not worth profiling this, just assume it's likely object (should be likely function but ValueType does not track // functions currently, so using ObjectType::Object instead) dstVal = NewGenericValue(ValueType::GetObject(ObjectType::Object).ToLikely(), dst); if(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyNativeArray() && instr->HasBailOutInfo()) { // The lowerer is not going to generate a fast path for this case. Remove any bailouts that require the fast // path. Note that the removed bailouts should not be necessary for correctness. IR::BailOutKind bailOutKind = instr->GetBailOutKind(); if(bailOutKind & IR::BailOutOnArrayAccessHelperCall) { bailOutKind -= IR::BailOutOnArrayAccessHelperCall; } if(bailOutKind == IR::BailOutOnImplicitCallsPreOp) { bailOutKind -= IR::BailOutOnImplicitCallsPreOp; } if(bailOutKind) { instr->SetBailOutKind(bailOutKind); } else { instr->ClearBailOutInfo(); } } return dstVal; case Js::OpCode::StElemI_A: case Js::OpCode::StElemI_A_Strict: dstVal = this->ValueNumberTransferDst(instr, src1Val); break; case Js::OpCode::LdLen_A: if (instr->IsProfiledInstr()) { const ValueType profiledValueType(instr->AsProfiledInstr()->u.FldInfo().valueType); if(!(profiledValueType.IsLikelyInt() && dst->AsRegOpnd()->m_sym->m_isNotNumber)) { return this->NewGenericValue(profiledValueType, dst); } } break; case Js::OpCode::BrOnEmpty: case Js::OpCode::BrOnNotEmpty: Assert(dst->IsRegOpnd()); Assert(dst->GetValueType().IsString()); return this->NewGenericValue(ValueType::String, dst); case Js::OpCode::IsInst: case Js::OpCode::LdTrue: case Js::OpCode::LdFalse: case Js::OpCode::CmEq_A: case Js::OpCode::CmSrEq_A: case Js::OpCode::CmNeq_A: case Js::OpCode::CmSrNeq_A: case Js::OpCode::CmLe_A: case Js::OpCode::CmUnLe_A: case Js::OpCode::CmLt_A: case Js::OpCode::CmUnLt_A: case Js::OpCode::CmGe_A: case Js::OpCode::CmUnGe_A: case Js::OpCode::CmGt_A: case Js::OpCode::CmUnGt_A: return this->NewGenericValue(ValueType::Boolean, dst); case Js::OpCode::LdUndef: return this->NewGenericValue(ValueType::Undefined, dst); case Js::OpCode::LdC_A_Null: return this->NewGenericValue(ValueType::Null, dst); case Js::OpCode::LdThis: if (!PHASE_OFF(Js::OptTagChecksPhase, this->func) && (src1ValueInfo == nullptr || src1ValueInfo->IsUninitialized())) { return this->NewGenericValue(ValueType::GetObject(ObjectType::Object).ToLikely().SetCanBeTaggedValue(false), dst); } break; case Js::OpCode::Typeof: case Js::OpCode::TypeofElem: return this->NewGenericValue(ValueType::String, dst); case Js::OpCode::InitLocalClosure: Assert(instr->GetDst()); Assert(instr->GetDst()->IsRegOpnd()); IR::RegOpnd *regOpnd = instr->GetDst()->AsRegOpnd(); StackSym *opndStackSym = regOpnd->m_sym; Assert(opndStackSym != nullptr); ObjectSymInfo *objectSymInfo = opndStackSym->m_objectInfo; Assert(objectSymInfo != nullptr); for (PropertySym *localVarSlotList = objectSymInfo->m_propertySymList; localVarSlotList; localVarSlotList = localVarSlotList->m_nextInStackSymList) { this->slotSyms->Set(localVarSlotList->m_id); } break; } if (dstVal == nullptr) { return this->NewGenericValue(dst->GetValueType(), dst); } return CurrentBlockData()->SetValue(dstVal, dst); } Value * GlobOpt::ValueNumberLdElemDst(IR::Instr **pInstr, Value *srcVal) { IR::Instr *&instr = *pInstr; IR::Opnd *dst = instr->GetDst(); Value *dstVal = nullptr; int32 newMin, newMax; ValueInfo *srcValueInfo = (srcVal ? srcVal->GetValueInfo() : nullptr); ValueType profiledElementType; if (instr->IsProfiledInstr()) { profiledElementType = instr->AsProfiledInstr()->u.ldElemInfo->GetElementType(); if(!(profiledElementType.IsLikelyInt() && dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->m_isNotNumber) && srcVal && srcValueInfo->IsUninitialized()) { if(IsLoopPrePass()) { dstVal = NewGenericValue(profiledElementType, dst); } else { // Assuming the profile data gives more precise value types based on the path it took at runtime, we // can improve the original value type. srcValueInfo->Type() = profiledElementType; instr->GetSrc1()->SetValueType(profiledElementType); } } } IR::IndirOpnd *src = instr->GetSrc1()->AsIndirOpnd(); const ValueType baseValueType(src->GetBaseOpnd()->GetValueType()); if (instr->DoStackArgsOpt() || !( baseValueType.IsLikelyOptimizedTypedArray() || (baseValueType.IsLikelyNativeArray() && instr->IsProfiledInstr()) // Specialized native array lowering for LdElem requires that it is profiled. ) || (!this->DoTypedArrayTypeSpec() && baseValueType.IsLikelyOptimizedTypedArray()) || // Don't do type spec on native array with a history of accessing gaps, as this is a bailout (!this->DoNativeArrayTypeSpec() && baseValueType.IsLikelyNativeArray()) || !ShouldExpectConventionalArrayIndexValue(src)) { if(DoTypedArrayTypeSpec() && !IsLoopPrePass()) { GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access.\n")); if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func)) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; baseValueType.ToString(baseValueTypeStr); Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not type specialize, because %s.\n"), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode), baseValueTypeStr, instr->DoStackArgsOpt() ? _u("instruction uses the arguments object") : baseValueType.IsLikelyOptimizedTypedArray() ? _u("index is negative or likely not int") : _u("of array type")); Output::Flush(); } } if(!dstVal) { if(srcVal) { dstVal = this->ValueNumberTransferDst(instr, srcVal); } else { dstVal = NewGenericValue(profiledElementType, dst); } } return dstVal; } Assert(instr->GetSrc1()->IsIndirOpnd()); IRType toType = TyVar; IR::BailOutKind bailOutKind = IR::BailOutConventionalTypedArrayAccessOnly; switch(baseValueType.GetObjectType()) { case ObjectType::Int8Array: case ObjectType::Int8VirtualArray: case ObjectType::Int8MixedArray: newMin = Int8ConstMin; newMax = Int8ConstMax; goto IntArrayCommon; case ObjectType::Uint8Array: case ObjectType::Uint8VirtualArray: case ObjectType::Uint8MixedArray: case ObjectType::Uint8ClampedArray: case ObjectType::Uint8ClampedVirtualArray: case ObjectType::Uint8ClampedMixedArray: newMin = Uint8ConstMin; newMax = Uint8ConstMax; goto IntArrayCommon; case ObjectType::Int16Array: case ObjectType::Int16VirtualArray: case ObjectType::Int16MixedArray: newMin = Int16ConstMin; newMax = Int16ConstMax; goto IntArrayCommon; case ObjectType::Uint16Array: case ObjectType::Uint16VirtualArray: case ObjectType::Uint16MixedArray: newMin = Uint16ConstMin; newMax = Uint16ConstMax; goto IntArrayCommon; case ObjectType::Int32Array: case ObjectType::Int32VirtualArray: case ObjectType::Int32MixedArray: case ObjectType::Uint32Array: // int-specialized loads from uint32 arrays will bail out on values that don't fit in an int32 case ObjectType::Uint32VirtualArray: case ObjectType::Uint32MixedArray: Int32Array: newMin = Int32ConstMin; newMax = Int32ConstMax; goto IntArrayCommon; IntArrayCommon: Assert(dst->IsRegOpnd()); // If int type spec is disabled, it is ok to load int values as they can help float type spec, and merging int32 with float64 => float64. // But if float type spec is also disabled, we'll have problems because float64 merged with var => float64... if (!this->DoAggressiveIntTypeSpec() && !this->DoFloatTypeSpec()) { if (!dstVal) { if (srcVal) { dstVal = this->ValueNumberTransferDst(instr, srcVal); } else { dstVal = NewGenericValue(profiledElementType, dst); } } return dstVal; } if (!this->IsLoopPrePass()) { if (instr->HasBailOutInfo()) { const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind(); Assert( ( !(oldBailOutKind & ~IR::BailOutKindBits) || (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp ) && !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject))); if (bailOutKind == IR::BailOutConventionalTypedArrayAccessOnly) { // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction // bails out for the right reason. instr->SetBailOutKind( bailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall))); } else { // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit // calls to occur, so it must be merged in to eliminate generating the helper call Assert(bailOutKind == IR::BailOutConventionalNativeArrayAccessOnly); instr->SetBailOutKind(oldBailOutKind | bailOutKind); } } else { GenerateBailAtOperation(&instr, bailOutKind); } } TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, nullptr, nullptr, bailOutKind, newMin, newMax, &dstVal); toType = TyInt32; break; case ObjectType::Float32Array: case ObjectType::Float32VirtualArray: case ObjectType::Float32MixedArray: case ObjectType::Float64Array: case ObjectType::Float64VirtualArray: case ObjectType::Float64MixedArray: Float64Array: Assert(dst->IsRegOpnd()); // If float type spec is disabled, don't load float64 values if (!this->DoFloatTypeSpec()) { if (!dstVal) { if (srcVal) { dstVal = this->ValueNumberTransferDst(instr, srcVal); } else { dstVal = NewGenericValue(profiledElementType, dst); } } return dstVal; } if (!this->IsLoopPrePass()) { if (instr->HasBailOutInfo()) { const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind(); Assert( ( !(oldBailOutKind & ~IR::BailOutKindBits) || (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp ) && !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject))); if (bailOutKind == IR::BailOutConventionalTypedArrayAccessOnly) { // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction // bails out for the right reason. instr->SetBailOutKind( bailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall))); } else { // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit // calls to occur, so it must be merged in to eliminate generating the helper call Assert(bailOutKind == IR::BailOutConventionalNativeArrayAccessOnly); instr->SetBailOutKind(oldBailOutKind | bailOutKind); } } else { GenerateBailAtOperation(&instr, bailOutKind); } } TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, &dstVal); toType = TyFloat64; break; default: Assert(baseValueType.IsLikelyNativeArray()); bailOutKind = IR::BailOutConventionalNativeArrayAccessOnly; if(baseValueType.HasIntElements()) { goto Int32Array; } Assert(baseValueType.HasFloatElements()); goto Float64Array; } if(!dstVal) { dstVal = NewGenericValue(profiledElementType, dst); } Assert(toType != TyVar); GOPT_TRACE_INSTR(instr, _u("Type specialized array access.\n")); if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func)) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; baseValueType.ToString(baseValueTypeStr); char dstValTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; dstVal->GetValueInfo()->Type().ToString(dstValTypeStr); Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, type specialized to %s producing %S"), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode), baseValueTypeStr, toType == TyInt32 ? _u("int32") : _u("float64"), dstValTypeStr); #if DBG_DUMP Output::Print(_u(" (")); dstVal->Dump(); Output::Print(_u(").\n")); #else Output::Print(_u(".\n")); #endif Output::Flush(); } return dstVal; } ValueType GlobOpt::GetPrepassValueTypeForDst( const ValueType desiredValueType, IR::Instr *const instr, Value *const src1Value, Value *const src2Value, bool const isValueInfoPrecise, bool const isSafeToTransferInPrepass) const { // Values with definite types can be created in the loop prepass only when it is guaranteed that the value type will be the // same on any iteration of the loop. The heuristics currently used are: // - If the source sym is not live on the back-edge, then it acquires a new value for each iteration of the loop, so // that value type can be definite // - Consider: A better solution for this is to track values that originate in this loop, which can have definite value // types. That catches more cases, should look into that in the future. // - If the source sym has a constant value that doesn't change for the duration of the function // - The operation always results in a definite value type. For instance, signed bitwise operations always result in an // int32, conv_num and ++ always result in a number, etc. // - For operations that always result in an int32, the resulting int range is precise only if the source syms pass // the above heuristics. Otherwise, the range must be expanded to the full int32 range. Assert(IsLoopPrePass()); Assert(instr); if(!isValueInfoPrecise) { if(!desiredValueType.IsDefinite()) { return isSafeToTransferInPrepass ? desiredValueType : desiredValueType.SetCanBeTaggedValue(true); } // If the desired value type is not precise, the value type of the destination is derived from the value types of the // sources. Since the value type of a source sym is not definite, the destination value type also cannot be definite. if(desiredValueType.IsInt() && OpCodeAttr::IsInt32(instr->m_opcode)) { // The op always produces an int32, but not always a tagged int return ValueType::GetInt(desiredValueType.IsLikelyTaggedInt()); } if(desiredValueType.IsNumber() && OpCodeAttr::ProducesNumber(instr->m_opcode)) { // The op always produces a number, but not always an int return desiredValueType.ToDefiniteAnyNumber(); } // Note: ToLikely() also sets CanBeTaggedValue return desiredValueType.ToLikely(); } return desiredValueType; } bool GlobOpt::IsPrepassSrcValueInfoPrecise(IR::Instr *const instr, Value *const src1Value, Value *const src2Value, bool * isSafeToTransferInPrepass) const { return (!instr->GetSrc1() || IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Value, isSafeToTransferInPrepass)) && (!instr->GetSrc2() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Value, isSafeToTransferInPrepass)); } bool GlobOpt::IsPrepassSrcValueInfoPrecise(IR::Opnd *const src, Value *const srcValue, bool * isSafeToTransferInPrepass) const { Assert(IsLoopPrePass()); Assert(src); if (isSafeToTransferInPrepass) { *isSafeToTransferInPrepass = false; } if (src->IsAddrOpnd() && srcValue->GetValueInfo()->GetSymStore() && srcValue->GetValueInfo()->GetSymStore()->IsStackSym() && srcValue->GetValueInfo()->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable()) { if (isSafeToTransferInPrepass) { *isSafeToTransferInPrepass = false; } return true; } if (!src->IsRegOpnd() || !srcValue) { return false; } ValueInfo *const srcValueInfo = srcValue->GetValueInfo(); bool isValueInfoDefinite = srcValueInfo->IsDefinite(); StackSym * srcSym = src->AsRegOpnd()->m_sym; bool isSafeToTransfer = IsSafeToTransferInPrepass(srcSym, srcValueInfo); if (isSafeToTransferInPrepass) { *isSafeToTransferInPrepass = isSafeToTransfer; } return isValueInfoDefinite && isSafeToTransfer; } bool GlobOpt::IsSafeToTransferInPrepass(StackSym * const srcSym, ValueInfo *const srcValueInfo) const { int32 intConstantValue; return srcSym->IsFromByteCodeConstantTable() || ( srcValueInfo->TryGetIntConstantValue(&intConstantValue) && !Js::TaggedInt::IsOverflow(intConstantValue) && GetTaggedIntConstantStackSym(intConstantValue) == srcSym ) || !currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(srcSym->m_id) || !currentBlock->loop->IsSymAssignedToInSelfOrParents(srcSym); } bool GlobOpt::SafeToCopyPropInPrepass(StackSym * const originalSym, StackSym * const copySym, Value *const value) const { Assert(this->currentBlock->globOptData.GetCopyPropSym(originalSym, value) == copySym); // In the following example, to copy-prop s2 into s1, it is not enough to check if s1 and s2 are safe to transfer. // In fact, both s1 and s2 are safe to transfer, but it is not legal to copy prop s2 into s1. // // s1 = s2 // $Loop: // s3 = s1 // s2 = s4 // Br $Loop // // In general, requirements for copy-propping in prepass are more restricted than those for transferring values. // For copy prop in prepass, if the original sym is live on back-edge, then the copy-prop sym should not be written to // in the loop (or its parents) ValueInfo* const valueInfo = value->GetValueInfo(); return IsSafeToTransferInPrepass(originalSym, valueInfo) && IsSafeToTransferInPrepass(copySym, valueInfo) && (!currentBlock->loop->regAlloc.liveOnBackEdgeSyms->Test(originalSym->m_id) || !currentBlock->loop->IsSymAssignedToInSelfOrParents(copySym)); } Value *GlobOpt::CreateDstUntransferredIntValue( const int32 min, const int32 max, IR::Instr *const instr, Value *const src1Value, Value *const src2Value) { Assert(instr); Assert(instr->GetDst()); Assert(OpCodeAttr::ProducesNumber(instr->m_opcode) || (instr->m_opcode == Js::OpCode::Add_A && src1Value->GetValueInfo()->IsNumber() && src2Value->GetValueInfo()->IsNumber())); ValueType valueType(ValueType::GetInt(IntConstantBounds(min, max).IsLikelyTaggable())); Assert(valueType.IsInt()); bool isValueInfoPrecise; if(IsLoopPrePass()) { isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value); valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, isValueInfoPrecise); } else { isValueInfoPrecise = true; } IR::Opnd *const dst = instr->GetDst(); if(isValueInfoPrecise) { Assert(valueType == ValueType::GetInt(IntConstantBounds(min, max).IsLikelyTaggable())); Assert(!(dst->IsRegOpnd() && dst->AsRegOpnd()->m_sym->IsTypeSpec())); return NewIntRangeValue(min, max, false, dst); } return NewGenericValue(valueType, dst); } Value * GlobOpt::CreateDstUntransferredValue( const ValueType desiredValueType, IR::Instr *const instr, Value *const src1Value, Value *const src2Value) { Assert(instr); Assert(instr->GetDst()); Assert(!desiredValueType.IsInt()); // use CreateDstUntransferredIntValue instead ValueType valueType(desiredValueType); if(IsLoopPrePass()) { valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value)); } return NewGenericValue(valueType, instr->GetDst()); } Value * GlobOpt::ValueNumberTransferDst(IR::Instr *const instr, Value * src1Val) { Value *dstVal = this->IsLoopPrePass() ? this->ValueNumberTransferDstInPrepass(instr, src1Val) : src1Val; // Don't copy-prop a temp over a user symbol. This is likely to extend the temp's lifetime, as the user symbol // is more likely to already have later references. // REVIEW: Enabling this does cause perf issues... #if 0 if (dstVal != src1Val) { return dstVal; } Sym *dstSym = dst->GetStackSym(); if (dstVal && dstSym && dstSym->IsStackSym() && !dstSym->AsStackSym()->m_isBytecodeTmp) { Sym *dstValSym = dstVal->GetValueInfo()->GetSymStore(); if (dstValSym && dstValSym->AsStackSym()->m_isBytecodeTmp /* src->GetIsDead()*/) { dstVal->GetValueInfo()->SetSymStore(dstSym); } } #endif return dstVal; } bool GlobOpt::IsSafeToTransferInPrePass(IR::Opnd *src, Value *srcValue) { if (src->IsRegOpnd()) { StackSym *srcSym = src->AsRegOpnd()->m_sym; if (srcSym->IsFromByteCodeConstantTable()) { return true; } ValueInfo *srcValueInfo = srcValue->GetValueInfo(); int32 srcIntConstantValue; if (srcValueInfo->TryGetIntConstantValue(&srcIntConstantValue) && !Js::TaggedInt::IsOverflow(srcIntConstantValue) && GetTaggedIntConstantStackSym(srcIntConstantValue) == srcSym) { return true; } } return false; } Value * GlobOpt::ValueNumberTransferDstInPrepass(IR::Instr *const instr, Value *const src1Val) { Value *dstVal = nullptr; if (!src1Val) { return nullptr; } bool isValueInfoPrecise; ValueInfo *const src1ValueInfo = src1Val->GetValueInfo(); // TODO: This conflicts with new values created by the type specialization code // We should re-enable if we change that code to avoid the new values. #if 0 if (this->IsSafeToTransferInPrePass(instr->GetSrc1(), src1Val)) { return src1Val; } if (this->IsPREInstrCandidateLoad(instr->m_opcode) && instr->GetDst()) { StackSym *dstSym = instr->GetDst()->AsRegOpnd()->m_sym; for (Loop *curLoop = this->currentBlock->loop; curLoop; curLoop = curLoop->parent) { if (curLoop->fieldPRESymStore->Test(dstSym->m_id)) { return src1Val; } } } if (instr->GetDst()->IsRegOpnd()) { StackSym *stackSym = instr->GetDst()->AsRegOpnd()->m_sym; if (stackSym->IsSingleDef() || this->IsLive(stackSym, this->prePassLoop->landingPad)) { IntConstantBounds src1IntConstantBounds; if (src1ValueInfo->TryGetIntConstantBounds(&src1IntConstantBounds) && !( src1IntConstantBounds.LowerBound() == INT32_MIN && src1IntConstantBounds.UpperBound() == INT32_MAX )) { const ValueType valueType( GetPrepassValueTypeForDst(src1ValueInfo->Type(), instr, src1Val, nullptr, &isValueInfoPrecise)); if (isValueInfoPrecise) { return src1Val; } } else { return src1Val; } } } #endif // Src1's value could change later in the loop, so the value wouldn't be the same for each // iteration. Since we don't iterate over loops "while (!changed)", go conservative on the // first pass when transferring a value that is live on the back-edge. // In prepass we are going to copy the value but with a different value number // for aggressive int type spec. bool isSafeToTransferInPrepass = false; isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Val, nullptr, &isSafeToTransferInPrepass); const ValueType valueType(GetPrepassValueTypeForDst(src1ValueInfo->Type(), instr, src1Val, nullptr, isValueInfoPrecise, isSafeToTransferInPrepass)); if(isValueInfoPrecise || isSafeToTransferInPrepass) { Assert(valueType == src1ValueInfo->Type()); if (!PHASE_OFF1(Js::AVTInPrePassPhase)) { dstVal = src1Val; } else { dstVal = CopyValue(src1Val); TrackCopiedValueForKills(dstVal); } } else if (valueType == src1ValueInfo->Type() && src1ValueInfo->IsGeneric()) // this else branch is probably not needed { Assert(valueType == src1ValueInfo->Type()); dstVal = CopyValue(src1Val); TrackCopiedValueForKills(dstVal); } else { dstVal = NewGenericValue(valueType); dstVal->GetValueInfo()->SetSymStore(src1ValueInfo->GetSymStore()); } return dstVal; } void GlobOpt::PropagateIntRangeForNot(int32 minimum, int32 maximum, int32 *pNewMin, int32* pNewMax) { int32 tmp; Int32Math::Not(minimum, pNewMin); *pNewMax = *pNewMin; Int32Math::Not(maximum, &tmp); *pNewMin = min(*pNewMin, tmp); *pNewMax = max(*pNewMax, tmp); } void GlobOpt::PropagateIntRangeBinary(IR::Instr *instr, int32 min1, int32 max1, int32 min2, int32 max2, int32 *pNewMin, int32* pNewMax) { int32 min, max, tmp, tmp2; min = INT32_MIN; max = INT32_MAX; switch (instr->m_opcode) { case Js::OpCode::Xor_A: case Js::OpCode::Or_A: // Find range with highest high order bit tmp = ::max((uint32)min1, (uint32)max1); tmp2 = ::max((uint32)min2, (uint32)max2); if ((uint32)tmp > (uint32)tmp2) { max = tmp; } else { max = tmp2; } if (max < 0) { min = INT32_MIN; // REVIEW: conservative... max = INT32_MAX; } else { // Turn values like 0x1010 into 0x1111 max = 1 << Math::Log2(max); max = (uint32)(max << 1) - 1; min = 0; } break; case Js::OpCode::And_A: if (min1 == INT32_MIN && min2 == INT32_MIN) { // Shortcut break; } // Find range with lowest higher bit tmp = ::max((uint32)min1, (uint32)max1); tmp2 = ::max((uint32)min2, (uint32)max2); if ((uint32)tmp < (uint32)tmp2) { min = min1; max = max1; } else { min = min2; max = max2; } // To compute max, look if min has higher high bit if ((uint32)min > (uint32)max) { max = min; } // If max is negative, max let's assume it could be -1, so result in MAX_INT if (max < 0) { max = INT32_MAX; } // If min is positive, the resulting min is zero if (min >= 0) { min = 0; } else { min = INT32_MIN; } break; case Js::OpCode::Shl_A: { // Shift count if (min2 != max2 && ((uint32)min2 > 0x1F || (uint32)max2 > 0x1F)) { min2 = 0; max2 = 0x1F; } else { min2 &= 0x1F; max2 &= 0x1F; } int32 min1FreeTopBitCount = min1 ? (sizeof(int32) * 8) - (Math::Log2(min1) + 1) : (sizeof(int32) * 8); int32 max1FreeTopBitCount = max1 ? (sizeof(int32) * 8) - (Math::Log2(max1) + 1) : (sizeof(int32) * 8); if (min1FreeTopBitCount <= max2 || max1FreeTopBitCount <= max2) { // If the shift is going to touch the sign bit return the max range min = INT32_MIN; max = INT32_MAX; } else { // Compute max // Turn values like 0x1010 into 0x1111 if (min1) { min1 = 1 << Math::Log2(min1); min1 = (min1 << 1) - 1; } if (max1) { max1 = 1 << Math::Log2(max1); max1 = (uint32)(max1 << 1) - 1; } if (max1 > 0) { int32 nrTopBits = (sizeof(int32) * 8) - Math::Log2(max1); if (nrTopBits < ::min(max2, 30)) max = INT32_MAX; else max = ::max((max1 << ::min(max2, 30)) & ~0x80000000, (min1 << min2) & ~0x80000000); } else { max = (max1 << min2) & ~0x80000000; } // Compute min if (min1 < 0) { min = ::min(min1 << max2, max1 << max2); } else { min = ::min(min1 << min2, max1 << max2); } // Turn values like 0x1110 into 0x1000 if (min) { min = 1 << Math::Log2(min); } } } break; case Js::OpCode::Shr_A: // Shift count if (min2 != max2 && ((uint32)min2 > 0x1F || (uint32)max2 > 0x1F)) { min2 = 0; max2 = 0x1F; } else { min2 &= 0x1F; max2 &= 0x1F; } // Compute max if (max1 < 0) { max = max1 >> max2; } else { max = max1 >> min2; } // Compute min if (min1 < 0) { min = min1 >> min2; } else { min = min1 >> max2; } break; case Js::OpCode::ShrU_A: // shift count is constant zero if ((min2 == max2) && (max2 & 0x1f) == 0) { // We can't encode uint32 result, so it has to be used as int32 only or the original value is positive. Assert(instr->ignoreIntOverflow || min1 >= 0); // We can transfer the signed int32 range. min = min1; max = max1; break; } const IntConstantBounds src2NewBounds = IntConstantBounds(min2, max2).And_0x1f(); // Zero is only allowed if result is always a signed int32 or always used as a signed int32 Assert(min1 >= 0 || instr->ignoreIntOverflow || !src2NewBounds.Contains(0)); min2 = src2NewBounds.LowerBound(); max2 = src2NewBounds.UpperBound(); Assert(min2 <= max2); // zero shift count is only allowed if result is used as int32 and/or value is positive Assert(min2 > 0 || instr->ignoreIntOverflow || min1 >= 0); uint32 umin1 = (uint32)min1; uint32 umax1 = (uint32)max1; if (umin1 > umax1) { uint32 temp = umax1; umax1 = umin1; umin1 = temp; } Assert(min2 >= 0 && max2 < 32); // Compute max if (min1 < 0) { umax1 = UINT32_MAX; } max = umax1 >> min2; // Compute min if (min1 <= 0 && max1 >=0) { min = 0; } else { min = umin1 >> max2; } // We should be able to fit uint32 range as int32 Assert(instr->ignoreIntOverflow || (min >= 0 && max >= 0) ); if (min > max) { // can only happen if shift count can be zero Assert(min2 == 0 && (instr->ignoreIntOverflow || min1 >= 0)); min = Int32ConstMin; max = Int32ConstMax; } break; } *pNewMin = min; *pNewMax = max; } IR::Instr * GlobOpt::TypeSpecialization( IR::Instr *instr, Value **pSrc1Val, Value **pSrc2Val, Value **pDstVal, bool *redoTypeSpecRef, bool *const forceInvariantHoistingRef) { Value *&src1Val = *pSrc1Val; Value *&src2Val = *pSrc2Val; *redoTypeSpecRef = false; Assert(!*forceInvariantHoistingRef); this->ignoredIntOverflowForCurrentInstr = false; this->ignoredNegativeZeroForCurrentInstr = false; // - Int32 values that can't be tagged are created as float constant values instead because a JavascriptNumber var is needed // for that value at runtime. For the purposes of type specialization, recover the int32 values so that they will be // treated as ints. // - If int overflow does not matter for the instruction, we can additionally treat uint32 values as int32 values because // the value resulting from the operation will eventually be converted to int32 anyway Value *const src1OriginalVal = src1Val; Value *const src2OriginalVal = src2Val; if(!instr->ShouldCheckForIntOverflow()) { if(src1Val && src1Val->GetValueInfo()->IsFloatConstant()) { int32 int32Value; bool isInt32; if(Js::JavascriptNumber::TryGetInt32OrUInt32Value( src1Val->GetValueInfo()->AsFloatConstant()->FloatValue(), &int32Value, &isInt32)) { src1Val = GetIntConstantValue(int32Value, instr); if(!isInt32) { this->ignoredIntOverflowForCurrentInstr = true; } } } if(src2Val && src2Val->GetValueInfo()->IsFloatConstant()) { int32 int32Value; bool isInt32; if(Js::JavascriptNumber::TryGetInt32OrUInt32Value( src2Val->GetValueInfo()->AsFloatConstant()->FloatValue(), &int32Value, &isInt32)) { src2Val = GetIntConstantValue(int32Value, instr); if(!isInt32) { this->ignoredIntOverflowForCurrentInstr = true; } } } } const AutoRestoreVal autoRestoreSrc1Val(src1OriginalVal, &src1Val); const AutoRestoreVal autoRestoreSrc2Val(src2OriginalVal, &src2Val); if (src1Val && instr->GetSrc2() == nullptr) { // Unary // Note make sure that native array StElemI gets to TypeSpecializeStElem. Do this for typed arrays, too? int32 intConstantValue; if (!this->IsLoopPrePass() && !instr->IsBranchInstr() && src1Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) && !( // Nothing to fold for element stores. Go into type specialization to see if they can at least be specialized. instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict || instr->m_opcode == Js::OpCode::StElemC || instr->m_opcode == Js::OpCode::MultiBr || instr->m_opcode == Js::OpCode::InlineArrayPop )) { if (OptConstFoldUnary(&instr, intConstantValue, src1Val == src1OriginalVal, pDstVal)) { return instr; } } else if (this->TypeSpecializeUnary( &instr, &src1Val, pDstVal, src1OriginalVal, redoTypeSpecRef, forceInvariantHoistingRef)) { return instr; } else if(*redoTypeSpecRef) { return instr; } } else if (instr->GetSrc2() && !instr->IsBranchInstr()) { // Binary if (!this->IsLoopPrePass()) { if (GetIsAsmJSFunc()) { if (CONFIG_FLAG(WasmFold)) { bool success = instr->GetSrc1()->IsInt64() ? this->OptConstFoldBinaryWasm(&instr, src1Val, src2Val, pDstVal) : this->OptConstFoldBinaryWasm(&instr, src1Val, src2Val, pDstVal); if (success) { return instr; } } } else { // OptConstFoldBinary doesn't do type spec, so only deal with things we are sure are int (IntConstant and IntRange) // and not just likely ints TypeSpecializeBinary will deal with type specializing them and fold them again IntConstantBounds src1IntConstantBounds, src2IntConstantBounds; if (src1Val && src1Val->GetValueInfo()->TryGetIntConstantBounds(&src1IntConstantBounds)) { if (src2Val && src2Val->GetValueInfo()->TryGetIntConstantBounds(&src2IntConstantBounds)) { if (this->OptConstFoldBinary(&instr, src1IntConstantBounds, src2IntConstantBounds, pDstVal)) { return instr; } } } } } } if (instr->GetSrc2() && this->TypeSpecializeBinary(&instr, pSrc1Val, pSrc2Val, pDstVal, src1OriginalVal, src2OriginalVal, redoTypeSpecRef)) { if (!this->IsLoopPrePass() && instr->m_opcode != Js::OpCode::Nop && instr->m_opcode != Js::OpCode::Br && // We may have const fold a branch // Cannot const-peep if the result of the operation is required for a bailout check !(instr->HasBailOutInfo() && instr->GetBailOutKind() & IR::BailOutOnResultConditions)) { if (src1Val && src1Val->GetValueInfo()->HasIntConstantValue()) { if (this->OptConstPeep(instr, instr->GetSrc1(), pDstVal, src1Val->GetValueInfo())) { return instr; } } else if (src2Val && src2Val->GetValueInfo()->HasIntConstantValue()) { if (this->OptConstPeep(instr, instr->GetSrc2(), pDstVal, src2Val->GetValueInfo())) { return instr; } } } return instr; } else if(*redoTypeSpecRef) { return instr; } if (instr->IsBranchInstr() && !this->IsLoopPrePass()) { if (this->OptConstFoldBranch(instr, src1Val, src2Val, pDstVal)) { return instr; } } // We didn't type specialize, make sure the srcs are unspecialized IR::Opnd *src1 = instr->GetSrc1(); if (src1) { instr = this->ToVarUses(instr, src1, false, src1Val); IR::Opnd *src2 = instr->GetSrc2(); if (src2) { instr = this->ToVarUses(instr, src2, false, src2Val); } } IR::Opnd *dst = instr->GetDst(); if (dst) { instr = this->ToVarUses(instr, dst, true, nullptr); // Handling for instructions other than built-ins that may require only dst type specialization // should be added here. if(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode) && !GetIsAsmJSFunc()) // don't need to do typespec for asmjs { this->TypeSpecializeInlineBuiltInDst(&instr, pDstVal); return instr; } // Clear the int specialized bit on the dst. if (dst->IsRegOpnd()) { IR::RegOpnd *dstRegOpnd = dst->AsRegOpnd(); if (!dstRegOpnd->m_sym->IsTypeSpec()) { this->ToVarRegOpnd(dstRegOpnd, this->currentBlock); } else if (dstRegOpnd->m_sym->IsInt32()) { this->ToInt32Dst(instr, dstRegOpnd, this->currentBlock); } else if (dstRegOpnd->m_sym->IsUInt32() && GetIsAsmJSFunc()) { this->ToUInt32Dst(instr, dstRegOpnd, this->currentBlock); } else if (dstRegOpnd->m_sym->IsFloat64()) { this->ToFloat64Dst(instr, dstRegOpnd, this->currentBlock); } } else if (dst->IsSymOpnd() && dst->AsSymOpnd()->m_sym->IsStackSym()) { this->ToVarStackSym(dst->AsSymOpnd()->m_sym->AsStackSym(), this->currentBlock); } } return instr; } bool GlobOpt::OptConstPeep(IR::Instr *instr, IR::Opnd *constSrc, Value **pDstVal, ValueInfo *valuInfo) { int32 value; IR::Opnd *src; IR::Opnd *nonConstSrc = (constSrc == instr->GetSrc1() ? instr->GetSrc2() : instr->GetSrc1()); // Try to find the value from value info first if (valuInfo->TryGetIntConstantValue(&value)) { } else if (constSrc->IsAddrOpnd()) { IR::AddrOpnd *addrOpnd = constSrc->AsAddrOpnd(); #ifdef _M_X64 Assert(addrOpnd->IsVar() || Math::FitsInDWord((size_t)addrOpnd->m_address)); #else Assert(sizeof(value) == sizeof(addrOpnd->m_address)); #endif if (addrOpnd->IsVar()) { value = Js::TaggedInt::ToInt32(addrOpnd->m_address); } else { // We asserted that the address will fit in a DWORD above value = ::Math::PointerCastToIntegral(constSrc->AsAddrOpnd()->m_address); } } else if (constSrc->IsIntConstOpnd()) { value = constSrc->AsIntConstOpnd()->AsInt32(); } else { return false; } switch(instr->m_opcode) { // Can't do all Add_A because of string concats. // Sub_A cannot be transformed to a NEG_A because 0 - 0 != -0 case Js::OpCode::Add_A: src = nonConstSrc; if (!src->GetValueType().IsInt()) { // 0 + -0 != -0 // "Foo" + 0 != "Foo return false; } // fall-through case Js::OpCode::Add_I4: if (value != 0) { return false; } if (constSrc == instr->GetSrc1()) { src = instr->GetSrc2(); } else { src = instr->GetSrc1(); } break; case Js::OpCode::Mul_A: case Js::OpCode::Mul_I4: if (value == 0) { // -0 * 0 != 0 return false; } else if (value == 1) { src = nonConstSrc; } else { return false; } break; case Js::OpCode::Div_A: if (value == 1 && constSrc == instr->GetSrc2()) { src = instr->GetSrc1(); } else { return false; } break; case Js::OpCode::Or_I4: if (value == -1) { src = constSrc; } else if (value == 0) { src = nonConstSrc; } else { return false; } break; case Js::OpCode::And_I4: if (value == -1) { src = nonConstSrc; } else if (value == 0) { src = constSrc; } else { return false; } break; case Js::OpCode::Shl_I4: case Js::OpCode::ShrU_I4: case Js::OpCode::Shr_I4: if (value != 0 || constSrc != instr->GetSrc2()) { return false; } src = instr->GetSrc1(); break; default: return false; } this->CaptureByteCodeSymUses(instr); if (src == instr->GetSrc1()) { instr->FreeSrc2(); } else { Assert(src == instr->GetSrc2()); instr->ReplaceSrc1(instr->UnlinkSrc2()); } instr->m_opcode = Js::OpCode::Ld_A; InvalidateInductionVariables(instr); return true; } Js::Var // TODO: michhol OOP JIT, shouldn't play with Vars GlobOpt::GetConstantVar(IR::Opnd *opnd, Value *val) { ValueInfo *valueInfo = val->GetValueInfo(); if (valueInfo->IsVarConstant() && valueInfo->IsPrimitive()) { return valueInfo->AsVarConstant()->VarValue(); } if (opnd->IsAddrOpnd()) { IR::AddrOpnd *addrOpnd = opnd->AsAddrOpnd(); if (addrOpnd->IsVar()) { return addrOpnd->m_address; } } else if (opnd->IsIntConstOpnd()) { if (!Js::TaggedInt::IsOverflow(opnd->AsIntConstOpnd()->AsInt32())) { return Js::TaggedInt::ToVarUnchecked(opnd->AsIntConstOpnd()->AsInt32()); } } #if FLOATVAR else if (opnd->IsFloatConstOpnd()) { return Js::JavascriptNumber::ToVar(opnd->AsFloatConstOpnd()->m_value); } #endif else if (opnd->IsRegOpnd() && opnd->AsRegOpnd()->m_sym->IsSingleDef()) { if (valueInfo->IsBoolean()) { IR::Instr * defInstr = opnd->AsRegOpnd()->m_sym->GetInstrDef(); if (defInstr->m_opcode != Js::OpCode::Ld_A || !defInstr->GetSrc1()->IsAddrOpnd()) { return nullptr; } Assert(defInstr->GetSrc1()->AsAddrOpnd()->IsVar()); return defInstr->GetSrc1()->AsAddrOpnd()->m_address; } else if (valueInfo->IsUndefined()) { return (Js::Var)this->func->GetScriptContextInfo()->GetUndefinedAddr(); } else if (valueInfo->IsNull()) { return (Js::Var)this->func->GetScriptContextInfo()->GetNullAddr(); } #if FLOATVAR else if (valueInfo->IsFloat()) { IR::Instr * defInstr = opnd->AsRegOpnd()->m_sym->GetInstrDef(); if ((defInstr->m_opcode == Js::OpCode::LdC_F8_R8 || defInstr->m_opcode == Js::OpCode::LdC_A_R8) && defInstr->GetSrc1()->IsFloatConstOpnd()) { return Js::JavascriptNumber::ToVar(defInstr->GetSrc1()->AsFloatConstOpnd()->m_value); } } #endif } return nullptr; } namespace { bool TryCompIntAndFloat(bool * result, Js::Var left, Js::Var right) { if (Js::TaggedInt::Is(left)) { // If both are tagged ints we should not get here. Assert(!Js::TaggedInt::Is(right)); if (Js::JavascriptNumber::Is_NoTaggedIntCheck(right)) { double value = Js::JavascriptNumber::GetValue(right); *result = (Js::TaggedInt::ToInt32(left) == value); return true; } } return false; } bool Op_JitEq(bool * result, Value * src1Val, Value * src2Val, Js::Var src1Var, Js::Var src2Var, Func * func, bool isStrict) { Assert(src1Val != nullptr && src2Val != nullptr); Assert(src1Var != nullptr && src2Var != nullptr); if (src1Var == src2Var) { if (Js::TaggedInt::Is(src1Var)) { *result = true; return true; } if (!isStrict && src1Val->GetValueInfo()->IsNotFloat()) { // If the vars are equal and they are not NaN, non-strict equal returns true. Not float guarantees not NaN. *result = true; return true; } #if FLOATVAR if (Js::JavascriptNumber::Is_NoTaggedIntCheck(src1Var)) { *result = !Js::JavascriptNumber::IsNan(Js::JavascriptNumber::GetValue(src1Var)); return true; } #endif if (src1Var == reinterpret_cast(func->GetScriptContextInfo()->GetTrueAddr()) || src1Var == reinterpret_cast(func->GetScriptContextInfo()->GetFalseAddr()) || src1Var == reinterpret_cast(func->GetScriptContextInfo()->GetNullAddr()) || src1Var == reinterpret_cast(func->GetScriptContextInfo()->GetUndefinedAddr())) { *result = true; return true; } // Other var comparisons require the runtime to prove. return false; } #if FLOATVAR if (TryCompIntAndFloat(result, src1Var, src2Var) || TryCompIntAndFloat(result, src2Var, src1Var)) { return true; } #endif return false; } bool Op_JitNeq(bool * result, Value * src1Val, Value * src2Val, Js::Var src1Var, Js::Var src2Var, Func * func, bool isStrict) { if (Op_JitEq(result, src1Val, src2Val, src1Var, src2Var, func, isStrict)) { *result = !*result; return true; } return false; } bool BoolAndIntStaticAndTypeMismatch(Value* src1Val, Value* src2Val, Js::Var src1Var, Js::Var src2Var) { ValueInfo *src1ValInfo = src1Val->GetValueInfo(); ValueInfo *src2ValInfo = src2Val->GetValueInfo(); return (src1ValInfo->IsNumber() && src1Var && src2ValInfo->IsBoolean() && src1Var != Js::TaggedInt::ToVarUnchecked(0) && src1Var != Js::TaggedInt::ToVarUnchecked(1)) || (src2ValInfo->IsNumber() && src2Var && src1ValInfo->IsBoolean() && src2Var != Js::TaggedInt::ToVarUnchecked(0) && src2Var != Js::TaggedInt::ToVarUnchecked(1)); } } bool GlobOpt::CanProveConditionalBranch(IR::Instr *instr, Value *src1Val, Value *src2Val, Js::Var src1Var, Js::Var src2Var, bool *result) { auto AreSourcesEqual = [&](Value * val1, Value * val2, bool undefinedCmp) -> bool { // NaN !== NaN, and objects can have valueOf/toString if (val1->IsEqualTo(val2)) { if (val1->GetValueInfo()->IsUndefined()) { return undefinedCmp; } ValueInfo * valInfo = val1->GetValueInfo(); return !valInfo->HasBeenUndefined() && valInfo->IsPrimitive() && valInfo->IsNotFloat(); } return false; }; // Make sure GetConstantVar only returns primitives. // TODO: OOP JIT, enabled these asserts //Assert(!src1Var || !Js::JavascriptOperators::IsObject(src1Var)); //Assert(!src2Var || !Js::JavascriptOperators::IsObject(src2Var)); int64 left64, right64; int32 left, right; int32 constVal; switch (instr->m_opcode) { #define BRANCHSIGNED(OPCODE,CMP,TYPE,UNSIGNEDNESS,UNDEFINEDCMP) \ case Js::OpCode::##OPCODE: \ if (src1Val && src2Val) \ { \ if (src1Val->GetValueInfo()->TryGetIntConstantValue(&left, UNSIGNEDNESS) && \ src2Val->GetValueInfo()->TryGetIntConstantValue(&right, UNSIGNEDNESS)) \ { \ *result = (TYPE)left CMP(TYPE)right; \ } \ if (src1Val->GetValueInfo()->TryGetInt64ConstantValue(&left64, UNSIGNEDNESS) && \ src2Val->GetValueInfo()->TryGetInt64ConstantValue(&right64, UNSIGNEDNESS)) \ { \ *result = (TYPE)left64 CMP(TYPE)right64; \ } \ else if (AreSourcesEqual(src1Val, src2Val, UNDEFINEDCMP)) \ { \ *result = 0 CMP 0; \ } \ else \ { \ return false; \ } \ } \ else \ { \ return false; \ } \ break; BRANCHSIGNED(BrEq_I4, == , int64, false, true) BRANCHSIGNED(BrGe_I4, >= , int64, false, false) BRANCHSIGNED(BrGt_I4, > , int64, false, false) BRANCHSIGNED(BrLt_I4, < , int64, false, false) BRANCHSIGNED(BrLe_I4, <= , int64, false, false) BRANCHSIGNED(BrNeq_I4, != , int64, false, false) BRANCHSIGNED(BrUnGe_I4, >= , uint64, true, false) BRANCHSIGNED(BrUnGt_I4, > , uint64, true, false) BRANCHSIGNED(BrUnLt_I4, < , uint64, true, false) BRANCHSIGNED(BrUnLe_I4, <= , uint64, true, false) #undef BRANCHSIGNED #define BRANCH(OPCODE,CMP,VARCMPFUNC,UNDEFINEDCMP) \ case Js::OpCode::##OPCODE: \ if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) && \ src2Val->GetValueInfo()->TryGetIntConstantValue(&right)) \ { \ *result = left CMP right; \ } \ else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, UNDEFINEDCMP)) \ { \ *result = 0 CMP 0; \ } \ else if (src1Var && src2Var) \ { \ if (func->IsOOPJIT() || !CONFIG_FLAG(OOPJITMissingOpts)) \ { \ return false; \ } \ *result = VARCMPFUNC(src1Var, src2Var, this->func->GetScriptContext()); \ } \ else \ { \ return false; \ } \ break; BRANCH(BrGe_A, >= , Js::JavascriptOperators::GreaterEqual, /*undefinedEquality*/ false) BRANCH(BrNotGe_A, <, !Js::JavascriptOperators::GreaterEqual, false) BRANCH(BrLt_A, <, Js::JavascriptOperators::Less, false) BRANCH(BrNotLt_A, >= , !Js::JavascriptOperators::Less, false) BRANCH(BrGt_A, >, Js::JavascriptOperators::Greater, false) BRANCH(BrNotGt_A, <= , !Js::JavascriptOperators::Greater, false) BRANCH(BrLe_A, <= , Js::JavascriptOperators::LessEqual, false) BRANCH(BrNotLe_A, >, !Js::JavascriptOperators::LessEqual, false) #undef BRANCH case Js::OpCode::BrEq_A: case Js::OpCode::BrNotNeq_A: if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) && src2Val->GetValueInfo()->TryGetIntConstantValue(&right)) { *result = left == right; } else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, true)) { *result = true; } else if (!src1Var || !src2Var) { if (BoolAndIntStaticAndTypeMismatch(src1Val, src2Val, src1Var, src2Var)) { *result = false; } else { return false; } } else { if (!Op_JitEq(result, src1Val, src2Val, src1Var, src2Var, this->func, false /* isStrict */)) { return false; } } break; case Js::OpCode::BrNeq_A: case Js::OpCode::BrNotEq_A: if (src1Val && src2Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&left) && src2Val->GetValueInfo()->TryGetIntConstantValue(&right)) { *result = left != right; } else if (src1Val && src2Val && AreSourcesEqual(src1Val, src2Val, true)) { *result = false; } else if (!src1Var || !src2Var) { if (BoolAndIntStaticAndTypeMismatch(src1Val, src2Val, src1Var, src2Var)) { *result = true; } else { return false; } } else { if (!Op_JitNeq(result, src1Val, src2Val, src1Var, src2Var, this->func, false /* isStrict */)) { return false; } } break; case Js::OpCode::BrSrEq_A: case Js::OpCode::BrSrNotNeq_A: if (!src1Var || !src2Var) { ValueInfo *src1ValInfo = src1Val->GetValueInfo(); ValueInfo *src2ValInfo = src2Val->GetValueInfo(); if ( (src1ValInfo->IsUndefined() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenUndefined()) || (src1ValInfo->IsNull() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNull()) || (src1ValInfo->IsBoolean() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenBoolean()) || (src1ValInfo->IsNumber() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNumber()) || (src1ValInfo->IsString() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenString()) || (src2ValInfo->IsUndefined() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenUndefined()) || (src2ValInfo->IsNull() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNull()) || (src2ValInfo->IsBoolean() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenBoolean()) || (src2ValInfo->IsNumber() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNumber()) || (src2ValInfo->IsString() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenString()) ) { *result = false; } else if (AreSourcesEqual(src1Val, src2Val, true)) { *result = true; } else { return false; } } else { if (!Op_JitEq(result, src1Val, src2Val, src1Var, src2Var, this->func, true /* isStrict */)) { return false; } } break; case Js::OpCode::BrSrNeq_A: case Js::OpCode::BrSrNotEq_A: if (!src1Var || !src2Var) { ValueInfo *src1ValInfo = src1Val->GetValueInfo(); ValueInfo *src2ValInfo = src2Val->GetValueInfo(); if ( (src1ValInfo->IsUndefined() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenUndefined()) || (src1ValInfo->IsNull() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNull()) || (src1ValInfo->IsBoolean() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenBoolean()) || (src1ValInfo->IsNumber() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenNumber()) || (src1ValInfo->IsString() && src2ValInfo->IsDefinite() && !src2ValInfo->HasBeenString()) || (src2ValInfo->IsUndefined() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenUndefined()) || (src2ValInfo->IsNull() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNull()) || (src2ValInfo->IsBoolean() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenBoolean()) || (src2ValInfo->IsNumber() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenNumber()) || (src2ValInfo->IsString() && src1ValInfo->IsDefinite() && !src1ValInfo->HasBeenString()) ) { *result = true; } else if (AreSourcesEqual(src1Val, src2Val, true)) { *result = false; } else { return false; } } else { if (!Op_JitNeq(result, src1Val, src2Val, src1Var, src2Var, this->func, true /* isStrict */)) { return false; } } break; case Js::OpCode::BrFalse_A: case Js::OpCode::BrTrue_A: { ValueInfo *const src1ValueInfo = src1Val->GetValueInfo(); if (src1ValueInfo->IsNull() || src1ValueInfo->IsUndefined()) { *result = instr->m_opcode == Js::OpCode::BrFalse_A; break; } if (src1ValueInfo->IsObject() && src1ValueInfo->GetObjectType() > ObjectType::Object) { // Specific object types that are tracked are equivalent to 'true' *result = instr->m_opcode == Js::OpCode::BrTrue_A; break; } if (!src1Var) { return false; } // Set *result = (evaluates true) and negate it later for BrFalse if (src1Var == reinterpret_cast(this->func->GetScriptContextInfo()->GetTrueAddr())) { *result = true; } else if (src1Var == reinterpret_cast(this->func->GetScriptContextInfo()->GetFalseAddr())) { *result = false; } else if (Js::TaggedInt::Is(src1Var)) { *result = (src1Var != reinterpret_cast(Js::AtomTag_IntPtr)); } #if FLOATVAR else if (Js::JavascriptNumber::Is_NoTaggedIntCheck(src1Var)) { double value = Js::JavascriptNumber::GetValue(src1Var); *result = (!Js::JavascriptNumber::IsNan(value)) && (!Js::JavascriptNumber::IsZero(value)); } #endif else { return false; } if (instr->m_opcode == Js::OpCode::BrFalse_A) { *result = !(*result); } break; } case Js::OpCode::BrFalse_I4: { constVal = 0; if (!src1Val->GetValueInfo()->TryGetIntConstantValue(&constVal)) { return false; } *result = constVal == 0; break; } case Js::OpCode::BrOnObject_A: { ValueInfo *const src1ValueInfo = src1Val->GetValueInfo(); if (!src1ValueInfo->IsDefinite()) { return false; } if (src1ValueInfo->IsPrimitive()) { *result = false; } else { if (src1ValueInfo->HasBeenPrimitive()) { return false; } *result = true; } break; } default: return false; } return true; } bool GlobOpt::OptConstFoldBranch(IR::Instr *instr, Value *src1Val, Value*src2Val, Value **pDstVal) { if (!src1Val) { return false; } Js::Var src1Var = this->GetConstantVar(instr->GetSrc1(), src1Val); Js::Var src2Var = nullptr; if (instr->GetSrc2()) { if (!src2Val) { return false; } src2Var = this->GetConstantVar(instr->GetSrc2(), src2Val); } bool result; if (!CanProveConditionalBranch(instr, src1Val, src2Val, src1Var, src2Var, &result)) { return false; } this->OptConstFoldBr(!!result, instr); return true; } bool GlobOpt::OptConstFoldUnary( IR::Instr * *pInstr, const int32 intConstantValue, const bool isUsingOriginalSrc1Value, Value **pDstVal) { IR::Instr * &instr = *pInstr; int32 value = 0; IR::Opnd *constOpnd; bool isInt = true; bool doSetDstVal = true; FloatConstType fValue = 0.0; if (!DoConstFold()) { return false; } if (instr->GetDst() && !instr->GetDst()->IsRegOpnd()) { return false; } switch(instr->m_opcode) { case Js::OpCode::Neg_A: if (intConstantValue == 0) { // Could fold to -0.0 return false; } if (Int32Math::Neg(intConstantValue, &value)) { return false; } break; case Js::OpCode::Not_A: Int32Math::Not(intConstantValue, &value); break; case Js::OpCode::Ld_A: if (instr->HasBailOutInfo()) { //The profile data for switch expr can be string and in GlobOpt we realize it is an int. if(instr->GetBailOutKind() == IR::BailOutExpectingString) { throw Js::RejitException(RejitReason::DisableSwitchOptExpectingString); } Assert(instr->GetBailOutKind() == IR::BailOutExpectingInteger); instr->ClearBailOutInfo(); } value = intConstantValue; if(isUsingOriginalSrc1Value) { doSetDstVal = false; // Let OptDst do it by copying src1Val } break; case Js::OpCode::Conv_Num: case Js::OpCode::LdC_A_I4: value = intConstantValue; if(isUsingOriginalSrc1Value) { doSetDstVal = false; // Let OptDst do it by copying src1Val } break; case Js::OpCode::Incr_A: if (Int32Math::Inc(intConstantValue, &value)) { return false; } break; case Js::OpCode::Decr_A: if (Int32Math::Dec(intConstantValue, &value)) { return false; } break; case Js::OpCode::InlineMathAcos: fValue = Js::Math::Acos((double)intConstantValue); isInt = false; break; case Js::OpCode::InlineMathAsin: fValue = Js::Math::Asin((double)intConstantValue); isInt = false; break; case Js::OpCode::InlineMathAtan: fValue = Js::Math::Atan((double)intConstantValue); isInt = false; break; case Js::OpCode::InlineMathCos: fValue = Js::Math::Cos((double)intConstantValue); isInt = false; break; case Js::OpCode::InlineMathExp: fValue = Js::Math::Exp((double)intConstantValue); isInt = false; break; case Js::OpCode::InlineMathLog: fValue = Js::Math::Log((double)intConstantValue); isInt = false; break; case Js::OpCode::InlineMathSin: fValue = Js::Math::Sin((double)intConstantValue); isInt = false; break; case Js::OpCode::InlineMathSqrt: fValue = ::sqrt((double)intConstantValue); isInt = false; break; case Js::OpCode::InlineMathTan: fValue = ::tan((double)intConstantValue); isInt = false; break; case Js::OpCode::InlineMathFround: fValue = (double) (float) intConstantValue; isInt = false; break; case Js::OpCode::InlineMathAbs: if (intConstantValue == INT32_MIN) { if (instr->GetDst()->IsInt32()) { // if dst is an int (e.g. in asm.js), we should coerce it, not convert to float value = static_cast(2147483648U); } else { // Rejit with AggressiveIntTypeSpecDisabled for Math.abs(INT32_MIN) because it causes dst // to be float type which could be different with previous type spec result in LoopPrePass throw Js::RejitException(RejitReason::AggressiveIntTypeSpecDisabled); } } else { value = ::abs(intConstantValue); } break; case Js::OpCode::InlineMathClz: DWORD clz; if (_BitScanReverse(&clz, intConstantValue)) { value = 31 - clz; } else { value = 32; } instr->ClearBailOutInfo(); break; case Js::OpCode::Ctz: Assert(func->GetJITFunctionBody()->IsWasmFunction()); Assert(!instr->HasBailOutInfo()); DWORD ctz; if (_BitScanForward(&ctz, intConstantValue)) { value = ctz; } else { value = 32; } break; case Js::OpCode::InlineMathFloor: value = intConstantValue; instr->ClearBailOutInfo(); break; case Js::OpCode::InlineMathCeil: value = intConstantValue; instr->ClearBailOutInfo(); break; case Js::OpCode::InlineMathRound: value = intConstantValue; instr->ClearBailOutInfo(); break; case Js::OpCode::ToVar: if (Js::TaggedInt::IsOverflow(intConstantValue)) { return false; } else { value = intConstantValue; instr->ClearBailOutInfo(); break; } default: return false; } this->CaptureByteCodeSymUses(instr); Assert(!instr->HasBailOutInfo()); // If we are, in fact, successful in constant folding the instruction, there is no point in having the bailoutinfo around anymore. // Make sure that it is cleared if it was initially present. if (!isInt) { value = (int32)fValue; if (fValue == (double)value) { isInt = true; } } if (isInt) { constOpnd = IR::IntConstOpnd::New(value, TyInt32, instr->m_func); GOPT_TRACE(_u("Constant folding to %d\n"), value); } else { constOpnd = IR::FloatConstOpnd::New(fValue, TyFloat64, instr->m_func); GOPT_TRACE(_u("Constant folding to %f\n"), fValue); } instr->ReplaceSrc1(constOpnd); this->OptSrc(constOpnd, &instr); IR::Opnd *dst = instr->GetDst(); Assert(dst->IsRegOpnd()); StackSym *dstSym = dst->AsRegOpnd()->m_sym; if (isInt) { if (dstSym->IsSingleDef()) { dstSym->SetIsIntConst(value); } if (doSetDstVal) { *pDstVal = GetIntConstantValue(value, instr, dst); } if (IsTypeSpecPhaseOff(this->func)) { instr->m_opcode = Js::OpCode::LdC_A_I4; this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock); } else { instr->m_opcode = Js::OpCode::Ld_I4; this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock); StackSym * currDstSym = instr->GetDst()->AsRegOpnd()->m_sym; if (currDstSym->IsSingleDef()) { currDstSym->SetIsIntConst(value); } } } else { *pDstVal = NewFloatConstantValue(fValue, dst); if (IsTypeSpecPhaseOff(this->func)) { instr->m_opcode = Js::OpCode::LdC_A_R8; this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock); } else { instr->m_opcode = Js::OpCode::LdC_F8_R8; this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock); } } InvalidateInductionVariables(instr); return true; } //------------------------------------------------------------------------------------------------------ // Type specialization //------------------------------------------------------------------------------------------------------ bool GlobOpt::IsWorthSpecializingToInt32DueToSrc(IR::Opnd *const src, Value *const val) { Assert(src); Assert(val); ValueInfo *valueInfo = val->GetValueInfo(); Assert(valueInfo->IsLikelyInt()); // If it is not known that the operand is definitely an int, the operand is not already type-specialized, and it's not live // in the loop landing pad (if we're in a loop), it's probably not worth type-specializing this instruction. The common case // where type-specializing this would be bad is where the operations are entirely on properties or array elements, where the // ratio of FromVars and ToVars to the number of actual operations is high, and the conversions would dominate the time // spent. On the other hand, if we're using a function formal parameter more than once, it would probably be worth // type-specializing it, hence the IsDead check on the operands. return valueInfo->IsInt() || valueInfo->HasIntConstantValue(true) || !src->GetIsDead() || !src->IsRegOpnd() || CurrentBlockData()->IsInt32TypeSpecialized(src->AsRegOpnd()->m_sym) || (this->currentBlock->loop && this->currentBlock->loop->landingPad->globOptData.IsLive(src->AsRegOpnd()->m_sym)); } bool GlobOpt::IsWorthSpecializingToInt32DueToDst(IR::Opnd *const dst) { Assert(dst); const auto sym = dst->AsRegOpnd()->m_sym; return CurrentBlockData()->IsInt32TypeSpecialized(sym) || (this->currentBlock->loop && this->currentBlock->loop->landingPad->globOptData.IsLive(sym)); } bool GlobOpt::IsWorthSpecializingToInt32(IR::Instr *const instr, Value *const src1Val, Value *const src2Val) { Assert(instr); const auto src1 = instr->GetSrc1(); const auto src2 = instr->GetSrc2(); // In addition to checking each operand and the destination, if for any reason we only have to do a maximum of two // conversions instead of the worst-case 3 conversions, it's probably worth specializing. if (IsWorthSpecializingToInt32DueToSrc(src1, src1Val) || (src2Val && IsWorthSpecializingToInt32DueToSrc(src2, src2Val))) { return true; } IR::Opnd *dst = instr->GetDst(); if (!dst || IsWorthSpecializingToInt32DueToDst(dst)) { return true; } if (dst->IsEqual(src1) || (src2Val && (dst->IsEqual(src2) || src1->IsEqual(src2)))) { return true; } IR::Instr *instrNext = instr->GetNextRealInstrOrLabel(); // Skip useless Ld_A's do { switch (instrNext->m_opcode) { case Js::OpCode::Ld_A: if (!dst->IsEqual(instrNext->GetSrc1())) { goto done; } dst = instrNext->GetDst(); break; case Js::OpCode::LdFld: case Js::OpCode::LdRootFld: case Js::OpCode::LdRootFldForTypeOf: case Js::OpCode::LdFldForTypeOf: case Js::OpCode::LdElemI_A: case Js::OpCode::ByteCodeUses: break; default: goto done; } instrNext = instrNext->GetNextRealInstrOrLabel(); } while (true); done: // If the next instr could also be type specialized, then it is probably worth it. if ((instrNext->GetSrc1() && dst->IsEqual(instrNext->GetSrc1())) || (instrNext->GetSrc2() && dst->IsEqual(instrNext->GetSrc2()))) { switch (instrNext->m_opcode) { case Js::OpCode::Add_A: case Js::OpCode::Sub_A: case Js::OpCode::Mul_A: case Js::OpCode::Div_A: case Js::OpCode::Rem_A: case Js::OpCode::Xor_A: case Js::OpCode::And_A: case Js::OpCode::Or_A: case Js::OpCode::Shl_A: case Js::OpCode::Shr_A: case Js::OpCode::Incr_A: case Js::OpCode::Decr_A: case Js::OpCode::Neg_A: case Js::OpCode::Not_A: case Js::OpCode::Conv_Num: case Js::OpCode::BrEq_I4: case Js::OpCode::BrTrue_I4: case Js::OpCode::BrFalse_I4: case Js::OpCode::BrGe_I4: case Js::OpCode::BrGt_I4: case Js::OpCode::BrLt_I4: case Js::OpCode::BrLe_I4: case Js::OpCode::BrNeq_I4: return true; } } return false; } bool GlobOpt::TypeSpecializeNumberUnary(IR::Instr *instr, Value *src1Val, Value **pDstVal) { Assert(src1Val->GetValueInfo()->IsNumber()); if (this->IsLoopPrePass()) { return false; } switch (instr->m_opcode) { case Js::OpCode::Conv_Num: // Optimize Conv_Num away since we know this is a number instr->m_opcode = Js::OpCode::Ld_A; return false; } return false; } bool GlobOpt::TypeSpecializeUnary( IR::Instr **pInstr, Value **pSrc1Val, Value **pDstVal, Value *const src1OriginalVal, bool *redoTypeSpecRef, bool *const forceInvariantHoistingRef) { Assert(pSrc1Val); Value *&src1Val = *pSrc1Val; Assert(src1Val); // We don't need to do typespec for asmjs if (IsTypeSpecPhaseOff(this->func) || GetIsAsmJSFunc()) { return false; } IR::Instr *&instr = *pInstr; int32 min, max; // Inline built-ins explicitly specify how srcs/dst must be specialized. if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode)) { TypeSpecializeInlineBuiltInUnary(pInstr, &src1Val, pDstVal, src1OriginalVal, redoTypeSpecRef); return true; } // Consider: If type spec wasn't completely done, make sure that we don't type-spec the dst 2nd time. if(instr->m_opcode == Js::OpCode::LdLen_A && TypeSpecializeLdLen(&instr, &src1Val, pDstVal, forceInvariantHoistingRef)) { return true; } if (!src1Val->GetValueInfo()->GetIntValMinMax(&min, &max, this->DoAggressiveIntTypeSpec())) { src1Val = src1OriginalVal; if (src1Val->GetValueInfo()->IsLikelyFloat()) { // Try to type specialize to float return this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal); } else if (src1Val->GetValueInfo()->IsNumber()) { return TypeSpecializeNumberUnary(instr, src1Val, pDstVal); } return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal); } return this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, min, max, src1OriginalVal, redoTypeSpecRef); } // Returns true if the built-in requested type specialization, and no further action needed, // otherwise returns false. void GlobOpt::TypeSpecializeInlineBuiltInUnary(IR::Instr **pInstr, Value **pSrc1Val, Value **pDstVal, Value *const src1OriginalVal, bool *redoTypeSpecRef) { IR::Instr *&instr = *pInstr; Assert(pSrc1Val); Value *&src1Val = *pSrc1Val; Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode)); Js::BuiltinFunction builtInId = Js::JavascriptLibrary::GetBuiltInInlineCandidateId(instr->m_opcode); // From actual instr, not profile based. Assert(builtInId != Js::BuiltinFunction::None); // Consider using different bailout for float/int FromVars, so that when the arg cannot be converted to number we don't disable // type spec for other parts of the big function but rather just don't inline that built-in instr. // E.g. could do that if the value is not likelyInt/likelyFloat. Js::BuiltInFlags builtInFlags = Js::JavascriptLibrary::GetFlagsForBuiltIn(builtInId); bool areAllArgsAlwaysFloat = (builtInFlags & Js::BuiltInFlags::BIF_Args) == Js::BuiltInFlags::BIF_TypeSpecUnaryToFloat; if (areAllArgsAlwaysFloat) { // InlineMathAcos, InlineMathAsin, InlineMathAtan, InlineMathCos, InlineMathExp, InlineMathLog, InlineMathSin, InlineMathSqrt, InlineMathTan. Assert(this->DoFloatTypeSpec()); // Type-spec the src. src1Val = src1OriginalVal; bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, /* skipDst = */ true); AssertMsg(retVal, "For inline built-ins the args have to be type-specialized to float, but something failed during the process."); // Type-spec the dst. this->TypeSpecializeFloatDst(instr, nullptr, src1Val, nullptr, pDstVal); } else if (instr->m_opcode == Js::OpCode::InlineMathAbs) { // Consider the case when the value is unknown - because of bailout in abs we may disable type spec for the whole function which is too much. // First, try int. int minVal, maxVal; bool shouldTypeSpecToInt = src1Val->GetValueInfo()->GetIntValMinMax(&minVal, &maxVal, /* doAggressiveIntTypeSpec = */ true); if (shouldTypeSpecToInt) { Assert(this->DoAggressiveIntTypeSpec()); bool retVal = this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, minVal, maxVal, src1OriginalVal, redoTypeSpecRef, true); AssertMsg(retVal, "For inline built-ins the args have to be type-specialized (int), but something failed during the process."); if (!this->IsLoopPrePass()) { // Create bailout for INT_MIN which does not have corresponding int value on the positive side. // Check int range: if we know the range is out of overflow, we do not need the bail out at all. if (minVal == INT32_MIN) { GenerateBailAtOperation(&instr, IR::BailOnIntMin); } } // Account for ::abs(INT_MIN) == INT_MIN (which is less than 0). maxVal = ::max( ::abs(Int32Math::NearestInRangeTo(minVal, INT_MIN + 1, INT_MAX)), ::abs(Int32Math::NearestInRangeTo(maxVal, INT_MIN + 1, INT_MAX))); minVal = minVal >= 0 ? minVal : 0; this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, nullptr, IR::BailOutInvalid, minVal, maxVal, pDstVal); } else { // If we couldn't do int, do float. Assert(this->DoFloatTypeSpec()); src1Val = src1OriginalVal; bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, true); AssertMsg(retVal, "For inline built-ins the args have to be type-specialized (float), but something failed during the process."); this->TypeSpecializeFloatDst(instr, nullptr, src1Val, nullptr, pDstVal); } } else if (instr->m_opcode == Js::OpCode::InlineMathFloor || instr->m_opcode == Js::OpCode::InlineMathCeil || instr->m_opcode == Js::OpCode::InlineMathRound) { // Type specialize src to float src1Val = src1OriginalVal; bool retVal = this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal, /* skipDst = */ true); AssertMsg(retVal, "For inline Math.floor and Math.ceil the src has to be type-specialized to float, but something failed during the process."); // Type specialize dst to int this->TypeSpecializeIntDst( instr, instr->m_opcode, nullptr, src1Val, nullptr, IR::BailOutInvalid, INT32_MIN, INT32_MAX, pDstVal); } else if(instr->m_opcode == Js::OpCode::InlineArrayPop) { IR::Opnd *const thisOpnd = instr->GetSrc1(); Assert(thisOpnd); // Ensure src1 (Array) is a var this->ToVarUses(instr, thisOpnd, false, src1Val); if(!this->IsLoopPrePass() && thisOpnd->GetValueType().IsLikelyNativeArray()) { // We bail out, if there is illegal access or a mismatch in the Native array type that is optimized for, during the run time. GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly); } if(!instr->GetDst()) { return; } // Try Type Specializing the element (return item from Pop) based on the array's profile data. if(thisOpnd->GetValueType().IsLikelyNativeIntArray()) { this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, nullptr, nullptr, IR::BailOutInvalid, INT32_MIN, INT32_MAX, pDstVal); } else if(thisOpnd->GetValueType().IsLikelyNativeFloatArray()) { this->TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, pDstVal); } else { // We reached here so the Element is not yet type specialized. Ensure element is a var if(instr->GetDst()->IsRegOpnd()) { this->ToVarRegOpnd(instr->GetDst()->AsRegOpnd(), currentBlock); } } } else if (instr->m_opcode == Js::OpCode::InlineMathClz) { Assert(this->DoAggressiveIntTypeSpec()); Assert(this->DoLossyIntTypeSpec()); //Type specialize to int bool retVal = this->TypeSpecializeIntUnary(pInstr, &src1Val, pDstVal, INT32_MIN, INT32_MAX, src1OriginalVal, redoTypeSpecRef); AssertMsg(retVal, "For clz32, the arg has to be type-specialized to int."); } else { AssertMsg(FALSE, "Unsupported built-in!"); } } void GlobOpt::TypeSpecializeInlineBuiltInBinary(IR::Instr **pInstr, Value *src1Val, Value* src2Val, Value **pDstVal, Value *const src1OriginalVal, Value *const src2OriginalVal) { IR::Instr *&instr = *pInstr; Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode)); switch(instr->m_opcode) { case Js::OpCode::InlineMathAtan2: { Js::BuiltinFunction builtInId = Js::JavascriptLibrary::GetBuiltInInlineCandidateId(instr->m_opcode); // From actual instr, not profile based. Js::BuiltInFlags builtInFlags = Js::JavascriptLibrary::GetFlagsForBuiltIn(builtInId); bool areAllArgsAlwaysFloat = (builtInFlags & Js::BuiltInFlags::BIF_TypeSpecAllToFloat) != 0; Assert(areAllArgsAlwaysFloat); Assert(this->DoFloatTypeSpec()); // Type-spec the src1, src2 and dst. src1Val = src1OriginalVal; src2Val = src2OriginalVal; bool retVal = this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal); AssertMsg(retVal, "For pow and atnan2 the args have to be type-specialized to float, but something failed during the process."); break; } case Js::OpCode::InlineMathPow: { #ifndef _M_ARM32_OR_ARM64 if (src2Val->GetValueInfo()->IsLikelyInt()) { bool lossy = false; this->ToInt32(instr, instr->GetSrc2(), this->currentBlock, src2Val, nullptr, lossy); IR::Opnd* src1 = instr->GetSrc1(); int32 valueMin, valueMax; if (src1Val->GetValueInfo()->IsLikelyInt() && this->DoPowIntIntTypeSpec() && src2Val->GetValueInfo()->GetIntValMinMax(&valueMin, &valueMax, this->DoAggressiveIntTypeSpec()) && valueMin >= 0) { this->ToInt32(instr, src1, this->currentBlock, src1Val, nullptr, lossy); this->TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, src2Val, IR::BailOutInvalid, INT32_MIN, INT32_MAX, pDstVal); if(!this->IsLoopPrePass()) { GenerateBailAtOperation(&instr, IR::BailOutOnPowIntIntOverflow); } } else { this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, IR::BailOutPrimitiveButString); TypeSpecializeFloatDst(instr, nullptr, src1Val, src2Val, pDstVal); } } else { #endif this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal); #ifndef _M_ARM32_OR_ARM64 } #endif break; } case Js::OpCode::InlineMathImul: { Assert(this->DoAggressiveIntTypeSpec()); Assert(this->DoLossyIntTypeSpec()); //Type specialize to int bool retVal = this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, INT32_MIN, INT32_MAX, false /* skipDst */); AssertMsg(retVal, "For imul, the args have to be type-specialized to int but something failed during the process."); break; } case Js::OpCode::InlineMathMin: case Js::OpCode::InlineMathMax: { if(src1Val->GetValueInfo()->IsLikelyInt() && src2Val->GetValueInfo()->IsLikelyInt()) { // Compute resulting range info int32 min1 = INT32_MIN; int32 max1 = INT32_MAX; int32 min2 = INT32_MIN; int32 max2 = INT32_MAX; int32 newMin, newMax; Assert(this->DoAggressiveIntTypeSpec()); src1Val->GetValueInfo()->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec()); src2Val->GetValueInfo()->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec()); if (instr->m_opcode == Js::OpCode::InlineMathMin) { newMin = min(min1, min2); newMax = min(max1, max2); } else { Assert(instr->m_opcode == Js::OpCode::InlineMathMax); newMin = max(min1, min2); newMax = max(max1, max2); } // Type specialize to int bool retVal = this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, newMin, newMax, false /* skipDst */); AssertMsg(retVal, "For min and max, the args have to be type-specialized to int if any one of the sources is an int, but something failed during the process."); } // Couldn't type specialize to int, type specialize to float else { Assert(this->DoFloatTypeSpec()); src1Val = src1OriginalVal; src2Val = src2OriginalVal; bool retVal = this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal); AssertMsg(retVal, "For min and max, the args have to be type-specialized to float if any one of the sources is a float, but something failed during the process."); } break; } case Js::OpCode::InlineArrayPush: { IR::Opnd *const thisOpnd = instr->GetSrc1(); Assert(thisOpnd); if(instr->GetDst() && instr->GetDst()->IsRegOpnd()) { // Set the dst as live here, as the built-ins return early from the TypeSpecialization functions - before the dst is marked as live. // Also, we are not specializing the dst separately and we are skipping the dst to be handled when we specialize the instruction above. this->ToVarRegOpnd(instr->GetDst()->AsRegOpnd(), currentBlock); } // Ensure src1 (Array) is a var this->ToVarUses(instr, thisOpnd, false, src1Val); if(!this->IsLoopPrePass()) { if(thisOpnd->GetValueType().IsLikelyNativeArray()) { // We bail out, if there is illegal access or a mismatch in the Native array type that is optimized for, during run time. GenerateBailAtOperation(&instr, IR::BailOutConventionalNativeArrayAccessOnly); } else { GenerateBailAtOperation(&instr, IR::BailOutOnImplicitCallsPreOp); } } // Try Type Specializing the element based on the array's profile data. if(thisOpnd->GetValueType().IsLikelyNativeFloatArray()) { src1Val = src1OriginalVal; src2Val = src2OriginalVal; } if((thisOpnd->GetValueType().IsLikelyNativeIntArray() && this->TypeSpecializeIntBinary(pInstr, src1Val, src2Val, pDstVal, INT32_MIN, INT32_MAX, true)) || (thisOpnd->GetValueType().IsLikelyNativeFloatArray() && this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal))) { break; } // The Element is not yet type specialized. Ensure element is a var this->ToVarUses(instr, instr->GetSrc2(), false, src2Val); break; } } } void GlobOpt::TypeSpecializeInlineBuiltInDst(IR::Instr **pInstr, Value **pDstVal) { IR::Instr *&instr = *pInstr; Assert(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode)); if (instr->m_opcode == Js::OpCode::InlineMathRandom) { Assert(this->DoFloatTypeSpec()); // Type specialize dst to float this->TypeSpecializeFloatDst(instr, nullptr, nullptr, nullptr, pDstVal); } } bool GlobOpt::TryTypeSpecializeUnaryToFloatHelper(IR::Instr** pInstr, Value** pSrc1Val, Value* const src1OriginalVal, Value **pDstVal) { // It has been determined that this instruction cannot be int-specialized. We need to determine whether to attempt to // float-specialize the instruction, or leave it unspecialized. #if !INT32VAR Value*& src1Val = *pSrc1Val; if(src1Val->GetValueInfo()->IsLikelyUntaggedInt()) { // An input range is completely outside the range of an int31. Even if the operation may overflow, it is // unlikely to overflow on these operations, so we leave it unspecialized on 64-bit platforms. However, on // 32-bit platforms, the value is untaggable and will be a JavascriptNumber, which is significantly slower to // use in an unspecialized operation compared to a tagged int. So, try to float-specialize the instruction. src1Val = src1OriginalVal; return this->TypeSpecializeFloatUnary(pInstr, src1Val, pDstVal); } #endif return false; } bool GlobOpt::TypeSpecializeIntBinary(IR::Instr **pInstr, Value *src1Val, Value *src2Val, Value **pDstVal, int32 min, int32 max, bool skipDst /* = false */) { // Consider moving the code for int type spec-ing binary functions here. IR::Instr *&instr = *pInstr; bool lossy = false; if(OpCodeAttr::IsInlineBuiltIn(instr->m_opcode)) { if(instr->m_opcode == Js::OpCode::InlineArrayPush) { int32 intConstantValue; bool isIntConstMissingItem = src2Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue); if(isIntConstMissingItem) { isIntConstMissingItem = Js::SparseArraySegment::IsMissingItem(&intConstantValue); } // Don't specialize if the element is not likelyInt or an IntConst which is a missing item value. if(!(src2Val->GetValueInfo()->IsLikelyInt()) || isIntConstMissingItem) { return false; } // We don't want to specialize both the source operands, though it is a binary instr. IR::Opnd * elementOpnd = instr->GetSrc2(); this->ToInt32(instr, elementOpnd, this->currentBlock, src2Val, nullptr, lossy); } else { IR::Opnd *src1 = instr->GetSrc1(); this->ToInt32(instr, src1, this->currentBlock, src1Val, nullptr, lossy); IR::Opnd *src2 = instr->GetSrc2(); this->ToInt32(instr, src2, this->currentBlock, src2Val, nullptr, lossy); } if(!skipDst) { IR::Opnd *dst = instr->GetDst(); if (dst) { TypeSpecializeIntDst(instr, instr->m_opcode, nullptr, src1Val, src2Val, IR::BailOutInvalid, min, max, pDstVal); } } return true; } else { AssertMsg(false, "Yet to move code for other binary functions here"); return false; } } bool GlobOpt::TypeSpecializeIntUnary( IR::Instr **pInstr, Value **pSrc1Val, Value **pDstVal, int32 min, int32 max, Value *const src1OriginalVal, bool *redoTypeSpecRef, bool skipDst /* = false */) { IR::Instr *&instr = *pInstr; Assert(pSrc1Val); Value *&src1Val = *pSrc1Val; bool isTransfer = false; Js::OpCode opcode; int32 newMin, newMax; bool lossy = false; IR::BailOutKind bailOutKind = IR::BailOutInvalid; bool ignoredIntOverflow = this->ignoredIntOverflowForCurrentInstr; bool ignoredNegativeZero = false; bool checkTypeSpecWorth = false; if(instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber) { return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal); } AddSubConstantInfo addSubConstantInfo; switch(instr->m_opcode) { case Js::OpCode::Ld_A: if (instr->GetSrc1()->IsRegOpnd()) { StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym; if (CurrentBlockData()->IsInt32TypeSpecialized(sym) == false) { // Type specializing an Ld_A isn't worth it, unless the src // is already type specialized. return false; } } newMin = min; newMax = max; opcode = Js::OpCode::Ld_I4; isTransfer = true; break; case Js::OpCode::Conv_Num: newMin = min; newMax = max; opcode = Js::OpCode::Ld_I4; isTransfer = true; break; case Js::OpCode::LdC_A_I4: newMin = newMax = instr->GetSrc1()->AsIntConstOpnd()->AsInt32(); opcode = Js::OpCode::Ld_I4; break; case Js::OpCode::Neg_A: if (min <= 0 && max >= 0) { if(instr->ShouldCheckForNegativeZero()) { // -0 matters since the sym is not a local, or is used in a way in which -0 would differ from +0 if(!DoAggressiveIntTypeSpec()) { // May result in -0 // Consider adding a dynamic check for src1 == 0 return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal); } if(min == 0 && max == 0) { // Always results in -0 return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal); } bailOutKind |= IR::BailOutOnNegativeZero; } else { ignoredNegativeZero = true; } } if (Int32Math::Neg(min, &newMax)) { if(instr->ShouldCheckForIntOverflow()) { if(!DoAggressiveIntTypeSpec()) { // May overflow return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal); } if(min == max) { // Always overflows return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal); } bailOutKind |= IR::BailOutOnOverflow; newMax = INT32_MAX; } else { ignoredIntOverflow = true; } } if (Int32Math::Neg(max, &newMin)) { if(instr->ShouldCheckForIntOverflow()) { if(!DoAggressiveIntTypeSpec()) { // May overflow return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal); } bailOutKind |= IR::BailOutOnOverflow; newMin = INT32_MAX; } else { ignoredIntOverflow = true; } } if(!instr->ShouldCheckForIntOverflow() && newMin > newMax) { // When ignoring overflow, the range needs to account for overflow. Since MIN_INT is the only int32 value that // overflows on Neg, and the value resulting from overflow is also MIN_INT, if calculating only the new min or new // max overflowed but not both, then the new min will be greater than the new max. In that case we need to consider // the full range of int32s as possible resulting values. newMin = INT32_MIN; newMax = INT32_MAX; } opcode = Js::OpCode::Neg_I4; checkTypeSpecWorth = true; break; case Js::OpCode::Not_A: if(!DoLossyIntTypeSpec()) { return false; } this->PropagateIntRangeForNot(min, max, &newMin, &newMax); opcode = Js::OpCode::Not_I4; lossy = true; break; case Js::OpCode::Incr_A: do // while(false) { const auto CannotOverflowBasedOnRelativeBounds = [&]() { const ValueInfo *const src1ValueInfo = src1Val->GetValueInfo(); return (src1ValueInfo->IsInt() || DoAggressiveIntTypeSpec()) && src1ValueInfo->IsIntBounded() && src1ValueInfo->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(1); }; if (Int32Math::Inc(min, &newMin)) { if(CannotOverflowBasedOnRelativeBounds()) { newMin = INT32_MAX; } else if(instr->ShouldCheckForIntOverflow()) { // Always overflows return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal); } else { // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints, // we use the full range of int32s. ignoredIntOverflow = true; newMin = INT32_MIN; newMax = INT32_MAX; break; } } if (Int32Math::Inc(max, &newMax)) { if(CannotOverflowBasedOnRelativeBounds()) { newMax = INT32_MAX; } else if(instr->ShouldCheckForIntOverflow()) { if(!DoAggressiveIntTypeSpec()) { // May overflow return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal); } bailOutKind |= IR::BailOutOnOverflow; newMax = INT32_MAX; } else { // See comment about ignoring overflow above ignoredIntOverflow = true; newMin = INT32_MIN; newMax = INT32_MAX; break; } } } while(false); if(!ignoredIntOverflow && instr->GetSrc1()->IsRegOpnd()) { addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min == max, 1); } opcode = Js::OpCode::Add_I4; if (!this->IsLoopPrePass()) { instr->SetSrc2(IR::IntConstOpnd::New(1, TyInt32, instr->m_func)); } checkTypeSpecWorth = true; break; case Js::OpCode::Decr_A: do // while(false) { const auto CannotOverflowBasedOnRelativeBounds = [&]() { const ValueInfo *const src1ValueInfo = src1Val->GetValueInfo(); return (src1ValueInfo->IsInt() || DoAggressiveIntTypeSpec()) && src1ValueInfo->IsIntBounded() && src1ValueInfo->AsIntBounded()->Bounds()->SubCannotOverflowBasedOnRelativeBounds(1); }; if (Int32Math::Dec(max, &newMax)) { if(CannotOverflowBasedOnRelativeBounds()) { newMax = INT32_MIN; } else if(instr->ShouldCheckForIntOverflow()) { // Always overflows return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal); } else { // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints, we // use the full range of int32s. ignoredIntOverflow = true; newMin = INT32_MIN; newMax = INT32_MAX; break; } } if (Int32Math::Dec(min, &newMin)) { if(CannotOverflowBasedOnRelativeBounds()) { newMin = INT32_MIN; } else if(instr->ShouldCheckForIntOverflow()) { if(!DoAggressiveIntTypeSpec()) { // May overflow return TryTypeSpecializeUnaryToFloatHelper(pInstr, &src1Val, src1OriginalVal, pDstVal); } bailOutKind |= IR::BailOutOnOverflow; newMin = INT32_MIN; } else { // See comment about ignoring overflow above ignoredIntOverflow = true; newMin = INT32_MIN; newMax = INT32_MAX; break; } } } while(false); if(!ignoredIntOverflow && instr->GetSrc1()->IsRegOpnd()) { addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min == max, -1); } opcode = Js::OpCode::Sub_I4; if (!this->IsLoopPrePass()) { instr->SetSrc2(IR::IntConstOpnd::New(1, TyInt32, instr->m_func)); } checkTypeSpecWorth = true; break; case Js::OpCode::BrFalse_A: case Js::OpCode::BrTrue_A: { if(DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrFalse(instr, src1Val, min, max)) { return true; } bool specialize = true; if (!src1Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc1()->IsRegOpnd()) { StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym; if (CurrentBlockData()->IsInt32TypeSpecialized(sym) == false) { // Type specializing a BrTrue_A/BrFalse_A isn't worth it, unless the src // is already type specialized specialize = false; } } if(instr->m_opcode == Js::OpCode::BrTrue_A) { UpdateIntBoundsForNotEqualBranch(src1Val, nullptr, 0); opcode = Js::OpCode::BrTrue_I4; } else { UpdateIntBoundsForEqualBranch(src1Val, nullptr, 0); opcode = Js::OpCode::BrFalse_I4; } if(!specialize) { return false; } newMin = 2; newMax = 1; // We'll assert if we make a range where min > max break; } case Js::OpCode::MultiBr: newMin = min; newMax = max; opcode = instr->m_opcode; break; case Js::OpCode::StElemI_A: case Js::OpCode::StElemI_A_Strict: case Js::OpCode::StElemC: if(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsLikelyAnyArrayWithNativeFloatValues()) { src1Val = src1OriginalVal; } return TypeSpecializeStElem(pInstr, src1Val, pDstVal); case Js::OpCode::NewScArray: case Js::OpCode::NewScArrayWithMissingValues: case Js::OpCode::InitFld: case Js::OpCode::InitRootFld: case Js::OpCode::StSlot: case Js::OpCode::StSlotChkUndecl: #if !FLOATVAR case Js::OpCode::StSlotBoxTemp: #endif case Js::OpCode::StFld: case Js::OpCode::StRootFld: case Js::OpCode::StFldStrict: case Js::OpCode::StRootFldStrict: case Js::OpCode::ArgOut_A: case Js::OpCode::ArgOut_A_Inline: case Js::OpCode::ArgOut_A_FixupForStackArgs: case Js::OpCode::ArgOut_A_Dynamic: case Js::OpCode::ArgOut_A_FromStackArgs: case Js::OpCode::ArgOut_A_SpreadArg: // For this one we need to implement type specialization //case Js::OpCode::ArgOut_A_InlineBuiltIn: case Js::OpCode::Ret: case Js::OpCode::LdElemUndef: case Js::OpCode::LdElemUndefScoped: return false; default: if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode)) { newMin = min; newMax = max; opcode = instr->m_opcode; break; // Note: we must keep checkTypeSpecWorth = false to make sure we never return false from this function. } return false; } // If this instruction is in a range of instructions where int overflow does not matter, we will still specialize it (won't // leave it unspecialized based on heuristics), since it is most likely worth specializing, and the dst value needs to be // guaranteed to be an int if(checkTypeSpecWorth && !ignoredIntOverflow && !ignoredNegativeZero && instr->ShouldCheckForIntOverflow() && !IsWorthSpecializingToInt32(instr, src1Val)) { // Even though type specialization is being skipped since it may not be worth it, the proper value should still be // maintained so that the result may be type specialized later. An int value is not created for the dst in any of // the following cases. // - A bailout check is necessary to specialize this instruction. The bailout check is what guarantees the result to be // an int, but since we're not going to specialize this instruction, there won't be a bailout check. // - Aggressive int type specialization is disabled and we're in a loop prepass. We're conservative on dst values in // that case, especially if the dst sym is live on the back-edge. if(bailOutKind == IR::BailOutInvalid && instr->GetDst() && (DoAggressiveIntTypeSpec() || !this->IsLoopPrePass())) { *pDstVal = CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, nullptr); } if(instr->GetSrc2()) { instr->FreeSrc2(); } return false; } this->ignoredIntOverflowForCurrentInstr = ignoredIntOverflow; this->ignoredNegativeZeroForCurrentInstr = ignoredNegativeZero; { // Try CSE again before modifying the IR, in case some attributes are required for successful CSE Value *src1IndirIndexVal = nullptr; Value *src2Val = nullptr; if(CSEOptimize(currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal, true /* intMathExprOnly */)) { *redoTypeSpecRef = true; return false; } } const Js::OpCode originalOpCode = instr->m_opcode; if (!this->IsLoopPrePass()) { // No re-write on prepass instr->m_opcode = opcode; } Value *src1ValueToSpecialize = src1Val; if(lossy) { // Lossy conversions to int32 must be done based on the original source values. For instance, if one of the values is a // float constant with a value that fits in a uint32 but not an int32, and the instruction can ignore int overflow, the // source value for the purposes of int specialization would have been changed to an int constant value by ignoring // overflow. If we were to specialize the sym using the int constant value, it would be treated as a lossless // conversion, but since there may be subsequent uses of the same float constant value that may not ignore overflow, // this must be treated as a lossy conversion by specializing the sym using the original float constant value. src1ValueToSpecialize = src1OriginalVal; } // Make sure the srcs are specialized IR::Opnd *src1 = instr->GetSrc1(); this->ToInt32(instr, src1, this->currentBlock, src1ValueToSpecialize, nullptr, lossy); if(bailOutKind != IR::BailOutInvalid && !this->IsLoopPrePass()) { GenerateBailAtOperation(&instr, bailOutKind); } if (!skipDst) { IR::Opnd *dst = instr->GetDst(); if (dst) { AssertMsg(!(isTransfer && !this->IsLoopPrePass()) || min == newMin && max == newMax, "If this is just a copy, old/new min/max should be the same"); TypeSpecializeIntDst( instr, originalOpCode, isTransfer ? src1Val : nullptr, src1Val, nullptr, bailOutKind, newMin, newMax, pDstVal, addSubConstantInfo.HasInfo() ? &addSubConstantInfo : nullptr); } } if(bailOutKind == IR::BailOutInvalid) { GOPT_TRACE(_u("Type specialized to INT\n")); #if ENABLE_DEBUG_CONFIG_OPTIONS if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase)) { Output::Print(_u("Type specialized to INT: ")); Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode)); } #endif } else { GOPT_TRACE(_u("Type specialized to INT with bailout on:\n")); if(bailOutKind & IR::BailOutOnOverflow) { GOPT_TRACE(_u(" Overflow\n")); #if ENABLE_DEBUG_CONFIG_OPTIONS if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase)) { Output::Print(_u("Type specialized to INT with bailout (%S): "), "Overflow"); Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode)); } #endif } if(bailOutKind & IR::BailOutOnNegativeZero) { GOPT_TRACE(_u(" Zero\n")); #if ENABLE_DEBUG_CONFIG_OPTIONS if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase)) { Output::Print(_u("Type specialized to INT with bailout (%S): "), "Zero"); Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode)); } #endif } } return true; } void GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, int32 newMin, int32 newMax, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo) { this->TypeSpecializeIntDst(instr, originalOpCode, valToTransfer, src1Value, src2Value, bailOutKind, ValueType::GetInt(IntConstantBounds(newMin, newMax).IsLikelyTaggable()), newMin, newMax, pDstVal, addSubConstantInfo); } void GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, ValueType valueType, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo) { this->TypeSpecializeIntDst(instr, originalOpCode, valToTransfer, src1Value, src2Value, bailOutKind, valueType, 0, 0, pDstVal, addSubConstantInfo); } void GlobOpt::TypeSpecializeIntDst(IR::Instr* instr, Js::OpCode originalOpCode, Value* valToTransfer, Value *const src1Value, Value *const src2Value, const IR::BailOutKind bailOutKind, ValueType valueType, int32 newMin, int32 newMax, Value** pDstVal, const AddSubConstantInfo *const addSubConstantInfo) { Assert(valueType.IsInt() || (valueType.IsNumber() && valueType.IsLikelyInt() && newMin == 0 && newMax == 0)); Assert(!valToTransfer || valToTransfer == src1Value); Assert(!addSubConstantInfo || addSubConstantInfo->HasInfo()); IR::Opnd *dst = instr->GetDst(); Assert(dst); bool isValueInfoPrecise; if(IsLoopPrePass()) { isValueInfoPrecise = IsPrepassSrcValueInfoPrecise(instr, src1Value, src2Value); valueType = GetPrepassValueTypeForDst(valueType, instr, src1Value, src2Value, isValueInfoPrecise); } else { isValueInfoPrecise = true; } // If dst has a circular reference in a loop, it probably won't get specialized. Don't mark the dst as type-specialized on // the pre-pass. With aggressive int spec though, it will take care of bailing out if necessary so there's no need to assume // that the dst will be a var even if it's live on the back-edge. Also if the op always produces an int32, then there's no // ambiguity in the dst's value type even in the prepass. if (!DoAggressiveIntTypeSpec() && this->IsLoopPrePass() && !valueType.IsInt()) { if (dst->IsRegOpnd()) { this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock); } return; } const IntBounds *dstBounds = nullptr; if(addSubConstantInfo && !addSubConstantInfo->SrcValueIsLikelyConstant() && DoTrackRelativeIntBounds()) { Assert(!ignoredIntOverflowForCurrentInstr); // Track bounds for add or sub with a constant. For instance, consider (b = a + 2). The value of 'b' should track that // it is equal to (the value of 'a') + 2. Additionally, the value of 'b' should inherit the bounds of 'a', offset by // the constant value. if(!valueType.IsInt() || !isValueInfoPrecise) { newMin = INT32_MIN; newMax = INT32_MAX; } dstBounds = IntBounds::Add( addSubConstantInfo->SrcValue(), addSubConstantInfo->Offset(), isValueInfoPrecise, IntConstantBounds(newMin, newMax), alloc); } // Src1's value could change later in the loop, so the value wouldn't be the same for each // iteration. Since we don't iterate over loops "while (!changed)", go conservative on the // pre-pass. if (valToTransfer) { // If this is just a copy, no need for creating a new value. Assert(!addSubConstantInfo); *pDstVal = this->ValueNumberTransferDst(instr, valToTransfer); CurrentBlockData()->InsertNewValue(*pDstVal, dst); } else if (valueType.IsInt() && isValueInfoPrecise) { bool wasNegativeZeroPreventedByBailout = false; if(newMin <= 0 && newMax >= 0) { switch(originalOpCode) { case Js::OpCode::Add_A: // -0 + -0 == -0 Assert(src1Value); Assert(src2Value); wasNegativeZeroPreventedByBailout = src1Value->GetValueInfo()->WasNegativeZeroPreventedByBailout() && src2Value->GetValueInfo()->WasNegativeZeroPreventedByBailout(); break; case Js::OpCode::Sub_A: // -0 - 0 == -0 Assert(src1Value); wasNegativeZeroPreventedByBailout = src1Value->GetValueInfo()->WasNegativeZeroPreventedByBailout(); break; case Js::OpCode::Neg_A: case Js::OpCode::Mul_A: case Js::OpCode::Div_A: case Js::OpCode::Rem_A: wasNegativeZeroPreventedByBailout = !!(bailOutKind & IR::BailOutOnNegativeZero); break; } } *pDstVal = dstBounds ? NewIntBoundedValue(valueType, dstBounds, wasNegativeZeroPreventedByBailout, nullptr) : NewIntRangeValue(newMin, newMax, wasNegativeZeroPreventedByBailout, nullptr); } else { *pDstVal = dstBounds ? NewIntBoundedValue(valueType, dstBounds, false, nullptr) : NewGenericValue(valueType); } if(addSubConstantInfo || updateInductionVariableValueNumber) { TrackIntSpecializedAddSubConstant(instr, addSubConstantInfo, *pDstVal, !!dstBounds); } CurrentBlockData()->SetValue(*pDstVal, dst); AssertMsg(dst->IsRegOpnd(), "What else?"); this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock); } bool GlobOpt::TypeSpecializeBinary(IR::Instr **pInstr, Value **pSrc1Val, Value **pSrc2Val, Value **pDstVal, Value *const src1OriginalVal, Value *const src2OriginalVal, bool *redoTypeSpecRef) { IR::Instr *&instr = *pInstr; int32 min1 = INT32_MIN, max1 = INT32_MAX, min2 = INT32_MIN, max2 = INT32_MAX, newMin, newMax, tmp; Js::OpCode opcode; Value *&src1Val = *pSrc1Val; Value *&src2Val = *pSrc2Val; // We don't need to do typespec for asmjs if (IsTypeSpecPhaseOff(this->func) || GetIsAsmJSFunc()) { return false; } if (OpCodeAttr::IsInlineBuiltIn(instr->m_opcode)) { this->TypeSpecializeInlineBuiltInBinary(pInstr, src1Val, src2Val, pDstVal, src1OriginalVal, src2OriginalVal); return true; } if (src1Val) { src1Val->GetValueInfo()->GetIntValMinMax(&min1, &max1, this->DoAggressiveIntTypeSpec()); } if (src2Val) { src2Val->GetValueInfo()->GetIntValMinMax(&min2, &max2, this->DoAggressiveIntTypeSpec()); } // Type specialize binary operators to int32 bool src1Lossy = true; bool src2Lossy = true; IR::BailOutKind bailOutKind = IR::BailOutInvalid; bool ignoredIntOverflow = this->ignoredIntOverflowForCurrentInstr; bool ignoredNegativeZero = false; bool skipSrc2 = false; bool skipDst = false; bool needsBoolConv = false; AddSubConstantInfo addSubConstantInfo; switch (instr->m_opcode) { case Js::OpCode::Or_A: if (!DoLossyIntTypeSpec()) { return false; } this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax); opcode = Js::OpCode::Or_I4; break; case Js::OpCode::And_A: if (!DoLossyIntTypeSpec()) { return false; } this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax); opcode = Js::OpCode::And_I4; break; case Js::OpCode::Xor_A: if (!DoLossyIntTypeSpec()) { return false; } this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax); opcode = Js::OpCode::Xor_I4; break; case Js::OpCode::Shl_A: if (!DoLossyIntTypeSpec()) { return false; } this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax); opcode = Js::OpCode::Shl_I4; break; case Js::OpCode::Shr_A: if (!DoLossyIntTypeSpec()) { return false; } this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax); opcode = Js::OpCode::Shr_I4; break; case Js::OpCode::ShrU_A: if (!DoLossyIntTypeSpec()) { return false; } if (min1 < 0 && IntConstantBounds(min2, max2).And_0x1f().Contains(0)) { // Src1 may be too large to represent as a signed int32, and src2 may be zero. Unless the resulting value is only // used as a signed int32 (hence allowing us to ignore the result's sign), don't specialize the instruction. if (!instr->ignoreIntOverflow) return false; ignoredIntOverflow = true; } this->PropagateIntRangeBinary(instr, min1, max1, min2, max2, &newMin, &newMax); opcode = Js::OpCode::ShrU_I4; break; case Js::OpCode::BrUnLe_A: // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely // int already, so require that both sources are likely int for folding. if (DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrUnsignedGreaterThan(instr, false, src1Val, min1, max1, src2Val, min2, max2)) { return true; } if (min1 >= 0 && min2 >= 0) { // Only handle positive values since this is unsigned... // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults // (INT32_MIN, INT32_MAX), so we're good. Assert(src1Val); Assert(src1Val->GetValueInfo()->IsLikelyInt()); Assert(src2Val); Assert(src2Val->GetValueInfo()->IsLikelyInt()); UpdateIntBoundsForLessThanOrEqualBranch(src1Val, src2Val); } if (!DoLossyIntTypeSpec()) { return false; } newMin = newMax = 0; opcode = Js::OpCode::BrUnLe_I4; break; case Js::OpCode::BrUnLt_A: // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely // int already, so require that both sources are likely int for folding. if (DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrUnsignedLessThan(instr, true, src1Val, min1, max1, src2Val, min2, max2)) { return true; } if (min1 >= 0 && min2 >= 0) { // Only handle positive values since this is unsigned... // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults // (INT32_MIN, INT32_MAX), so we're good. Assert(src1Val); Assert(src1Val->GetValueInfo()->IsLikelyInt()); Assert(src2Val); Assert(src2Val->GetValueInfo()->IsLikelyInt()); UpdateIntBoundsForLessThanBranch(src1Val, src2Val); } if (!DoLossyIntTypeSpec()) { return false; } newMin = newMax = 0; opcode = Js::OpCode::BrUnLt_I4; break; case Js::OpCode::BrUnGe_A: // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely // int already, so require that both sources are likely int for folding. if (DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrUnsignedLessThan(instr, false, src1Val, min1, max1, src2Val, min2, max2)) { return true; } if (min1 >= 0 && min2 >= 0) { // Only handle positive values since this is unsigned... // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults // (INT32_MIN, INT32_MAX), so we're good. Assert(src1Val); Assert(src1Val->GetValueInfo()->IsLikelyInt()); Assert(src2Val); Assert(src2Val->GetValueInfo()->IsLikelyInt()); UpdateIntBoundsForGreaterThanOrEqualBranch(src1Val, src2Val); } if (!DoLossyIntTypeSpec()) { return false; } newMin = newMax = 0; opcode = Js::OpCode::BrUnGe_I4; break; case Js::OpCode::BrUnGt_A: // Folding the branch based on bounds will attempt a lossless int32 conversion of the sources if they are not definitely // int already, so require that both sources are likely int for folding. if (DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrUnsignedGreaterThan(instr, true, src1Val, min1, max1, src2Val, min2, max2)) { return true; } if (min1 >= 0 && min2 >= 0) { // Only handle positive values since this is unsigned... // Bounds are tracked only for likely int values. Only likely int values may have bounds that are not the defaults // (INT32_MIN, INT32_MAX), so we're good. Assert(src1Val); Assert(src1Val->GetValueInfo()->IsLikelyInt()); Assert(src2Val); Assert(src2Val->GetValueInfo()->IsLikelyInt()); UpdateIntBoundsForGreaterThanBranch(src1Val, src2Val); } if (!DoLossyIntTypeSpec()) { return false; } newMin = newMax = 0; opcode = Js::OpCode::BrUnGt_I4; break; case Js::OpCode::CmUnLe_A: if (!DoLossyIntTypeSpec()) { return false; } newMin = 0; newMax = 1; opcode = Js::OpCode::CmUnLe_I4; needsBoolConv = true; break; case Js::OpCode::CmUnLt_A: if (!DoLossyIntTypeSpec()) { return false; } newMin = 0; newMax = 1; opcode = Js::OpCode::CmUnLt_I4; needsBoolConv = true; break; case Js::OpCode::CmUnGe_A: if (!DoLossyIntTypeSpec()) { return false; } newMin = 0; newMax = 1; opcode = Js::OpCode::CmUnGe_I4; needsBoolConv = true; break; case Js::OpCode::CmUnGt_A: if (!DoLossyIntTypeSpec()) { return false; } newMin = 0; newMax = 1; opcode = Js::OpCode::CmUnGt_I4; needsBoolConv = true; break; case Js::OpCode::Expo_A: { src1Val = src1OriginalVal; src2Val = src2OriginalVal; return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal); } case Js::OpCode::Div_A: { ValueType specializedValueType = GetDivValueType(instr, src1Val, src2Val, true); if (specializedValueType.IsFloat()) { // Either result is float or 1/x or cst1/cst2 where cst1%cst2 != 0 // Note: We should really constant fold cst1%cst2... src1Val = src1OriginalVal; src2Val = src2OriginalVal; return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal); } #ifdef _M_ARM if (!AutoSystemInfo::Data.ArmDivAvailable()) { return false; } #endif if (specializedValueType.IsInt()) { if (max2 == 0x80000000 || (min2 == 0 && max2 == 00)) { return false; } if (min1 == 0x80000000 && min2 <= -1 && max2 >= -1) { // Prevent integer overflow, as div by zero or MIN_INT / -1 will throw an exception // Or we know we are dividing by zero (which is weird to have because the profile data // say we got an int) bailOutKind = IR::BailOutOnDivOfMinInt; } src1Lossy = false; // Detect -0 on the sources src2Lossy = false; opcode = Js::OpCode::Div_I4; Assert(!instr->GetSrc1()->IsUnsigned()); bailOutKind |= IR::BailOnDivResultNotInt; if (max2 >= 0 && min2 <= 0) { // Need to check for divide by zero if the denominator range includes 0 bailOutKind |= IR::BailOutOnDivByZero; } if (max1 >= 0 && min1 <= 0) { // Numerator contains 0 so the result contains 0 newMin = 0; newMax = 0; if (min2 < 0) { // Denominator may be negative, so the result could be negative 0 if (instr->ShouldCheckForNegativeZero()) { bailOutKind |= IR::BailOutOnNegativeZero; } else { ignoredNegativeZero = true; } } } else { // Initialize to invalid value, one of the condition below will update it correctly newMin = INT_MAX; newMax = INT_MIN; } // Deal with the positive and negative range separately for both the numerator and the denominator, // and integrate to the overall min and max. // If the result is positive (positive/positive or negative/negative): // The min should be the smallest magnitude numerator (positive_Min1 | negative_Max1) // divided by --------------------------------------------------------------- // largest magnitude denominator (positive_Max2 | negative_Min2) // // The max should be the largest magnitude numerator (positive_Max1 | negative_Max1) // divided by --------------------------------------------------------------- // smallest magnitude denominator (positive_Min2 | negative_Max2) // If the result is negative (positive/negative or positive/negative): // The min should be the largest magnitude numerator (positive_Max1 | negative_Min1) // divided by --------------------------------------------------------------- // smallest magnitude denominator (negative_Max2 | positive_Min2) // // The max should be the smallest magnitude numerator (positive_Min1 | negative_Max1) // divided by --------------------------------------------------------------- // largest magnitude denominator (negative_Min2 | positive_Max2) // Consider: The range can be slightly more precise if we take care of the rounding if (max1 > 0) { // Take only the positive numerator range int32 positive_Min1 = max(1, min1); int32 positive_Max1 = max1; if (max2 > 0) { // Take only the positive denominator range int32 positive_Min2 = max(1, min2); int32 positive_Max2 = max2; // Positive / Positive int32 quadrant1_Min = positive_Min1 <= positive_Max2? 1 : positive_Min1 / positive_Max2; int32 quadrant1_Max = positive_Max1 <= positive_Min2? 1 : positive_Max1 / positive_Min2; Assert(1 <= quadrant1_Min && quadrant1_Min <= quadrant1_Max); // The result should positive newMin = min(newMin, quadrant1_Min); newMax = max(newMax, quadrant1_Max); } if (min2 < 0) { // Take only the negative denominator range int32 negative_Min2 = min2; int32 negative_Max2 = min(-1, max2); // Positive / Negative int32 quadrant2_Min = -positive_Max1 >= negative_Max2? -1 : positive_Max1 / negative_Max2; int32 quadrant2_Max = -positive_Min1 >= negative_Min2? -1 : positive_Min1 / negative_Min2; // The result should negative Assert(quadrant2_Min <= quadrant2_Max && quadrant2_Max <= -1); newMin = min(newMin, quadrant2_Min); newMax = max(newMax, quadrant2_Max); } } if (min1 < 0) { // Take only the native numerator range int32 negative_Min1 = min1; int32 negative_Max1 = min(-1, max1); if (max2 > 0) { // Take only the positive denominator range int32 positive_Min2 = max(1, min2); int32 positive_Max2 = max2; // Negative / Positive int32 quadrant4_Min = negative_Min1 >= -positive_Min2? -1 : negative_Min1 / positive_Min2; int32 quadrant4_Max = negative_Max1 >= -positive_Max2? -1 : negative_Max1 / positive_Max2; // The result should negative Assert(quadrant4_Min <= quadrant4_Max && quadrant4_Max <= -1); newMin = min(newMin, quadrant4_Min); newMax = max(newMax, quadrant4_Max); } if (min2 < 0) { // Take only the negative denominator range int32 negative_Min2 = min2; int32 negative_Max2 = min(-1, max2); int32 quadrant3_Min; int32 quadrant3_Max; // Negative / Negative if (negative_Max1 == 0x80000000 && negative_Min2 == -1) { quadrant3_Min = negative_Max1 >= negative_Min2? 1 : (negative_Max1+1) / negative_Min2; } else { quadrant3_Min = negative_Max1 >= negative_Min2? 1 : negative_Max1 / negative_Min2; } if (negative_Min1 == 0x80000000 && negative_Max2 == -1) { quadrant3_Max = negative_Min1 >= negative_Max2? 1 : (negative_Min1+1) / negative_Max2; } else { quadrant3_Max = negative_Min1 >= negative_Max2? 1 : negative_Min1 / negative_Max2; } // The result should positive Assert(1 <= quadrant3_Min && quadrant3_Min <= quadrant3_Max); newMin = min(newMin, quadrant3_Min); newMax = max(newMax, quadrant3_Max); } } Assert(newMin <= newMax); // Continue to int type spec break; } } // fall-through default: { const bool involesLargeInt32 = (src1Val && src1Val->GetValueInfo()->IsLikelyUntaggedInt()) || (src2Val && src2Val->GetValueInfo()->IsLikelyUntaggedInt()); const auto trySpecializeToFloat = [&](const bool mayOverflow) -> bool { // It has been determined that this instruction cannot be int-specialized. Need to determine whether to attempt // to float-specialize the instruction, or leave it unspecialized. if((involesLargeInt32 #if INT32VAR && mayOverflow #endif ) || (instr->m_opcode == Js::OpCode::Mul_A && !this->DoAggressiveMulIntTypeSpec()) ) { // An input range is completely outside the range of an int31 and the operation is likely to overflow. // Additionally, on 32-bit platforms, the value is untaggable and will be a JavascriptNumber, which is // significantly slower to use in an unspecialized operation compared to a tagged int. So, try to // float-specialize the instruction. src1Val = src1OriginalVal; src2Val = src2OriginalVal; return TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal); } return false; }; if (instr->m_opcode != Js::OpCode::ArgOut_A_InlineBuiltIn) { if ((src1Val && src1Val->GetValueInfo()->IsLikelyFloat()) || (src2Val && src2Val->GetValueInfo()->IsLikelyFloat())) { // Try to type specialize to float src1Val = src1OriginalVal; src2Val = src2OriginalVal; return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal); } if (src1Val == nullptr || src2Val == nullptr || !src1Val->GetValueInfo()->IsLikelyInt() || !src2Val->GetValueInfo()->IsLikelyInt() || ( !DoAggressiveIntTypeSpec() && ( !(src1Val->GetValueInfo()->IsInt() || CurrentBlockData()->IsSwitchInt32TypeSpecialized(instr)) || !src2Val->GetValueInfo()->IsInt() ) ) || (instr->GetSrc1()->IsRegOpnd() && instr->GetSrc1()->AsRegOpnd()->m_sym->m_isNotNumber) || (instr->GetSrc2()->IsRegOpnd() && instr->GetSrc2()->AsRegOpnd()->m_sym->m_isNotNumber)) { return trySpecializeToFloat(true); } } // Try to type specialize to int32 // If one of the values is a float constant with a value that fits in a uint32 but not an int32, // and the instruction can ignore int overflow, the source value for the purposes of int specialization // would have been changed to an int constant value by ignoring overflow. But, the conversion is still lossy. if (!(src1OriginalVal && src1OriginalVal->GetValueInfo()->IsFloatConstant() && src1Val && src1Val->GetValueInfo()->HasIntConstantValue())) { src1Lossy = false; } if (!(src2OriginalVal && src2OriginalVal->GetValueInfo()->IsFloatConstant() && src2Val && src2Val->GetValueInfo()->HasIntConstantValue())) { src2Lossy = false; } switch(instr->m_opcode) { case Js::OpCode::ArgOut_A_InlineBuiltIn: // If the src is already type-specialized, if we don't type-specialize ArgOut_A_InlineBuiltIn instr, we'll get additional ToVar. // So, to avoid that, type-specialize the ArgOut_A_InlineBuiltIn instr. // Else we don't need to type-specialize the instr, we are fine with src being Var. if (instr->GetSrc1()->IsRegOpnd()) { StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym; if (CurrentBlockData()->IsInt32TypeSpecialized(sym)) { opcode = instr->m_opcode; skipDst = true; // We should keep dst as is, otherwise the link opnd for next ArgOut/InlineBuiltInStart would be broken. skipSrc2 = true; // src2 is linkOpnd. We don't need to type-specialize it. newMin = min1; newMax = max1; // Values don't matter, these are unused. goto LOutsideSwitch; // Continue to int-type-specialize. } else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym)) { src1Val = src1OriginalVal; src2Val = src2OriginalVal; return this->TypeSpecializeFloatBinary(instr, src1Val, src2Val, pDstVal); } } return false; case Js::OpCode::Add_A: do // while(false) { const auto CannotOverflowBasedOnRelativeBounds = [&](int32 *const constantValueRef) { Assert(constantValueRef); if(min2 == max2 && src1Val->GetValueInfo()->IsIntBounded() && src1Val->GetValueInfo()->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(min2)) { *constantValueRef = min2; return true; } else if( min1 == max1 && src2Val->GetValueInfo()->IsIntBounded() && src2Val->GetValueInfo()->AsIntBounded()->Bounds()->AddCannotOverflowBasedOnRelativeBounds(min1)) { *constantValueRef = min1; return true; } return false; }; if (Int32Math::Add(min1, min2, &newMin)) { int32 constantSrcValue; if(CannotOverflowBasedOnRelativeBounds(&constantSrcValue)) { newMin = constantSrcValue >= 0 ? INT32_MAX : INT32_MIN; } else if(instr->ShouldCheckForIntOverflow()) { if(involesLargeInt32 || !DoAggressiveIntTypeSpec()) { // May overflow return trySpecializeToFloat(true); } bailOutKind |= IR::BailOutOnOverflow; newMin = min1 < 0 ? INT32_MIN : INT32_MAX; } else { // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since // overflow causes the value to wrap around, and we don't have a way to specify a lower and upper // range of ints, we use the full range of int32s. ignoredIntOverflow = true; newMin = INT32_MIN; newMax = INT32_MAX; break; } } if (Int32Math::Add(max1, max2, &newMax)) { int32 constantSrcValue; if(CannotOverflowBasedOnRelativeBounds(&constantSrcValue)) { newMax = constantSrcValue >= 0 ? INT32_MAX : INT32_MIN; } else if(instr->ShouldCheckForIntOverflow()) { if(involesLargeInt32 || !DoAggressiveIntTypeSpec()) { // May overflow return trySpecializeToFloat(true); } bailOutKind |= IR::BailOutOnOverflow; newMax = max1 < 0 ? INT32_MIN : INT32_MAX; } else { // See comment about ignoring overflow above ignoredIntOverflow = true; newMin = INT32_MIN; newMax = INT32_MAX; break; } } if(bailOutKind & IR::BailOutOnOverflow) { Assert(bailOutKind == IR::BailOutOnOverflow); Assert(instr->ShouldCheckForIntOverflow()); int32 temp; if(Int32Math::Add( Int32Math::NearestInRangeTo(0, min1, max1), Int32Math::NearestInRangeTo(0, min2, max2), &temp)) { // Always overflows return trySpecializeToFloat(true); } } } while(false); if (!this->IsLoopPrePass() && newMin == newMax && bailOutKind == IR::BailOutInvalid) { // Take care of Add with zero here, since we know we're dealing with 2 numbers. this->CaptureByteCodeSymUses(instr); IR::Opnd *src; bool isAddZero = true; int32 intConstantValue; if (src1Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) && intConstantValue == 0) { src = instr->UnlinkSrc2(); instr->FreeSrc1(); } else if (src2Val->GetValueInfo()->TryGetIntConstantValue(&intConstantValue) && intConstantValue == 0) { src = instr->UnlinkSrc1(); instr->FreeSrc2(); } else { // This should have been handled by const folding, unless: // - A source's value was substituted with a different value here, which is after const folding happened // - A value is not definitely int, but once converted to definite int, it would be zero due to a // condition in the source code such as if(a === 0). Ideally, we would specialize the sources and // remove the add, but doesn't seem too important for now. Assert( !DoConstFold() || src1Val != src1OriginalVal || src2Val != src2OriginalVal || !src1Val->GetValueInfo()->IsInt() || !src2Val->GetValueInfo()->IsInt()); isAddZero = false; src = nullptr; } if (isAddZero) { IR::Instr *newInstr = IR::Instr::New(Js::OpCode::Ld_A, instr->UnlinkDst(), src, instr->m_func); newInstr->SetByteCodeOffset(instr); instr->m_opcode = Js::OpCode::Nop; this->currentBlock->InsertInstrAfter(newInstr, instr); return true; } } if(!ignoredIntOverflow) { if(min2 == max2 && (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val)) && instr->GetSrc1()->IsRegOpnd()) { addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min1 == max1, min2); } else if( min1 == max1 && (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Val)) && instr->GetSrc2()->IsRegOpnd()) { addSubConstantInfo.Set(instr->GetSrc2()->AsRegOpnd()->m_sym, src2Val, min2 == max2, min1); } } opcode = Js::OpCode::Add_I4; break; case Js::OpCode::Sub_A: do // while(false) { const auto CannotOverflowBasedOnRelativeBounds = [&]() { return min2 == max2 && src1Val->GetValueInfo()->IsIntBounded() && src1Val->GetValueInfo()->AsIntBounded()->Bounds()->SubCannotOverflowBasedOnRelativeBounds(min2); }; if (Int32Math::Sub(min1, max2, &newMin)) { if(CannotOverflowBasedOnRelativeBounds()) { Assert(min2 == max2); newMin = min2 >= 0 ? INT32_MIN : INT32_MAX; } else if(instr->ShouldCheckForIntOverflow()) { if(involesLargeInt32 || !DoAggressiveIntTypeSpec()) { // May overflow return trySpecializeToFloat(true); } bailOutKind |= IR::BailOutOnOverflow; newMin = min1 < 0 ? INT32_MIN : INT32_MAX; } else { // When ignoring overflow, the range needs to account for overflow. For any Add or Sub, since overflow // causes the value to wrap around, and we don't have a way to specify a lower and upper range of ints, // we use the full range of int32s. ignoredIntOverflow = true; newMin = INT32_MIN; newMax = INT32_MAX; break; } } if (Int32Math::Sub(max1, min2, &newMax)) { if(CannotOverflowBasedOnRelativeBounds()) { Assert(min2 == max2); newMax = min2 >= 0 ? INT32_MIN: INT32_MAX; } else if(instr->ShouldCheckForIntOverflow()) { if(involesLargeInt32 || !DoAggressiveIntTypeSpec()) { // May overflow return trySpecializeToFloat(true); } bailOutKind |= IR::BailOutOnOverflow; newMax = max1 < 0 ? INT32_MIN : INT32_MAX; } else { // See comment about ignoring overflow above ignoredIntOverflow = true; newMin = INT32_MIN; newMax = INT32_MAX; break; } } if(bailOutKind & IR::BailOutOnOverflow) { Assert(bailOutKind == IR::BailOutOnOverflow); Assert(instr->ShouldCheckForIntOverflow()); int32 temp; if(Int32Math::Sub( Int32Math::NearestInRangeTo(-1, min1, max1), Int32Math::NearestInRangeTo(0, min2, max2), &temp)) { // Always overflows return trySpecializeToFloat(true); } } } while(false); if(!ignoredIntOverflow && min2 == max2 && min2 != INT32_MIN && (!IsLoopPrePass() || IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val)) && instr->GetSrc1()->IsRegOpnd()) { addSubConstantInfo.Set(instr->GetSrc1()->AsRegOpnd()->m_sym, src1Val, min1 == max1, -min2); } opcode = Js::OpCode::Sub_I4; break; case Js::OpCode::Mul_A: { bool isConservativeMulInt = !DoAggressiveMulIntTypeSpec() || !DoAggressiveIntTypeSpec(); // Be conservative about predicting Mul overflow in prepass. // Operands that are live on back edge may be denied lossless-conversion to int32 and // trigger rejit with AggressiveIntTypeSpec off. // Besides multiplying a variable in a loop can overflow in just a few iterations even in simple cases like v *= 2 // So, make sure we definitely know the source max/min values, otherwise assume the full range. if (isConservativeMulInt && IsLoopPrePass()) { if (!IsPrepassSrcValueInfoPrecise(instr->GetSrc1(), src1Val)) { max1 = INT32_MAX; min1 = INT32_MIN; } if (!IsPrepassSrcValueInfoPrecise(instr->GetSrc2(), src2Val)) { max2 = INT32_MAX; min2 = INT32_MIN; } } if (Int32Math::Mul(min1, min2, &newMin)) { if (involesLargeInt32 || isConservativeMulInt) { // May overflow return trySpecializeToFloat(true); } bailOutKind |= IR::BailOutOnMulOverflow; newMin = (min1 < 0) ^ (min2 < 0) ? INT32_MIN : INT32_MAX; } newMax = newMin; if (Int32Math::Mul(max1, max2, &tmp)) { if (involesLargeInt32 || isConservativeMulInt) { // May overflow return trySpecializeToFloat(true); } bailOutKind |= IR::BailOutOnMulOverflow; tmp = (max1 < 0) ^ (max2 < 0) ? INT32_MIN : INT32_MAX; } newMin = min(newMin, tmp); newMax = max(newMax, tmp); if (Int32Math::Mul(min1, max2, &tmp)) { if (involesLargeInt32 || isConservativeMulInt) { // May overflow return trySpecializeToFloat(true); } bailOutKind |= IR::BailOutOnMulOverflow; tmp = (min1 < 0) ^ (max2 < 0) ? INT32_MIN : INT32_MAX; } newMin = min(newMin, tmp); newMax = max(newMax, tmp); if (Int32Math::Mul(max1, min2, &tmp)) { if (involesLargeInt32 || isConservativeMulInt) { // May overflow return trySpecializeToFloat(true); } bailOutKind |= IR::BailOutOnMulOverflow; tmp = (max1 < 0) ^ (min2 < 0) ? INT32_MIN : INT32_MAX; } newMin = min(newMin, tmp); newMax = max(newMax, tmp); if (bailOutKind & IR::BailOutOnMulOverflow) { // CSE only if two MULs have the same overflow check behavior. // Currently this is set to be ignore int32 overflow, but not 53-bit, or int32 overflow matters. if (!instr->ShouldCheckFor32BitOverflow() && instr->ShouldCheckForNon32BitOverflow()) { // If we allow int to overflow then there can be anything in the resulting int newMin = IntConstMin; newMax = IntConstMax; ignoredIntOverflow = true; } int32 temp, overflowValue; if (Int32Math::Mul( Int32Math::NearestInRangeTo(0, min1, max1), Int32Math::NearestInRangeTo(0, min2, max2), &temp, &overflowValue)) { Assert(instr->ignoreOverflowBitCount >= 32); int overflowMatters = 64 - instr->ignoreOverflowBitCount; if (!ignoredIntOverflow || // Use shift to check high bits in case its negative ((overflowValue << overflowMatters) >> overflowMatters) != overflowValue ) { // Always overflows return trySpecializeToFloat(true); } } } if (newMin <= 0 && newMax >= 0 && // New range crosses zero (min1 < 0 || min2 < 0) && // An operand's range contains a negative integer !(min1 > 0 || min2 > 0) && // Neither operand's range contains only positive integers !instr->GetSrc1()->IsEqual(instr->GetSrc2())) // The operands don't have the same value { if (instr->ShouldCheckForNegativeZero()) { // -0 matters since the sym is not a local, or is used in a way in which -0 would differ from +0 if (!DoAggressiveIntTypeSpec()) { // May result in -0 return trySpecializeToFloat(false); } if (((min1 == 0 && max1 == 0) || (min2 == 0 && max2 == 0)) && (max1 < 0 || max2 < 0)) { // Always results in -0 return trySpecializeToFloat(false); } bailOutKind |= IR::BailOutOnNegativeZero; } else { ignoredNegativeZero = true; } } opcode = Js::OpCode::Mul_I4; break; } case Js::OpCode::Rem_A: { IR::Opnd* src2 = instr->GetSrc2(); if (!this->IsLoopPrePass() && min2 == max2 && min1 >= 0) { int32 value = min2; if (value == (1 << Math::Log2(value)) && src2->IsAddrOpnd()) { Assert(src2->AsAddrOpnd()->IsVar()); instr->m_opcode = Js::OpCode::And_A; src2->AsAddrOpnd()->SetAddress(Js::TaggedInt::ToVarUnchecked(value - 1), IR::AddrOpndKindConstantVar); *pSrc2Val = GetIntConstantValue(value - 1, instr); src2Val = *pSrc2Val; return this->TypeSpecializeBinary(&instr, pSrc1Val, pSrc2Val, pDstVal, src1OriginalVal, src2Val, redoTypeSpecRef); } } #ifdef _M_ARM if (!AutoSystemInfo::Data.ArmDivAvailable()) { return false; } #endif if (min1 < 0) { // The most negative it can be is min1, unless limited by min2/max2 int32 negMaxAbs2; if (min2 == INT32_MIN) { negMaxAbs2 = INT32_MIN; } else { negMaxAbs2 = -max(abs(min2), abs(max2)) + 1; } newMin = max(min1, negMaxAbs2); } else { newMin = 0; } bool isModByPowerOf2 = (instr->IsProfiledInstr() && instr->m_func->HasProfileInfo() && instr->m_func->GetReadOnlyProfileInfo()->IsModulusOpByPowerOf2(static_cast(instr->AsProfiledInstr()->u.profileId))); if(isModByPowerOf2) { Assert(bailOutKind == IR::BailOutInvalid); bailOutKind = IR::BailOnModByPowerOf2; newMin = 0; } else { if (min2 <= 0 && max2 >= 0) { // Consider: We could handle the zero case with a check and bailout... return false; } if (min1 == 0x80000000 && (min2 <= -1 && max2 >= -1)) { // Prevent integer overflow, as div by zero or MIN_INT / -1 will throw an exception return false; } if (min1 < 0) { if(instr->ShouldCheckForNegativeZero()) { if (!DoAggressiveIntTypeSpec()) { return false; } bailOutKind |= IR::BailOutOnNegativeZero; } else { ignoredNegativeZero = true; } } } { int32 absMax2; if (min2 == INT32_MIN) { // abs(INT32_MIN) == INT32_MAX because of overflow absMax2 = INT32_MAX; } else { absMax2 = max(abs(min2), abs(max2)) - 1; } newMax = min(absMax2, max(max1, 0)); newMax = max(newMin, newMax); } opcode = Js::OpCode::Rem_I4; Assert(!instr->GetSrc1()->IsUnsigned()); break; } case Js::OpCode::CmEq_A: case Js::OpCode::CmSrEq_A: if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val)) { return false; } newMin = 0; newMax = 1; opcode = Js::OpCode::CmEq_I4; needsBoolConv = true; break; case Js::OpCode::CmNeq_A: case Js::OpCode::CmSrNeq_A: if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val)) { return false; } newMin = 0; newMax = 1; opcode = Js::OpCode::CmNeq_I4; needsBoolConv = true; break; case Js::OpCode::CmLe_A: if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val)) { return false; } newMin = 0; newMax = 1; opcode = Js::OpCode::CmLe_I4; needsBoolConv = true; break; case Js::OpCode::CmLt_A: if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val)) { return false; } newMin = 0; newMax = 1; opcode = Js::OpCode::CmLt_I4; needsBoolConv = true; break; case Js::OpCode::CmGe_A: if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val)) { return false; } newMin = 0; newMax = 1; opcode = Js::OpCode::CmGe_I4; needsBoolConv = true; break; case Js::OpCode::CmGt_A: if (!IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val)) { return false; } newMin = 0; newMax = 1; opcode = Js::OpCode::CmGt_I4; needsBoolConv = true; break; case Js::OpCode::BrSrEq_A: case Js::OpCode::BrEq_A: case Js::OpCode::BrNotNeq_A: case Js::OpCode::BrSrNotNeq_A: { if(DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrEqual(instr, true, src1Val, min1, max1, src2Val, min2, max2)) { return true; } const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val); UpdateIntBoundsForEqualBranch(src1Val, src2Val); if(!specialize) { return false; } opcode = Js::OpCode::BrEq_I4; // We'll get a warning if we don't assign a value to these... // We'll assert if we use them and make a range where min > max newMin = 2; newMax = 1; break; } case Js::OpCode::BrSrNeq_A: case Js::OpCode::BrNeq_A: case Js::OpCode::BrSrNotEq_A: case Js::OpCode::BrNotEq_A: { if(DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrEqual(instr, false, src1Val, min1, max1, src2Val, min2, max2)) { return true; } const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val); UpdateIntBoundsForNotEqualBranch(src1Val, src2Val); if(!specialize) { return false; } opcode = Js::OpCode::BrNeq_I4; // We'll get a warning if we don't assign a value to these... // We'll assert if we use them and make a range where min > max newMin = 2; newMax = 1; break; } case Js::OpCode::BrGt_A: case Js::OpCode::BrNotLe_A: { if(DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrGreaterThan(instr, true, src1Val, min1, max1, src2Val, min2, max2)) { return true; } const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val); UpdateIntBoundsForGreaterThanBranch(src1Val, src2Val); if(!specialize) { return false; } opcode = Js::OpCode::BrGt_I4; // We'll get a warning if we don't assign a value to these... // We'll assert if we use them and make a range where min > max newMin = 2; newMax = 1; break; } case Js::OpCode::BrGe_A: case Js::OpCode::BrNotLt_A: { if(DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrGreaterThanOrEqual(instr, true, src1Val, min1, max1, src2Val, min2, max2)) { return true; } const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val); UpdateIntBoundsForGreaterThanOrEqualBranch(src1Val, src2Val); if(!specialize) { return false; } opcode = Js::OpCode::BrGe_I4; // We'll get a warning if we don't assign a value to these... // We'll assert if we use them and make a range where min > max newMin = 2; newMax = 1; break; } case Js::OpCode::BrLt_A: case Js::OpCode::BrNotGe_A: { if(DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrGreaterThanOrEqual(instr, false, src1Val, min1, max1, src2Val, min2, max2)) { return true; } const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val); UpdateIntBoundsForLessThanBranch(src1Val, src2Val); if(!specialize) { return false; } opcode = Js::OpCode::BrLt_I4; // We'll get a warning if we don't assign a value to these... // We'll assert if we use them and make a range where min > max newMin = 2; newMax = 1; break; } case Js::OpCode::BrLe_A: case Js::OpCode::BrNotGt_A: { if(DoConstFold() && !IsLoopPrePass() && TryOptConstFoldBrGreaterThan(instr, false, src1Val, min1, max1, src2Val, min2, max2)) { return true; } const bool specialize = IsWorthSpecializingToInt32Branch(instr, src1Val, src2Val); UpdateIntBoundsForLessThanOrEqualBranch(src1Val, src2Val); if(!specialize) { return false; } opcode = Js::OpCode::BrLe_I4; // We'll get a warning if we don't assign a value to these... // We'll assert if we use them and make a range where min > max newMin = 2; newMax = 1; break; } default: return false; } // If this instruction is in a range of instructions where int overflow does not matter, we will still specialize it // (won't leave it unspecialized based on heuristics), since it is most likely worth specializing, and the dst value // needs to be guaranteed to be an int if(!ignoredIntOverflow && !ignoredNegativeZero && !needsBoolConv && instr->ShouldCheckForIntOverflow() && !IsWorthSpecializingToInt32(instr, src1Val, src2Val)) { // Even though type specialization is being skipped since it may not be worth it, the proper value should still be // maintained so that the result may be type specialized later. An int value is not created for the dst in any of // the following cases. // - A bailout check is necessary to specialize this instruction. The bailout check is what guarantees the result to // be an int, but since we're not going to specialize this instruction, there won't be a bailout check. // - Aggressive int type specialization is disabled and we're in a loop prepass. We're conservative on dst values in // that case, especially if the dst sym is live on the back-edge. if(bailOutKind == IR::BailOutInvalid && instr->GetDst() && src1Val->GetValueInfo()->IsInt() && src2Val->GetValueInfo()->IsInt() && (DoAggressiveIntTypeSpec() || !this->IsLoopPrePass())) { *pDstVal = CreateDstUntransferredIntValue(newMin, newMax, instr, src1Val, src2Val); } return false; } } // case default } // switch LOutsideSwitch: this->ignoredIntOverflowForCurrentInstr = ignoredIntOverflow; this->ignoredNegativeZeroForCurrentInstr = ignoredNegativeZero; { // Try CSE again before modifying the IR, in case some attributes are required for successful CSE Value *src1IndirIndexVal = nullptr; if(CSEOptimize(currentBlock, &instr, &src1Val, &src2Val, &src1IndirIndexVal, true /* intMathExprOnly */)) { *redoTypeSpecRef = true; return false; } } const Js::OpCode originalOpCode = instr->m_opcode; if (!this->IsLoopPrePass()) { // No re-write on prepass instr->m_opcode = opcode; } Value *src1ValueToSpecialize = src1Val, *src2ValueToSpecialize = src2Val; // Lossy conversions to int32 must be done based on the original source values. For instance, if one of the values is a // float constant with a value that fits in a uint32 but not an int32, and the instruction can ignore int overflow, the // source value for the purposes of int specialization would have been changed to an int constant value by ignoring // overflow. If we were to specialize the sym using the int constant value, it would be treated as a lossless // conversion, but since there may be subsequent uses of the same float constant value that may not ignore overflow, // this must be treated as a lossy conversion by specializing the sym using the original float constant value. if(src1Lossy) { src1ValueToSpecialize = src1OriginalVal; } if (src2Lossy) { src2ValueToSpecialize = src2OriginalVal; } // Make sure the srcs are specialized IR::Opnd* src1 = instr->GetSrc1(); this->ToInt32(instr, src1, this->currentBlock, src1ValueToSpecialize, nullptr, src1Lossy); if (!skipSrc2) { IR::Opnd* src2 = instr->GetSrc2(); this->ToInt32(instr, src2, this->currentBlock, src2ValueToSpecialize, nullptr, src2Lossy); } if(bailOutKind != IR::BailOutInvalid && !this->IsLoopPrePass()) { GenerateBailAtOperation(&instr, bailOutKind); } if (!skipDst && instr->GetDst()) { if (needsBoolConv) { IR::RegOpnd *varDst; if (this->IsLoopPrePass()) { varDst = instr->GetDst()->AsRegOpnd(); this->ToVarRegOpnd(varDst, this->currentBlock); } else { // Generate: // t1.i = CmCC t2.i, t3.i // t1.v = Conv_bool t1.i // // If the only uses of t1 are ints, the conv_bool will get dead-stored TypeSpecializeIntDst(instr, originalOpCode, nullptr, src1Val, src2Val, bailOutKind, newMin, newMax, pDstVal); IR::RegOpnd *intDst = instr->GetDst()->AsRegOpnd(); intDst->SetIsJITOptimizedReg(true); varDst = IR::RegOpnd::New(intDst->m_sym->GetVarEquivSym(this->func), TyVar, this->func); IR::Instr *convBoolInstr = IR::Instr::New(Js::OpCode::Conv_Bool, varDst, intDst, this->func); // In some cases (e.g. unsigned compare peep code), a comparison will use variables // other than the ones initially intended for it, if we can determine that we would // arrive at the same result. This means that we get a ByteCodeUses operation after // the actual comparison. Since Inserting the Conv_bool just after the compare, and // just before the ByteCodeUses, would cause issues later on with register lifetime // calculation, we want to insert the Conv_bool after the whole compare instruction // block. IR::Instr *putAfter = instr; while (putAfter->m_next && putAfter->m_next->IsByteCodeUsesInstrFor(instr)) { putAfter = putAfter->m_next; } putAfter->InsertAfter(convBoolInstr); convBoolInstr->SetByteCodeOffset(instr); this->ToVarRegOpnd(varDst, this->currentBlock); CurrentBlockData()->liveInt32Syms->Set(varDst->m_sym->m_id); CurrentBlockData()->liveLossyInt32Syms->Set(varDst->m_sym->m_id); } *pDstVal = this->NewGenericValue(ValueType::Boolean, varDst); } else { TypeSpecializeIntDst( instr, originalOpCode, nullptr, src1Val, src2Val, bailOutKind, newMin, newMax, pDstVal, addSubConstantInfo.HasInfo() ? &addSubConstantInfo : nullptr); } } if(bailOutKind == IR::BailOutInvalid) { GOPT_TRACE(_u("Type specialized to INT\n")); #if ENABLE_DEBUG_CONFIG_OPTIONS if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase)) { Output::Print(_u("Type specialized to INT: ")); Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode)); } #endif } else { GOPT_TRACE(_u("Type specialized to INT with bailout on:\n")); if(bailOutKind & (IR::BailOutOnOverflow | IR::BailOutOnMulOverflow) ) { GOPT_TRACE(_u(" Overflow\n")); #if ENABLE_DEBUG_CONFIG_OPTIONS if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase)) { Output::Print(_u("Type specialized to INT with bailout (%S): "), "Overflow"); Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode)); } #endif } if(bailOutKind & IR::BailOutOnNegativeZero) { GOPT_TRACE(_u(" Zero\n")); #if ENABLE_DEBUG_CONFIG_OPTIONS if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::AggressiveIntTypeSpecPhase)) { Output::Print(_u("Type specialized to INT with bailout (%S): "), "Zero"); Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode)); } #endif } } return true; } bool GlobOpt::IsWorthSpecializingToInt32Branch(IR::Instr const * instr, Value const * src1Val, Value const * src2Val) const { if (!src1Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc1()->IsRegOpnd()) { StackSym const *sym1 = instr->GetSrc1()->AsRegOpnd()->m_sym; if (CurrentBlockData()->IsInt32TypeSpecialized(sym1) == false) { if (!src2Val->GetValueInfo()->HasIntConstantValue() && instr->GetSrc2()->IsRegOpnd()) { StackSym const *sym2 = instr->GetSrc2()->AsRegOpnd()->m_sym; if (CurrentBlockData()->IsInt32TypeSpecialized(sym2) == false) { // Type specializing a Br itself isn't worth it, unless one src // is already type specialized return false; } } } } return true; } bool GlobOpt::TryOptConstFoldBrFalse( IR::Instr *const instr, Value *const srcValue, const int32 min, const int32 max) { Assert(instr); Assert(instr->m_opcode == Js::OpCode::BrFalse_A || instr->m_opcode == Js::OpCode::BrTrue_A); Assert(srcValue); if(!(DoAggressiveIntTypeSpec() ? srcValue->GetValueInfo()->IsLikelyInt() : srcValue->GetValueInfo()->IsInt())) { return false; } if(ValueInfo::IsEqualTo(srcValue, min, max, nullptr, 0, 0)) { OptConstFoldBr(instr->m_opcode == Js::OpCode::BrFalse_A, instr, srcValue); return true; } if(ValueInfo::IsNotEqualTo(srcValue, min, max, nullptr, 0, 0)) { OptConstFoldBr(instr->m_opcode == Js::OpCode::BrTrue_A, instr, srcValue); return true; } return false; } bool GlobOpt::TryOptConstFoldBrEqual( IR::Instr *const instr, const bool branchOnEqual, Value *const src1Value, const int32 min1, const int32 max1, Value *const src2Value, const int32 min2, const int32 max2) { Assert(instr); Assert(src1Value); Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt()); Assert(src2Value); Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt()); if(ValueInfo::IsEqualTo(src1Value, min1, max1, src2Value, min2, max2)) { OptConstFoldBr(branchOnEqual, instr, src1Value, src2Value); return true; } if(ValueInfo::IsNotEqualTo(src1Value, min1, max1, src2Value, min2, max2)) { OptConstFoldBr(!branchOnEqual, instr, src1Value, src2Value); return true; } return false; } bool GlobOpt::TryOptConstFoldBrGreaterThan( IR::Instr *const instr, const bool branchOnGreaterThan, Value *const src1Value, const int32 min1, const int32 max1, Value *const src2Value, const int32 min2, const int32 max2) { Assert(instr); Assert(src1Value); Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt()); Assert(src2Value); Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt()); if(ValueInfo::IsGreaterThan(src1Value, min1, max1, src2Value, min2, max2)) { OptConstFoldBr(branchOnGreaterThan, instr, src1Value, src2Value); return true; } if(ValueInfo::IsLessThanOrEqualTo(src1Value, min1, max1, src2Value, min2, max2)) { OptConstFoldBr(!branchOnGreaterThan, instr, src1Value, src2Value); return true; } return false; } bool GlobOpt::TryOptConstFoldBrGreaterThanOrEqual( IR::Instr *const instr, const bool branchOnGreaterThanOrEqual, Value *const src1Value, const int32 min1, const int32 max1, Value *const src2Value, const int32 min2, const int32 max2) { Assert(instr); Assert(src1Value); Assert(DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt()); Assert(src2Value); Assert(DoAggressiveIntTypeSpec() ? src2Value->GetValueInfo()->IsLikelyInt() : src2Value->GetValueInfo()->IsInt()); if(ValueInfo::IsGreaterThanOrEqualTo(src1Value, min1, max1, src2Value, min2, max2)) { OptConstFoldBr(branchOnGreaterThanOrEqual, instr, src1Value, src2Value); return true; } if(ValueInfo::IsLessThan(src1Value, min1, max1, src2Value, min2, max2)) { OptConstFoldBr(!branchOnGreaterThanOrEqual, instr, src1Value, src2Value); return true; } return false; } bool GlobOpt::TryOptConstFoldBrUnsignedLessThan( IR::Instr *const instr, const bool branchOnLessThan, Value *const src1Value, const int32 min1, const int32 max1, Value *const src2Value, const int32 min2, const int32 max2) { Assert(DoConstFold()); Assert(!IsLoopPrePass()); if(!src1Value || !src2Value || !( DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() && src2Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt() && src2Value->GetValueInfo()->IsInt() )) { return false; } uint uMin1 = (min1 < 0 ? (max1 < 0 ? min((uint)min1, (uint)max1) : 0) : min1); uint uMax1 = max((uint)min1, (uint)max1); uint uMin2 = (min2 < 0 ? (max2 < 0 ? min((uint)min2, (uint)max2) : 0) : min2); uint uMax2 = max((uint)min2, (uint)max2); if (uMax1 < uMin2) { // Range 1 is always lesser than Range 2 OptConstFoldBr(branchOnLessThan, instr, src1Value, src2Value); return true; } if (uMin1 >= uMax2) { // Range 2 is always lesser than Range 1 OptConstFoldBr(!branchOnLessThan, instr, src1Value, src2Value); return true; } return false; } bool GlobOpt::TryOptConstFoldBrUnsignedGreaterThan( IR::Instr *const instr, const bool branchOnGreaterThan, Value *const src1Value, const int32 min1, const int32 max1, Value *const src2Value, const int32 min2, const int32 max2) { Assert(DoConstFold()); Assert(!IsLoopPrePass()); if(!src1Value || !src2Value || !( DoAggressiveIntTypeSpec() ? src1Value->GetValueInfo()->IsLikelyInt() && src2Value->GetValueInfo()->IsLikelyInt() : src1Value->GetValueInfo()->IsInt() && src2Value->GetValueInfo()->IsInt() )) { return false; } uint uMin1 = (min1 < 0 ? (max1 < 0 ? min((uint)min1, (uint)max1) : 0) : min1); uint uMax1 = max((uint)min1, (uint)max1); uint uMin2 = (min2 < 0 ? (max2 < 0 ? min((uint)min2, (uint)max2) : 0) : min2); uint uMax2 = max((uint)min2, (uint)max2); if (uMin1 > uMax2) { // Range 1 is always greater than Range 2 OptConstFoldBr(branchOnGreaterThan, instr, src1Value, src2Value); return true; } if (uMax1 <= uMin2) { // Range 2 is always greater than Range 1 OptConstFoldBr(!branchOnGreaterThan, instr, src1Value, src2Value); return true; } return false; } void GlobOpt::SetPathDependentInfo(const bool conditionToBranch, const PathDependentInfo &info) { Assert(this->currentBlock->GetSuccList()->Count() == 2); IR::Instr * fallthrough = this->currentBlock->GetNext()->GetFirstInstr(); FOREACH_SLISTBASECOUNTED_ENTRY(FlowEdge*, edge, this->currentBlock->GetSuccList()) { if (conditionToBranch == (edge->GetSucc()->GetFirstInstr() != fallthrough)) { edge->SetPathDependentInfo(info, alloc); return; } } NEXT_SLISTBASECOUNTED_ENTRY; // In case flowgraph peeps is disabled, we could have conditional branch to next instr Assert(this->func->HasTry() || PHASE_OFF(Js::FGPeepsPhase, this->func)); } PathDependentInfoToRestore GlobOpt::UpdatePathDependentInfo(PathDependentInfo *const info) { Assert(info); if(!info->HasInfo()) { return PathDependentInfoToRestore(); } decltype(&GlobOpt::UpdateIntBoundsForEqual) UpdateIntBoundsForLeftValue, UpdateIntBoundsForRightValue; switch(info->Relationship()) { case PathDependentRelationship::Equal: UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForEqual; UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForEqual; break; case PathDependentRelationship::NotEqual: UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForNotEqual; UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForNotEqual; break; case PathDependentRelationship::GreaterThanOrEqual: UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForGreaterThanOrEqual; UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForLessThanOrEqual; break; case PathDependentRelationship::GreaterThan: UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForGreaterThan; UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForLessThan; break; case PathDependentRelationship::LessThanOrEqual: UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForLessThanOrEqual; UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForGreaterThanOrEqual; break; case PathDependentRelationship::LessThan: UpdateIntBoundsForLeftValue = &GlobOpt::UpdateIntBoundsForLessThan; UpdateIntBoundsForRightValue = &GlobOpt::UpdateIntBoundsForGreaterThan; break; default: Assert(false); __assume(false); } ValueInfo *leftValueInfo = info->LeftValue()->GetValueInfo(); IntConstantBounds leftConstantBounds; AssertVerify(leftValueInfo->TryGetIntConstantBounds(&leftConstantBounds, true)); ValueInfo *rightValueInfo; IntConstantBounds rightConstantBounds; if(info->RightValue()) { rightValueInfo = info->RightValue()->GetValueInfo(); AssertVerify(rightValueInfo->TryGetIntConstantBounds(&rightConstantBounds, true)); } else { rightValueInfo = nullptr; rightConstantBounds = IntConstantBounds(info->RightConstantValue(), info->RightConstantValue()); } ValueInfo *const newLeftValueInfo = (this->*UpdateIntBoundsForLeftValue)( info->LeftValue(), leftConstantBounds, info->RightValue(), rightConstantBounds, true); if(newLeftValueInfo) { ChangeValueInfo(nullptr, info->LeftValue(), newLeftValueInfo); AssertVerify(newLeftValueInfo->TryGetIntConstantBounds(&leftConstantBounds, true)); } else { leftValueInfo = nullptr; } ValueInfo *const newRightValueInfo = (this->*UpdateIntBoundsForRightValue)( info->RightValue(), rightConstantBounds, info->LeftValue(), leftConstantBounds, true); if(newRightValueInfo) { ChangeValueInfo(nullptr, info->RightValue(), newRightValueInfo); } else { rightValueInfo = nullptr; } return PathDependentInfoToRestore(leftValueInfo, rightValueInfo); } void GlobOpt::RestorePathDependentInfo(PathDependentInfo *const info, const PathDependentInfoToRestore infoToRestore) { Assert(info); if(infoToRestore.LeftValueInfo()) { Assert(info->LeftValue()); ChangeValueInfo(nullptr, info->LeftValue(), infoToRestore.LeftValueInfo()); } if(infoToRestore.RightValueInfo()) { Assert(info->RightValue()); ChangeValueInfo(nullptr, info->RightValue(), infoToRestore.RightValueInfo()); } } bool GlobOpt::TypeSpecializeFloatUnary(IR::Instr **pInstr, Value *src1Val, Value **pDstVal, bool skipDst /* = false */) { IR::Instr *&instr = *pInstr; IR::Opnd *src1; IR::Opnd *dst; Js::OpCode opcode = instr->m_opcode; Value *valueToTransfer = nullptr; Assert(src1Val && src1Val->GetValueInfo()->IsLikelyNumber() || OpCodeAttr::IsInlineBuiltIn(instr->m_opcode)); if (!this->DoFloatTypeSpec()) { return false; } // For inline built-ins we need to do type specialization. Check upfront to avoid duplicating same case labels. if (!OpCodeAttr::IsInlineBuiltIn(instr->m_opcode)) { switch (opcode) { case Js::OpCode::ArgOut_A_InlineBuiltIn: skipDst = true; // fall-through case Js::OpCode::Ld_A: case Js::OpCode::BrTrue_A: case Js::OpCode::BrFalse_A: if (instr->GetSrc1()->IsRegOpnd()) { StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym; if (CurrentBlockData()->IsFloat64TypeSpecialized(sym) == false) { // Type specializing an Ld_A isn't worth it, unless the src // is already type specialized return false; } } if (instr->m_opcode == Js::OpCode::Ld_A) { valueToTransfer = src1Val; } break; case Js::OpCode::Neg_A: break; case Js::OpCode::Conv_Num: Assert(src1Val); opcode = Js::OpCode::Ld_A; valueToTransfer = src1Val; if (!src1Val->GetValueInfo()->IsNumber()) { StackSym *sym = instr->GetSrc1()->AsRegOpnd()->m_sym; valueToTransfer = NewGenericValue(ValueType::Float, instr->GetDst()->GetStackSym()); if (CurrentBlockData()->IsFloat64TypeSpecialized(sym) == false) { // Set the dst as a nonDeadStore. We want to keep the Ld_A to prevent the FromVar from // being dead-stored, as it could cause implicit calls. dst = instr->GetDst(); dst->AsRegOpnd()->m_dontDeadStore = true; } } break; case Js::OpCode::StElemI_A: case Js::OpCode::StElemI_A_Strict: case Js::OpCode::StElemC: return TypeSpecializeStElem(pInstr, src1Val, pDstVal); default: return false; } } // Make sure the srcs are specialized src1 = instr->GetSrc1(); // Use original val when calling toFloat64 as this is what we'll use to try hoisting the fromVar if we're in a loop. this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, IR::BailOutPrimitiveButString); if (!skipDst) { dst = instr->GetDst(); if (dst) { this->TypeSpecializeFloatDst(instr, valueToTransfer, src1Val, nullptr, pDstVal); if (!this->IsLoopPrePass()) { instr->m_opcode = opcode; } } } GOPT_TRACE_INSTR(instr, _u("Type specialized to FLOAT: ")); #if ENABLE_DEBUG_CONFIG_OPTIONS if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FloatTypeSpecPhase)) { Output::Print(_u("Type specialized to FLOAT: ")); Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode)); } #endif return true; } // Unconditionally type-spec dst to float. void GlobOpt::TypeSpecializeFloatDst(IR::Instr *instr, Value *valToTransfer, Value *const src1Value, Value *const src2Value, Value **pDstVal) { IR::Opnd* dst = instr->GetDst(); Assert(dst); AssertMsg(dst->IsRegOpnd(), "What else?"); this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock); if(valToTransfer) { *pDstVal = this->ValueNumberTransferDst(instr, valToTransfer); CurrentBlockData()->InsertNewValue(*pDstVal, dst); } else { *pDstVal = CreateDstUntransferredValue(ValueType::Float, instr, src1Value, src2Value); } } bool GlobOpt::TypeSpecializeLdLen( IR::Instr * *const instrRef, Value * *const src1ValueRef, Value * *const dstValueRef, bool *const forceInvariantHoistingRef) { Assert(instrRef); IR::Instr *&instr = *instrRef; Assert(instr); Assert(instr->m_opcode == Js::OpCode::LdLen_A); Assert(src1ValueRef); Value *&src1Value = *src1ValueRef; Assert(dstValueRef); Value *&dstValue = *dstValueRef; Assert(forceInvariantHoistingRef); bool &forceInvariantHoisting = *forceInvariantHoistingRef; if(!DoLdLenIntSpec(instr, instr->GetSrc1()->GetValueType())) { return false; } IR::BailOutKind bailOutKind = IR::BailOutOnIrregularLength; if(!IsLoopPrePass()) { IR::RegOpnd *const baseOpnd = instr->GetSrc1()->AsRegOpnd(); if(baseOpnd->IsArrayRegOpnd()) { StackSym *const lengthSym = baseOpnd->AsArrayRegOpnd()->LengthSym(); if(lengthSym) { CaptureByteCodeSymUses(instr); instr->m_opcode = Js::OpCode::Ld_I4; instr->ReplaceSrc1(IR::RegOpnd::New(lengthSym, lengthSym->GetType(), func)); instr->ClearBailOutInfo(); // Find the hoisted length value Value *const lengthValue = CurrentBlockData()->FindValue(lengthSym); Assert(lengthValue); src1Value = lengthValue; ValueInfo *const lengthValueInfo = lengthValue->GetValueInfo(); IntConstantBounds lengthConstantBounds; AssertVerify(lengthValueInfo->TryGetIntConstantBounds(&lengthConstantBounds)); Assert(lengthConstantBounds.LowerBound() >= 0); if (lengthValueInfo->GetSymStore() == lengthSym) { // When type specializing the dst below, we will end up inserting lengthSym.u32 as symstore for a var // Clear the symstore here, so that we dont end up with problems with copyprop later on lengthValueInfo->SetSymStore(nullptr); } // Int-specialize, and transfer the value to the dst TypeSpecializeIntDst( instr, Js::OpCode::LdLen_A, src1Value, src1Value, nullptr, bailOutKind, lengthConstantBounds.LowerBound(), lengthConstantBounds.UpperBound(), &dstValue); // Try to force hoisting the Ld_I4 so that the length will have an invariant sym store that can be // copy-propped. Invariant hoisting does not automatically hoist Ld_I4. forceInvariantHoisting = true; return true; } } if (instr->HasBailOutInfo()) { Assert(instr->GetBailOutKind() == IR::BailOutMarkTempObject); bailOutKind = IR::BailOutOnIrregularLength | IR::BailOutMarkTempObject; instr->SetBailOutKind(bailOutKind); } else { Assert(bailOutKind == IR::BailOutOnIrregularLength); GenerateBailAtOperation(&instr, bailOutKind); } } TypeSpecializeIntDst( instr, Js::OpCode::LdLen_A, nullptr, nullptr, nullptr, bailOutKind, 0, INT32_MAX, &dstValue); return true; } bool GlobOpt::TypeSpecializeFloatBinary(IR::Instr *instr, Value *src1Val, Value *src2Val, Value **pDstVal) { IR::Opnd *src1; IR::Opnd *src2; IR::Opnd *dst; bool allowUndefinedOrNullSrc1 = true; bool allowUndefinedOrNullSrc2 = true; bool skipSrc1 = false; bool skipSrc2 = false; bool skipDst = false; bool convertDstToBool = false; if (!this->DoFloatTypeSpec()) { return false; } // For inline built-ins we need to do type specialization. Check upfront to avoid duplicating same case labels. if (!OpCodeAttr::IsInlineBuiltIn(instr->m_opcode)) { switch (instr->m_opcode) { case Js::OpCode::Sub_A: case Js::OpCode::Mul_A: case Js::OpCode::Div_A: case Js::OpCode::Expo_A: // Avoid if one source is known not to be a number. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber()) { return false; } break; case Js::OpCode::BrSrEq_A: case Js::OpCode::BrSrNeq_A: case Js::OpCode::BrEq_A: case Js::OpCode::BrNeq_A: case Js::OpCode::BrSrNotEq_A: case Js::OpCode::BrNotEq_A: case Js::OpCode::BrSrNotNeq_A: case Js::OpCode::BrNotNeq_A: // Avoid if one source is known not to be a number. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber()) { return false; } // Undef == Undef, but +Undef != +Undef // 0.0 != null, but 0.0 == +null // // So Bailout on anything but numbers for both src1 and src2 allowUndefinedOrNullSrc1 = false; allowUndefinedOrNullSrc2 = false; break; case Js::OpCode::BrGt_A: case Js::OpCode::BrGe_A: case Js::OpCode::BrLt_A: case Js::OpCode::BrLe_A: case Js::OpCode::BrNotGt_A: case Js::OpCode::BrNotGe_A: case Js::OpCode::BrNotLt_A: case Js::OpCode::BrNotLe_A: // Avoid if one source is known not to be a number. if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber()) { return false; } break; case Js::OpCode::Add_A: // For Add, we need both sources to be Numbers, otherwise it could be a string concat if (!src1Val || !src2Val || !(src1Val->GetValueInfo()->IsLikelyNumber() && src2Val->GetValueInfo()->IsLikelyNumber())) { return false; } break; case Js::OpCode::ArgOut_A_InlineBuiltIn: skipSrc2 = true; skipDst = true; break; case Js::OpCode::CmEq_A: case Js::OpCode::CmSrEq_A: case Js::OpCode::CmNeq_A: case Js::OpCode::CmSrNeq_A: { if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber()) { return false; } allowUndefinedOrNullSrc1 = false; allowUndefinedOrNullSrc2 = false; convertDstToBool = true; break; } case Js::OpCode::CmLe_A: case Js::OpCode::CmLt_A: case Js::OpCode::CmGe_A: case Js::OpCode::CmGt_A: { if (src1Val->GetValueInfo()->IsNotNumber() || src2Val->GetValueInfo()->IsNotNumber()) { return false; } convertDstToBool = true; break; } default: return false; } } else { switch (instr->m_opcode) { case Js::OpCode::InlineArrayPush: bool isFloatConstMissingItem = src2Val->GetValueInfo()->IsFloatConstant(); if(isFloatConstMissingItem) { FloatConstType floatValue = src2Val->GetValueInfo()->AsFloatConstant()->FloatValue(); isFloatConstMissingItem = Js::SparseArraySegment::IsMissingItem(&floatValue); } // Don't specialize if the element is not likelyNumber - we will surely bailout if(!(src2Val->GetValueInfo()->IsLikelyNumber()) || isFloatConstMissingItem) { return false; } // Only specialize the Second source - element skipSrc1 = true; skipDst = true; allowUndefinedOrNullSrc2 = false; break; } } // Make sure the srcs are specialized if(!skipSrc1) { src1 = instr->GetSrc1(); this->ToFloat64(instr, src1, this->currentBlock, src1Val, nullptr, (allowUndefinedOrNullSrc1 ? IR::BailOutPrimitiveButString : IR::BailOutNumberOnly)); } if (!skipSrc2) { src2 = instr->GetSrc2(); this->ToFloat64(instr, src2, this->currentBlock, src2Val, nullptr, (allowUndefinedOrNullSrc2 ? IR::BailOutPrimitiveButString : IR::BailOutNumberOnly)); } if (!skipDst) { dst = instr->GetDst(); if (dst) { if (convertDstToBool) { *pDstVal = CreateDstUntransferredValue(ValueType::Boolean, instr, src1Val, src2Val); ToVarRegOpnd(dst->AsRegOpnd(), currentBlock); } else { *pDstVal = CreateDstUntransferredValue(ValueType::Float, instr, src1Val, src2Val); AssertMsg(dst->IsRegOpnd(), "What else?"); this->ToFloat64Dst(instr, dst->AsRegOpnd(), this->currentBlock); } } } GOPT_TRACE_INSTR(instr, _u("Type specialized to FLOAT: ")); #if ENABLE_DEBUG_CONFIG_OPTIONS if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::FloatTypeSpecPhase)) { Output::Print(_u("Type specialized to FLOAT: ")); Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode)); } #endif return true; } bool GlobOpt::TypeSpecializeStElem(IR::Instr ** pInstr, Value *src1Val, Value **pDstVal) { IR::Instr *&instr = *pInstr; IR::RegOpnd *baseOpnd = instr->GetDst()->AsIndirOpnd()->GetBaseOpnd(); ValueType baseValueType(baseOpnd->GetValueType()); if (instr->DoStackArgsOpt() || (!this->DoTypedArrayTypeSpec() && baseValueType.IsLikelyOptimizedTypedArray()) || (!this->DoNativeArrayTypeSpec() && baseValueType.IsLikelyNativeArray()) || !(baseValueType.IsLikelyOptimizedTypedArray() || baseValueType.IsLikelyNativeArray())) { GOPT_TRACE_INSTR(instr, _u("Didn't type specialize array access, because typed array type specialization is disabled, or base is not an optimized typed array.\n")); if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func)) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; baseValueType.ToString(baseValueTypeStr); Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because %s.\n"), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode), baseValueTypeStr, instr->DoStackArgsOpt() ? _u("instruction uses the arguments object") : _u("typed array type specialization is disabled, or base is not an optimized typed array")); Output::Flush(); } return false; } Assert(instr->GetSrc1()->IsRegOpnd() || (src1Val && src1Val->GetValueInfo()->HasIntConstantValue())); StackSym *sym = instr->GetSrc1()->IsRegOpnd() ? instr->GetSrc1()->AsRegOpnd()->m_sym : nullptr; // Only type specialize the source of store element if the source symbol is already type specialized to int or float. if (sym) { if (baseValueType.IsLikelyNativeArray()) { // Gently coerce these src's into native if it seems likely to work. // Otherwise we can't use the fast path to store. // But don't try to put a float-specialized number into an int array this way. if (!( CurrentBlockData()->IsInt32TypeSpecialized(sym) || ( src1Val && ( DoAggressiveIntTypeSpec() ? src1Val->GetValueInfo()->IsLikelyInt() : src1Val->GetValueInfo()->IsInt() ) ) )) { if (!( CurrentBlockData()->IsFloat64TypeSpecialized(sym) || (src1Val && src1Val->GetValueInfo()->IsLikelyNumber()) ) || baseValueType.HasIntElements()) { return false; } } } else if (!CurrentBlockData()->IsInt32TypeSpecialized(sym) && !CurrentBlockData()->IsFloat64TypeSpecialized(sym)) { GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because src is not type specialized.\n")); if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func)) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; baseValueType.ToString(baseValueTypeStr); Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because src is not specialized.\n"), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode), baseValueTypeStr); Output::Flush(); } return false; } } int32 src1IntConstantValue; if(baseValueType.IsLikelyNativeIntArray() && src1Val && src1Val->GetValueInfo()->TryGetIntConstantValue(&src1IntConstantValue)) { if(Js::SparseArraySegment::IsMissingItem(&src1IntConstantValue)) { return false; } } // Note: doing ToVarUses to make sure we do get the int32 version of the index before trying to access its value in // ShouldExpectConventionalArrayIndexValue. Not sure why that never gave us a problem before. Assert(instr->GetDst()->IsIndirOpnd()); IR::IndirOpnd *dst = instr->GetDst()->AsIndirOpnd(); // Make sure we use the int32 version of the index operand symbol, if available. Otherwise, ensure the var symbol is live (by // potentially inserting a ToVar). this->ToVarUses(instr, dst, /* isDst = */ true, nullptr); if (!ShouldExpectConventionalArrayIndexValue(dst)) { GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because index is negative or likely not int.\n")); if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func)) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; baseValueType.ToString(baseValueTypeStr); Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not specialize because index is negative or likely not int.\n"), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode), baseValueTypeStr); Output::Flush(); } return false; } IRType toType = TyVar; bool isLossyAllowed = true; IR::BailOutKind arrayBailOutKind = IR::BailOutConventionalTypedArrayAccessOnly; switch(baseValueType.GetObjectType()) { case ObjectType::Int8Array: case ObjectType::Uint8Array: case ObjectType::Int16Array: case ObjectType::Uint16Array: case ObjectType::Int32Array: case ObjectType::Int8VirtualArray: case ObjectType::Uint8VirtualArray: case ObjectType::Int16VirtualArray: case ObjectType::Uint16VirtualArray: case ObjectType::Int32VirtualArray: case ObjectType::Int8MixedArray: case ObjectType::Uint8MixedArray: case ObjectType::Int16MixedArray: case ObjectType::Uint16MixedArray: case ObjectType::Int32MixedArray: Int32Array: if (this->DoAggressiveIntTypeSpec() || this->DoFloatTypeSpec()) { toType = TyInt32; } break; case ObjectType::Uint32Array: case ObjectType::Uint32VirtualArray: case ObjectType::Uint32MixedArray: // Uint32Arrays may store values that overflow int32. If the value being stored comes from a symbol that's // already losslessly type specialized to int32, we'll use it. Otherwise, if we only have a float64 specialized // value, we don't want to force bailout if it doesn't fit in int32. Instead, we'll emit conversion in the // lowerer, and handle overflow, if necessary. if (!sym || CurrentBlockData()->IsInt32TypeSpecialized(sym)) { toType = TyInt32; } else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym)) { toType = TyFloat64; } break; case ObjectType::Float32Array: case ObjectType::Float64Array: case ObjectType::Float32VirtualArray: case ObjectType::Float32MixedArray: case ObjectType::Float64VirtualArray: case ObjectType::Float64MixedArray: Float64Array: if (this->DoFloatTypeSpec()) { toType = TyFloat64; } break; case ObjectType::Uint8ClampedArray: case ObjectType::Uint8ClampedVirtualArray: case ObjectType::Uint8ClampedMixedArray: // Uint8ClampedArray requires rounding (as opposed to truncation) of floating point values. If source symbol is // float type specialized, type specialize this instruction to float as well, and handle rounding in the // lowerer. if (!sym || CurrentBlockData()->IsInt32TypeSpecialized(sym)) { toType = TyInt32; isLossyAllowed = false; } else if (CurrentBlockData()->IsFloat64TypeSpecialized(sym)) { toType = TyFloat64; } break; default: Assert(baseValueType.IsLikelyNativeArray()); isLossyAllowed = false; arrayBailOutKind = IR::BailOutConventionalNativeArrayAccessOnly; if(baseValueType.HasIntElements()) { goto Int32Array; } Assert(baseValueType.HasFloatElements()); goto Float64Array; } if (toType != TyVar) { GOPT_TRACE_INSTR(instr, _u("Type specialized array access.\n")); if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func)) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; baseValueType.ToString(baseValueTypeStr); Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, type specialized to %s.\n"), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode), baseValueTypeStr, toType == TyInt32 ? _u("int32") : _u("float64")); Output::Flush(); } IR::BailOutKind bailOutKind = ((toType == TyInt32) ? IR::BailOutIntOnly : IR::BailOutNumberOnly); this->ToTypeSpecUse(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, toType, bailOutKind, /* lossy = */ isLossyAllowed); if (!this->IsLoopPrePass()) { bool bConvertToBailoutInstr = true; // Definite StElemC doesn't need bailout, because it can't fail or cause conversion. if (instr->m_opcode == Js::OpCode::StElemC && baseValueType.IsObject()) { if (baseValueType.HasIntElements()) { //Native int array requires a missing element check & bailout int32 min = INT32_MIN; int32 max = INT32_MAX; if (src1Val->GetValueInfo()->GetIntValMinMax(&min, &max, false)) { bConvertToBailoutInstr = ((min <= Js::JavascriptNativeIntArray::MissingItem) && (max >= Js::JavascriptNativeIntArray::MissingItem)); } } else { bConvertToBailoutInstr = false; } } if (bConvertToBailoutInstr) { if(instr->HasBailOutInfo()) { const IR::BailOutKind oldBailOutKind = instr->GetBailOutKind(); Assert( ( !(oldBailOutKind & ~IR::BailOutKindBits) || (oldBailOutKind & ~IR::BailOutKindBits) == IR::BailOutOnImplicitCallsPreOp ) && !(oldBailOutKind & IR::BailOutKindBits & ~(IR::BailOutOnArrayAccessHelperCall | IR::BailOutMarkTempObject))); if(arrayBailOutKind == IR::BailOutConventionalTypedArrayAccessOnly) { // BailOutConventionalTypedArrayAccessOnly also bails out if the array access is outside the head // segment bounds, and guarantees no implicit calls. Override the bailout kind so that the instruction // bails out for the right reason. instr->SetBailOutKind( arrayBailOutKind | (oldBailOutKind & (IR::BailOutKindBits - IR::BailOutOnArrayAccessHelperCall))); } else { // BailOutConventionalNativeArrayAccessOnly by itself may generate a helper call, and may cause implicit // calls to occur, so it must be merged in to eliminate generating the helper call. Assert(arrayBailOutKind == IR::BailOutConventionalNativeArrayAccessOnly); instr->SetBailOutKind(oldBailOutKind | arrayBailOutKind); } } else { GenerateBailAtOperation(&instr, arrayBailOutKind); } } } } else { GOPT_TRACE_INSTR(instr, _u("Didn't specialize array access, because the source was not already specialized.\n")); if (PHASE_TRACE(Js::TypedArrayTypeSpecPhase, this->func)) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; char baseValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; baseValueType.ToString(baseValueTypeStr); Output::Print(_u("Typed Array Optimization: function: %s (%s): instr: %s, base value type: %S, did not type specialize, because of array type.\n"), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode), baseValueTypeStr); Output::Flush(); } } return toType != TyVar; } IR::Instr * GlobOpt::ToVarUses(IR::Instr *instr, IR::Opnd *opnd, bool isDst, Value *val) { Sym *sym; switch (opnd->GetKind()) { case IR::OpndKindReg: if (!isDst && !CurrentBlockData()->liveVarSyms->Test(opnd->AsRegOpnd()->m_sym->m_id)) { instr = this->ToVar(instr, opnd->AsRegOpnd(), this->currentBlock, val, true); } break; case IR::OpndKindSym: sym = opnd->AsSymOpnd()->m_sym; if (sym->IsPropertySym() && !CurrentBlockData()->liveVarSyms->Test(sym->AsPropertySym()->m_stackSym->m_id) && sym->AsPropertySym()->m_stackSym->IsVar()) { StackSym *propertyBase = sym->AsPropertySym()->m_stackSym; IR::RegOpnd *newOpnd = IR::RegOpnd::New(propertyBase, TyVar, instr->m_func); instr = this->ToVar(instr, newOpnd, this->currentBlock, CurrentBlockData()->FindValue(propertyBase), true); } break; case IR::OpndKindIndir: IR::RegOpnd *baseOpnd = opnd->AsIndirOpnd()->GetBaseOpnd(); if (!CurrentBlockData()->liveVarSyms->Test(baseOpnd->m_sym->m_id)) { instr = this->ToVar(instr, baseOpnd, this->currentBlock, CurrentBlockData()->FindValue(baseOpnd->m_sym), true); } IR::RegOpnd *indexOpnd = opnd->AsIndirOpnd()->GetIndexOpnd(); if (indexOpnd && !indexOpnd->m_sym->IsTypeSpec()) { instr = ToTypeSpecIndex(instr, indexOpnd, opnd->AsIndirOpnd()); } break; } return instr; } IR::Instr * GlobOpt::ToTypeSpecIndex(IR::Instr * instr, IR::RegOpnd * indexOpnd, IR::IndirOpnd * indirOpnd) { Assert(indirOpnd != nullptr || indexOpnd == instr->GetSrc1()); bool isGetterOrSetter = instr->m_opcode == Js::OpCode::InitGetElemI || instr->m_opcode == Js::OpCode::InitSetElemI || instr->m_opcode == Js::OpCode::InitClassMemberGetComputedName || instr->m_opcode == Js::OpCode::InitClassMemberSetComputedName; if (!isGetterOrSetter // typespec is disabled for getters, setters && (indexOpnd->GetValueType().IsInt() ? !IsTypeSpecPhaseOff(func) : indexOpnd->GetValueType().IsLikelyInt() && DoAggressiveIntTypeSpec()) && !GetIsAsmJSFunc()) // typespec is disabled for asmjs { StackSym *const indexVarSym = indexOpnd->m_sym; Value *const indexValue = CurrentBlockData()->FindValue(indexVarSym); Assert(indexValue); Assert(indexValue->GetValueInfo()->IsLikelyInt()); ToInt32(instr, indexOpnd, currentBlock, indexValue, indirOpnd, false); Assert(indexValue->GetValueInfo()->IsInt() || IsLoopPrePass()); if (!IsLoopPrePass()) { IR::Opnd * intOpnd = indirOpnd ? indirOpnd->GetIndexOpnd() : instr->GetSrc1(); if (intOpnd != nullptr) { Assert(!intOpnd->IsRegOpnd() || intOpnd->AsRegOpnd()->m_sym->IsTypeSpec()); IntConstantBounds indexConstantBounds; AssertVerify(indexValue->GetValueInfo()->TryGetIntConstantBounds(&indexConstantBounds)); if (ValueInfo::IsGreaterThanOrEqualTo( indexValue, indexConstantBounds.LowerBound(), indexConstantBounds.UpperBound(), nullptr, 0, 0)) { intOpnd->SetType(TyUint32); } } } } else if (!CurrentBlockData()->liveVarSyms->Test(indexOpnd->m_sym->m_id)) { instr = this->ToVar(instr, indexOpnd, this->currentBlock, CurrentBlockData()->FindValue(indexOpnd->m_sym), true); } return instr; } IR::Instr * GlobOpt::ToVar(IR::Instr *instr, IR::RegOpnd *regOpnd, BasicBlock *block, Value *value, bool needsUpdate) { IR::Instr *newInstr; StackSym *varSym = regOpnd->m_sym; if (IsTypeSpecPhaseOff(this->func)) { return instr; } if (this->IsLoopPrePass()) { block->globOptData.liveVarSyms->Set(varSym->m_id); return instr; } if (block->globOptData.liveVarSyms->Test(varSym->m_id)) { // Already live, nothing to do return instr; } if (!varSym->IsVar()) { Assert(!varSym->IsTypeSpec()); // Leave non-vars alone. return instr; } Assert(block->globOptData.IsTypeSpecialized(varSym)); if (!value) { value = block->globOptData.FindValue(varSym); } ValueInfo *valueInfo = value ? value->GetValueInfo() : nullptr; if(valueInfo && valueInfo->IsInt()) { // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the // lossy state. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); } IRType fromType = TyIllegal; StackSym *typeSpecSym = nullptr; if (block->globOptData.liveInt32Syms->Test(varSym->m_id) && !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id)) { fromType = TyInt32; typeSpecSym = varSym->GetInt32EquivSym(this->func); Assert(valueInfo); Assert(valueInfo->IsInt()); } else if (block->globOptData.liveFloat64Syms->Test(varSym->m_id)) { fromType = TyFloat64; typeSpecSym = varSym->GetFloat64EquivSym(this->func); // Ensure that all bailout FromVars that generate a value for this type-specialized sym will bail out on any non-number // value, even ones that have already been generated before. Float-specialized non-number values cannot be converted // back to Var since they will not go back to the original non-number value. The dead-store pass will update the bailout // kind on already-generated FromVars based on this bit. typeSpecSym->m_requiresBailOnNotNumber = true; // A previous float conversion may have used BailOutPrimitiveButString, which does not change the value type to say // definitely float, since it can also be a non-string primitive. The convert back to Var though, will cause that // bailout kind to be changed to BailOutNumberOnly in the dead-store phase, so from the point of the initial conversion // to float, that the value is definitely number. Since we don't know where the FromVar is, change the value type here. if(valueInfo) { if(!valueInfo->IsNumber()) { valueInfo = valueInfo->SpecializeToFloat64(alloc); ChangeValueInfo(block, value, valueInfo); regOpnd->SetValueType(valueInfo->Type()); } } else { value = NewGenericValue(ValueType::Float); valueInfo = value->GetValueInfo(); block->globOptData.SetValue(value, varSym); regOpnd->SetValueType(valueInfo->Type()); } } else { Assert(UNREACHED); } AssertOrFailFast(valueInfo); int32 intConstantValue; if (valueInfo->TryGetIntConstantValue(&intConstantValue)) { // Lower will tag or create a number directly newInstr = IR::Instr::New(Js::OpCode::LdC_A_I4, regOpnd, IR::IntConstOpnd::New(intConstantValue, TyInt32, instr->m_func), instr->m_func); } else { IR::RegOpnd * regNew = IR::RegOpnd::New(typeSpecSym, fromType, instr->m_func); Js::OpCode opcode = Js::OpCode::ToVar; regNew->SetIsJITOptimizedReg(true); newInstr = IR::Instr::New(opcode, regOpnd, regNew, instr->m_func); } newInstr->SetByteCodeOffset(instr); newInstr->GetDst()->AsRegOpnd()->SetIsJITOptimizedReg(true); ValueType valueType = valueInfo->Type(); if(fromType == TyInt32) { #if !INT32VAR // All 32-bit ints are taggable on 64-bit architectures IntConstantBounds constantBounds; AssertVerify(valueInfo->TryGetIntConstantBounds(&constantBounds)); if(constantBounds.IsTaggable()) #endif { // The value is within the taggable range, so set the opnd value types to TaggedInt to avoid the overflow check valueType = ValueType::GetTaggedInt(); } } newInstr->GetDst()->SetValueType(valueType); newInstr->GetSrc1()->SetValueType(valueType); IR::Instr *insertAfterInstr = instr->m_prev; if (instr == block->GetLastInstr() && (instr->IsBranchInstr() || instr->m_opcode == Js::OpCode::BailTarget)) { // Don't insert code between the branch and the preceding ByteCodeUses instrs... while(insertAfterInstr->m_opcode == Js::OpCode::ByteCodeUses) { insertAfterInstr = insertAfterInstr->m_prev; } } block->InsertInstrAfter(newInstr, insertAfterInstr); block->globOptData.liveVarSyms->Set(varSym->m_id); GOPT_TRACE_OPND(regOpnd, _u("Converting to var\n")); if (block->loop) { Assert(!this->IsLoopPrePass()); this->TryHoistInvariant(newInstr, block, value, value, nullptr, false); } if (needsUpdate) { // Make sure that the kill effect of the ToVar instruction is tracked and that the kill of a property // type is reflected in the current instruction. this->ProcessKills(newInstr); this->ValueNumberObjectType(newInstr->GetDst(), newInstr); if (instr->GetSrc1() && instr->GetSrc1()->IsSymOpnd() && instr->GetSrc1()->AsSymOpnd()->IsPropertySymOpnd()) { // Reprocess the load source. We need to reset the PropertySymOpnd fields first. IR::PropertySymOpnd *propertySymOpnd = instr->GetSrc1()->AsPropertySymOpnd(); if (propertySymOpnd->IsTypeCheckSeqCandidate()) { propertySymOpnd->SetTypeChecked(false); propertySymOpnd->SetTypeAvailable(false); propertySymOpnd->SetWriteGuardChecked(false); } this->FinishOptPropOp(instr, propertySymOpnd); instr = this->SetTypeCheckBailOut(instr->GetSrc1(), instr, nullptr); } } return instr; } IR::Instr * GlobOpt::ToInt32(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, bool lossy) { return this->ToTypeSpecUse(instr, opnd, block, val, indir, TyInt32, IR::BailOutIntOnly, lossy); } IR::Instr * GlobOpt::ToFloat64(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, IR::BailOutKind bailOutKind) { return this->ToTypeSpecUse(instr, opnd, block, val, indir, TyFloat64, bailOutKind); } IR::Instr * GlobOpt::ToTypeSpecUse(IR::Instr *instr, IR::Opnd *opnd, BasicBlock *block, Value *val, IR::IndirOpnd *indir, IRType toType, IR::BailOutKind bailOutKind, bool lossy, IR::Instr *insertBeforeInstr) { Assert(bailOutKind != IR::BailOutInvalid); IR::Instr *newInstr; if (!val && opnd->IsRegOpnd()) { val = block->globOptData.FindValue(opnd->AsRegOpnd()->m_sym); } ValueInfo *valueInfo = val ? val->GetValueInfo() : nullptr; bool needReplaceSrc = false; bool updateBlockLastInstr = false; if (instr) { needReplaceSrc = true; if (!insertBeforeInstr) { insertBeforeInstr = instr; } } else if (!insertBeforeInstr) { // Insert it at the end of the block insertBeforeInstr = block->GetLastInstr(); if (insertBeforeInstr->IsBranchInstr() || insertBeforeInstr->m_opcode == Js::OpCode::BailTarget) { // Don't insert code between the branch and the preceding ByteCodeUses instrs... while(insertBeforeInstr->m_prev->m_opcode == Js::OpCode::ByteCodeUses) { insertBeforeInstr = insertBeforeInstr->m_prev; } } else { insertBeforeInstr = insertBeforeInstr->m_next; updateBlockLastInstr = true; } } // Int constant values will be propagated into the instruction. For ArgOut_A_InlineBuiltIn, there's no benefit from // const-propping, so those are excluded. if (opnd->IsRegOpnd() && !( valueInfo && (valueInfo->HasIntConstantValue() || valueInfo->IsFloatConstant()) && (!instr || instr->m_opcode != Js::OpCode::ArgOut_A_InlineBuiltIn) )) { IR::RegOpnd *regSrc = opnd->AsRegOpnd(); StackSym *varSym = regSrc->m_sym; Js::OpCode opcode = Js::OpCode::FromVar; if (varSym->IsTypeSpec() || !block->globOptData.liveVarSyms->Test(varSym->m_id)) { // Conversion between int32 and float64 if (varSym->IsTypeSpec()) { varSym = varSym->GetVarEquivSym(this->func); } opcode = Js::OpCode::Conv_Prim; } Assert(block->globOptData.liveVarSyms->Test(varSym->m_id) || block->globOptData.IsTypeSpecialized(varSym)); StackSym *typeSpecSym = nullptr; BOOL isLive = FALSE; BVSparse *livenessBv = nullptr; if(valueInfo && valueInfo->IsInt()) { // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the // lossy state. block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); } if (toType == TyInt32) { // Need to determine whether the conversion is actually lossy or lossless. If the value is an int, then it's a // lossless conversion despite the type of conversion requested. The liveness of the converted int32 sym needs to be // set to reflect the actual type of conversion done. Also, a lossless conversion needs the value to determine // whether the conversion may need to bail out. Assert(valueInfo); if(valueInfo->IsInt()) { lossy = false; } else { Assert(IsLoopPrePass() || !block->globOptData.IsInt32TypeSpecialized(varSym)); } livenessBv = block->globOptData.liveInt32Syms; isLive = livenessBv->Test(varSym->m_id) && (lossy || !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id)); if (this->IsLoopPrePass()) { if (!isLive) { livenessBv->Set(varSym->m_id); if (lossy) { block->globOptData.liveLossyInt32Syms->Set(varSym->m_id); } else { block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); } } return instr; } typeSpecSym = varSym->GetInt32EquivSym(this->func); if (!isLive) { if (!opnd->IsVar() || !block->globOptData.liveVarSyms->Test(varSym->m_id) || (block->globOptData.liveFloat64Syms->Test(varSym->m_id) && valueInfo && valueInfo->IsLikelyFloat())) { Assert(block->globOptData.liveFloat64Syms->Test(varSym->m_id)); if(!lossy && !valueInfo->IsInt()) { // Shouldn't try to do a lossless conversion from float64 to int32 when the value is not known to be an // int. There are cases where we need more than two passes over loops to flush out all dependencies. // It's possible for the loop prepass to think that a sym s1 remains an int because it acquires the // value of another sym s2 that is an int in the prepass at that time. However, s2 can become a float // later in the loop body, in which case s1 would become a float on the second iteration of the loop. By // that time, we would have already committed to having s1 live as a lossless int on entry into the // loop, and we end up having to compensate by doing a lossless conversion from float to int, which will // need a bailout and will most likely bail out. // // If s2 becomes a var instead of a float, then the compensation is legal although not ideal. After // enough bailouts, rejit would be triggered with aggressive int type spec turned off. For the // float-to-int conversion though, there's no point in emitting a bailout because we already know that // the value is a float and has high probability of bailing out (whereas a var has a chance to be a // tagged int), and so currently lossless conversion from float to int with bailout is not supported. // // So, treating this case as a compile-time bailout. The exception will trigger the jit work item to be // restarted with aggressive int type specialization disabled. if(bailOutKind == IR::BailOutExpectingInteger) { Assert(IsSwitchOptEnabledForIntTypeSpec()); throw Js::RejitException(RejitReason::DisableSwitchOptExpectingInteger); } else { Assert(DoAggressiveIntTypeSpec()); if(PHASE_TRACE(Js::BailOutPhase, this->func)) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; Output::Print( _u("BailOut (compile-time): function: %s (%s) varSym: "), this->func->GetJITFunctionBody()->GetDisplayName(), this->func->GetDebugNumberSet(debugStringBuffer), varSym->m_id); #if DBG_DUMP varSym->Dump(); #else Output::Print(_u("s%u"), varSym->m_id); #endif if(varSym->HasByteCodeRegSlot()) { Output::Print(_u(" byteCodeReg: R%u"), varSym->GetByteCodeRegSlot()); } Output::Print(_u(" (lossless conversion from float64 to int32)\n")); Output::Flush(); } if(!DoAggressiveIntTypeSpec()) { // Aggressive int type specialization is already off for some reason. Prevent trying to rejit again // because it won't help and the same thing will happen again. Just abort jitting this function. if(PHASE_TRACE(Js::BailOutPhase, this->func)) { Output::Print(_u(" Aborting JIT because AggressiveIntTypeSpec is already off\n")); Output::Flush(); } throw Js::OperationAbortedException(); } throw Js::RejitException(RejitReason::AggressiveIntTypeSpecDisabled); } } if(opnd->IsVar()) { regSrc->SetType(TyFloat64); regSrc->m_sym = varSym->GetFloat64EquivSym(this->func); opcode = Js::OpCode::Conv_Prim; } else { Assert(regSrc->IsFloat64()); Assert(regSrc->m_sym->IsFloat64()); Assert(opcode == Js::OpCode::Conv_Prim); } } } GOPT_TRACE_OPND(regSrc, _u("Converting to int32\n")); } else if (toType == TyFloat64) { // float64 typeSpecSym = varSym->GetFloat64EquivSym(this->func); if(!IsLoopPrePass() && typeSpecSym->m_requiresBailOnNotNumber && block->globOptData.IsFloat64TypeSpecialized(varSym)) { // This conversion is already protected by a BailOutNumberOnly bailout (or at least it will be after the // dead-store phase). Since 'requiresBailOnNotNumber' is not flow-based, change the value to definitely float. if(valueInfo) { if(!valueInfo->IsNumber()) { valueInfo = valueInfo->SpecializeToFloat64(alloc); ChangeValueInfo(block, val, valueInfo); opnd->SetValueType(valueInfo->Type()); } } else { val = NewGenericValue(ValueType::Float); valueInfo = val->GetValueInfo(); block->globOptData.SetValue(val, varSym); opnd->SetValueType(valueInfo->Type()); } } if(bailOutKind == IR::BailOutNumberOnly) { if(!IsLoopPrePass()) { // Ensure that all bailout FromVars that generate a value for this type-specialized sym will bail out on any // non-number value, even ones that have already been generated before. The dead-store pass will update the // bailout kind on already-generated FromVars based on this bit. typeSpecSym->m_requiresBailOnNotNumber = true; } } else if(typeSpecSym->m_requiresBailOnNotNumber) { Assert(bailOutKind == IR::BailOutPrimitiveButString); bailOutKind = IR::BailOutNumberOnly; } livenessBv = block->globOptData.liveFloat64Syms; isLive = livenessBv->Test(varSym->m_id); if (this->IsLoopPrePass()) { if(!isLive) { livenessBv->Set(varSym->m_id); } if (this->OptIsInvariant(opnd, block, this->prePassLoop, val, false, true)) { this->prePassLoop->forceFloat64SymsOnEntry->Set(varSym->m_id); } else { Sym *symStore = (valueInfo ? valueInfo->GetSymStore() : NULL); if (symStore && symStore != varSym && this->OptIsInvariant(symStore, block, this->prePassLoop, block->globOptData.FindValue(symStore), false, true)) { // If symStore is assigned to sym and we want sym to be type-specialized, for symStore to be specialized // outside the loop. this->prePassLoop->forceFloat64SymsOnEntry->Set(symStore->m_id); } } return instr; } if (!isLive && regSrc->IsVar()) { if (!block->globOptData.liveVarSyms->Test(varSym->m_id) || ( block->globOptData.liveInt32Syms->Test(varSym->m_id) && !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id) && valueInfo && valueInfo->IsLikelyInt() )) { Assert(block->globOptData.liveInt32Syms->Test(varSym->m_id)); Assert(!block->globOptData.liveLossyInt32Syms->Test(varSym->m_id)); // Shouldn't try to convert a lossy int32 to anything regSrc->SetType(TyInt32); regSrc->m_sym = varSym->GetInt32EquivSym(this->func); opcode = Js::OpCode::Conv_Prim; } } GOPT_TRACE_OPND(regSrc, _u("Converting to float64\n")); } bool needLoad = false; if (needReplaceSrc) { bool wasDead = regSrc->GetIsDead(); // needReplaceSrc means we are type specializing a use, and need to replace the src on the instr if (!isLive) { needLoad = true; // ReplaceSrc will delete it. regSrc = regSrc->Copy(instr->m_func)->AsRegOpnd(); } IR::RegOpnd * regNew = IR::RegOpnd::New(typeSpecSym, toType, instr->m_func); if(valueInfo) { regNew->SetValueType(valueInfo->Type()); regNew->m_wasNegativeZeroPreventedByBailout = valueInfo->WasNegativeZeroPreventedByBailout(); } regNew->SetIsDead(wasDead); regNew->SetIsJITOptimizedReg(true); this->CaptureByteCodeSymUses(instr); if (indir == nullptr) { instr->ReplaceSrc(opnd, regNew); } else { indir->ReplaceIndexOpnd(regNew); } opnd = regNew; if (!needLoad) { Assert(isLive); return instr; } } else { // We just need to insert a load of a type spec sym if(isLive) { return instr; } // Insert it before the specified instruction instr = insertBeforeInstr; } IR::RegOpnd *regDst = IR::RegOpnd::New(typeSpecSym, toType, instr->m_func); bool isBailout = false; bool isHoisted = false; bool isInLandingPad = (block->next && !block->next->isDeleted && block->next->isLoopHeader); if (isInLandingPad) { Loop *loop = block->next->loop; Assert(loop && loop->landingPad == block); Assert(loop->bailOutInfo); } if (opcode == Js::OpCode::FromVar) { if (toType == TyInt32) { Assert(valueInfo); if (lossy) { if (!valueInfo->IsPrimitive() && !block->globOptData.IsTypeSpecialized(varSym)) { // Lossy conversions to int32 on non-primitive values may have implicit calls to toString or valueOf, which // may be overridden to have a side effect. The side effect needs to happen every time the conversion is // supposed to happen, so the resulting lossy int32 value cannot be reused. Bail out on implicit calls. Assert(DoLossyIntTypeSpec()); bailOutKind = IR::BailOutOnNotPrimitive; isBailout = true; } } else if (!valueInfo->IsInt()) { // The operand is likely an int (hence the request to convert to int), so bail out if it's not an int. Only // bail out if a lossless conversion to int is requested. Lossy conversions to int such as in (a | 0) don't // need to bail out. if (bailOutKind == IR::BailOutExpectingInteger) { Assert(IsSwitchOptEnabledForIntTypeSpec()); } else { Assert(DoAggressiveIntTypeSpec()); } isBailout = true; } } else if (toType == TyFloat64 && (!valueInfo || !valueInfo->IsNumber())) { // Bailout if converting vars to float if we can't prove they are floats: // x = str + float; -> need to bailout if str is a string // // x = obj * 0.1; // y = obj * 0.2; -> if obj has valueof, we'll only call valueof once on the FromVar conversion... Assert(bailOutKind != IR::BailOutInvalid); isBailout = true; } } if (isBailout) { if (isInLandingPad) { Loop *loop = block->next->loop; this->EnsureBailTarget(loop); instr = loop->bailOutInfo->bailOutInstr; updateBlockLastInstr = false; newInstr = IR::BailOutInstr::New(opcode, bailOutKind, loop->bailOutInfo, instr->m_func); newInstr->SetDst(regDst); newInstr->SetSrc1(regSrc); } else { newInstr = IR::BailOutInstr::New(opcode, regDst, regSrc, bailOutKind, instr, instr->m_func); } } else { newInstr = IR::Instr::New(opcode, regDst, regSrc, instr->m_func); } newInstr->SetByteCodeOffset(instr); instr->InsertBefore(newInstr); if (updateBlockLastInstr) { block->SetLastInstr(newInstr); } regDst->SetIsJITOptimizedReg(true); newInstr->GetSrc1()->AsRegOpnd()->SetIsJITOptimizedReg(true); ValueInfo *const oldValueInfo = valueInfo; if(valueInfo) { newInstr->GetSrc1()->SetValueType(valueInfo->Type()); } if(isBailout) { Assert(opcode == Js::OpCode::FromVar); if(toType == TyInt32) { Assert(valueInfo); if(!lossy) { Assert(bailOutKind == IR::BailOutIntOnly || bailOutKind == IR::BailOutExpectingInteger); valueInfo = valueInfo->SpecializeToInt32(alloc, isPerformingLoopBackEdgeCompensation); ChangeValueInfo(nullptr, val, valueInfo); int32 intConstantValue; if(indir && needReplaceSrc && valueInfo->TryGetIntConstantValue(&intConstantValue)) { // A likely-int value can have constant bounds due to conditional branches narrowing its range. Now that // the sym has been proven to be an int, the likely-int value, after specialization, will be constant. // Replace the index opnd in the indir with an offset. Assert(opnd == indir->GetIndexOpnd()); Assert(indir->GetScale() == 0); indir->UnlinkIndexOpnd()->Free(instr->m_func); opnd = nullptr; indir->SetOffset(intConstantValue); } } } else if (toType == TyFloat64) { if(bailOutKind == IR::BailOutNumberOnly) { if(valueInfo) { valueInfo = valueInfo->SpecializeToFloat64(alloc); ChangeValueInfo(block, val, valueInfo); } else { val = NewGenericValue(ValueType::Float); valueInfo = val->GetValueInfo(); block->globOptData.SetValue(val, varSym); } } } else { Assert(UNREACHED); } } if(valueInfo) { newInstr->GetDst()->SetValueType(valueInfo->Type()); if(needReplaceSrc && opnd) { opnd->SetValueType(valueInfo->Type()); } } if (block->loop) { Assert(!this->IsLoopPrePass()); isHoisted = this->TryHoistInvariant(newInstr, block, val, val, nullptr, false, lossy, false, bailOutKind); } if (isBailout) { if (!isHoisted && !isInLandingPad) { if(valueInfo) { // Since this is a pre-op bailout, the old value info should be used for the purposes of bailout. For // instance, the value info could be LikelyInt but with a constant range. Once specialized to int, the value // info would be an int constant. However, the int constant is only guaranteed if the value is actually an // int, which this conversion is verifying, so bailout cannot assume the constant value. if(oldValueInfo) { val->SetValueInfo(oldValueInfo); } else { block->globOptData.ClearSymValue(varSym); } } // Fill in bail out info if the FromVar is a bailout instr, and it wasn't hoisted as invariant. // If it was hoisted, the invariant code will fill out the bailout info with the loop landing pad bailout info. this->FillBailOutInfo(block, newInstr); if(valueInfo) { // Restore the new value info after filling the bailout info if(oldValueInfo) { val->SetValueInfo(valueInfo); } else { block->globOptData.SetValue(val, varSym); } } } } // Now that we've captured the liveness in the bailout info, we can mark this as live. // This type specialized sym isn't live if the FromVar bails out. livenessBv->Set(varSym->m_id); if(toType == TyInt32) { if(lossy) { block->globOptData.liveLossyInt32Syms->Set(varSym->m_id); } else { block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); } } } else { Assert(valueInfo); if(opnd->IsRegOpnd() && valueInfo->IsInt()) { // If two syms have the same value, one is lossy-int-specialized, and then the other is int-specialized, the value // would have been updated to definitely int. Upon using the lossy-int-specialized sym later, it would be flagged as // lossy while the value is definitely int. Since the bit-vectors are based on the sym and not the value, update the // lossy state. block->globOptData.liveLossyInt32Syms->Clear(opnd->AsRegOpnd()->m_sym->m_id); if(toType == TyInt32) { lossy = false; } } if (this->IsLoopPrePass()) { if(opnd->IsRegOpnd()) { StackSym *const sym = opnd->AsRegOpnd()->m_sym; if(toType == TyInt32) { Assert(!sym->IsTypeSpec()); block->globOptData.liveInt32Syms->Set(sym->m_id); if(lossy) { block->globOptData.liveLossyInt32Syms->Set(sym->m_id); } else { block->globOptData.liveLossyInt32Syms->Clear(sym->m_id); } } else { Assert(toType == TyFloat64); AnalysisAssert(instr); StackSym *const varSym = sym->IsTypeSpec() ? sym->GetVarEquivSym(instr->m_func) : sym; block->globOptData.liveFloat64Syms->Set(varSym->m_id); } } return instr; } if (!needReplaceSrc) { instr = insertBeforeInstr; } IR::Opnd *constOpnd; int32 intConstantValue; if(valueInfo->TryGetIntConstantValue(&intConstantValue)) { if(toType == TyInt32) { constOpnd = IR::IntConstOpnd::New(intConstantValue, TyInt32, instr->m_func); } else { Assert(toType == TyFloat64); constOpnd = IR::FloatConstOpnd::New(static_cast(intConstantValue), TyFloat64, instr->m_func); } } else if(valueInfo->IsFloatConstant()) { const FloatConstType floatValue = valueInfo->AsFloatConstant()->FloatValue(); if(toType == TyInt32) { // In some loop scenarios, a sym can be specialized to int32 on loop entry // during the prepass and then subsequentely specialized to float within // the loop, leading to an attempted lossy conversion from float64 to int32 // on the backedge. For these cases, disable aggressive int type specialization // and try again. if (!lossy) { AssertOrFailFast(DoAggressiveIntTypeSpec()); throw Js::RejitException(RejitReason::AggressiveIntTypeSpecDisabled); } constOpnd = IR::IntConstOpnd::New( Js::JavascriptMath::ToInt32(floatValue), TyInt32, instr->m_func); } else { Assert(toType == TyFloat64); constOpnd = IR::FloatConstOpnd::New(floatValue, TyFloat64, instr->m_func); } } else { Assert(opnd->IsVar()); Assert(opnd->IsAddrOpnd()); AssertMsg(opnd->AsAddrOpnd()->IsVar(), "We only expect to see addr that are var before lower."); // Don't need to capture uses, we are only replacing an addr opnd if(toType == TyInt32) { constOpnd = IR::IntConstOpnd::New(Js::TaggedInt::ToInt32(opnd->AsAddrOpnd()->m_address), TyInt32, instr->m_func); } else { Assert(toType == TyFloat64); constOpnd = IR::FloatConstOpnd::New(Js::TaggedInt::ToDouble(opnd->AsAddrOpnd()->m_address), TyFloat64, instr->m_func); } } if (toType == TyInt32) { if (needReplaceSrc) { CaptureByteCodeSymUses(instr); if(indir) { Assert(opnd == indir->GetIndexOpnd()); Assert(indir->GetScale() == 0); indir->UnlinkIndexOpnd()->Free(instr->m_func); indir->SetOffset(constOpnd->AsIntConstOpnd()->AsInt32()); } else { instr->ReplaceSrc(opnd, constOpnd); } } else { StackSym *varSym = opnd->AsRegOpnd()->m_sym; if(varSym->IsTypeSpec()) { varSym = varSym->GetVarEquivSym(nullptr); Assert(varSym); } if(block->globOptData.liveInt32Syms->TestAndSet(varSym->m_id)) { Assert(!!block->globOptData.liveLossyInt32Syms->Test(varSym->m_id) == lossy); } else { if(lossy) { block->globOptData.liveLossyInt32Syms->Set(varSym->m_id); } StackSym *int32Sym = varSym->GetInt32EquivSym(instr->m_func); IR::RegOpnd *int32Reg = IR::RegOpnd::New(int32Sym, TyInt32, instr->m_func); int32Reg->SetIsJITOptimizedReg(true); newInstr = IR::Instr::New(Js::OpCode::Ld_I4, int32Reg, constOpnd, instr->m_func); newInstr->SetByteCodeOffset(instr); instr->InsertBefore(newInstr); if (updateBlockLastInstr) { block->SetLastInstr(newInstr); } } } } else { StackSym *floatSym; bool newFloatSym = false; StackSym* varSym; if (opnd->IsRegOpnd()) { varSym = opnd->AsRegOpnd()->m_sym; if (varSym->IsTypeSpec()) { varSym = varSym->GetVarEquivSym(nullptr); Assert(varSym); } floatSym = varSym->GetFloat64EquivSym(instr->m_func); } else { varSym = block->globOptData.GetCopyPropSym(nullptr, val); if(!varSym) { // Clear the symstore to ensure it's set below to this new symbol this->SetSymStoreDirect(val->GetValueInfo(), nullptr); varSym = StackSym::New(TyVar, instr->m_func); newFloatSym = true; } floatSym = varSym->GetFloat64EquivSym(instr->m_func); } IR::RegOpnd *floatReg = IR::RegOpnd::New(floatSym, TyFloat64, instr->m_func); floatReg->SetIsJITOptimizedReg(true); // If the value is not live - let's load it. if(!block->globOptData.liveFloat64Syms->TestAndSet(varSym->m_id)) { newInstr = IR::Instr::New(Js::OpCode::LdC_F8_R8, floatReg, constOpnd, instr->m_func); newInstr->SetByteCodeOffset(instr); instr->InsertBefore(newInstr); if (updateBlockLastInstr) { block->SetLastInstr(newInstr); } if(newFloatSym) { block->globOptData.SetValue(val, varSym); } // Src is always invariant, but check if the dst is, and then hoist. if (block->loop && ( (newFloatSym && block->loop->CanHoistInvariants()) || this->OptIsInvariant(floatReg, block, block->loop, val, false, false) )) { Assert(!this->IsLoopPrePass()); this->OptHoistInvariant(newInstr, block, block->loop, val, val, nullptr, false); } } if (needReplaceSrc) { CaptureByteCodeSymUses(instr); instr->ReplaceSrc(opnd, floatReg); } } return instr; } return newInstr; } void GlobOpt::ToVarRegOpnd(IR::RegOpnd *dst, BasicBlock *block) { ToVarStackSym(dst->m_sym, block); } void GlobOpt::ToVarStackSym(StackSym *varSym, BasicBlock *block) { //added another check for sym , in case of asmjs there is mostly no var syms and hence added a new check to see if it is the primary sym Assert(!varSym->IsTypeSpec()); block->globOptData.liveVarSyms->Set(varSym->m_id); block->globOptData.liveInt32Syms->Clear(varSym->m_id); block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); block->globOptData.liveFloat64Syms->Clear(varSym->m_id); } void GlobOpt::ToInt32Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block) { StackSym *varSym = dst->m_sym; Assert(!varSym->IsTypeSpec()); if (!this->IsLoopPrePass() && varSym->IsVar()) { StackSym *int32Sym = varSym->GetInt32EquivSym(instr->m_func); // Use UnlinkDst / SetDst to make sure isSingleDef is tracked properly, // since we'll just be hammering the symbol. dst = instr->UnlinkDst()->AsRegOpnd(); dst->m_sym = int32Sym; dst->SetType(TyInt32); instr->SetDst(dst); } block->globOptData.liveInt32Syms->Set(varSym->m_id); block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); // The store makes it lossless block->globOptData.liveVarSyms->Clear(varSym->m_id); block->globOptData.liveFloat64Syms->Clear(varSym->m_id); } void GlobOpt::ToUInt32Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block) { // We should be calling only for asmjs function Assert(GetIsAsmJSFunc()); StackSym *varSym = dst->m_sym; Assert(!varSym->IsTypeSpec()); block->globOptData.liveInt32Syms->Set(varSym->m_id); block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); // The store makes it lossless block->globOptData.liveVarSyms->Clear(varSym->m_id); block->globOptData.liveFloat64Syms->Clear(varSym->m_id); } void GlobOpt::ToFloat64Dst(IR::Instr *instr, IR::RegOpnd *dst, BasicBlock *block) { StackSym *varSym = dst->m_sym; Assert(!varSym->IsTypeSpec()); if (!this->IsLoopPrePass() && varSym->IsVar()) { StackSym *float64Sym = varSym->GetFloat64EquivSym(this->func); // Use UnlinkDst / SetDst to make sure isSingleDef is tracked properly, // since we'll just be hammering the symbol. dst = instr->UnlinkDst()->AsRegOpnd(); dst->m_sym = float64Sym; dst->SetType(TyFloat64); instr->SetDst(dst); } block->globOptData.liveFloat64Syms->Set(varSym->m_id); block->globOptData.liveVarSyms->Clear(varSym->m_id); block->globOptData.liveInt32Syms->Clear(varSym->m_id); block->globOptData.liveLossyInt32Syms->Clear(varSym->m_id); } static void SetIsConstFlag(StackSym* dstSym, int64 value) { Assert(dstSym); dstSym->SetIsInt64Const(); } static void SetIsConstFlag(StackSym* dstSym, int value) { Assert(dstSym); dstSym->SetIsIntConst(value); } static IR::Opnd* CreateIntConstOpnd(IR::Instr* instr, int64 value) { return (IR::Opnd*)IR::Int64ConstOpnd::New(value, instr->GetDst()->GetType(), instr->m_func); } static IR::Opnd* CreateIntConstOpnd(IR::Instr* instr, int value) { IntConstType constVal; if (instr->GetDst()->IsUnsigned()) { // we should zero extend in case of uint constVal = (uint32)value; } else { constVal = value; } return (IR::Opnd*)IR::IntConstOpnd::New(constVal, instr->GetDst()->GetType(), instr->m_func); } template IR::Opnd* GlobOpt::ReplaceWConst(IR::Instr **pInstr, T value, Value **pDstVal) { IR::Instr * &instr = *pInstr; IR::Opnd * constOpnd = CreateIntConstOpnd(instr, value); instr->ReplaceSrc1(constOpnd); instr->FreeSrc2(); this->OptSrc(constOpnd, &instr); IR::Opnd *dst = instr->GetDst(); StackSym *dstSym = dst->AsRegOpnd()->m_sym; if (dstSym->IsSingleDef()) { SetIsConstFlag(dstSym, value); } GOPT_TRACE_INSTR(instr, _u("Constant folding to %d: \n"), value); *pDstVal = GetIntConstantValue(value, instr, dst); return dst; } template bool GlobOpt::OptConstFoldBinaryWasm( IR::Instr** pInstr, const Value* src1, const Value* src2, Value **pDstVal) { IR::Instr* &instr = *pInstr; if (!DoConstFold()) { return false; } T src1IntConstantValue, src2IntConstantValue; if (!src1 || !src1->GetValueInfo()->TryGetIntConstantValue(&src1IntConstantValue, false) || //a bit sketchy: false for int32 means likelyInt = false !src2 || !src2->GetValueInfo()->TryGetIntConstantValue(&src2IntConstantValue, false) //and unsigned = false for int64 ) { return false; } int64 tmpValueOut; if (!instr->BinaryCalculatorT(src1IntConstantValue, src2IntConstantValue, &tmpValueOut, func->GetJITFunctionBody()->IsWasmFunction())) { return false; } this->CaptureByteCodeSymUses(instr); IR::Opnd *dst = (instr->GetDst()->IsInt64()) ? //dst can be int32 for int64 comparison operators ReplaceWConst(pInstr, tmpValueOut, pDstVal) : ReplaceWConst(pInstr, (int)tmpValueOut, pDstVal); instr->m_opcode = Js::OpCode::Ld_I4; this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock); return true; } bool GlobOpt::OptConstFoldBinary( IR::Instr * *pInstr, const IntConstantBounds &src1IntConstantBounds, const IntConstantBounds &src2IntConstantBounds, Value **pDstVal) { IR::Instr * &instr = *pInstr; int32 value; IR::IntConstOpnd *constOpnd; if (!DoConstFold()) { return false; } int32 src1IntConstantValue = -1; int32 src2IntConstantValue = -1; int32 src1MaxIntConstantValue = -1; int32 src2MaxIntConstantValue = -1; int32 src1MinIntConstantValue = -1; int32 src2MinIntConstantValue = -1; if (instr->IsBranchInstr()) { src1MinIntConstantValue = src1IntConstantBounds.LowerBound(); src1MaxIntConstantValue = src1IntConstantBounds.UpperBound(); src2MinIntConstantValue = src2IntConstantBounds.LowerBound(); src2MaxIntConstantValue = src2IntConstantBounds.UpperBound(); } else if (src1IntConstantBounds.IsConstant() && src2IntConstantBounds.IsConstant()) { src1IntConstantValue = src1IntConstantBounds.LowerBound(); src2IntConstantValue = src2IntConstantBounds.LowerBound(); } else { return false; } IntConstType tmpValueOut; if (!instr->BinaryCalculator(src1IntConstantValue, src2IntConstantValue, &tmpValueOut, TyInt32) || !Math::FitsInDWord(tmpValueOut)) { return false; } value = (int32)tmpValueOut; this->CaptureByteCodeSymUses(instr); constOpnd = IR::IntConstOpnd::New(value, TyInt32, instr->m_func); instr->ReplaceSrc1(constOpnd); instr->FreeSrc2(); this->OptSrc(constOpnd, &instr); IR::Opnd *dst = instr->GetDst(); Assert(dst->IsRegOpnd()); StackSym *dstSym = dst->AsRegOpnd()->m_sym; if (dstSym->IsSingleDef()) { dstSym->SetIsIntConst(value); } GOPT_TRACE_INSTR(instr, _u("Constant folding to %d: \n"), value); *pDstVal = GetIntConstantValue(value, instr, dst); if (IsTypeSpecPhaseOff(this->func)) { instr->m_opcode = Js::OpCode::LdC_A_I4; this->ToVarRegOpnd(dst->AsRegOpnd(), this->currentBlock); } else { instr->m_opcode = Js::OpCode::Ld_I4; this->ToInt32Dst(instr, dst->AsRegOpnd(), this->currentBlock); } InvalidateInductionVariables(instr); return true; } void GlobOpt::OptConstFoldBr(bool test, IR::Instr *instr, Value * src1Val, Value * src2Val) { GOPT_TRACE_INSTR(instr, _u("Constant folding to branch: ")); BasicBlock *deadBlock; if (src1Val) { this->ToInt32(instr, instr->GetSrc1(), this->currentBlock, src1Val, nullptr, false); } if (src2Val) { this->ToInt32(instr, instr->GetSrc2(), this->currentBlock, src2Val, nullptr, false); } this->CaptureByteCodeSymUses(instr); if (test) { instr->m_opcode = Js::OpCode::Br; instr->FreeSrc1(); if(instr->GetSrc2()) { instr->FreeSrc2(); } deadBlock = instr->m_next->AsLabelInstr()->GetBasicBlock(); } else { AssertMsg(instr->m_next->IsLabelInstr(), "Next instr of branch should be a label..."); if(instr->AsBranchInstr()->IsMultiBranch()) { return; } deadBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock(); instr->FreeSrc1(); if(instr->GetSrc2()) { instr->FreeSrc2(); } instr->m_opcode = Js::OpCode::Nop; } // Loop back edge: we would have already decremented data use count for the tail block when we processed the loop header. if (!(this->currentBlock->loop && this->currentBlock->loop->GetHeadBlock() == deadBlock)) { this->currentBlock->DecrementDataUseCount(); } this->currentBlock->RemoveDeadSucc(deadBlock, this->func->m_fg); if (deadBlock->GetPredList()->Count() == 0) { deadBlock->SetDataUseCount(0); } } void GlobOpt::ChangeValueType( BasicBlock *const block, Value *const value, const ValueType newValueType, const bool preserveSubclassInfo, const bool allowIncompatibleType) const { Assert(value); // Why are we trying to change the value type of the type sym value? Asserting here to make sure we don't deep copy the type sym's value info. Assert(!value->GetValueInfo()->IsJsType()); ValueInfo *const valueInfo = value->GetValueInfo(); const ValueType valueType(valueInfo->Type()); if(valueType == newValueType && (preserveSubclassInfo || valueInfo->IsGeneric())) { return; } // ArrayValueInfo has information specific to the array type, so make sure that doesn't change Assert( !preserveSubclassInfo || !valueInfo->IsArrayValueInfo() || newValueType.IsObject() && newValueType.GetObjectType() == valueInfo->GetObjectType()); Assert(!valueInfo->GetSymStore() || !valueInfo->GetSymStore()->IsStackSym() || !valueInfo->GetSymStore()->AsStackSym()->IsFromByteCodeConstantTable()); ValueInfo *const newValueInfo = preserveSubclassInfo ? valueInfo->Copy(alloc) : valueInfo->CopyWithGenericStructureKind(alloc); newValueInfo->Type() = newValueType; ChangeValueInfo(block, value, newValueInfo, allowIncompatibleType); } void GlobOpt::ChangeValueInfo(BasicBlock *const block, Value *const value, ValueInfo *const newValueInfo, const bool allowIncompatibleType, const bool compensated) const { Assert(value); Assert(newValueInfo); // The value type must be changed to something more specific or something more generic. For instance, it would be changed to // something more specific if the current value type is LikelyArray and checks have been done to ensure that it's an array, // and it would be changed to something more generic if a call kills the Array value type and it must be treated as // LikelyArray going forward. // There are cases where we change the type because of different profile information, and because of rejit, these profile information // may conflict. Need to allow incompatible type in those cause. However, the old type should be indefinite. Assert((allowIncompatibleType && !value->GetValueInfo()->IsDefinite()) || AreValueInfosCompatible(newValueInfo, value->GetValueInfo())); // ArrayValueInfo has information specific to the array type, so make sure that doesn't change Assert( !value->GetValueInfo()->IsArrayValueInfo() || !newValueInfo->IsArrayValueInfo() || newValueInfo->GetObjectType() == value->GetValueInfo()->GetObjectType()); if(block) { TrackValueInfoChangeForKills(block, value, newValueInfo, compensated); } value->SetValueInfo(newValueInfo); } bool GlobOpt::AreValueInfosCompatible(const ValueInfo *const v0, const ValueInfo *const v1) const { Assert(v0); Assert(v1); if(v0->IsUninitialized() || v1->IsUninitialized()) { return true; } const bool doAggressiveIntTypeSpec = DoAggressiveIntTypeSpec(); if(doAggressiveIntTypeSpec && (v0->IsInt() || v1->IsInt())) { // Int specialization in some uncommon loop cases involving dependencies, needs to allow specializing values of // arbitrary types, even values that are definitely not int, to compensate for aggressive assumptions made by a loop // prepass return true; } if ((v0->Type()).IsMixedTypedArrayPair(v1->Type()) || (v1->Type()).IsMixedTypedArrayPair(v0->Type())) { return true; } const bool doFloatTypeSpec = DoFloatTypeSpec(); if(doFloatTypeSpec && (v0->IsFloat() || v1->IsFloat())) { // Float specialization allows specializing values of arbitrary types, even values that are definitely not float return true; } const bool doArrayMissingValueCheckHoist = DoArrayMissingValueCheckHoist(); const bool doNativeArrayTypeSpec = DoNativeArrayTypeSpec(); const auto AreValueTypesCompatible = [=](const ValueType t0, const ValueType t1) { return t0.IsSubsetOf(t1, doAggressiveIntTypeSpec, doFloatTypeSpec, doArrayMissingValueCheckHoist, doNativeArrayTypeSpec) || t1.IsSubsetOf(t0, doAggressiveIntTypeSpec, doFloatTypeSpec, doArrayMissingValueCheckHoist, doNativeArrayTypeSpec); }; const ValueType t0(v0->Type().ToDefinite()), t1(v1->Type().ToDefinite()); if(t0.IsLikelyObject() && t1.IsLikelyObject()) { // Check compatibility for the primitive portions and the object portions of the value types separately if(AreValueTypesCompatible(t0.ToDefiniteObject(), t1.ToDefiniteObject()) && ( !t0.HasBeenPrimitive() || !t1.HasBeenPrimitive() || AreValueTypesCompatible(t0.ToDefinitePrimitiveSubset(), t1.ToDefinitePrimitiveSubset()) )) { return true; } } else if(AreValueTypesCompatible(t0, t1)) { return true; } const FloatConstantValueInfo *floatConstantValueInfo; const ValueInfo *likelyIntValueinfo; if(v0->IsFloatConstant() && v1->IsLikelyInt()) { floatConstantValueInfo = v0->AsFloatConstant(); likelyIntValueinfo = v1; } else if(v0->IsLikelyInt() && v1->IsFloatConstant()) { floatConstantValueInfo = v1->AsFloatConstant(); likelyIntValueinfo = v0; } else { return false; } // A float constant value with a value that is actually an int is a subset of a likely-int value. // Ideally, we should create an int constant value for this up front, such that IsInt() also returns true. There // were other issues with that, should see if that can be done. int32 int32Value; return Js::JavascriptNumber::TryGetInt32Value(floatConstantValueInfo->FloatValue(), &int32Value) && (!likelyIntValueinfo->IsLikelyTaggedInt() || !Js::TaggedInt::IsOverflow(int32Value)); } #if DBG void GlobOpt::VerifyArrayValueInfoForTracking( const ValueInfo *const valueInfo, const bool isJsArray, const BasicBlock *const block, const bool ignoreKnownImplicitCalls) const { Assert(valueInfo); Assert(valueInfo->IsAnyOptimizedArray()); Assert(isJsArray == valueInfo->IsArrayOrObjectWithArray()); Assert(!isJsArray == valueInfo->IsOptimizedTypedArray()); Assert(block); Loop *implicitCallsLoop; if(block->next && !block->next->isDeleted && block->next->isLoopHeader) { // Since a loop's landing pad does not have user code, determine whether disabling implicit calls is allowed in the // landing pad based on the loop for which this block is the landing pad. implicitCallsLoop = block->next->loop; Assert(implicitCallsLoop); Assert(implicitCallsLoop->landingPad == block); } else { implicitCallsLoop = block->loop; } Assert( !isJsArray || DoArrayCheckHoist(valueInfo->Type(), implicitCallsLoop) || ( ignoreKnownImplicitCalls && !(implicitCallsLoop ? ImplicitCallFlagsAllowOpts(implicitCallsLoop) : ImplicitCallFlagsAllowOpts(func)) )); Assert(!(isJsArray && valueInfo->HasNoMissingValues() && !DoArrayMissingValueCheckHoist())); Assert( !( valueInfo->IsArrayValueInfo() && ( valueInfo->AsArrayValueInfo()->HeadSegmentSym() || valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym() ) && !DoArraySegmentHoist(valueInfo->Type()) )); #if 0 // We can't assert here that there is only a head segment length sym if hoisting is allowed in the current block, // because we may have propagated the sym forward out of a loop, and hoisting may be allowed inside but not // outside the loop. Assert( isJsArray || !valueInfo->IsArrayValueInfo() || !valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym() || DoTypedArraySegmentLengthHoist(implicitCallsLoop) || ignoreKnownImplicitCalls || (implicitCallsLoop ? ImplicitCallFlagsAllowOpts(implicitCallsLoop) : ImplicitCallFlagsAllowOpts(func)) ); #endif Assert( !( isJsArray && valueInfo->IsArrayValueInfo() && valueInfo->AsArrayValueInfo()->LengthSym() && !DoArrayLengthHoist() )); } #endif void GlobOpt::TrackNewValueForKills(Value *const value) { Assert(value); if(!value->GetValueInfo()->IsAnyOptimizedArray()) { return; } DoTrackNewValueForKills(value); } void GlobOpt::DoTrackNewValueForKills(Value *const value) { Assert(value); ValueInfo *const valueInfo = value->GetValueInfo(); Assert(valueInfo->IsAnyOptimizedArray()); Assert(!valueInfo->IsArrayValueInfo()); // The value and value info here are new, so it's okay to modify the value info in-place Assert(!valueInfo->GetSymStore()); const bool isJsArray = valueInfo->IsArrayOrObjectWithArray(); Assert(!isJsArray == valueInfo->IsOptimizedTypedArray()); const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray(); Loop *implicitCallsLoop; if(currentBlock->next && !currentBlock->next->isDeleted && currentBlock->next->isLoopHeader) { // Since a loop's landing pad does not have user code, determine whether disabling implicit calls is allowed in the // landing pad based on the loop for which this block is the landing pad. implicitCallsLoop = currentBlock->next->loop; Assert(implicitCallsLoop); Assert(implicitCallsLoop->landingPad == currentBlock); } else { implicitCallsLoop = currentBlock->loop; } if(isJsArray || isVirtualTypedArray) { if(!DoArrayCheckHoist(valueInfo->Type(), implicitCallsLoop)) { // Array opts are disabled for this value type, so treat it as an indefinite value type going forward valueInfo->Type() = valueInfo->Type().ToLikely(); return; } if(isJsArray && valueInfo->HasNoMissingValues() && !DoArrayMissingValueCheckHoist()) { valueInfo->Type() = valueInfo->Type().SetHasNoMissingValues(false); } } #if DBG VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock); #endif if(!isJsArray && !isVirtualTypedArray) { return; } // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can // revert the value type to a likely version. CurrentBlockData()->valuesToKillOnCalls->Add(value); } void GlobOpt::TrackCopiedValueForKills(Value *const value) { Assert(value); if(!value->GetValueInfo()->IsAnyOptimizedArray()) { return; } DoTrackCopiedValueForKills(value); } void GlobOpt::DoTrackCopiedValueForKills(Value *const value) { Assert(value); ValueInfo *const valueInfo = value->GetValueInfo(); Assert(valueInfo->IsAnyOptimizedArray()); const bool isJsArray = valueInfo->IsArrayOrObjectWithArray(); Assert(!isJsArray == valueInfo->IsOptimizedTypedArray()); const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray(); #if DBG VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock); #endif if(!isJsArray && !isVirtualTypedArray && !(valueInfo->IsArrayValueInfo() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym())) { return; } // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can // revert the value type to a likely version. CurrentBlockData()->valuesToKillOnCalls->Add(value); } void GlobOpt::TrackMergedValueForKills( Value *const value, GlobOptBlockData *const blockData, BVSparse *const mergedValueTypesTrackedForKills) const { Assert(value); if(!value->GetValueInfo()->IsAnyOptimizedArray()) { return; } DoTrackMergedValueForKills(value, blockData, mergedValueTypesTrackedForKills); } void GlobOpt::DoTrackMergedValueForKills( Value *const value, GlobOptBlockData *const blockData, BVSparse *const mergedValueTypesTrackedForKills) const { Assert(value); Assert(blockData); ValueInfo *valueInfo = value->GetValueInfo(); Assert(valueInfo->IsAnyOptimizedArray()); const bool isJsArray = valueInfo->IsArrayOrObjectWithArray(); Assert(!isJsArray == valueInfo->IsOptimizedTypedArray()); const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray(); #if DBG VerifyArrayValueInfoForTracking(valueInfo, isJsArray, currentBlock, true); #endif if(!isJsArray && !isVirtualTypedArray && !(valueInfo->IsArrayValueInfo() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym())) { return; } // Can't assume going forward that it will definitely be an array without disabling implicit calls, because the // array may be transformed into an ES5 array. Since array opts are enabled, implicit calls can be disabled, and we can // treat it as a definite value type going forward, but the value needs to be tracked so that something like a call can // revert the value type to a likely version. if(!mergedValueTypesTrackedForKills || !mergedValueTypesTrackedForKills->TestAndSet(value->GetValueNumber())) { blockData->valuesToKillOnCalls->Add(value); } } void GlobOpt::TrackValueInfoChangeForKills(BasicBlock *const block, Value *const value, ValueInfo *const newValueInfo, const bool compensated) const { Assert(block); Assert(value); Assert(newValueInfo); ValueInfo *const oldValueInfo = value->GetValueInfo(); #if DBG if(oldValueInfo->IsAnyOptimizedArray()) { VerifyArrayValueInfoForTracking(oldValueInfo, oldValueInfo->IsArrayOrObjectWithArray(), block, compensated); } #endif const bool trackOldValueInfo = oldValueInfo->IsArrayOrObjectWithArray() || oldValueInfo->IsOptimizedVirtualTypedArray() || ( oldValueInfo->IsOptimizedTypedArray() && oldValueInfo->IsArrayValueInfo() && oldValueInfo->AsArrayValueInfo()->HeadSegmentLengthSym() ); Assert(trackOldValueInfo == block->globOptData.valuesToKillOnCalls->ContainsKey(value)); #if DBG if(newValueInfo->IsAnyOptimizedArray()) { VerifyArrayValueInfoForTracking(newValueInfo, newValueInfo->IsArrayOrObjectWithArray(), block, compensated); } #endif const bool trackNewValueInfo = newValueInfo->IsArrayOrObjectWithArray() || newValueInfo->IsOptimizedVirtualTypedArray() || ( newValueInfo->IsOptimizedTypedArray() && newValueInfo->IsArrayValueInfo() && newValueInfo->AsArrayValueInfo()->HeadSegmentLengthSym() ); if(trackOldValueInfo == trackNewValueInfo) { return; } if(trackNewValueInfo) { block->globOptData.valuesToKillOnCalls->Add(value); } else { block->globOptData.valuesToKillOnCalls->Remove(value); } } void GlobOpt::ProcessValueKills(IR::Instr *const instr) { Assert(instr); ValueSet *const valuesToKillOnCalls = CurrentBlockData()->valuesToKillOnCalls; if(!IsLoopPrePass() && valuesToKillOnCalls->Count() == 0) { return; } const JsArrayKills kills = CheckJsArrayKills(instr); Assert(!kills.KillsArrayHeadSegments() || kills.KillsArrayHeadSegmentLengths()); if(IsLoopPrePass()) { rootLoopPrePass->jsArrayKills = rootLoopPrePass->jsArrayKills.Merge(kills); Assert( !rootLoopPrePass->parent || rootLoopPrePass->jsArrayKills.AreSubsetOf(rootLoopPrePass->parent->jsArrayKills)); if(kills.KillsAllArrays()) { rootLoopPrePass->needImplicitCallBailoutChecksForJsArrayCheckHoist = false; } if(valuesToKillOnCalls->Count() == 0) { return; } } if(kills.KillsAllArrays()) { Assert(kills.KillsTypedArrayHeadSegmentLengths()); // - Calls need to kill the value types of values in the following list. For instance, calls can transform a JS array // into an ES5 array, so any definitely-array value types need to be killed. Also, VirtualTypeArrays do not have // bounds checks; this can be problematic if the array is detached, so check to ensure that it is a virtual array. // Update the value types to likley to ensure a bailout that asserts Array type is generated. // - Calls also need to kill typed array head segment lengths. A typed array's array buffer may be transferred to a web // worker, in which case the typed array's length is set to zero. for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext()) { Value *const value = it.CurrentValue(); ValueInfo *const valueInfo = value->GetValueInfo(); Assert( valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray() || valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()); if (valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray()) { ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false); continue; } ChangeValueInfo( nullptr, value, valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true)); } valuesToKillOnCalls->Clear(); return; } if(kills.KillsArraysWithNoMissingValues()) { // Some operations may kill arrays with no missing values in unlikely circumstances. Convert their value types to likely // versions so that the checks have to be redone. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext()) { Value *const value = it.CurrentValue(); ValueInfo *const valueInfo = value->GetValueInfo(); Assert( valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray() || valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()); if(!valueInfo->IsArrayOrObjectWithArray() || !valueInfo->HasNoMissingValues()) { continue; } ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false); it.RemoveCurrent(); } } else if(kills.KillsObjectArraysWithNoMissingValues()) { // Some operations may kill objects with arrays-with-no-missing-values in unlikely circumstances. Convert their value types to likely // versions so that the checks have to be redone. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext()) { Value *const value = it.CurrentValue(); ValueInfo *const valueInfo = value->GetValueInfo(); Assert( valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray() || valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()); if(!valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsArray() || !valueInfo->HasNoMissingValues()) { continue; } ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false); it.RemoveCurrent(); } } if(kills.KillsNativeArrays()) { // Some operations may kill native arrays in (what should be) unlikely circumstances. Convert their value types to // likely versions so that the checks have to be redone. for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext()) { Value *const value = it.CurrentValue(); ValueInfo *const valueInfo = value->GetValueInfo(); Assert( valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray() || valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()); if(!valueInfo->IsArrayOrObjectWithArray() || valueInfo->HasVarElements()) { continue; } ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false); it.RemoveCurrent(); } } const bool likelyKillsJsArraysWithNoMissingValues = IsOperationThatLikelyKillsJsArraysWithNoMissingValues(instr); if(!kills.KillsArrayHeadSegmentLengths()) { Assert(!kills.KillsArrayHeadSegments()); if(!likelyKillsJsArraysWithNoMissingValues && !kills.KillsArrayLengths()) { return; } } for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext()) { Value *const value = it.CurrentValue(); ValueInfo *valueInfo = value->GetValueInfo(); Assert( valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray() || valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()); if(!valueInfo->IsArrayOrObjectWithArray()) { continue; } if(likelyKillsJsArraysWithNoMissingValues && valueInfo->HasNoMissingValues()) { ChangeValueType(nullptr, value, valueInfo->Type().SetHasNoMissingValues(false), true); valueInfo = value->GetValueInfo(); } if(!valueInfo->IsArrayValueInfo()) { continue; } ArrayValueInfo *const arrayValueInfo = valueInfo->AsArrayValueInfo(); const bool removeHeadSegment = kills.KillsArrayHeadSegments() && arrayValueInfo->HeadSegmentSym(); const bool removeHeadSegmentLength = kills.KillsArrayHeadSegmentLengths() && arrayValueInfo->HeadSegmentLengthSym(); const bool removeLength = kills.KillsArrayLengths() && arrayValueInfo->LengthSym(); if(removeHeadSegment || removeHeadSegmentLength || removeLength) { ChangeValueInfo( nullptr, value, arrayValueInfo->Copy(alloc, !removeHeadSegment, !removeHeadSegmentLength, !removeLength)); valueInfo = value->GetValueInfo(); } } } void GlobOpt::ProcessValueKills(BasicBlock *const block, GlobOptBlockData *const blockData) { Assert(block); Assert(blockData); ValueSet *const valuesToKillOnCalls = blockData->valuesToKillOnCalls; if(!IsLoopPrePass() && valuesToKillOnCalls->Count() == 0) { return; } // If the current block or loop has implicit calls, kill all definitely-array value types, as using that info will cause // implicit calls to be disabled, resulting in unnecessary bailouts const bool killValuesOnImplicitCalls = (block->loop ? !this->ImplicitCallFlagsAllowOpts(block->loop) : !this->ImplicitCallFlagsAllowOpts(func)); if (!killValuesOnImplicitCalls) { return; } if(IsLoopPrePass() && block->loop == rootLoopPrePass) { AnalysisAssert(rootLoopPrePass); for (Loop * loop = rootLoopPrePass; loop != nullptr; loop = loop->parent) { loop->jsArrayKills.SetKillsAllArrays(); } Assert(!rootLoopPrePass->parent || rootLoopPrePass->jsArrayKills.AreSubsetOf(rootLoopPrePass->parent->jsArrayKills)); if(valuesToKillOnCalls->Count() == 0) { return; } } for(auto it = valuesToKillOnCalls->GetIterator(); it.IsValid(); it.MoveNext()) { Value *const value = it.CurrentValue(); ValueInfo *const valueInfo = value->GetValueInfo(); Assert( valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray() || valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()); if(valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray()) { ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false); continue; } ChangeValueInfo( nullptr, value, valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true)); } valuesToKillOnCalls->Clear(); } void GlobOpt::ProcessValueKillsForLoopHeaderAfterBackEdgeMerge(BasicBlock *const block, GlobOptBlockData *const blockData) { Assert(block); Assert(block->isLoopHeader); Assert(blockData); ValueSet *const valuesToKillOnCalls = blockData->valuesToKillOnCalls; if(valuesToKillOnCalls->Count() == 0) { return; } const JsArrayKills loopKills(block->loop->jsArrayKills); for(auto it = valuesToKillOnCalls->GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext()) { Value *const value = it.CurrentValue(); ValueInfo *valueInfo = value->GetValueInfo(); Assert( valueInfo->IsArrayOrObjectWithArray() || valueInfo->IsOptimizedVirtualTypedArray() || valueInfo->IsOptimizedTypedArray() && valueInfo->AsArrayValueInfo()->HeadSegmentLengthSym()); const bool isJsArray = valueInfo->IsArrayOrObjectWithArray(); Assert(!isJsArray == valueInfo->IsOptimizedTypedArray()); const bool isVirtualTypedArray = valueInfo->IsOptimizedVirtualTypedArray(); if((isJsArray || isVirtualTypedArray) ? loopKills.KillsValueType(valueInfo->Type()) : loopKills.KillsTypedArrayHeadSegmentLengths()) { // Hoisting array checks and other related things for this type is disabled for the loop due to the kill, as // compensation code is currently not added on back-edges. When merging values from a back-edge, the array value // type cannot be definite, as that may require adding compensation code on the back-edge if the optimization pass // chooses to not optimize the array. if(isJsArray || isVirtualTypedArray) { ChangeValueType(nullptr, value, valueInfo->Type().ToLikely(), false); } else { ChangeValueInfo( nullptr, value, valueInfo->AsArrayValueInfo()->Copy(alloc, true, false /* copyHeadSegmentLength */, true)); } it.RemoveCurrent(); continue; } if(!isJsArray || !valueInfo->IsArrayValueInfo()) { continue; } // Similarly, if the loop contains an operation that kills JS array segments, don't make the segment or other related // syms available initially inside the loop ArrayValueInfo *const arrayValueInfo = valueInfo->AsArrayValueInfo(); const bool removeHeadSegment = loopKills.KillsArrayHeadSegments() && arrayValueInfo->HeadSegmentSym(); const bool removeHeadSegmentLength = loopKills.KillsArrayHeadSegmentLengths() && arrayValueInfo->HeadSegmentLengthSym(); const bool removeLength = loopKills.KillsArrayLengths() && arrayValueInfo->LengthSym(); if(removeHeadSegment || removeHeadSegmentLength || removeLength) { ChangeValueInfo( nullptr, value, arrayValueInfo->Copy(alloc, !removeHeadSegment, !removeHeadSegmentLength, !removeLength)); valueInfo = value->GetValueInfo(); } } } bool GlobOpt::NeedBailOnImplicitCallForLiveValues(BasicBlock const * const block, const bool isForwardPass) const { if(isForwardPass) { return block->globOptData.valuesToKillOnCalls->Count() != 0; } if(block->noImplicitCallUses->IsEmpty()) { Assert(block->noImplicitCallNoMissingValuesUses->IsEmpty()); Assert(block->noImplicitCallNativeArrayUses->IsEmpty()); Assert(block->noImplicitCallJsArrayHeadSegmentSymUses->IsEmpty()); Assert(block->noImplicitCallArrayLengthSymUses->IsEmpty()); return false; } return true; } IR::Instr* GlobOpt::CreateBoundsCheckInstr(IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset, Func* func) { IR::Instr* instr = IR::Instr::New(Js::OpCode::BoundCheck, func); return AttachBoundsCheckData(instr, lowerBound, upperBound, offset); } IR::Instr* GlobOpt::CreateBoundsCheckInstr(IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset, IR::BailOutKind bailoutkind, BailOutInfo* bailoutInfo, Func * func) { IR::Instr* instr = IR::BailOutInstr::New(Js::OpCode::BoundCheck, bailoutkind, bailoutInfo, func); return AttachBoundsCheckData(instr, lowerBound, upperBound, offset); } IR::Instr* GlobOpt::AttachBoundsCheckData(IR::Instr* instr, IR::Opnd* lowerBound, IR::Opnd* upperBound, int offset) { instr->SetSrc1(lowerBound); instr->SetSrc2(upperBound); if (offset != 0) { instr->SetDst(IR::IntConstOpnd::New(offset, TyInt32, instr->m_func)); } return instr; } void GlobOpt::OptArraySrc(IR::Instr ** const instrRef, Value ** src1Val, Value ** src2Val) { Assert(instrRef != nullptr); ArraySrcOpt arraySrcOpt(this, instrRef, src1Val, src2Val); arraySrcOpt.Optimize(); } void GlobOpt::ProcessNoImplicitCallArrayUses(IR::RegOpnd * baseOpnd, IR::ArrayRegOpnd * baseArrayOpnd, IR::Instr * instr, bool isLikelyJsArray, bool useNoMissingValues) { if (isLikelyJsArray) { // Insert an instruction to indicate to the dead-store pass that implicit calls need to be kept disabled until this // instruction. Operations other than LdElem, StElem and IsIn don't benefit much from arrays having no missing values, // so no need to ensure that the array still has no missing values. For a particular array, if none of the accesses // benefit much from the no-missing-values information, it may be beneficial to avoid checking for no missing // values, especially in the case for a single array access, where the cost of the check could be relatively // significant. An StElem has to do additional checks in the common path if the array may have missing values, and // a StElem that operates on an array that has no missing values is more likely to keep the no-missing-values info // on the array more precise, so it still benefits a little from the no-missing-values info. this->CaptureNoImplicitCallUses(baseOpnd, isLikelyJsArray); } else if (baseArrayOpnd && baseArrayOpnd->HeadSegmentLengthSym()) { // A typed array's array buffer may be transferred to a web worker as part of an implicit call, in which case the typed // array's length is set to zero. Insert an instruction to indicate to the dead-store pass that implicit calls need to // be disabled until this instruction. IR::RegOpnd *const headSegmentLengthOpnd = IR::RegOpnd::New( baseArrayOpnd->HeadSegmentLengthSym(), baseArrayOpnd->HeadSegmentLengthSym()->GetType(), instr->m_func); const IR::AutoReuseOpnd autoReuseHeadSegmentLengthOpnd(headSegmentLengthOpnd, instr->m_func); this->CaptureNoImplicitCallUses(headSegmentLengthOpnd, false); } } void GlobOpt::OptStackArgLenAndConst(IR::Instr* instr, Value** src1Val) { if (!PHASE_OFF(Js::StackArgLenConstOptPhase, instr->m_func) && instr->m_func->IsStackArgsEnabled() && instr->usesStackArgumentsObject && instr->IsInlined()) { IR::Opnd* src1 = instr->GetSrc1(); auto replaceInstr = [&](IR::Opnd* newopnd, Js::OpCode opcode) { if (PHASE_TESTTRACE(Js::StackArgLenConstOptPhase, instr->m_func)) { Output::Print(_u("Inlined function %s have replaced opcode %s with opcode %s for stack arg optimization. \n"), instr->m_func->GetJITFunctionBody()->GetDisplayName(), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode), Js::OpCodeUtil::GetOpCodeName(opcode)); Output::Flush(); } this->CaptureByteCodeSymUses(instr); instr->m_opcode = opcode; instr->ReplaceSrc1(newopnd); if (instr->HasBailOutInfo()) { instr->ClearBailOutInfo(); } if (instr->IsProfiledInstr()) { Assert(opcode == Js::OpCode::Ld_A || opcode == Js::OpCode::Typeof); instr->AsProfiledInstr()->u.FldInfo().valueType = ValueType::Uninitialized; } *src1Val = this->OptSrc(instr->GetSrc1(), &instr); instr->m_func->hasArgLenAndConstOpt = true; }; Assert(CurrentBlockData()->IsArgumentsOpnd(src1)); switch(instr->m_opcode) { case Js::OpCode::LdLen_A: { IR::AddrOpnd* newopnd = IR::AddrOpnd::New(Js::TaggedInt::ToVarUnchecked(instr->m_func->actualCount - 1), IR::AddrOpndKindConstantVar, instr->m_func); replaceInstr(newopnd, Js::OpCode::Ld_A); break; } case Js::OpCode::LdElemI_A: case Js::OpCode::TypeofElem: { IR::IndirOpnd* indirOpndSrc1 = src1->AsIndirOpnd(); if (!indirOpndSrc1->GetIndexOpnd()) { int argIndex = indirOpndSrc1->GetOffset() + 1; IR::Instr* defInstr = nullptr; IR::Instr* inlineeStart = instr->m_func->GetInlineeStart(); inlineeStart->IterateArgInstrs([&](IR::Instr* argInstr) { StackSym *argSym = argInstr->GetDst()->AsSymOpnd()->m_sym->AsStackSym(); if (argSym->GetArgSlotNum() - 1 == argIndex) { defInstr = argInstr; return true; } return false; }); Js::OpCode replacementOpcode; if (instr->m_opcode == Js::OpCode::TypeofElem) { replacementOpcode = Js::OpCode::Typeof; } else { replacementOpcode = Js::OpCode::Ld_A; } // If we cannot find the right instruction. I.E. When calling arguments[2] and no arguments were passed to the func if (defInstr == nullptr) { IR::Opnd * undefined = IR::AddrOpnd::New(instr->m_func->GetScriptContextInfo()->GetUndefinedAddr(), IR::AddrOpndKindDynamicVar, instr->m_func, true); undefined->SetValueType(ValueType::Undefined); replaceInstr(undefined, replacementOpcode); } else { replaceInstr(defInstr->GetSrc1(), replacementOpcode); } } else { instr->m_func->unoptimizableArgumentsObjReference++; } break; } } } } void GlobOpt::CaptureNoImplicitCallUses( IR::Opnd *opnd, const bool usesNoMissingValuesInfo, IR::Instr *const includeCurrentInstr) { Assert(!IsLoopPrePass()); Assert(noImplicitCallUsesToInsert); Assert(opnd); // The opnd may be deleted later, so make a copy to ensure it is alive for inserting NoImplicitCallUses later opnd = opnd->Copy(func); if(!usesNoMissingValuesInfo) { const ValueType valueType(opnd->GetValueType()); if(valueType.IsArrayOrObjectWithArray() && valueType.HasNoMissingValues()) { // Inserting NoImplicitCallUses for an opnd with a definitely-array-with-no-missing-values value type means that the // instruction following it uses the information that the array has no missing values in some way, for instance, it // may omit missing value checks. Based on that, the dead-store phase in turn ensures that the necessary bailouts // are inserted to ensure that the array still has no missing values until the following instruction. Since // 'usesNoMissingValuesInfo' is false, change the value type to indicate to the dead-store phase that the following // instruction does not use the no-missing-values information. opnd->SetValueType(valueType.SetHasNoMissingValues(false)); } } if(includeCurrentInstr) { IR::Instr *const noImplicitCallUses = IR::PragmaInstr::New(Js::OpCode::NoImplicitCallUses, 0, includeCurrentInstr->m_func); noImplicitCallUses->SetSrc1(opnd); noImplicitCallUses->GetSrc1()->SetIsJITOptimizedReg(true); includeCurrentInstr->InsertAfter(noImplicitCallUses); return; } noImplicitCallUsesToInsert->Add(opnd); } void GlobOpt::InsertNoImplicitCallUses(IR::Instr *const instr) { Assert(noImplicitCallUsesToInsert); const int n = noImplicitCallUsesToInsert->Count(); if(n == 0) { return; } IR::Instr *const insertBeforeInstr = instr->GetInsertBeforeByteCodeUsesInstr(); for(int i = 0; i < n;) { IR::Instr *const noImplicitCallUses = IR::PragmaInstr::New(Js::OpCode::NoImplicitCallUses, 0, instr->m_func); noImplicitCallUses->SetSrc1(noImplicitCallUsesToInsert->Item(i)); noImplicitCallUses->GetSrc1()->SetIsJITOptimizedReg(true); ++i; if(i < n) { noImplicitCallUses->SetSrc2(noImplicitCallUsesToInsert->Item(i)); noImplicitCallUses->GetSrc2()->SetIsJITOptimizedReg(true); ++i; } noImplicitCallUses->SetByteCodeOffset(instr); insertBeforeInstr->InsertBefore(noImplicitCallUses); } noImplicitCallUsesToInsert->Clear(); } void GlobOpt::PrepareLoopArrayCheckHoist() { if(IsLoopPrePass() || !currentBlock->loop || !currentBlock->isLoopHeader || !currentBlock->loop->parent) { return; } if(currentBlock->loop->parent->needImplicitCallBailoutChecksForJsArrayCheckHoist) { // If the parent loop is an array check elimination candidate, so is the current loop. Even though the current loop may // not have array accesses, if the parent loop hoists array checks, the current loop also needs implicit call checks. currentBlock->loop->needImplicitCallBailoutChecksForJsArrayCheckHoist = true; } } JsArrayKills GlobOpt::CheckJsArrayKills(IR::Instr *const instr) { Assert(instr); JsArrayKills kills; if(instr->UsesAllFields()) { // Calls can (but are unlikely to) change a javascript array into an ES5 array, which may have different behavior for // index properties. kills.SetKillsAllArrays(); return kills; } const bool doArrayMissingValueCheckHoist = DoArrayMissingValueCheckHoist(); const bool doNativeArrayTypeSpec = DoNativeArrayTypeSpec(); const bool doArraySegmentHoist = DoArraySegmentHoist(ValueType::GetObject(ObjectType::Array)); Assert(doArraySegmentHoist == DoArraySegmentHoist(ValueType::GetObject(ObjectType::ObjectWithArray))); const bool doArrayLengthHoist = DoArrayLengthHoist(); if(!doArrayMissingValueCheckHoist && !doNativeArrayTypeSpec && !doArraySegmentHoist && !doArrayLengthHoist) { return kills; } // The following operations may create missing values in an array in an unlikely circumstance. Even though they don't kill // the fact that the 'this' parameter is an array (when implicit calls are disabled), we don't have a way to say the value // type is definitely array but it likely has no missing values. So, these will kill the definite value type as well, making // it likely array, such that the array checks will have to be redone. const bool useValueTypes = !IsLoopPrePass(); // Source value types are not guaranteed to be correct in a loop prepass switch(instr->m_opcode) { case Js::OpCode::StElemC: case Js::OpCode::StElemI_A: case Js::OpCode::StElemI_A_Strict: { Assert(instr->GetDst()); if(!instr->GetDst()->IsIndirOpnd()) { break; } const ValueType baseValueType = useValueTypes ? instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType() : ValueType::Uninitialized; if(useValueTypes && baseValueType.IsNotArrayOrObjectWithArray()) { break; } if(instr->IsProfiledInstr()) { const Js::StElemInfo *const stElemInfo = instr->AsProfiledInstr()->u.stElemInfo; if(doArraySegmentHoist && stElemInfo->LikelyStoresOutsideHeadSegmentBounds()) { kills.SetKillsArrayHeadSegments(); kills.SetKillsArrayHeadSegmentLengths(); } if(doArrayLengthHoist && !(useValueTypes && baseValueType.IsNotArray()) && stElemInfo->LikelyStoresOutsideArrayBounds()) { kills.SetKillsArrayLengths(); } } break; } case Js::OpCode::DeleteElemI_A: case Js::OpCode::DeleteElemIStrict_A: Assert(instr->GetSrc1()); if(!instr->GetSrc1()->IsIndirOpnd() || (useValueTypes && instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType().IsNotArrayOrObjectWithArray())) { break; } if(doArrayMissingValueCheckHoist) { kills.SetKillsArraysWithNoMissingValues(); } if(doArraySegmentHoist) { kills.SetKillsArrayHeadSegmentLengths(); } break; case Js::OpCode::ConsoleScopedStFld: case Js::OpCode::ConsoleScopedStFldStrict: case Js::OpCode::ScopedStFld: case Js::OpCode::ScopedStFldStrict: case Js::OpCode::StFld: case Js::OpCode::StFldStrict: case Js::OpCode::StSuperFld: case Js::OpCode::StSuperFldStrict: { Assert(instr->GetDst()); if(!doArraySegmentHoist && !doArrayLengthHoist) { break; } IR::SymOpnd *const symDst = instr->GetDst()->AsSymOpnd(); if(!symDst->IsPropertySymOpnd()) { break; } IR::PropertySymOpnd *const dst = symDst->AsPropertySymOpnd(); if(dst->m_sym->AsPropertySym()->m_propertyId != Js::PropertyIds::length) { break; } if(useValueTypes && dst->GetPropertyOwnerValueType().IsNotArray()) { // Setting the 'length' property of an object that is not an array, even if it has an internal array, does // not kill the head segment or head segment length of any arrays. break; } if(doArraySegmentHoist) { kills.SetKillsArrayHeadSegmentLengths(); } if(doArrayLengthHoist) { kills.SetKillsArrayLengths(); } break; } case Js::OpCode::InlineArrayPush: { Assert(instr->GetSrc2()); IR::Opnd *const arrayOpnd = instr->GetSrc1(); Assert(arrayOpnd); const ValueType arrayValueType(arrayOpnd->GetValueType()); if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray())) { break; } if(doArrayMissingValueCheckHoist) { kills.SetKillsArraysWithNoMissingValues(); } if(doArraySegmentHoist) { kills.SetKillsArrayHeadSegments(); kills.SetKillsArrayHeadSegmentLengths(); } if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray())) { kills.SetKillsArrayLengths(); } // Don't kill NativeArray, if there is no mismatch between array's type and element's type. if(doNativeArrayTypeSpec && !(useValueTypes && arrayValueType.IsNativeArray() && ((arrayValueType.IsLikelyNativeIntArray() && instr->GetSrc2()->IsInt32()) || (arrayValueType.IsLikelyNativeFloatArray() && instr->GetSrc2()->IsFloat())) ) && !(useValueTypes && arrayValueType.IsNotNativeArray())) { kills.SetKillsNativeArrays(); } break; } case Js::OpCode::InlineArrayPop: { IR::Opnd *const arrayOpnd = instr->GetSrc1(); Assert(arrayOpnd); const ValueType arrayValueType(arrayOpnd->GetValueType()); if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray())) { break; } if(doArraySegmentHoist) { kills.SetKillsArrayHeadSegmentLengths(); } if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray())) { kills.SetKillsArrayLengths(); } if(doArrayMissingValueCheckHoist && !(useValueTypes && arrayValueType.IsArray())) { kills.SetKillsObjectArraysWithNoMissingValues(); } break; } case Js::OpCode::CallDirect: { Assert(instr->GetSrc1()); // Find the 'this' parameter and check if it's possible for it to be an array IR::Opnd *const arrayOpnd = instr->FindCallArgumentOpnd(1); Assert(arrayOpnd); const ValueType arrayValueType(arrayOpnd->GetValueType()); if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray())) { break; } const IR::JnHelperMethod helperMethod = instr->GetSrc1()->AsHelperCallOpnd()->m_fnHelper; if(doArrayMissingValueCheckHoist) { switch(helperMethod) { case IR::HelperArray_Reverse: case IR::HelperArray_Shift: case IR::HelperArray_Splice: case IR::HelperArray_Unshift: kills.SetKillsArraysWithNoMissingValues(); break; } } if(doArraySegmentHoist) { switch(helperMethod) { case IR::HelperArray_Reverse: case IR::HelperArray_Shift: case IR::HelperArray_Splice: case IR::HelperArray_Unshift: case IR::HelperArray_Concat: kills.SetKillsArrayHeadSegments(); kills.SetKillsArrayHeadSegmentLengths(); break; } } if(doArrayLengthHoist && !(useValueTypes && arrayValueType.IsNotArray())) { switch(helperMethod) { case IR::HelperArray_Shift: case IR::HelperArray_Splice: case IR::HelperArray_Unshift: kills.SetKillsArrayLengths(); break; } } if(doNativeArrayTypeSpec && !(useValueTypes && arrayValueType.IsNotNativeArray())) { switch(helperMethod) { case IR::HelperArray_Reverse: case IR::HelperArray_Shift: case IR::HelperArray_Slice: // Currently not inlined. //case IR::HelperArray_Sort: case IR::HelperArray_Splice: case IR::HelperArray_Unshift: case IR::HelperArray_Concat: kills.SetKillsNativeArrays(); break; } } break; } case Js::OpCode::InitProto: { // Find the 'this' parameter and check if it's possible for it to be an array IR::Opnd *const arrayOpnd = instr->GetSrc1(); Assert(arrayOpnd); const ValueType arrayValueType(arrayOpnd->GetValueType()); if(!arrayOpnd->IsRegOpnd() || (useValueTypes && arrayValueType.IsNotArrayOrObjectWithArray())) { break; } if(doNativeArrayTypeSpec && !(useValueTypes && arrayValueType.IsNotNativeArray())) { kills.SetKillsNativeArrays(); } break; } case Js::OpCode::NewClassProto: Assert(instr->GetSrc1()); if (IR::AddrOpnd::IsEqualAddr(instr->GetSrc1(), (void*)func->GetScriptContextInfo()->GetObjectPrototypeAddr())) { // No extends operand, the proto parent is the Object prototype break; } // Fall through case Js::OpCode::NewScObjectNoCtor: case Js::OpCode::NewScObjectNoCtorFull: if(doNativeArrayTypeSpec) { // Class/object construction can make something a prototype kills.SetKillsNativeArrays(); } break; } return kills; } GlobOptBlockData const * GlobOpt::CurrentBlockData() const { return &this->currentBlock->globOptData; } GlobOptBlockData * GlobOpt::CurrentBlockData() { return &this->currentBlock->globOptData; } void GlobOpt::CommitCapturedValuesCandidate() { GlobOptBlockData * globOptData = CurrentBlockData(); globOptData->changedSyms->ClearAll(); if (!this->changedSymsAfterIncBailoutCandidate->IsEmpty()) { // // some symbols are changed after the values for current bailout have been // captured (GlobOpt::CapturedValues), need to restore such symbols as changed // for following incremental bailout construction, or we will miss capturing // values for later bailout // // swap changedSyms and changedSymsAfterIncBailoutCandidate // because both are from this->alloc BVSparse * tempBvSwap = globOptData->changedSyms; globOptData->changedSyms = this->changedSymsAfterIncBailoutCandidate; this->changedSymsAfterIncBailoutCandidate = tempBvSwap; } if (globOptData->capturedValues) { globOptData->capturedValues->DecrementRefCount(); } globOptData->capturedValues = globOptData->capturedValuesCandidate; // null out capturedValuesCandidate to stop tracking symbols change for it globOptData->capturedValuesCandidate = nullptr; } bool GlobOpt::IsOperationThatLikelyKillsJsArraysWithNoMissingValues(IR::Instr *const instr) { // StElem is profiled with information indicating whether it will likely create a missing value in the array. In that case, // we prefer to kill the no-missing-values information in the value so that we don't bail out in a likely circumstance. return (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict) && DoArrayMissingValueCheckHoist() && instr->IsProfiledInstr() && instr->AsProfiledInstr()->u.stElemInfo->LikelyCreatesMissingValue(); } bool GlobOpt::NeedBailOnImplicitCallForArrayCheckHoist(BasicBlock const * const block, const bool isForwardPass) const { Assert(block); return isForwardPass && block->loop && block->loop->needImplicitCallBailoutChecksForJsArrayCheckHoist; } bool GlobOpt::PrepareForIgnoringIntOverflow(IR::Instr *const instr) { Assert(instr); const bool isBoundary = instr->m_opcode == Js::OpCode::NoIntOverflowBoundary; // Update the instruction's "int overflow matters" flag based on whether we are currently allowing ignoring int overflows. // Some operations convert their srcs to int32s, those can still ignore int overflow. if(instr->ignoreIntOverflowInRange) { instr->ignoreIntOverflowInRange = !intOverflowCurrentlyMattersInRange || OpCodeAttr::IsInt32(instr->m_opcode); } if(!intOverflowDoesNotMatterRange) { Assert(intOverflowCurrentlyMattersInRange); // There are no more ranges of instructions where int overflow does not matter, in this block. return isBoundary; } if(instr == intOverflowDoesNotMatterRange->LastInstr()) { Assert(isBoundary); // Reached the last instruction in the range intOverflowCurrentlyMattersInRange = true; intOverflowDoesNotMatterRange = intOverflowDoesNotMatterRange->Next(); return isBoundary; } if(!intOverflowCurrentlyMattersInRange) { return isBoundary; } if(instr != intOverflowDoesNotMatterRange->FirstInstr()) { // Have not reached the next range return isBoundary; } Assert(isBoundary); // This is the first instruction in a range of instructions where int overflow does not matter. There can be many inputs to // instructions in the range, some of which are inputs to the range itself (that is, the values are not defined in the // range). Ignoring int overflow is only valid for int operations, so we need to ensure that all inputs to the range are // int (not "likely int") before ignoring any overflows in the range. Ensuring that a sym with a "likely int" value is an // int requires a bail-out. These bail-out check need to happen before any overflows are ignored, otherwise it's too late. // The backward pass tracked all inputs into the range. Iterate over them and verify the values, and insert lossless // conversions to int as necessary, before the first instruction in the range. If for any reason all values cannot be // guaranteed to be ints, the optimization will be disabled for this range. intOverflowCurrentlyMattersInRange = false; { BVSparse tempBv1(tempAlloc); BVSparse tempBv2(tempAlloc); { // Just renaming the temp BVs for this section to indicate how they're used so that it makes sense BVSparse &symsToExclude = tempBv1; BVSparse &symsToInclude = tempBv2; #if DBG_DUMP SymID couldNotConvertSymId = 0; #endif FOREACH_BITSET_IN_SPARSEBV(id, intOverflowDoesNotMatterRange->SymsRequiredToBeInt()) { Sym *const sym = func->m_symTable->Find(id); Assert(sym); // Some instructions with property syms are also tracked by the backward pass, and may be included in the range // (LdSlot for instance). These property syms don't get their values until either copy-prop resolves a value for // them, or a new value is created once the use of the property sym is reached. In either case, we're not that // far yet, so we need to find the future value of the property sym by evaluating copy-prop in reverse. Value *const value = sym->IsStackSym() ? CurrentBlockData()->FindValue(sym) : CurrentBlockData()->FindFuturePropertyValue(sym->AsPropertySym()); if(!value) { #if DBG_DUMP couldNotConvertSymId = id; #endif intOverflowCurrentlyMattersInRange = true; BREAK_BITSET_IN_SPARSEBV; } const bool isInt32OrUInt32Float = value->GetValueInfo()->IsFloatConstant() && Js::JavascriptNumber::IsInt32OrUInt32(value->GetValueInfo()->AsFloatConstant()->FloatValue()); if(value->GetValueInfo()->IsInt() || isInt32OrUInt32Float) { if(!IsLoopPrePass()) { // Input values that are already int can be excluded from int-specialization. We can treat unsigned // int32 values as int32 values (ignoring the overflow), since the values will only be used inside the // range where overflow does not matter. symsToExclude.Set(sym->m_id); } continue; } if(!DoAggressiveIntTypeSpec() || !value->GetValueInfo()->IsLikelyInt()) { // When aggressive int specialization is off, syms with "likely int" values cannot be forced to int since // int bail-out checks are not allowed in that mode. Similarly, with aggressive int specialization on, it // wouldn't make sense to force non-"likely int" values to int since it would almost guarantee a bail-out at // runtime. In both cases, just disable ignoring overflow for this range. #if DBG_DUMP couldNotConvertSymId = id; #endif intOverflowCurrentlyMattersInRange = true; BREAK_BITSET_IN_SPARSEBV; } if(IsLoopPrePass()) { // The loop prepass does not modify bit-vectors. Since it doesn't add bail-out checks, it also does not need // to specialize anything up-front. It only needs to be consistent in how it determines whether to allow // ignoring overflow for a range, based on the values of inputs into the range. continue; } // Since input syms are tracked in the backward pass, where there is no value tracking, it will not be aware of // copy-prop. If a copy-prop sym is available, it will be used instead, so exclude the original sym and include // the copy-prop sym for specialization. StackSym *const copyPropSym = CurrentBlockData()->GetCopyPropSym(sym, value); if(copyPropSym) { symsToExclude.Set(sym->m_id); Assert(!symsToExclude.Test(copyPropSym->m_id)); const bool needsToBeLossless = !intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Test(sym->m_id); if(intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Test(copyPropSym->m_id) || symsToInclude.TestAndSet(copyPropSym->m_id)) { // The copy-prop sym is already included if(needsToBeLossless) { // The original sym needs to be lossless, so make the copy-prop sym lossless as well. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Clear(copyPropSym->m_id); } } else if(!needsToBeLossless) { // The copy-prop sym was not included before, and the original sym can be lossy, so make it lossy. intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Set(copyPropSym->m_id); } } else if(!sym->IsStackSym()) { // Only stack syms can be converted to int, and copy-prop syms are stack syms. If a copy-prop sym was not // found for the property sym, we can't ignore overflows in this range. #if DBG_DUMP couldNotConvertSymId = id; #endif intOverflowCurrentlyMattersInRange = true; BREAK_BITSET_IN_SPARSEBV; } } NEXT_BITSET_IN_SPARSEBV; if(intOverflowCurrentlyMattersInRange) { #if DBG_DUMP if(PHASE_TRACE(Js::TrackCompoundedIntOverflowPhase, func) && !IsLoopPrePass()) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; Output::Print( _u("TrackCompoundedIntOverflow - Top function: %s (%s), Phase: %s, Block: %u, Disabled ignoring overflows\n"), func->GetJITFunctionBody()->GetDisplayName(), func->GetDebugNumberSet(debugStringBuffer), Js::PhaseNames[Js::ForwardPhase], currentBlock->GetBlockNum()); Output::Print(_u(" Input sym could not be turned into an int: %u\n"), couldNotConvertSymId); Output::Print(_u(" First instr: ")); instr->m_next->Dump(); Output::Flush(); } #endif intOverflowDoesNotMatterRange = intOverflowDoesNotMatterRange->Next(); return isBoundary; } if(IsLoopPrePass()) { return isBoundary; } // Update the syms to specialize after enumeration intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(&symsToExclude); intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Minus(&symsToExclude); intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Or(&symsToInclude); } { // Exclude syms that are already live as lossless int32, and exclude lossy conversions of syms that are already live // as lossy int32. // symsToExclude = liveInt32Syms - liveLossyInt32Syms // syms live as lossless int // lossySymsToExclude = symsRequiredToBeLossyInt & liveLossyInt32Syms; // syms we want as lossy int that are already live as lossy int // symsToExclude |= lossySymsToExclude // symsRequiredToBeInt -= symsToExclude // symsRequiredToBeLossyInt -= symsToExclude BVSparse &symsToExclude = tempBv1; BVSparse &lossySymsToExclude = tempBv2; symsToExclude.Minus(CurrentBlockData()->liveInt32Syms, CurrentBlockData()->liveLossyInt32Syms); lossySymsToExclude.And( intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt(), CurrentBlockData()->liveLossyInt32Syms); symsToExclude.Or(&lossySymsToExclude); intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(&symsToExclude); intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Minus(&symsToExclude); } #if DBG { // Verify that the syms to be converted are live // liveSyms = liveInt32Syms | liveFloat64Syms | liveVarSyms // deadSymsRequiredToBeInt = symsRequiredToBeInt - liveSyms BVSparse &liveSyms = tempBv1; BVSparse &deadSymsRequiredToBeInt = tempBv2; liveSyms.Or(CurrentBlockData()->liveInt32Syms, CurrentBlockData()->liveFloat64Syms); liveSyms.Or(CurrentBlockData()->liveVarSyms); deadSymsRequiredToBeInt.Minus(intOverflowDoesNotMatterRange->SymsRequiredToBeInt(), &liveSyms); Assert(deadSymsRequiredToBeInt.IsEmpty()); } #endif } // Int-specialize the syms before the first instruction of the range (the current instruction) intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Minus(intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()); #if DBG_DUMP if(PHASE_TRACE(Js::TrackCompoundedIntOverflowPhase, func)) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; Output::Print( _u("TrackCompoundedIntOverflow - Top function: %s (%s), Phase: %s, Block: %u\n"), func->GetJITFunctionBody()->GetDisplayName(), func->GetDebugNumberSet(debugStringBuffer), Js::PhaseNames[Js::ForwardPhase], currentBlock->GetBlockNum()); Output::Print(_u(" Input syms to be int-specialized (lossless): ")); intOverflowDoesNotMatterRange->SymsRequiredToBeInt()->Dump(); Output::Print(_u(" Input syms to be converted to int (lossy): ")); intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt()->Dump(); Output::Print(_u(" First instr: ")); instr->m_next->Dump(); Output::Flush(); } #endif ToInt32(intOverflowDoesNotMatterRange->SymsRequiredToBeInt(), currentBlock, false /* lossy */, instr); ToInt32(intOverflowDoesNotMatterRange->SymsRequiredToBeLossyInt(), currentBlock, true /* lossy */, instr); return isBoundary; } void GlobOpt::VerifyIntSpecForIgnoringIntOverflow(IR::Instr *const instr) { if(intOverflowCurrentlyMattersInRange || IsLoopPrePass()) { return; } Assert(instr->m_opcode != Js::OpCode::Mul_I4 || (instr->m_opcode == Js::OpCode::Mul_I4 && !instr->ShouldCheckFor32BitOverflow() && instr->ShouldCheckForNon32BitOverflow() )); // Instructions that are marked as "overflow doesn't matter" in the range must guarantee that they operate on int values and // result in int values, for ignoring overflow to be valid. So, int-specialization is required for such instructions in the // range. Ld_A is an exception because it only specializes if the src sym is available as a required specialized sym, and it // doesn't generate bailouts or cause ignoring int overflow to be invalid. // MULs are allowed to start a region and have BailOutInfo since they will bailout on non-32 bit overflow. if(instr->m_opcode == Js::OpCode::Ld_A || ((!instr->HasBailOutInfo() || instr->m_opcode == Js::OpCode::Mul_I4) && (!instr->GetDst() || instr->GetDst()->IsInt32()) && (!instr->GetSrc1() || instr->GetSrc1()->IsInt32()) && (!instr->GetSrc2() || instr->GetSrc2()->IsInt32()))) { return; } if (!instr->HasBailOutInfo() && !instr->HasAnySideEffects()) { return; } // This can happen for Neg_A if it needs to bail out on negative zero, and perhaps other cases as well. It's too late to fix // the problem (overflows may already be ignored), so handle it by bailing out at compile-time and disabling tracking int // overflow. Assert(!func->IsTrackCompoundedIntOverflowDisabled()); if(PHASE_TRACE(Js::BailOutPhase, this->func)) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; Output::Print( _u("BailOut (compile-time): function: %s (%s) instr: "), func->GetJITFunctionBody()->GetDisplayName(), func->GetDebugNumberSet(debugStringBuffer)); #if DBG_DUMP instr->Dump(); #else Output::Print(_u("%s "), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode)); #endif Output::Print(_u("(overflow does not matter but could not int-spec or needed bailout)\n")); Output::Flush(); } if(func->IsTrackCompoundedIntOverflowDisabled()) { // Tracking int overflows is already off for some reason. Prevent trying to rejit again because it won't help and the // same thing will happen again and cause an infinite loop. Just abort jitting this function. if(PHASE_TRACE(Js::BailOutPhase, this->func)) { Output::Print(_u(" Aborting JIT because TrackIntOverflow is already off\n")); Output::Flush(); } throw Js::OperationAbortedException(); } throw Js::RejitException(RejitReason::TrackIntOverflowDisabled); } // It makes lowering easier if it can assume that the first src is never a constant, // at least for commutative operators. For non-commutative, just hoist the constant. void GlobOpt::PreLowerCanonicalize(IR::Instr *instr, Value **pSrc1Val, Value **pSrc2Val) { IR::Opnd *dst = instr->GetDst(); IR::Opnd *src1 = instr->GetSrc1(); IR::Opnd *src2 = instr->GetSrc2(); if (src1->IsImmediateOpnd()) { // Swap for dst, src } else if (src2 && dst && src2->IsRegOpnd()) { if (src2->GetIsDead() && !src1->GetIsDead() && !src1->IsEqual(dst)) { // Swap if src2 is dead, as the reg can be reuse for the dst for opEqs like on x86 (ADD r1, r2) } else if (src2->IsEqual(dst)) { // Helps lowering of opEqs } else { return; } // Make sure we don't swap 2 srcs with valueOf calls. if (OpCodeAttr::OpndHasImplicitCall(instr->m_opcode)) { if (instr->IsBranchInstr()) { if (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive()) { return; } } else if (!src1->GetValueType().IsPrimitive() && !src2->GetValueType().IsPrimitive()) { return; } } } else { return; } Js::OpCode opcode = instr->m_opcode; switch (opcode) { case Js::OpCode::And_A: case Js::OpCode::Mul_A: case Js::OpCode::Or_A: case Js::OpCode::Xor_A: case Js::OpCode::And_I4: case Js::OpCode::Mul_I4: case Js::OpCode::Or_I4: case Js::OpCode::Xor_I4: case Js::OpCode::Add_I4: swap_srcs: if (!instr->GetSrc2()->IsImmediateOpnd()) { instr->m_opcode = opcode; instr->SwapOpnds(); Value *tempVal = *pSrc1Val; *pSrc1Val = *pSrc2Val; *pSrc2Val = tempVal; return; } break; case Js::OpCode::BrSrEq_A: case Js::OpCode::BrSrNotNeq_A: case Js::OpCode::BrEq_I4: goto swap_srcs; case Js::OpCode::BrSrNeq_A: case Js::OpCode::BrNeq_A: case Js::OpCode::BrSrNotEq_A: case Js::OpCode::BrNotEq_A: case Js::OpCode::BrNeq_I4: goto swap_srcs; case Js::OpCode::BrGe_A: opcode = Js::OpCode::BrLe_A; goto swap_srcs; case Js::OpCode::BrNotGe_A: opcode = Js::OpCode::BrNotLe_A; goto swap_srcs; case Js::OpCode::BrGe_I4: opcode = Js::OpCode::BrLe_I4; goto swap_srcs; case Js::OpCode::BrGt_A: opcode = Js::OpCode::BrLt_A; goto swap_srcs; case Js::OpCode::BrNotGt_A: opcode = Js::OpCode::BrNotLt_A; goto swap_srcs; case Js::OpCode::BrGt_I4: opcode = Js::OpCode::BrLt_I4; goto swap_srcs; case Js::OpCode::BrLe_A: opcode = Js::OpCode::BrGe_A; goto swap_srcs; case Js::OpCode::BrNotLe_A: opcode = Js::OpCode::BrNotGe_A; goto swap_srcs; case Js::OpCode::BrLe_I4: opcode = Js::OpCode::BrGe_I4; goto swap_srcs; case Js::OpCode::BrLt_A: opcode = Js::OpCode::BrGt_A; goto swap_srcs; case Js::OpCode::BrNotLt_A: opcode = Js::OpCode::BrNotGt_A; goto swap_srcs; case Js::OpCode::BrLt_I4: opcode = Js::OpCode::BrGt_I4; goto swap_srcs; case Js::OpCode::BrEq_A: case Js::OpCode::BrNotNeq_A: case Js::OpCode::CmEq_A: case Js::OpCode::CmNeq_A: // this == "" not the same as "" == this... if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive())) { return; } goto swap_srcs; case Js::OpCode::CmGe_A: if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive())) { return; } opcode = Js::OpCode::CmLe_A; goto swap_srcs; case Js::OpCode::CmGt_A: if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive())) { return; } opcode = Js::OpCode::CmLt_A; goto swap_srcs; case Js::OpCode::CmLe_A: if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive())) { return; } opcode = Js::OpCode::CmGe_A; goto swap_srcs; case Js::OpCode::CmLt_A: if (!src1->IsImmediateOpnd() && (!src1->GetValueType().IsPrimitive() || !src2->GetValueType().IsPrimitive())) { return; } opcode = Js::OpCode::CmGt_A; goto swap_srcs; case Js::OpCode::CallI: case Js::OpCode::CallIFixed: case Js::OpCode::NewScObject: case Js::OpCode::NewScObjectSpread: case Js::OpCode::NewScObjArray: case Js::OpCode::NewScObjArraySpread: case Js::OpCode::NewScObjectNoCtor: // Don't insert load to register if the function operand is a fixed function. if (instr->HasFixedFunctionAddressTarget()) { return; } break; // Can't do add because <32 + "Hello"> isn't equal to <"Hello" + 32> // Lower can do the swap. Other op-codes listed below don't need immediate source hoisting, as the fast paths handle it, // or the lowering handles the hoisting. case Js::OpCode::Add_A: if (src1->IsFloat()) { goto swap_srcs; } return; case Js::OpCode::Sub_I4: case Js::OpCode::Neg_I4: case Js::OpCode::Not_I4: case Js::OpCode::NewScFunc: case Js::OpCode::NewScGenFunc: case Js::OpCode::NewScFuncHomeObj: case Js::OpCode::NewScGenFuncHomeObj: case Js::OpCode::NewScArray: case Js::OpCode::NewScIntArray: case Js::OpCode::NewScFltArray: case Js::OpCode::NewScArrayWithMissingValues: case Js::OpCode::NewRegEx: case Js::OpCode::Ld_A: case Js::OpCode::Ld_I4: case Js::OpCode::ThrowRuntimeError: case Js::OpCode::TrapIfMinIntOverNegOne: case Js::OpCode::TrapIfTruncOverflow: case Js::OpCode::TrapIfZero: case Js::OpCode::TrapIfUnalignedAccess: case Js::OpCode::FromVar: case Js::OpCode::Conv_Prim: case Js::OpCode::Conv_Prim_Sat: case Js::OpCode::LdC_A_I4: case Js::OpCode::LdStr: case Js::OpCode::InitFld: case Js::OpCode::InitRootFld: case Js::OpCode::StartCall: case Js::OpCode::ArgOut_A: case Js::OpCode::ArgOut_A_Inline: case Js::OpCode::ArgOut_A_Dynamic: case Js::OpCode::ArgOut_A_FromStackArgs: case Js::OpCode::ArgOut_A_InlineBuiltIn: case Js::OpCode::ArgOut_A_InlineSpecialized: case Js::OpCode::ArgOut_A_SpreadArg: case Js::OpCode::InlineeEnd: case Js::OpCode::EndCallForPolymorphicInlinee: case Js::OpCode::InlineeMetaArg: case Js::OpCode::InlineBuiltInEnd: case Js::OpCode::InlineNonTrackingBuiltInEnd: case Js::OpCode::CallHelper: case Js::OpCode::LdElemUndef: case Js::OpCode::LdElemUndefScoped: case Js::OpCode::RuntimeTypeError: case Js::OpCode::RuntimeReferenceError: case Js::OpCode::Ret: case Js::OpCode::NewScObjectSimple: case Js::OpCode::NewScObjectLiteral: case Js::OpCode::StFld: case Js::OpCode::StRootFld: case Js::OpCode::StSlot: case Js::OpCode::StSlotChkUndecl: case Js::OpCode::StElemC: case Js::OpCode::StArrSegElemC: case Js::OpCode::StElemI_A: case Js::OpCode::StElemI_A_Strict: case Js::OpCode::CallDirect: case Js::OpCode::BrNotHasSideEffects: case Js::OpCode::NewConcatStrMulti: case Js::OpCode::NewConcatStrMultiBE: case Js::OpCode::ExtendArg_A: case Js::OpCode::NewScopeSlots: case Js::OpCode::NewScopeSlotsWithoutPropIds: case Js::OpCode::NewStackScopeSlots: case Js::OpCode::IsInst: case Js::OpCode::BailOnEqual: case Js::OpCode::BailOnNotEqual: case Js::OpCode::StArrViewElem: return; } if (!src1->IsImmediateOpnd()) { return; } // The fast paths or lowering of the remaining instructions may not support handling immediate opnds for the first src. The // immediate src1 is hoisted here into a separate instruction. if (src1->IsIntConstOpnd()) { IR::Instr *newInstr = instr->HoistSrc1(Js::OpCode::Ld_I4); ToInt32Dst(newInstr, newInstr->GetDst()->AsRegOpnd(), this->currentBlock); } else if (src1->IsInt64ConstOpnd()) { instr->HoistSrc1(Js::OpCode::Ld_I4); } else { instr->HoistSrc1(Js::OpCode::Ld_A); } src1 = instr->GetSrc1(); src1->AsRegOpnd()->m_sym->SetIsConst(); } // Clear the ValueMap pf the values invalidated by this instr. void GlobOpt::ProcessKills(IR::Instr *instr) { if (instr->m_opcode == Js::OpCode::Yield) { this->CurrentBlockData()->KillStateForGeneratorYield(instr); } this->ProcessFieldKills(instr); this->ProcessValueKills(instr); this->ProcessArrayValueKills(instr); } bool GlobOpt::OptIsInvariant(IR::Opnd *src, BasicBlock *block, Loop *loop, Value *srcVal, bool isNotTypeSpecConv, bool allowNonPrimitives) { if(!loop->CanHoistInvariants()) { return false; } Sym *sym; switch(src->GetKind()) { case IR::OpndKindAddr: case IR::OpndKindFloatConst: case IR::OpndKindIntConst: return true; case IR::OpndKindReg: sym = src->AsRegOpnd()->m_sym; break; case IR::OpndKindSym: sym = src->AsSymOpnd()->m_sym; if (src->AsSymOpnd()->IsPropertySymOpnd()) { if (src->AsSymOpnd()->AsPropertySymOpnd()->IsTypeChecked()) { // We do not handle hoisting these yet. We might be hoisting this across the instr with the type check protecting this one. // And somehow, the dead-store pass now removes the type check on that instr later on... // For CheckFixedFld, there is no benefit hoisting these if they don't have a type check as they won't generate code. return false; } } break; case IR::OpndKindHelperCall: // Helper calls, like the private slot getter, can be invariant. // Consider moving more math builtin to invariant? return HelperMethodAttributes::IsInVariant(src->AsHelperCallOpnd()->m_fnHelper); default: return false; } return OptIsInvariant(sym, block, loop, srcVal, isNotTypeSpecConv, allowNonPrimitives); } bool GlobOpt::OptIsInvariant(Sym *sym, BasicBlock *block, Loop *loop, Value *srcVal, bool isNotTypeSpecConv, bool allowNonPrimitives, Value **loopHeadValRef) { Value *localLoopHeadVal; if(!loopHeadValRef) { loopHeadValRef = &localLoopHeadVal; } Value *&loopHeadVal = *loopHeadValRef; loopHeadVal = nullptr; if(!loop->CanHoistInvariants()) { return false; } if (sym->IsStackSym()) { if (sym->AsStackSym()->IsTypeSpec()) { StackSym *varSym = sym->AsStackSym()->GetVarEquivSym(this->func); // Make sure the int32/float64 version of this is available. // Note: We could handle this by converting the src, but usually the // conversion is hoistable if this is hoistable anyway. // In some weird cases it may not be however, so we'll bail out. if (sym->AsStackSym()->IsInt32()) { Assert(block->globOptData.liveInt32Syms->Test(varSym->m_id)); if (!loop->landingPad->globOptData.liveInt32Syms->Test(varSym->m_id) || (loop->landingPad->globOptData.liveLossyInt32Syms->Test(varSym->m_id) && !block->globOptData.liveLossyInt32Syms->Test(varSym->m_id))) { // Either the int32 sym is not live in the landing pad, or it's lossy in the landing pad and the // instruction's block is using the lossless version. In either case, the instruction cannot be hoisted // without doing a conversion of this operand. return false; } } else if (sym->AsStackSym()->IsFloat64()) { if (!loop->landingPad->globOptData.liveFloat64Syms->Test(varSym->m_id)) { return false; } } sym = sym->AsStackSym()->GetVarEquivSym(this->func); } else { // Make sure the var version of this is available. // Note: We could handle this by converting the src, but usually the // conversion is hoistable if this is hoistable anyway. // In some weird cases it may not be however, so we'll bail out. if (!loop->landingPad->globOptData.liveVarSyms->Test(sym->m_id)) { return false; } } } else if (sym->IsPropertySym()) { if (!loop->landingPad->globOptData.liveVarSyms->Test(sym->AsPropertySym()->m_stackSym->m_id)) { return false; } } else { return false; } // We rely on having a value. if (srcVal == NULL) { return false; } // A symbol is invariant if its current value is the same as it was upon entering the loop. loopHeadVal = loop->landingPad->globOptData.FindValue(sym); if (loopHeadVal == NULL || loopHeadVal->GetValueNumber() != srcVal->GetValueNumber()) { return false; } // Can't hoist non-primitives, unless we have safeguards against valueof/tostring. Additionally, we need to consider // the value annotations on the source *before* the loop: if we hoist this instruction outside the loop, we can't // necessarily rely on type annotations added (and enforced) earlier in the loop's body. // // It might look as though !loopHeadVal->GetValueInfo()->IsPrimitive() implies // !loop->landingPad->globOptData.IsTypeSpecialized(sym), but it turns out that this is not always the case. We // encountered a test case in which we had previously hoisted a FromVar (to float 64) instruction, but its bailout code was // BailoutPrimitiveButString, rather than BailoutNumberOnly, which would have allowed us to conclude that the dest was // definitely a float64. Instead, it was only *likely* a float64, causing IsPrimitive to return false. if (!allowNonPrimitives && !loopHeadVal->GetValueInfo()->IsPrimitive() && !loop->landingPad->globOptData.IsTypeSpecialized(sym)) { return false; } if(!isNotTypeSpecConv && loop->symsDefInLoop->Test(sym->m_id)) { // Typically, a sym is considered invariant if it has the same value in the current block and in the loop landing pad. // The sym may have had a different value earlier in the loop or on the back-edge, but as long as it's reassigned to its // value outside the loop, it would be considered invariant in this block. Consider that case: // s1 = s2[invariant] // // s1 = s2[invariant] // // s1 now has the same value as in the landing pad, and is considered invariant // s1 += s3 // // s1 is not invariant here, or on the back-edge // ++s3 // s3 is not invariant, so the add above cannot be hoisted // // // A problem occurs at the point of (s1 += s3) when: // - At (s1 = s2) inside the loop, s1 was made to be the sym store of that value. This by itself is legal, because // after that transfer, s1 and s2 have the same value. // - (s1 += s3) is type-specialized but s1 is not specialized in the loop header. This happens when s1 is not // specialized entering the loop, and since s1 is not used before it's defined in the loop, it's not specialized // on back-edges. // // With that, at (s1 += s3), the conversion of s1 to the type-specialized version would be hoisted because s1 is // invariant just before that instruction. Since this add is specialized, the specialized version of the sym is modified // in the loop without a reassignment at (s1 = s2) inside the loop, and (s1 += s3) would then use an incorrect value of // s1 (it would use the value of s1 from the previous loop iteration, instead of using the value of s2). // // The problem here, is that we cannot hoist the conversion of s1 into its specialized version across the assignment // (s1 = s2) inside the loop. So for the purposes of type specialization, don't consider a sym invariant if it has a def // inside the loop. return false; } // For values with an int range, require additionally that the range is the same as in the landing pad, as the range may // have been changed on this path based on branches, and int specialization and invariant hoisting may rely on the range // being the same. For type spec conversions, only require that if the value is an int constant in the current block, that // it is also an int constant with the same value in the landing pad. Other range differences don't matter for type spec. IntConstantBounds srcIntConstantBounds, loopHeadIntConstantBounds; if(srcVal->GetValueInfo()->TryGetIntConstantBounds(&srcIntConstantBounds) && (isNotTypeSpecConv || srcIntConstantBounds.IsConstant()) && ( !loopHeadVal->GetValueInfo()->TryGetIntConstantBounds(&loopHeadIntConstantBounds) || loopHeadIntConstantBounds.LowerBound() != srcIntConstantBounds.LowerBound() || loopHeadIntConstantBounds.UpperBound() != srcIntConstantBounds.UpperBound() )) { return false; } // Disabling this assert, because it does not hold true when we force specialize in the loop landing pad //Assert((!loopHeadVal->GetValueInfo()->IsPrimitive()) || srcVal->GetValueInfo()->IsLikelyPrimitive()); return true; } bool GlobOpt::OptIsInvariant( IR::Instr *instr, BasicBlock *block, Loop *loop, Value *src1Val, Value *src2Val, bool isNotTypeSpecConv, const bool forceInvariantHoisting) { if (!loop->CanHoistInvariants()) { return false; } if (!OpCodeAttr::CanCSE(instr->m_opcode)) { return false; } bool allowNonPrimitives = !OpCodeAttr::OpndHasImplicitCall(instr->m_opcode); switch(instr->m_opcode) { // Can't legally hoist these case Js::OpCode::LdLen_A: return false; //Can't Hoist BailOnNotStackArgs, as it is necessary as InlineArgsOptimization relies on this opcode //to decide whether to throw rejit exception or not. case Js::OpCode::BailOnNotStackArgs: return false; // Usually not worth hoisting these case Js::OpCode::Ld_A: case Js::OpCode::Ld_I4: case Js::OpCode::LdC_A_I4: if(!forceInvariantHoisting) { return false; } break; // Can't hoist these outside the function it's for. The LdArgumentsFromFrame for an inlinee depends on the inlinee meta arg // that holds the arguments object, which is only initialized at the start of the inlinee. So, can't hoist this outside the // inlinee. case Js::OpCode::LdArgumentsFromFrame: if(instr->m_func != loop->GetFunc()) { return false; } break; case Js::OpCode::FromVar: if (instr->HasBailOutInfo()) { allowNonPrimitives = true; } break; case Js::OpCode::CheckObjType: // Bug 11712101: If the operand is a field, ensure that its containing object type is invariant // before hoisting -- that is, don't hoist a CheckObjType over a DeleteFld on that object. // (CheckObjType only checks the operand and its immediate parent, so we don't need to go // any farther up the object graph.) Assert(instr->GetSrc1()); PropertySym *propertySym = instr->GetSrc1()->AsPropertySymOpnd()->GetPropertySym(); if (propertySym->HasObjectTypeSym()) { StackSym *objectTypeSym = propertySym->GetObjectTypeSym(); if (!this->OptIsInvariant(objectTypeSym, block, loop, this->CurrentBlockData()->FindValue(objectTypeSym), true, true)) { return false; } } break; } IR::Opnd *dst = instr->GetDst(); if (dst && !dst->IsRegOpnd()) { return false; } IR::Opnd *src1 = instr->GetSrc1(); if (src1) { if (!this->OptIsInvariant(src1, block, loop, src1Val, isNotTypeSpecConv, allowNonPrimitives)) { return false; } IR::Opnd *src2 = instr->GetSrc2(); if (src2) { if (!this->OptIsInvariant(src2, block, loop, src2Val, isNotTypeSpecConv, allowNonPrimitives)) { return false; } } } return true; } bool GlobOpt::OptDstIsInvariant(IR::RegOpnd *dst) { StackSym *dstSym = dst->m_sym; if (dstSym->IsTypeSpec()) { // The type-specialized sym may be single def, but not the original... dstSym = dstSym->GetVarEquivSym(this->func); } return (dstSym->m_isSingleDef); } void GlobOpt::OptHoistUpdateValueType( Loop* loop, IR::Instr* instr, IR::Opnd** srcOpndPtr /* All code paths that change src, should update srcOpndPtr*/, Value* opndVal) { if (opndVal == nullptr || instr->m_opcode == Js::OpCode::FromVar || srcOpndPtr == nullptr || *srcOpndPtr == nullptr) { return; } IR::Opnd* srcOpnd = *srcOpndPtr; Sym* opndSym = srcOpnd->GetSym();; if (opndSym) { BasicBlock* landingPad = loop->landingPad; Value* opndValueInLandingPad = landingPad->globOptData.FindValue(opndSym); Assert(opndVal->GetValueNumber() == opndValueInLandingPad->GetValueNumber()); ValueType opndValueTypeInLandingPad = opndValueInLandingPad->GetValueInfo()->Type(); if (srcOpnd->GetValueType() != opndValueTypeInLandingPad) { srcOpnd->SetValueType(opndValueTypeInLandingPad); if (instr->m_opcode == Js::OpCode::SetConcatStrMultiItemBE) { Assert(!opndSym->IsPropertySym()); Assert(!opndValueTypeInLandingPad.IsString()); Assert(instr->GetDst()); IR::RegOpnd* strOpnd = IR::RegOpnd::New(TyVar, instr->m_func); strOpnd->SetValueType(ValueType::String); strOpnd->SetValueTypeFixed(); IR::Instr* convPrimStrInstr = IR::Instr::New(Js::OpCode::Conv_PrimStr, strOpnd, srcOpnd->Use(instr->m_func), instr->m_func); instr->ReplaceSrc(srcOpnd, strOpnd); // Replace above will free srcOpnd, so reassign it *srcOpndPtr = srcOpnd = reinterpret_cast(strOpnd); // We add ConvPrim_Str in the landingpad, and since this instruction doesn't go through the checks in OptInstr, the bailout is never added // As we expand hoisting of instructions to new opcode, we need a better framework to handle such cases if (IsImplicitCallBailOutCurrentlyNeeded(convPrimStrInstr, opndValueInLandingPad, nullptr, landingPad, landingPad->globOptData.liveFields->IsEmpty(), true, true)) { EnsureBailTarget(loop); loop->bailOutInfo->bailOutInstr->InsertBefore(convPrimStrInstr); convPrimStrInstr = convPrimStrInstr->ConvertToBailOutInstr(convPrimStrInstr, IR::BailOutOnImplicitCallsPreOp, loop->bailOutInfo->bailOutOffset); convPrimStrInstr->ReplaceBailOutInfo(loop->bailOutInfo); } else { if (loop->bailOutInfo->bailOutInstr) { loop->bailOutInfo->bailOutInstr->InsertBefore(convPrimStrInstr); } else { landingPad->InsertAfter(convPrimStrInstr); } } // If we came here opndSym can't be PropertySym return; } } if (opndSym->IsPropertySym()) { // Also fix valueInfo on objPtr StackSym* opndObjPtrSym = opndSym->AsPropertySym()->m_stackSym; Value* opndObjPtrSymValInLandingPad = landingPad->globOptData.FindValue(opndObjPtrSym); ValueInfo* opndObjPtrSymValueInfoInLandingPad = opndObjPtrSymValInLandingPad->GetValueInfo(); srcOpnd->AsSymOpnd()->SetPropertyOwnerValueType(opndObjPtrSymValueInfoInLandingPad->Type()); } } } void GlobOpt::OptHoistInvariant( IR::Instr *instr, BasicBlock *block, Loop *loop, Value *dstVal, Value *const src1Val, Value *const src2Val, bool isNotTypeSpecConv, bool lossy, IR::BailOutKind bailoutKind) { BasicBlock *landingPad = loop->landingPad; IR::Opnd* src1 = instr->GetSrc1(); if (src1) { // We are hoisting this instruction possibly past other uses, which might invalidate the last use info. Clear it. OptHoistUpdateValueType(loop, instr, &src1, src1Val); if (src1->IsRegOpnd()) { src1->AsRegOpnd()->m_isTempLastUse = false; } IR::Opnd* src2 = instr->GetSrc2(); if (src2) { OptHoistUpdateValueType(loop, instr, &src2, src2Val); if (src2->IsRegOpnd()) { src2->AsRegOpnd()->m_isTempLastUse = false; } } } IR::RegOpnd *dst = instr->GetDst() ? instr->GetDst()->AsRegOpnd() : nullptr; if(dst) { switch (instr->m_opcode) { case Js::OpCode::CmEq_I4: case Js::OpCode::CmNeq_I4: case Js::OpCode::CmLt_I4: case Js::OpCode::CmLe_I4: case Js::OpCode::CmGt_I4: case Js::OpCode::CmGe_I4: case Js::OpCode::CmUnLt_I4: case Js::OpCode::CmUnLe_I4: case Js::OpCode::CmUnGt_I4: case Js::OpCode::CmUnGe_I4: // These operations are a special case. They generate a lossy int value, and the var sym is initialized using // Conv_Bool. A sym cannot be live only as a lossy int sym, the var needs to be live as well since the lossy int // sym cannot be used to convert to var. We don't know however, whether the Conv_Bool will be hoisted. The idea // currently is that the sym is only used on the path in which it is initialized inside the loop. So, don't // hoist any liveness info for the dst. if (!this->GetIsAsmJSFunc()) { lossy = true; } break; case Js::OpCode::FromVar: { StackSym* src1StackSym = IR::RegOpnd::TryGetStackSym(instr->GetSrc1()); if (instr->HasBailOutInfo()) { IR::BailOutKind instrBailoutKind = instr->GetBailOutKind(); Assert(instrBailoutKind == IR::BailOutIntOnly || instrBailoutKind == IR::BailOutExpectingInteger || instrBailoutKind == IR::BailOutOnNotPrimitive || instrBailoutKind == IR::BailOutNumberOnly || instrBailoutKind == IR::BailOutPrimitiveButString); } else if (src1StackSym && bailoutKind != IR::BailOutInvalid) { // We may be hoisting FromVar from a region where it didn't need a bailout (src1 had a definite value type) to a region // where it would. In such cases, the FromVar needs a bailout based on the value type of src1 in its new position. Assert(!src1StackSym->IsTypeSpec()); Value* landingPadSrc1val = landingPad->globOptData.FindValue(src1StackSym); Assert(src1Val->GetValueNumber() == landingPadSrc1val->GetValueNumber()); ValueInfo *src1ValueInfo = src1Val->GetValueInfo(); ValueInfo *landingPadSrc1ValueInfo = landingPadSrc1val->GetValueInfo(); IRType dstType = dst->GetType(); const auto AddBailOutToFromVar = [&]() { instr->GetSrc1()->SetValueType(landingPadSrc1val->GetValueInfo()->Type()); EnsureBailTarget(loop); if (block->IsLandingPad()) { instr = instr->ConvertToBailOutInstr(instr, bailoutKind, loop->bailOutInfo->bailOutOffset); } else { instr = instr->ConvertToBailOutInstr(instr, bailoutKind); } }; // A definite type in the source position and not a definite type in the destination (landing pad) // and no bailout on the instruction; we should put a bailout on the hoisted instruction. if (dstType == TyInt32) { if (lossy) { if ((src1ValueInfo->IsPrimitive() || block->globOptData.IsTypeSpecialized(src1StackSym)) && // didn't need a lossy type spec bailout in the source block (!landingPadSrc1ValueInfo->IsPrimitive() && !landingPad->globOptData.IsTypeSpecialized(src1StackSym))) // needs a lossy type spec bailout in the landing pad { bailoutKind = IR::BailOutOnNotPrimitive; AddBailOutToFromVar(); } } else if (src1ValueInfo->IsInt() && !landingPadSrc1ValueInfo->IsInt()) { AddBailOutToFromVar(); } } else if ((dstType == TyFloat64 && src1ValueInfo->IsNumber() && !landingPadSrc1ValueInfo->IsNumber())) { AddBailOutToFromVar(); } } break; } } if (dstVal == NULL) { dstVal = this->NewGenericValue(ValueType::Uninitialized, dst); } // ToVar/FromVar don't need a new dst because it has to be invariant if their src is invariant. bool dstDoesntNeedLoad = (!isNotTypeSpecConv && instr->m_opcode != Js::OpCode::LdC_A_I4); StackSym *varSym = dst->m_sym; if (varSym->IsTypeSpec()) { varSym = varSym->GetVarEquivSym(this->func); } Value *const landingPadDstVal = loop->landingPad->globOptData.FindValue(varSym); if(landingPadDstVal ? dstVal->GetValueNumber() != landingPadDstVal->GetValueNumber() : loop->symsDefInLoop->Test(varSym->m_id)) { // We need a temp for FromVar/ToVar if dst changes in the loop. dstDoesntNeedLoad = false; } if (!dstDoesntNeedLoad && this->OptDstIsInvariant(dst) == false) { // Keep dst in place, hoist instr using a new dst. instr->UnlinkDst(); // Set type specialization info correctly for this new sym StackSym *copyVarSym; IR::RegOpnd *copyReg; if (dst->m_sym->IsTypeSpec()) { copyVarSym = StackSym::New(TyVar, instr->m_func); StackSym *copySym = copyVarSym; if (dst->m_sym->IsInt32()) { if(lossy) { // The new sym would only be live as a lossy int since we're only hoisting the store to the int version // of the sym, and cannot be converted to var. It is not legal to have a sym only live as a lossy int, // so don't update liveness info for this sym. } else { block->globOptData.liveInt32Syms->Set(copyVarSym->m_id); } copySym = copySym->GetInt32EquivSym(instr->m_func); } else if (dst->m_sym->IsFloat64()) { block->globOptData.liveFloat64Syms->Set(copyVarSym->m_id); copySym = copySym->GetFloat64EquivSym(instr->m_func); } copyReg = IR::RegOpnd::New(copySym, copySym->GetType(), instr->m_func); } else { copyReg = IR::RegOpnd::New(dst->GetType(), instr->m_func); copyVarSym = copyReg->m_sym; block->globOptData.liveVarSyms->Set(copyVarSym->m_id); } copyReg->SetValueType(dst->GetValueType()); IR::Instr *copyInstr = IR::Instr::New(Js::OpCode::Ld_A, dst, copyReg, instr->m_func); copyInstr->SetByteCodeOffset(instr); instr->SetDst(copyReg); instr->InsertBefore(copyInstr); dst->m_sym->m_mayNotBeTempLastUse = true; if (instr->GetSrc1() && instr->GetSrc1()->IsImmediateOpnd()) { // Propagate IsIntConst if appropriate switch(instr->m_opcode) { case Js::OpCode::Ld_A: case Js::OpCode::Ld_I4: case Js::OpCode::LdC_A_I4: copyReg->m_sym->SetIsConst(); break; } } ValueInfo *dstValueInfo = dstVal->GetValueInfo(); if((!dstValueInfo->GetSymStore() || dstValueInfo->GetSymStore() == varSym) && !lossy) { // The destination's value may have been transferred from one of the invariant sources, in which case we should // keep the sym store intact, as that sym will likely have a better lifetime than this new copy sym. For // instance, if we're inside a conditioned block, because we don't make the copy sym live and set its value in // all preceding blocks, this sym would not be live after exiting this block, causing this value to not // participate in copy-prop after this block. this->SetSymStoreDirect(dstValueInfo, copyVarSym); } block->globOptData.InsertNewValue(dstVal, copyReg); dst = copyReg; } } // Move to landing pad block->UnlinkInstr(instr); if (loop->bailOutInfo->bailOutInstr) { loop->bailOutInfo->bailOutInstr->InsertBefore(instr); } else { landingPad->InsertAfter(instr); } GlobOpt::MarkNonByteCodeUsed(instr); if (instr->HasBailOutInfo() || instr->HasAuxBailOut()) { Assert(loop->bailOutInfo); EnsureBailTarget(loop); // Copy bailout info of loop top. instr->ReplaceBailOutInfo(loop->bailOutInfo); } if(!dst) { return; } // The bailout info's liveness for the dst sym is not updated in loop landing pads because bailout instructions previously // hoisted into the loop's landing pad may bail out before the current type of the dst sym became live (perhaps due to this // instruction). Since the landing pad will have a shared bailout point, the bailout info cannot assume that the current // type of the dst sym was live during every bailout hoisted into the landing pad. StackSym *const dstSym = dst->m_sym; StackSym *const dstVarSym = dstSym->IsTypeSpec() ? dstSym->GetVarEquivSym(nullptr) : dstSym; Assert(dstVarSym); if(isNotTypeSpecConv || !loop->landingPad->globOptData.IsLive(dstVarSym)) { // A new dst is being hoisted, or the same single-def dst that would not be live before this block. So, make it live and // update the value info with the same value info in this block. if(lossy) { // This is a lossy conversion to int. The instruction was given a new dst specifically for hoisting, so this new dst // will not be live as a var before this block. A sym cannot be live only as a lossy int sym, the var needs to be // live as well since the lossy int sym cannot be used to convert to var. Since the var version of the sym is not // going to be initialized, don't hoist any liveness info for the dst. The sym is only going to be used on the path // in which it is initialized inside the loop. Assert(dstSym->IsTypeSpec()); Assert(dstSym->IsInt32()); return; } // Check if the dst value was transferred from the src. If so, the value transfer needs to be replicated. bool isTransfer = dstVal == src1Val; StackSym *transferValueOfSym = nullptr; if(isTransfer) { Assert(instr->GetSrc1()); if(instr->GetSrc1()->IsRegOpnd()) { StackSym *src1Sym = instr->GetSrc1()->AsRegOpnd()->m_sym; if(src1Sym->IsTypeSpec()) { src1Sym = src1Sym->GetVarEquivSym(nullptr); Assert(src1Sym); } if(dstVal == block->globOptData.FindValue(src1Sym)) { transferValueOfSym = src1Sym; } } } // SIMD_JS if (instr->m_opcode == Js::OpCode::ExtendArg_A) { // Check if we should have CSE'ed this EA Assert(instr->GetSrc1()); // If the dstVal symstore is not the dst itself, then we copied the Value from another expression. if (dstVal->GetValueInfo()->GetSymStore() != instr->GetDst()->GetStackSym()) { isTransfer = true; transferValueOfSym = dstVal->GetValueInfo()->GetSymStore()->AsStackSym(); } } const ValueNumber dstValueNumber = dstVal->GetValueNumber(); ValueNumber dstNewValueNumber = InvalidValueNumber; for(InvariantBlockBackwardIterator it(this, block, loop->landingPad, nullptr); it.IsValid(); it.MoveNext()) { BasicBlock *const hoistBlock = it.Block(); GlobOptBlockData &hoistBlockData = hoistBlock->globOptData; Assert(!hoistBlockData.IsLive(dstVarSym)); hoistBlockData.MakeLive(dstSym, lossy); Value *newDstValue; do { if(isTransfer) { if(transferValueOfSym) { newDstValue = hoistBlockData.FindValue(transferValueOfSym); if(newDstValue && newDstValue->GetValueNumber() == dstValueNumber) { break; } } // It's a transfer, but we don't have a sym whose value number matches in the target block. Use a new value // number since we don't know if there is already a value with the current number for the target block. if(dstNewValueNumber == InvalidValueNumber) { dstNewValueNumber = NewValueNumber(); } newDstValue = CopyValue(dstVal, dstNewValueNumber); break; } newDstValue = CopyValue(dstVal, dstValueNumber); } while(false); hoistBlockData.SetValue(newDstValue, dstVarSym); } return; } #if DBG if(instr->GetSrc1()->IsRegOpnd()) // Type spec conversion may load a constant into a dst sym { StackSym *const srcSym = instr->GetSrc1()->AsRegOpnd()->m_sym; Assert(srcSym != dstSym); // Type spec conversion must be changing the type, so the syms must be different StackSym *const srcVarSym = srcSym->IsTypeSpec() ? srcSym->GetVarEquivSym(nullptr) : srcSym; Assert(srcVarSym == dstVarSym); // Type spec conversion must be between variants of the same var sym } #endif bool changeValueType = false, changeValueTypeToInt = false; if(dstSym->IsTypeSpec()) { if(dst->IsInt32()) { if(!lossy) { Assert( !instr->HasBailOutInfo() || instr->GetBailOutKind() == IR::BailOutIntOnly || instr->GetBailOutKind() == IR::BailOutExpectingInteger); changeValueType = changeValueTypeToInt = true; } } else if (dst->IsFloat64()) { if(instr->HasBailOutInfo() && instr->GetBailOutKind() == IR::BailOutNumberOnly) { changeValueType = true; } } } ValueInfo *previousValueInfoBeforeUpdate = nullptr, *previousValueInfoAfterUpdate = nullptr; for(InvariantBlockBackwardIterator it( this, block, loop->landingPad, dstVarSym, dstVal->GetValueNumber()); it.IsValid(); it.MoveNext()) { BasicBlock *const hoistBlock = it.Block(); GlobOptBlockData &hoistBlockData = hoistBlock->globOptData; #if DBG // TODO: There are some odd cases with field hoisting where the sym is invariant in only part of the loop and the info // does not flow through all blocks. Un-comment the verification below after PRE replaces field hoisting. //// Verify that the src sym is live as the required type, and that the conversion is valid //Assert(IsLive(dstVarSym, &hoistBlockData)); //if(instr->GetSrc1()->IsRegOpnd()) //{ // IR::RegOpnd *const src = instr->GetSrc1()->AsRegOpnd(); // StackSym *const srcSym = instr->GetSrc1()->AsRegOpnd()->m_sym; // if(srcSym->IsTypeSpec()) // { // if(src->IsInt32()) // { // Assert(hoistBlockData.liveInt32Syms->Test(dstVarSym->m_id)); // Assert(!hoistBlockData.liveLossyInt32Syms->Test(dstVarSym->m_id)); // shouldn't try to convert a lossy int32 to anything // } // else // { // Assert(src->IsFloat64()); // Assert(hoistBlockData.liveFloat64Syms->Test(dstVarSym->m_id)); // if(dstSym->IsTypeSpec() && dst->IsInt32()) // { // Assert(lossy); // shouldn't try to do a lossless conversion from float64 to int32 // } // } // } // else // { // Assert(hoistBlockData.liveVarSyms->Test(dstVarSym->m_id)); // } //} //if(dstSym->IsTypeSpec() && dst->IsInt32()) //{ // // If the sym is already specialized as required in the block to which we are attempting to hoist the conversion, // // that info should have flowed into this block // if(lossy) // { // Assert(!hoistBlockData.liveInt32Syms->Test(dstVarSym->m_id)); // } // else // { // Assert(!IsInt32TypeSpecialized(dstVarSym, hoistBlock)); // } //} #endif hoistBlockData.MakeLive(dstSym, lossy); if(!changeValueType) { continue; } Value *const hoistBlockValue = it.InvariantSymValue(); ValueInfo *const hoistBlockValueInfo = hoistBlockValue->GetValueInfo(); if(hoistBlockValueInfo == previousValueInfoBeforeUpdate) { if(hoistBlockValueInfo != previousValueInfoAfterUpdate) { HoistInvariantValueInfo(previousValueInfoAfterUpdate, hoistBlockValue, hoistBlock); } } else { previousValueInfoBeforeUpdate = hoistBlockValueInfo; ValueInfo *const newValueInfo = changeValueTypeToInt ? hoistBlockValueInfo->SpecializeToInt32(alloc) : hoistBlockValueInfo->SpecializeToFloat64(alloc); previousValueInfoAfterUpdate = newValueInfo; ChangeValueInfo(changeValueTypeToInt ? nullptr : hoistBlock, hoistBlockValue, newValueInfo); } } } bool GlobOpt::TryHoistInvariant( IR::Instr *instr, BasicBlock *block, Value *dstVal, Value *src1Val, Value *src2Val, bool isNotTypeSpecConv, const bool lossy, const bool forceInvariantHoisting, IR::BailOutKind bailoutKind) { Assert(!this->IsLoopPrePass()); if (OptIsInvariant(instr, block, block->loop, src1Val, src2Val, isNotTypeSpecConv, forceInvariantHoisting)) { #if DBG if (Js::Configuration::Global.flags.Trace.IsEnabled(Js::InvariantsPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId())) { Output::Print(_u(" **** INVARIANT *** ")); instr->Dump(); } #endif #if ENABLE_DEBUG_CONFIG_OPTIONS if (Js::Configuration::Global.flags.TestTrace.IsEnabled(Js::InvariantsPhase)) { Output::Print(_u(" **** INVARIANT *** ")); Output::Print(_u("%s \n"), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode)); } #endif Loop *loop = block->loop; // Try hoisting from to outer most loop while (loop->parent && OptIsInvariant(instr, block, loop->parent, src1Val, src2Val, isNotTypeSpecConv, forceInvariantHoisting)) { loop = loop->parent; } // Record the byte code use here since we are going to move this instruction up if (isNotTypeSpecConv) { InsertNoImplicitCallUses(instr); this->CaptureByteCodeSymUses(instr); this->InsertByteCodeUses(instr, true); } #if DBG else { PropertySym *propertySymUse = NULL; NoRecoverMemoryJitArenaAllocator tempAllocator(_u("BE-GlobOpt-Temp"), this->alloc->GetPageAllocator(), Js::Throw::OutOfMemory); BVSparse * tempByteCodeUse = JitAnew(&tempAllocator, BVSparse, &tempAllocator); GlobOpt::TrackByteCodeSymUsed(instr, tempByteCodeUse, &propertySymUse); Assert(tempByteCodeUse->Count() == 0 && propertySymUse == NULL); } #endif OptHoistInvariant(instr, block, loop, dstVal, src1Val, src2Val, isNotTypeSpecConv, lossy, bailoutKind); return true; } return false; } InvariantBlockBackwardIterator::InvariantBlockBackwardIterator( GlobOpt *const globOpt, BasicBlock *const exclusiveBeginBlock, BasicBlock *const inclusiveEndBlock, StackSym *const invariantSym, const ValueNumber invariantSymValueNumber, bool followFlow) : globOpt(globOpt), exclusiveEndBlock(inclusiveEndBlock->prev), invariantSym(invariantSym), invariantSymValueNumber(invariantSymValueNumber), block(exclusiveBeginBlock), blockBV(globOpt->tempAlloc), followFlow(followFlow) #if DBG , inclusiveEndBlock(inclusiveEndBlock) #endif { Assert(exclusiveBeginBlock); Assert(inclusiveEndBlock); Assert(!inclusiveEndBlock->isDeleted); Assert(exclusiveBeginBlock != inclusiveEndBlock); Assert(!invariantSym == (invariantSymValueNumber == InvalidValueNumber)); MoveNext(); } bool InvariantBlockBackwardIterator::IsValid() const { return block != exclusiveEndBlock; } void InvariantBlockBackwardIterator::MoveNext() { Assert(IsValid()); while(true) { #if DBG BasicBlock *const previouslyIteratedBlock = block; #endif block = block->prev; if(!IsValid()) { Assert(previouslyIteratedBlock == inclusiveEndBlock); break; } if (!this->UpdatePredBlockBV()) { continue; } if (!this->UpdatePredBlockBV()) { continue; } if(block->isDeleted) { continue; } if(!block->globOptData.HasData()) { // This block's info has already been merged with all of its successors continue; } if(!invariantSym) { break; } invariantSymValue = block->globOptData.FindValue(invariantSym); if(!invariantSymValue || invariantSymValue->GetValueNumber() != invariantSymValueNumber) { // BailOnNoProfile and throw blocks are not moved outside loops. A sym table cleanup on these paths may delete the // values. Field hoisting also has some odd cases where the hoisted stack sym is invariant in only part of the loop. continue; } break; } } bool InvariantBlockBackwardIterator::UpdatePredBlockBV() { if (!this->followFlow) { return true; } // Track blocks we've visited to ensure that we only iterate over predecessor blocks if (!this->blockBV.IsEmpty() && !this->blockBV.Test(this->block->GetBlockNum())) { return false; } FOREACH_SLISTBASECOUNTED_ENTRY(FlowEdge*, edge, this->block->GetPredList()) { this->blockBV.Set(edge->GetPred()->GetBlockNum()); } NEXT_SLISTBASECOUNTED_ENTRY; return true; } BasicBlock * InvariantBlockBackwardIterator::Block() const { Assert(IsValid()); return block; } Value * InvariantBlockBackwardIterator::InvariantSymValue() const { Assert(IsValid()); Assert(invariantSym); return invariantSymValue; } void GlobOpt::HoistInvariantValueInfo( ValueInfo *const invariantValueInfoToHoist, Value *const valueToUpdate, BasicBlock *const targetBlock) { Assert(invariantValueInfoToHoist); Assert(valueToUpdate); Assert(targetBlock); // Why are we trying to change the value type of the type sym value? Asserting here to make sure we don't deep copy the type sym's value info. Assert(!invariantValueInfoToHoist->IsJsType()); Sym *const symStore = valueToUpdate->GetValueInfo()->GetSymStore(); ValueInfo *newValueInfo; if(invariantValueInfoToHoist->GetSymStore() == symStore) { newValueInfo = invariantValueInfoToHoist; } else { newValueInfo = invariantValueInfoToHoist->Copy(alloc); this->SetSymStoreDirect(newValueInfo, symStore); } ChangeValueInfo(targetBlock, valueToUpdate, newValueInfo, true); } // static bool GlobOpt::DoInlineArgsOpt(Func const * func) { Func const * topFunc = func->GetTopFunc(); Assert(topFunc != func); bool doInlineArgsOpt = !PHASE_OFF(Js::InlineArgsOptPhase, topFunc) && !func->GetHasCalls() && !func->GetHasUnoptimizedArgumentsAccess() && func->m_canDoInlineArgsOpt; return doInlineArgsOpt; } bool GlobOpt::IsSwitchOptEnabled(Func const * func) { Assert(func->IsTopFunc()); return !PHASE_OFF(Js::SwitchOptPhase, func) && !func->IsSwitchOptDisabled() && func->DoGlobOpt(); } bool GlobOpt::IsSwitchOptEnabledForIntTypeSpec(Func const * func) { return IsSwitchOptEnabled(func) && !IsTypeSpecPhaseOff(func) && DoAggressiveIntTypeSpec(func); } bool GlobOpt::DoConstFold() const { return !PHASE_OFF(Js::ConstFoldPhase, func); } bool GlobOpt::IsTypeSpecPhaseOff(Func const *func) { return PHASE_OFF(Js::TypeSpecPhase, func) || func->IsJitInDebugMode(); } bool GlobOpt::DoTypeSpec() const { return doTypeSpec; } bool GlobOpt::DoAggressiveIntTypeSpec(Func const * func) { return !PHASE_OFF(Js::AggressiveIntTypeSpecPhase, func) && !IsTypeSpecPhaseOff(func) && !func->IsAggressiveIntTypeSpecDisabled(); } bool GlobOpt::DoAggressiveIntTypeSpec() const { return doAggressiveIntTypeSpec; } bool GlobOpt::DoAggressiveMulIntTypeSpec() const { return doAggressiveMulIntTypeSpec; } bool GlobOpt::DoDivIntTypeSpec() const { return doDivIntTypeSpec; } // static bool GlobOpt::DoLossyIntTypeSpec(Func const * func) { return !PHASE_OFF(Js::LossyIntTypeSpecPhase, func) && !IsTypeSpecPhaseOff(func) && (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsLossyIntTypeSpecDisabled()); } bool GlobOpt::DoLossyIntTypeSpec() const { return doLossyIntTypeSpec; } // static bool GlobOpt::DoFloatTypeSpec(Func const * func) { return !PHASE_OFF(Js::FloatTypeSpecPhase, func) && !IsTypeSpecPhaseOff(func) && (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsFloatTypeSpecDisabled()) && AutoSystemInfo::Data.SSE2Available(); } bool GlobOpt::DoFloatTypeSpec() const { return doFloatTypeSpec; } bool GlobOpt::DoStringTypeSpec(Func const * func) { return !PHASE_OFF(Js::StringTypeSpecPhase, func) && !IsTypeSpecPhaseOff(func); } // static bool GlobOpt::DoTypedArrayTypeSpec(Func const * func) { return !PHASE_OFF(Js::TypedArrayTypeSpecPhase, func) && !IsTypeSpecPhaseOff(func) && (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsTypedArrayTypeSpecDisabled(func->IsLoopBody())) #if defined(_M_IX86) && AutoSystemInfo::Data.SSE2Available() #endif ; } // static bool GlobOpt::DoNativeArrayTypeSpec(Func const * func) { return !PHASE_OFF(Js::NativeArrayPhase, func) && !IsTypeSpecPhaseOff(func) #if defined(_M_IX86) && AutoSystemInfo::Data.SSE2Available() #endif ; } bool GlobOpt::DoArrayCheckHoist(Func const * const func) { Assert(func->IsTopFunc()); return !PHASE_OFF(Js::ArrayCheckHoistPhase, func) && !func->IsArrayCheckHoistDisabled() && !func->IsJitInDebugMode(); // StElemI fast path is not allowed when in debug mode, so it cannot have bailout } bool GlobOpt::DoArrayCheckHoist() const { return doArrayCheckHoist; } bool GlobOpt::DoArrayCheckHoist(const ValueType baseValueType, Loop* loop, IR::Instr const * const instr) const { if(!DoArrayCheckHoist() || (instr && !IsLoopPrePass() && instr->DoStackArgsOpt())) { return false; } // This includes typed arrays, but not virtual typed arrays, whose vtable can change if the buffer goes away. // Note that in the virtual case the vtable check is the only way to catch this, since there's no bound check. if(!(baseValueType.IsLikelyArrayOrObjectWithArray() || baseValueType.IsLikelyOptimizedVirtualTypedArray()) || (loop ? ImplicitCallFlagsAllowOpts(loop) : ImplicitCallFlagsAllowOpts(func))) { return true; } // The function or loop does not allow disabling implicit calls, which is required to eliminate redundant JS array checks #if DBG_DUMP if((((loop ? loop->GetImplicitCallFlags() : func->m_fg->implicitCallFlags) & ~Js::ImplicitCall_External) == 0) && Js::Configuration::Global.flags.Trace.IsEnabled(Js::HostOptPhase)) { Output::Print(_u("DoArrayCheckHoist disabled for JS arrays because of external: ")); func->DumpFullFunctionName(); Output::Print(_u("\n")); Output::Flush(); } #endif return false; } bool GlobOpt::DoArrayMissingValueCheckHoist(Func const * const func) { return DoArrayCheckHoist(func) && !PHASE_OFF(Js::ArrayMissingValueCheckHoistPhase, func) && (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsArrayMissingValueCheckHoistDisabled(func->IsLoopBody())); } bool GlobOpt::DoArrayMissingValueCheckHoist() const { return doArrayMissingValueCheckHoist; } bool GlobOpt::DoArraySegmentHoist(const ValueType baseValueType, Func const * const func) { Assert(baseValueType.IsLikelyAnyOptimizedArray()); if(!DoArrayCheckHoist(func) || PHASE_OFF(Js::ArraySegmentHoistPhase, func)) { return false; } if(!baseValueType.IsLikelyArrayOrObjectWithArray()) { return true; } return !PHASE_OFF(Js::JsArraySegmentHoistPhase, func) && (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsJsArraySegmentHoistDisabled(func->IsLoopBody())); } bool GlobOpt::DoArraySegmentHoist(const ValueType baseValueType) const { Assert(baseValueType.IsLikelyAnyOptimizedArray()); return baseValueType.IsLikelyArrayOrObjectWithArray() ? doJsArraySegmentHoist : doArraySegmentHoist; } bool GlobOpt::DoTypedArraySegmentLengthHoist(Loop *const loop) const { if(!DoArraySegmentHoist(ValueType::GetObject(ObjectType::Int32Array))) { return false; } if(loop ? ImplicitCallFlagsAllowOpts(loop) : ImplicitCallFlagsAllowOpts(func)) { return true; } // The function or loop does not allow disabling implicit calls, which is required to eliminate redundant typed array // segment length loads. #if DBG_DUMP if((((loop ? loop->GetImplicitCallFlags() : func->m_fg->implicitCallFlags) & ~Js::ImplicitCall_External) == 0) && Js::Configuration::Global.flags.Trace.IsEnabled(Js::HostOptPhase)) { Output::Print(_u("DoArraySegmentLengthHoist disabled for typed arrays because of external: ")); func->DumpFullFunctionName(); Output::Print(_u("\n")); Output::Flush(); } #endif return false; } bool GlobOpt::DoArrayLengthHoist(Func const * const func) { return DoArrayCheckHoist(func) && !PHASE_OFF(Js::Phase::ArrayLengthHoistPhase, func) && (!func->HasProfileInfo() || !func->GetReadOnlyProfileInfo()->IsArrayLengthHoistDisabled(func->IsLoopBody())); } bool GlobOpt::DoArrayLengthHoist() const { return doArrayLengthHoist; } bool GlobOpt::DoEliminateArrayAccessHelperCall(Func *const func) { return DoArrayCheckHoist(func); } bool GlobOpt::DoEliminateArrayAccessHelperCall() const { return doEliminateArrayAccessHelperCall; } bool GlobOpt::DoLdLenIntSpec(IR::Instr * const instr, const ValueType baseValueType) { Assert(!instr || instr->m_opcode == Js::OpCode::LdLen_A); Assert(!instr || instr->GetDst()); Assert(!instr || instr->GetSrc1()); if(PHASE_OFF(Js::LdLenIntSpecPhase, func) || IsTypeSpecPhaseOff(func) || (func->HasProfileInfo() && func->GetReadOnlyProfileInfo()->IsLdLenIntSpecDisabled()) || (instr && !IsLoopPrePass() && instr->DoStackArgsOpt())) { return false; } if(instr && instr->IsProfiledInstr() && ( !instr->AsProfiledInstr()->u.FldInfo().valueType.IsLikelyInt() || instr->GetDst()->AsRegOpnd()->m_sym->m_isNotNumber )) { return false; } Assert(!instr || baseValueType == instr->GetSrc1()->GetValueType()); return baseValueType.HasBeenString() || (baseValueType.IsLikelyAnyOptimizedArray() && baseValueType.GetObjectType() != ObjectType::ObjectWithArray); } bool GlobOpt::DoPathDependentValues() const { return !PHASE_OFF(Js::Phase::PathDependentValuesPhase, func); } bool GlobOpt::DoTrackRelativeIntBounds() const { return doTrackRelativeIntBounds; } bool GlobOpt::DoBoundCheckElimination() const { return doBoundCheckElimination; } bool GlobOpt::DoBoundCheckHoist() const { return doBoundCheckHoist; } bool GlobOpt::DoLoopCountBasedBoundCheckHoist() const { return doLoopCountBasedBoundCheckHoist; } bool GlobOpt::DoPowIntIntTypeSpec() const { return doPowIntIntTypeSpec; } bool GlobOpt::DoTagChecks() const { return doTagChecks; } bool GlobOpt::TrackArgumentsObject() { if (PHASE_OFF(Js::StackArgOptPhase, this->func)) { this->CannotAllocateArgumentsObjectOnStack(nullptr); return false; } return func->GetHasStackArgs(); } void GlobOpt::CannotAllocateArgumentsObjectOnStack(Func * curFunc) { if (curFunc != nullptr && curFunc->hasArgLenAndConstOpt) { Assert(!curFunc->GetJITOutput()->GetOutputData()->disableStackArgOpt); curFunc->GetJITOutput()->GetOutputData()->disableStackArgOpt = true; throw Js::RejitException(RejitReason::DisableStackArgLenAndConstOpt); } func->SetHasStackArgs(false); #ifdef ENABLE_DEBUG_CONFIG_OPTIONS if (PHASE_TESTTRACE(Js::StackArgOptPhase, this->func)) { char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; Output::Print(_u("Stack args disabled for function %s(%s)\n"), func->GetJITFunctionBody()->GetDisplayName(), func->GetDebugNumberSet(debugStringBuffer)); Output::Flush(); } #endif } IR::Instr * GlobOpt::PreOptPeep(IR::Instr *instr) { if (OpCodeAttr::HasDeadFallThrough(instr->m_opcode)) { switch (instr->m_opcode) { case Js::OpCode::BailOnNoProfile: { // Handle BailOnNoProfile if (instr->HasBailOutInfo()) { if (!this->prePassLoop) { FillBailOutInfo(this->currentBlock, instr); } // Already processed. return instr; } // Convert to bailout instr IR::Instr *nextBytecodeOffsetInstr = instr->GetNextRealInstrOrLabel(); while(nextBytecodeOffsetInstr->GetByteCodeOffset() == Js::Constants::NoByteCodeOffset) { nextBytecodeOffsetInstr = nextBytecodeOffsetInstr->GetNextRealInstrOrLabel(); Assert(!nextBytecodeOffsetInstr->IsLabelInstr()); } instr = instr->ConvertToBailOutInstr(nextBytecodeOffsetInstr, IR::BailOutOnNoProfile); instr->ClearByteCodeOffset(); instr->SetByteCodeOffset(nextBytecodeOffsetInstr); if (!this->currentBlock->loop) { FillBailOutInfo(this->currentBlock, instr); } else { Assert(this->prePassLoop); } break; } case Js::OpCode::BailOnException: { Assert( ( this->func->HasTry() && this->func->DoOptimizeTry() && instr->m_prev->m_opcode == Js::OpCode::Catch && instr->m_prev->m_prev->IsLabelInstr() && instr->m_prev->m_prev->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeCatch ) || ( this->func->HasFinally() && this->func->DoOptimizeTry() && instr->m_prev->AsLabelInstr() && instr->m_prev->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeFinally ) ); break; } case Js::OpCode::BailOnEarlyExit: { Assert(this->func->HasFinally() && this->func->DoOptimizeTry()); break; } default: { if(this->currentBlock->loop && !this->IsLoopPrePass()) { return instr; } break; } } RemoveCodeAfterNoFallthroughInstr(instr); } return instr; } void GlobOpt::RemoveCodeAfterNoFallthroughInstr(IR::Instr *instr) { if (instr != this->currentBlock->GetLastInstr()) { // Remove dead code after bailout IR::Instr *instrDead = instr->m_next; IR::Instr *instrNext; for (; instrDead != this->currentBlock->GetLastInstr(); instrDead = instrNext) { instrNext = instrDead->m_next; if (instrNext->m_opcode == Js::OpCode::FunctionExit) { break; } this->func->m_fg->RemoveInstr(instrDead, this); } IR::Instr *instrNextBlock = instrDead->m_next; this->func->m_fg->RemoveInstr(instrDead, this); this->currentBlock->SetLastInstr(instrNextBlock->m_prev); } // Cleanup dead successors FOREACH_SUCCESSOR_BLOCK_EDITING(deadBlock, this->currentBlock, iter) { this->currentBlock->RemoveDeadSucc(deadBlock, this->func->m_fg); if (this->currentBlock->GetDataUseCount() > 0) { this->currentBlock->DecrementDataUseCount(); } } NEXT_SUCCESSOR_BLOCK_EDITING; } void GlobOpt::ProcessTryHandler(IR::Instr* instr) { Assert(instr->m_next->IsLabelInstr() && instr->m_next->AsLabelInstr()->GetRegion()->GetType() == RegionType::RegionTypeTry); Region* tryRegion = instr->m_next->AsLabelInstr()->GetRegion(); BVSparse * writeThroughSymbolsSet = tryRegion->writeThroughSymbolsSet; ToVar(writeThroughSymbolsSet, this->currentBlock); } bool GlobOpt::ProcessExceptionHandlingEdges(IR::Instr* instr) { Assert(instr->m_opcode == Js::OpCode::BrOnException || instr->m_opcode == Js::OpCode::BrOnNoException); if (instr->m_opcode == Js::OpCode::BrOnException) { if (instr->AsBranchInstr()->GetTarget()->GetRegion()->GetType() == RegionType::RegionTypeCatch) { // BrOnException was added to model flow from try region to the catch region to assist // the backward pass in propagating bytecode upward exposed info from the catch block // to the try, and to handle break blocks. Removing it here as it has served its purpose // and keeping it around might also have unintended effects while merging block data for // the catch block's predecessors. // Note that the Deadstore pass will still be able to propagate bytecode upward exposed info // because it doesn't skip dead blocks for that. this->RemoveFlowEdgeToCatchBlock(instr); this->currentBlock->RemoveInstr(instr); return true; } else { // We add BrOnException from a finally region to early exit, remove that since it has served its purpose return this->RemoveFlowEdgeToFinallyOnExceptionBlock(instr); } } else if (instr->m_opcode == Js::OpCode::BrOnNoException) { if (instr->AsBranchInstr()->GetTarget()->GetRegion()->GetType() == RegionType::RegionTypeCatch) { this->RemoveFlowEdgeToCatchBlock(instr); } else { this->RemoveFlowEdgeToFinallyOnExceptionBlock(instr); } } return false; } void GlobOpt::InsertToVarAtDefInTryRegion(IR::Instr * instr, IR::Opnd * dstOpnd) { if ((this->currentRegion->GetType() == RegionTypeTry || this->currentRegion->GetType() == RegionTypeFinally) && dstOpnd->IsRegOpnd() && dstOpnd->AsRegOpnd()->m_sym->HasByteCodeRegSlot()) { StackSym * sym = dstOpnd->AsRegOpnd()->m_sym; if (sym->IsVar()) { return; } StackSym * varSym = sym->GetVarEquivSym(nullptr); if ((this->currentRegion->GetType() == RegionTypeTry && this->currentRegion->writeThroughSymbolsSet->Test(varSym->m_id)) || ((this->currentRegion->GetType() == RegionTypeFinally && this->currentRegion->GetMatchingTryRegion()->writeThroughSymbolsSet->Test(varSym->m_id)))) { IR::RegOpnd * regOpnd = IR::RegOpnd::New(varSym, IRType::TyVar, instr->m_func); this->ToVar(instr->m_next, regOpnd, this->currentBlock, NULL, false); } } } void GlobOpt::RemoveFlowEdgeToCatchBlock(IR::Instr * instr) { Assert(instr->IsBranchInstr()); BasicBlock * catchBlock = nullptr; BasicBlock * predBlock = nullptr; if (instr->m_opcode == Js::OpCode::BrOnException) { catchBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock(); predBlock = this->currentBlock; } else { Assert(instr->m_opcode == Js::OpCode::BrOnNoException); IR::Instr * nextInstr = instr->GetNextRealInstrOrLabel(); Assert(nextInstr->IsLabelInstr()); IR::LabelInstr * nextLabel = nextInstr->AsLabelInstr(); if (nextLabel->GetRegion() && nextLabel->GetRegion()->GetType() == RegionTypeCatch) { catchBlock = nextLabel->GetBasicBlock(); predBlock = this->currentBlock; } else { Assert(nextLabel->m_next->IsBranchInstr() && nextLabel->m_next->AsBranchInstr()->IsUnconditional()); BasicBlock * nextBlock = nextLabel->GetBasicBlock(); IR::BranchInstr * branchToCatchBlock = nextLabel->m_next->AsBranchInstr(); IR::LabelInstr * catchBlockLabel = branchToCatchBlock->GetTarget(); Assert(catchBlockLabel->GetRegion()->GetType() == RegionTypeCatch); catchBlock = catchBlockLabel->GetBasicBlock(); predBlock = nextBlock; } } Assert(catchBlock); Assert(predBlock); if (this->func->m_fg->FindEdge(predBlock, catchBlock)) { predBlock->RemoveDeadSucc(catchBlock, this->func->m_fg); if (predBlock == this->currentBlock) { predBlock->DecrementDataUseCount(); } } } bool GlobOpt::RemoveFlowEdgeToFinallyOnExceptionBlock(IR::Instr * instr) { Assert(instr->IsBranchInstr()); if (instr->m_opcode == Js::OpCode::BrOnNoException && instr->AsBranchInstr()->m_brFinallyToEarlyExit) { // We add edge from finally to early exit block // We should not remove this edge // If a loop has continue, and we add edge in finally to continue // Break block removal can move all continues inside the loop to branch to the continue added within finally // If we get rid of this edge, then loop may loose all backedges // Ideally, doing tail duplication before globopt would enable us to remove these edges, but since we do it after globopt, keep it this way for now // See test1() in core/test/tryfinallytests.js return false; } BasicBlock * finallyBlock = nullptr; BasicBlock * predBlock = nullptr; if (instr->m_opcode == Js::OpCode::BrOnException) { finallyBlock = instr->AsBranchInstr()->GetTarget()->GetBasicBlock(); predBlock = this->currentBlock; } else { Assert(instr->m_opcode == Js::OpCode::BrOnNoException); IR::Instr * nextInstr = instr->GetNextRealInstrOrLabel(); Assert(nextInstr->IsLabelInstr()); IR::LabelInstr * nextLabel = nextInstr->AsLabelInstr(); if (nextLabel->GetRegion() && nextLabel->GetRegion()->GetType() == RegionTypeFinally) { finallyBlock = nextLabel->GetBasicBlock(); predBlock = this->currentBlock; } else { if (!(nextLabel->m_next->IsBranchInstr() && nextLabel->m_next->AsBranchInstr()->IsUnconditional())) { return false; } BasicBlock * nextBlock = nextLabel->GetBasicBlock(); IR::BranchInstr * branchTofinallyBlockOrEarlyExit = nextLabel->m_next->AsBranchInstr(); IR::LabelInstr * finallyBlockLabelOrEarlyExitLabel = branchTofinallyBlockOrEarlyExit->GetTarget(); finallyBlock = finallyBlockLabelOrEarlyExitLabel->GetBasicBlock(); predBlock = nextBlock; } } Assert(finallyBlock && predBlock); if (this->func->m_fg->FindEdge(predBlock, finallyBlock)) { predBlock->RemoveDeadSucc(finallyBlock, this->func->m_fg); if (instr->m_opcode == Js::OpCode::BrOnException) { this->currentBlock->RemoveInstr(instr); } if (finallyBlock->GetFirstInstr()->AsLabelInstr()->IsUnreferenced()) { // Traverse predBlocks of finallyBlock, if any of the preds have a different region, set m_hasNonBranchRef to true // If not, this label can get eliminated and an incorrect region from the predecessor can get propagated in lowered code // See test3() in tryfinallytests.js Region * finallyRegion = finallyBlock->GetFirstInstr()->AsLabelInstr()->GetRegion(); FOREACH_PREDECESSOR_BLOCK(pred, finallyBlock) { Region * predRegion = pred->GetFirstInstr()->AsLabelInstr()->GetRegion(); if (predRegion != finallyRegion) { finallyBlock->GetFirstInstr()->AsLabelInstr()->m_hasNonBranchRef = true; } } NEXT_PREDECESSOR_BLOCK; } if (predBlock == this->currentBlock) { predBlock->DecrementDataUseCount(); } } return true; } IR::Instr * GlobOpt::OptPeep(IR::Instr *instr, Value *src1Val, Value *src2Val) { IR::Opnd *dst, *src1, *src2; if (this->IsLoopPrePass()) { return instr; } switch (instr->m_opcode) { case Js::OpCode::DeadBrEqual: case Js::OpCode::DeadBrRelational: case Js::OpCode::DeadBrSrEqual: src1 = instr->GetSrc1(); src2 = instr->GetSrc2(); // These branches were turned into dead branches because they were unnecessary (branch to next, ...). // The DeadBr are necessary in case the evaluation of the sources have side-effects. // If we know for sure the srcs are primitive or have been type specialized, we don't need these instructions if (((src1Val && src1Val->GetValueInfo()->IsPrimitive()) || (src1->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src1->AsRegOpnd()->m_sym))) && ((src2Val && src2Val->GetValueInfo()->IsPrimitive()) || (src2->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src2->AsRegOpnd()->m_sym)))) { this->CaptureByteCodeSymUses(instr); instr->m_opcode = Js::OpCode::Nop; } break; case Js::OpCode::DeadBrOnHasProperty: src1 = instr->GetSrc1(); if (((src1Val && src1Val->GetValueInfo()->IsPrimitive()) || (src1->IsRegOpnd() && CurrentBlockData()->IsTypeSpecialized(src1->AsRegOpnd()->m_sym)))) { this->CaptureByteCodeSymUses(instr); instr->m_opcode = Js::OpCode::Nop; } break; case Js::OpCode::Ld_A: case Js::OpCode::Ld_I4: src1 = instr->GetSrc1(); dst = instr->GetDst(); if (dst->IsRegOpnd() && dst->IsEqual(src1)) { dst = instr->UnlinkDst(); if (!dst->GetIsJITOptimizedReg()) { IR::ByteCodeUsesInstr *bytecodeUse = IR::ByteCodeUsesInstr::New(instr); bytecodeUse->SetDst(dst); instr->InsertAfter(bytecodeUse); } instr->FreeSrc1(); instr->m_opcode = Js::OpCode::Nop; } break; } return instr; } void GlobOpt::OptimizeIndirUses(IR::IndirOpnd *indirOpnd, IR::Instr * *pInstr, Value **indirIndexValRef) { IR::Instr * &instr = *pInstr; Assert(!indirIndexValRef || !*indirIndexValRef); // Update value types and copy-prop the base OptSrc(indirOpnd->GetBaseOpnd(), &instr, nullptr, indirOpnd); IR::RegOpnd *indexOpnd = indirOpnd->GetIndexOpnd(); if (!indexOpnd) { return; } // Update value types and copy-prop the index Value *indexVal = OptSrc(indexOpnd, &instr, nullptr, indirOpnd); if(indirIndexValRef) { *indirIndexValRef = indexVal; } } bool GlobOpt::IsPREInstrCandidateLoad(Js::OpCode opcode) { switch (opcode) { case Js::OpCode::LdFld: case Js::OpCode::LdFldForTypeOf: case Js::OpCode::LdRootFld: case Js::OpCode::LdRootFldForTypeOf: case Js::OpCode::LdMethodFld: case Js::OpCode::LdRootMethodFld: case Js::OpCode::LdSlot: case Js::OpCode::LdSlotArr: return true; } return false; } bool GlobOpt::IsPREInstrSequenceCandidateLoad(Js::OpCode opcode) { switch (opcode) { default: return IsPREInstrCandidateLoad(opcode); case Js::OpCode::Ld_A: case Js::OpCode::BytecodeArgOutCapture: return true; } } bool GlobOpt::IsPREInstrCandidateStore(Js::OpCode opcode) { switch (opcode) { case Js::OpCode::StFld: case Js::OpCode::StRootFld: case Js::OpCode::StSlot: return true; } return false; } bool GlobOpt::ImplicitCallFlagsAllowOpts(Loop *loop) { return loop->GetImplicitCallFlags() != Js::ImplicitCall_HasNoInfo && (((loop->GetImplicitCallFlags() & ~Js::ImplicitCall_Accessor) | Js::ImplicitCall_None) == Js::ImplicitCall_None); } bool GlobOpt::ImplicitCallFlagsAllowOpts(Func const *func) { return func->m_fg->implicitCallFlags != Js::ImplicitCall_HasNoInfo && (((func->m_fg->implicitCallFlags & ~Js::ImplicitCall_Accessor) | Js::ImplicitCall_None) == Js::ImplicitCall_None); } #if DBG_DUMP void GlobOpt::Dump() const { this->DumpSymToValueMap(); } void GlobOpt::DumpSymToValueMap(BasicBlock const * block) const { Output::Print(_u("\n*** SymToValueMap ***\n")); block->globOptData.DumpSymToValueMap(); } void GlobOpt::DumpSymToValueMap() const { DumpSymToValueMap(this->currentBlock); } void GlobOpt::DumpSymVal(int index) { SymID id = index; extern Func *CurrentFunc; Sym *sym = this->func->m_symTable->Find(id); AssertMsg(sym, "Sym not found!!!"); Output::Print(_u("Sym: ")); sym->Dump(); Output::Print(_u("\t\tValueNumber: ")); Value * pValue = CurrentBlockData()->FindValueFromMapDirect(sym->m_id); pValue->Dump(); Output::Print(_u("\n")); } void GlobOpt::Trace(BasicBlock * block, bool before) const { bool globOptTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::GlobOptPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()); bool typeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::TypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()); bool floatTypeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FloatTypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()); bool fieldCopyPropTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldCopyPropPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()); bool objTypeSpecTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::ObjTypeSpecPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()); bool valueTableTrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::ValueTablePhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()); bool fieldPRETrace = Js::Configuration::Global.flags.Trace.IsEnabled(Js::FieldPREPhase, this->func->GetSourceContextId(), this->func->GetLocalFunctionId()); bool anyTrace = globOptTrace || typeSpecTrace || floatTypeSpecTrace || fieldCopyPropTrace || objTypeSpecTrace || valueTableTrace || fieldPRETrace; if (!anyTrace) { return; } if (fieldPRETrace && this->IsLoopPrePass()) { if (block->isLoopHeader && before) { Output::Print(_u("==== Loop Prepass block header #%-3d, Visiting Loop block head #%-3d\n"), this->prePassLoop->GetHeadBlock()->GetBlockNum(), block->GetBlockNum()); } } if (!typeSpecTrace && !floatTypeSpecTrace && !valueTableTrace && !Js::Configuration::Global.flags.Verbose) { return; } if (before) { Output::Print(_u("========================================================================\n")); Output::Print(_u("Begin OptBlock: Block #%-3d"), block->GetBlockNum()); if (block->loop) { Output::Print(_u(" Loop block header:%-3d currentLoop block head:%-3d %s"), block->loop->GetHeadBlock()->GetBlockNum(), this->prePassLoop ? this->prePassLoop->GetHeadBlock()->GetBlockNum() : 0, this->IsLoopPrePass() ? _u("PrePass") : _u("")); } Output::Print(_u("\n")); } else { Output::Print(_u("-----------------------------------------------------------------------\n")); Output::Print(_u("After OptBlock: Block #%-3d\n"), block->GetBlockNum()); } if ((typeSpecTrace || floatTypeSpecTrace) && !block->globOptData.liveVarSyms->IsEmpty()) { Output::Print(_u(" Live var syms: ")); block->globOptData.liveVarSyms->Dump(); } if (typeSpecTrace && !block->globOptData.liveInt32Syms->IsEmpty()) { Assert(this->tempBv->IsEmpty()); this->tempBv->Minus(block->globOptData.liveInt32Syms, block->globOptData.liveLossyInt32Syms); if(!this->tempBv->IsEmpty()) { Output::Print(_u(" Int32 type specialized (lossless) syms: ")); this->tempBv->Dump(); } this->tempBv->ClearAll(); if(!block->globOptData.liveLossyInt32Syms->IsEmpty()) { Output::Print(_u(" Int32 converted (lossy) syms: ")); block->globOptData.liveLossyInt32Syms->Dump(); } } if (floatTypeSpecTrace && !block->globOptData.liveFloat64Syms->IsEmpty()) { Output::Print(_u(" Float64 type specialized syms: ")); block->globOptData.liveFloat64Syms->Dump(); } if ((fieldCopyPropTrace || objTypeSpecTrace) && this->DoFieldCopyProp(block->loop) && !block->globOptData.liveFields->IsEmpty()) { Output::Print(_u(" Live field syms: ")); block->globOptData.liveFields->Dump(); } if (objTypeSpecTrace || valueTableTrace) { Output::Print(_u(" Value table:\n")); block->globOptData.DumpSymToValueMap(); } if (before) { Output::Print(_u("-----------------------------------------------------------------------\n")); \ } Output::Flush(); } void GlobOpt::TraceSettings() const { Output::Print(_u("GlobOpt Settings:\r\n")); Output::Print(_u(" FloatTypeSpec: %s\r\n"), this->DoFloatTypeSpec() ? _u("enabled") : _u("disabled")); Output::Print(_u(" AggressiveIntTypeSpec: %s\r\n"), this->DoAggressiveIntTypeSpec() ? _u("enabled") : _u("disabled")); Output::Print(_u(" LossyIntTypeSpec: %s\r\n"), this->DoLossyIntTypeSpec() ? _u("enabled") : _u("disabled")); Output::Print(_u(" ArrayCheckHoist: %s\r\n"), this->func->IsArrayCheckHoistDisabled() ? _u("disabled") : _u("enabled")); Output::Print(_u(" ImplicitCallFlags: %s\r\n"), Js::DynamicProfileInfo::GetImplicitCallFlagsString(this->func->m_fg->implicitCallFlags)); for (Loop * loop = this->func->m_fg->loopList; loop != NULL; loop = loop->next) { Output::Print(_u(" loop: %d, ImplicitCallFlags: %s\r\n"), loop->GetLoopNumber(), Js::DynamicProfileInfo::GetImplicitCallFlagsString(loop->GetImplicitCallFlags())); } Output::Flush(); } #endif // DBG_DUMP IR::Instr * GlobOpt::TrackMarkTempObject(IR::Instr * instrStart, IR::Instr * instrLast) { if (!this->func->GetHasMarkTempObjects()) { return instrLast; } IR::Instr * instr = instrStart; IR::Instr * instrEnd = instrLast->m_next; IR::Instr * lastInstr = nullptr; GlobOptBlockData& globOptData = *CurrentBlockData(); do { bool mayNeedBailOnImplicitCallsPreOp = !this->IsLoopPrePass() && instr->HasAnyImplicitCalls() && globOptData.maybeTempObjectSyms != nullptr; if (mayNeedBailOnImplicitCallsPreOp) { IR::Opnd * src1 = instr->GetSrc1(); if (src1) { instr = GenerateBailOutMarkTempObjectIfNeeded(instr, src1, false); IR::Opnd * src2 = instr->GetSrc2(); if (src2) { instr = GenerateBailOutMarkTempObjectIfNeeded(instr, src2, false); } } } IR::Opnd *dst = instr->GetDst(); if (dst) { if (dst->IsRegOpnd()) { TrackTempObjectSyms(instr, dst->AsRegOpnd()); } else if (mayNeedBailOnImplicitCallsPreOp) { instr = GenerateBailOutMarkTempObjectIfNeeded(instr, dst, true); } } lastInstr = instr; instr = instr->m_next; } while (instr != instrEnd); return lastInstr; } void GlobOpt::TrackTempObjectSyms(IR::Instr * instr, IR::RegOpnd * opnd) { // If it is marked as dstIsTempObject, we should have mark temped it, or type specialized it to Ld_I4. Assert(!instr->dstIsTempObject || ObjectTempVerify::CanMarkTemp(instr, nullptr)); GlobOptBlockData& globOptData = *CurrentBlockData(); bool canStoreTemp = false; bool maybeTemp = false; if (OpCodeAttr::TempObjectProducing(instr->m_opcode)) { maybeTemp = instr->dstIsTempObject; // We have to make sure that lower will always generate code to do stack allocation // before we can store any other stack instance onto it. Otherwise, we would not // walk object to box the stack property. canStoreTemp = instr->dstIsTempObject && ObjectTemp::CanStoreTemp(instr); } else if (OpCodeAttr::TempObjectTransfer(instr->m_opcode)) { // Need to check both sources, GetNewScObject has two srcs for transfer. // No need to get var equiv sym here as transfer of type spec value does not transfer a mark temp object. maybeTemp = globOptData.maybeTempObjectSyms && ( (instr->GetSrc1()->IsRegOpnd() && globOptData.maybeTempObjectSyms->Test(instr->GetSrc1()->AsRegOpnd()->m_sym->m_id)) || (instr->GetSrc2() && instr->GetSrc2()->IsRegOpnd() && globOptData.maybeTempObjectSyms->Test(instr->GetSrc2()->AsRegOpnd()->m_sym->m_id))); canStoreTemp = globOptData.canStoreTempObjectSyms && ( (instr->GetSrc1()->IsRegOpnd() && globOptData.canStoreTempObjectSyms->Test(instr->GetSrc1()->AsRegOpnd()->m_sym->m_id)) && (!instr->GetSrc2() || (instr->GetSrc2()->IsRegOpnd() && globOptData.canStoreTempObjectSyms->Test(instr->GetSrc2()->AsRegOpnd()->m_sym->m_id)))); AssertOrFailFast(!canStoreTemp || instr->dstIsTempObject); AssertOrFailFast(!maybeTemp || instr->dstIsTempObject); } // Need to get the var equiv sym as assignment of type specialized sym kill the var sym value anyway. StackSym * sym = opnd->m_sym; if (!sym->IsVar()) { sym = sym->GetVarEquivSym(nullptr); if (sym == nullptr) { return; } } SymID symId = sym->m_id; if (maybeTemp) { // Only var sym should be temp objects Assert(opnd->m_sym == sym); if (globOptData.maybeTempObjectSyms == nullptr) { globOptData.maybeTempObjectSyms = JitAnew(this->alloc, BVSparse, this->alloc); } globOptData.maybeTempObjectSyms->Set(symId); if (canStoreTemp) { if (instr->m_opcode == Js::OpCode::NewScObjectLiteral && !this->IsLoopPrePass()) { // For object literal, we install the final type up front. // If there are bailout before we finish initializing all the fields, we need to // zero out the rest if we stack allocate the literal, so that the boxing would not // try to box trash pointer in the properties. // Although object Literal initialization can be done lexically, BailOnNoProfile may cause some path // to disappear. Do it is flow base make it easier to stop propagate those entries. IR::IntConstOpnd * propertyArrayIdOpnd = instr->GetSrc1()->AsIntConstOpnd(); const Js::PropertyIdArray * propIds = instr->m_func->GetJITFunctionBody()->ReadPropertyIdArrayFromAuxData(propertyArrayIdOpnd->AsUint32()); // Duplicates are removed by parser Assert(!propIds->hadDuplicates); if (globOptData.stackLiteralInitFldDataMap == nullptr) { globOptData.stackLiteralInitFldDataMap = JitAnew(alloc, StackLiteralInitFldDataMap, alloc); } else { Assert(!globOptData.stackLiteralInitFldDataMap->ContainsKey(sym)); } StackLiteralInitFldData data = { propIds, 0}; globOptData.stackLiteralInitFldDataMap->AddNew(sym, data); } if (globOptData.canStoreTempObjectSyms == nullptr) { globOptData.canStoreTempObjectSyms = JitAnew(this->alloc, BVSparse, this->alloc); } globOptData.canStoreTempObjectSyms->Set(symId); } else if (globOptData.canStoreTempObjectSyms) { globOptData.canStoreTempObjectSyms->Clear(symId); } } else { Assert(!canStoreTemp); if (globOptData.maybeTempObjectSyms) { if (globOptData.canStoreTempObjectSyms) { globOptData.canStoreTempObjectSyms->Clear(symId); } globOptData.maybeTempObjectSyms->Clear(symId); } else { Assert(!globOptData.canStoreTempObjectSyms); } // The symbol is being assigned to, the sym shouldn't still be in the stackLiteralInitFldDataMap Assert(this->IsLoopPrePass() || globOptData.stackLiteralInitFldDataMap == nullptr || globOptData.stackLiteralInitFldDataMap->Count() == 0 || !globOptData.stackLiteralInitFldDataMap->ContainsKey(sym)); } } IR::Instr * GlobOpt::GenerateBailOutMarkTempObjectIfNeeded(IR::Instr * instr, IR::Opnd * opnd, bool isDst) { Assert(opnd); Assert(isDst == (opnd == instr->GetDst())); Assert(opnd != instr->GetDst() || !opnd->IsRegOpnd()); Assert(!this->IsLoopPrePass()); Assert(instr->HasAnyImplicitCalls()); // Only dst reg opnd opcode or ArgOut_A should have dstIsTempObject marked Assert(!isDst || !instr->dstIsTempObject || instr->m_opcode == Js::OpCode::ArgOut_A); // Post-op implicit call shouldn't have installed yet Assert(!instr->HasBailOutInfo() || (instr->GetBailOutKind() & IR::BailOutKindBits) != IR::BailOutOnImplicitCalls); GlobOptBlockData& globOptData = *CurrentBlockData(); Assert(globOptData.maybeTempObjectSyms != nullptr); IR::PropertySymOpnd * propertySymOpnd = nullptr; StackSym * stackSym = ObjectTemp::GetStackSym(opnd, &propertySymOpnd); // It is okay to not get the var equiv sym here, as use of a type specialized sym is not use of the temp object // so no need to add mark temp bailout. // TempObjectSysm doesn't contain any type spec sym, so we will get false here for all type spec sym. if (stackSym && globOptData.maybeTempObjectSyms->Test(stackSym->m_id)) { if (instr->HasBailOutInfo()) { instr->SetBailOutKind(instr->GetBailOutKind() | IR::BailOutMarkTempObject); instr->GetBailOutInfo()->canDeadStore = false; } else { // On insert the pre op bailout if it is not Direct field access do nothing, don't check the dst yet. // SetTypeCheckBailout will clear this out if it is direct field access. if (isDst || (instr->m_opcode == Js::OpCode::FromVar && !opnd->GetValueType().IsPrimitive()) || propertySymOpnd == nullptr || !propertySymOpnd->IsTypeCheckProtected()) { this->GenerateBailAtOperation(&instr, IR::BailOutMarkTempObject); instr->GetBailOutInfo()->canDeadStore = false; } else if (propertySymOpnd->MayHaveImplicitCall()) { this->GenerateBailAtOperation(&instr, IR::BailOutMarkTempObject); } } if (!opnd->IsRegOpnd() && (!isDst || (globOptData.canStoreTempObjectSyms && globOptData.canStoreTempObjectSyms->Test(stackSym->m_id)))) { // If this opnd is a dst, that means that the object pointer is a stack object, // and we can store temp object/number on it. // If the opnd is a src, that means that the object pointer may be a stack object // so the load may be a temp object/number and we need to track its use. // Don't mark start of indir as can store temp, because we don't actually know // what it is assigning to. if (!isDst || !opnd->IsIndirOpnd()) { opnd->SetCanStoreTemp(); } if (propertySymOpnd) { // Track initfld of stack literals if (isDst && instr->m_opcode == Js::OpCode::InitFld) { const Js::PropertyId propertyId = propertySymOpnd->m_sym->AsPropertySym()->m_propertyId; // We don't need to track numeric properties init if (!this->func->GetThreadContextInfo()->IsNumericProperty(propertyId)) { DebugOnly(bool found = false); globOptData.stackLiteralInitFldDataMap->RemoveIf(stackSym, [&](StackSym * key, StackLiteralInitFldData & data) { DebugOnly(found = true); Assert(key == stackSym); Assert(data.currentInitFldCount < data.propIds->count); if (data.propIds->elements[data.currentInitFldCount] != propertyId) { #if DBG bool duplicate = false; for (uint i = 0; i < data.currentInitFldCount; i++) { if (data.propIds->elements[i] == propertyId) { duplicate = true; break; } } Assert(duplicate); #endif // duplicate initialization return false; } bool finished = (++data.currentInitFldCount == data.propIds->count); #if DBG if (finished) { // We can still track the finished stack literal InitFld lexically. this->finishedStackLiteralInitFld->Set(stackSym->m_id); } #endif return finished; }); // We might still see InitFld even we have finished with all the property Id because // of duplicate entries at the end Assert(found || finishedStackLiteralInitFld->Test(stackSym->m_id)); } } } } } return instr; } LoopCount * GlobOpt::GetOrGenerateLoopCountForMemOp(Loop *loop) { LoopCount *loopCount = loop->loopCount; if (loopCount && !loopCount->HasGeneratedLoopCountSym()) { Assert(loop->bailOutInfo); EnsureBailTarget(loop); GenerateLoopCountPlusOne(loop, loopCount); } return loopCount; } IR::Opnd * GlobOpt::GenerateInductionVariableChangeForMemOp(Loop *loop, byte unroll, IR::Instr *insertBeforeInstr) { AssertOrFailFast(unroll != Js::Constants::InvalidLoopUnrollFactor); LoopCount *loopCount = loop->loopCount; IR::Opnd *sizeOpnd = nullptr; Assert(loopCount); Assert(loop->memOpInfo->inductionVariableOpndPerUnrollMap); if (loop->memOpInfo->inductionVariableOpndPerUnrollMap->TryGetValue(unroll, &sizeOpnd)) { return sizeOpnd; } Func *localFunc = loop->GetFunc(); const auto InsertInstr = [&](IR::Instr *instr) { if (insertBeforeInstr == nullptr) { loop->landingPad->InsertAfter(instr); } else { insertBeforeInstr->InsertBefore(instr); } }; if (loopCount->LoopCountMinusOneSym()) { IRType type = loopCount->LoopCountSym()->GetType(); // Loop count is off by one, so add one IR::RegOpnd *loopCountOpnd = IR::RegOpnd::New(loopCount->LoopCountSym(), type, localFunc); sizeOpnd = loopCountOpnd; if (unroll != 1) { sizeOpnd = IR::RegOpnd::New(TyUint32, this->func); IR::Opnd *unrollOpnd = IR::IntConstOpnd::New(unroll, type, localFunc); IR::Instr *inductionChangeMultiplier = IR::Instr::New( Js::OpCode::Mul_I4, sizeOpnd, loopCountOpnd, unrollOpnd, localFunc); InsertInstr(inductionChangeMultiplier); inductionChangeMultiplier->ConvertToBailOutInstr(loop->bailOutInfo, IR::BailOutOnOverflow); } } else { int32 loopCountMinusOnePlusOne; int32 size; if (Int32Math::Add(loopCount->LoopCountMinusOneConstantValue(), 1, &loopCountMinusOnePlusOne) || Int32Math::Mul(loopCountMinusOnePlusOne, unroll, &size)) { throw Js::RejitException(RejitReason::MemOpDisabled); } Assert(size > 0); sizeOpnd = IR::IntConstOpnd::New(size, IRType::TyUint32, localFunc); } loop->memOpInfo->inductionVariableOpndPerUnrollMap->Add(unroll, sizeOpnd); return sizeOpnd; } IR::RegOpnd* GlobOpt::GenerateStartIndexOpndForMemop(Loop *loop, IR::Opnd *indexOpnd, IR::Opnd *sizeOpnd, bool isInductionVariableChangeIncremental, bool bIndexAlreadyChanged, IR::Instr *insertBeforeInstr) { IR::RegOpnd *startIndexOpnd = nullptr; Func *localFunc = loop->GetFunc(); IRType type = indexOpnd->GetType(); const int cacheIndex = ((int)isInductionVariableChangeIncremental << 1) | (int)bIndexAlreadyChanged; if (loop->memOpInfo->startIndexOpndCache[cacheIndex]) { return loop->memOpInfo->startIndexOpndCache[cacheIndex]; } const auto InsertInstr = [&](IR::Instr *instr) { if (insertBeforeInstr == nullptr) { loop->landingPad->InsertAfter(instr); } else { insertBeforeInstr->InsertBefore(instr); } }; startIndexOpnd = IR::RegOpnd::New(type, localFunc); // If the 2 are different we can simply use indexOpnd if (isInductionVariableChangeIncremental != bIndexAlreadyChanged) { InsertInstr(IR::Instr::New(Js::OpCode::Ld_A, startIndexOpnd, indexOpnd, localFunc)); } else { // Otherwise add 1 to it InsertInstr(IR::Instr::New(Js::OpCode::Add_I4, startIndexOpnd, indexOpnd, IR::IntConstOpnd::New(1, type, localFunc, true), localFunc)); } if (!isInductionVariableChangeIncremental) { InsertInstr(IR::Instr::New(Js::OpCode::Sub_I4, startIndexOpnd, startIndexOpnd, sizeOpnd, localFunc)); } loop->memOpInfo->startIndexOpndCache[cacheIndex] = startIndexOpnd; return startIndexOpnd; } IR::Instr* GlobOpt::FindUpperBoundsCheckInstr(IR::Instr* fromInstr) { IR::Instr *upperBoundCheck = fromInstr; do { upperBoundCheck = upperBoundCheck->m_prev; Assert(upperBoundCheck); Assert(!upperBoundCheck->IsLabelInstr()); } while (upperBoundCheck->m_opcode != Js::OpCode::BoundCheck); return upperBoundCheck; } IR::Instr* GlobOpt::FindArraySegmentLoadInstr(IR::Instr* fromInstr) { IR::Instr *headSegmentLengthLoad = fromInstr; do { headSegmentLengthLoad = headSegmentLengthLoad->m_prev; Assert(headSegmentLengthLoad); Assert(!headSegmentLengthLoad->IsLabelInstr()); } while (headSegmentLengthLoad->m_opcode != Js::OpCode::LdIndir); return headSegmentLengthLoad; } void GlobOpt::RemoveMemOpSrcInstr(IR::Instr* memopInstr, IR::Instr* srcInstr, BasicBlock* block) { Assert(srcInstr && (srcInstr->m_opcode == Js::OpCode::LdElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A_Strict)); Assert(memopInstr && (memopInstr->m_opcode == Js::OpCode::Memcopy || memopInstr->m_opcode == Js::OpCode::Memset)); Assert(block); const bool isDst = srcInstr->m_opcode == Js::OpCode::StElemI_A || srcInstr->m_opcode == Js::OpCode::StElemI_A_Strict; IR::RegOpnd* opnd = (isDst ? memopInstr->GetDst() : memopInstr->GetSrc1())->AsIndirOpnd()->GetBaseOpnd(); IR::ArrayRegOpnd* arrayOpnd = opnd->IsArrayRegOpnd() ? opnd->AsArrayRegOpnd() : nullptr; IR::Instr* topInstr = srcInstr; if (srcInstr->extractedUpperBoundCheckWithoutHoisting) { IR::Instr *upperBoundCheck = FindUpperBoundsCheckInstr(srcInstr); Assert(upperBoundCheck && upperBoundCheck != srcInstr); topInstr = upperBoundCheck; } if (srcInstr->loadedArrayHeadSegmentLength && arrayOpnd && arrayOpnd->HeadSegmentLengthSym()) { IR::Instr *arrayLoadSegmentHeadLength = FindArraySegmentLoadInstr(topInstr); Assert(arrayLoadSegmentHeadLength); topInstr = arrayLoadSegmentHeadLength; arrayOpnd->RemoveHeadSegmentLengthSym(); } if (srcInstr->loadedArrayHeadSegment && arrayOpnd && arrayOpnd->HeadSegmentSym()) { IR::Instr *arrayLoadSegmentHead = FindArraySegmentLoadInstr(topInstr); Assert(arrayLoadSegmentHead); topInstr = arrayLoadSegmentHead; arrayOpnd->RemoveHeadSegmentSym(); } // If no bounds check are present, simply look up for instruction added for instrumentation if(topInstr == srcInstr) { bool checkPrev = true; while (checkPrev) { switch (topInstr->m_prev->m_opcode) { case Js::OpCode::BailOnNotArray: case Js::OpCode::NoImplicitCallUses: case Js::OpCode::ByteCodeUses: topInstr = topInstr->m_prev; checkPrev = !!topInstr->m_prev; break; default: checkPrev = false; break; } } } while (topInstr != srcInstr) { IR::Instr* removeInstr = topInstr; topInstr = topInstr->m_next; Assert( removeInstr->m_opcode == Js::OpCode::BailOnNotArray || removeInstr->m_opcode == Js::OpCode::NoImplicitCallUses || removeInstr->m_opcode == Js::OpCode::ByteCodeUses || removeInstr->m_opcode == Js::OpCode::LdIndir || removeInstr->m_opcode == Js::OpCode::BoundCheck ); if (removeInstr->m_opcode != Js::OpCode::ByteCodeUses) { block->RemoveInstr(removeInstr); } } this->ConvertToByteCodeUses(srcInstr); } void GlobOpt::GetMemOpSrcInfo(Loop* loop, IR::Instr* instr, IR::RegOpnd*& base, IR::RegOpnd*& index, IRType& arrayType) { Assert(instr && (instr->m_opcode == Js::OpCode::LdElemI_A || instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict)); IR::Opnd* arrayOpnd = instr->m_opcode == Js::OpCode::LdElemI_A ? instr->GetSrc1() : instr->GetDst(); Assert(arrayOpnd->IsIndirOpnd()); IR::IndirOpnd* indirArrayOpnd = arrayOpnd->AsIndirOpnd(); IR::RegOpnd* baseOpnd = (IR::RegOpnd*)indirArrayOpnd->GetBaseOpnd(); IR::RegOpnd* indexOpnd = (IR::RegOpnd*)indirArrayOpnd->GetIndexOpnd(); Assert(baseOpnd); Assert(indexOpnd); // Process Out Params base = baseOpnd; index = indexOpnd; arrayType = indirArrayOpnd->GetType(); } void GlobOpt::EmitMemop(Loop * loop, LoopCount *loopCount, const MemOpEmitData* emitData) { Assert(emitData); Assert(emitData->candidate); Assert(emitData->stElemInstr); Assert(emitData->stElemInstr->m_opcode == Js::OpCode::StElemI_A || emitData->stElemInstr->m_opcode == Js::OpCode::StElemI_A_Strict); IR::BailOutKind bailOutKind = emitData->bailOutKind; const byte unroll = emitData->inductionVar.unroll; Assert(unroll == 1); const bool isInductionVariableChangeIncremental = emitData->inductionVar.isIncremental; const bool bIndexAlreadyChanged = emitData->candidate->bIndexAlreadyChanged; IR::RegOpnd *baseOpnd = nullptr; IR::RegOpnd *indexOpnd = nullptr; IRType dstType; GetMemOpSrcInfo(loop, emitData->stElemInstr, baseOpnd, indexOpnd, dstType); Func *localFunc = loop->GetFunc(); // Handle bailout info EnsureBailTarget(loop); Assert(bailOutKind != IR::BailOutInvalid); // Keep only Array bits bailOuts. Consider handling these bailouts instead of simply ignoring them bailOutKind &= IR::BailOutForArrayBits; // Add our custom bailout to handle Op_MemCopy return value. bailOutKind |= IR::BailOutOnMemOpError; BailOutInfo *const bailOutInfo = loop->bailOutInfo; Assert(bailOutInfo); IR::Instr *insertBeforeInstr = bailOutInfo->bailOutInstr; Assert(insertBeforeInstr); IR::Opnd *sizeOpnd = GenerateInductionVariableChangeForMemOp(loop, unroll, insertBeforeInstr); IR::RegOpnd *startIndexOpnd = GenerateStartIndexOpndForMemop(loop, indexOpnd, sizeOpnd, isInductionVariableChangeIncremental, bIndexAlreadyChanged, insertBeforeInstr); IR::IndirOpnd* dstOpnd = IR::IndirOpnd::New(baseOpnd, startIndexOpnd, dstType, localFunc); IR::Opnd *src1; const bool isMemset = emitData->candidate->IsMemSet(); // Get the source according to the memop type if (isMemset) { MemSetEmitData* data = (MemSetEmitData*)emitData; const Loop::MemSetCandidate* candidate = data->candidate->AsMemSet(); if (candidate->srcSym) { IR::RegOpnd* regSrc = IR::RegOpnd::New(candidate->srcSym, candidate->srcSym->GetType(), func); regSrc->SetIsJITOptimizedReg(true); src1 = regSrc; } else { src1 = IR::AddrOpnd::New(candidate->constant.ToVar(localFunc), IR::AddrOpndKindConstantAddress, localFunc); } } else { Assert(emitData->candidate->IsMemCopy()); MemCopyEmitData* data = (MemCopyEmitData*)emitData; Assert(data->ldElemInstr); Assert(data->ldElemInstr->m_opcode == Js::OpCode::LdElemI_A); IR::RegOpnd *srcBaseOpnd = nullptr; IR::RegOpnd *srcIndexOpnd = nullptr; IRType srcType; GetMemOpSrcInfo(loop, data->ldElemInstr, srcBaseOpnd, srcIndexOpnd, srcType); Assert(GetVarSymID(srcIndexOpnd->GetStackSym()) == GetVarSymID(indexOpnd->GetStackSym())); src1 = IR::IndirOpnd::New(srcBaseOpnd, startIndexOpnd, srcType, localFunc); } // Generate memcopy IR::Instr* memopInstr = IR::BailOutInstr::New(isMemset ? Js::OpCode::Memset : Js::OpCode::Memcopy, bailOutKind, bailOutInfo, localFunc); memopInstr->SetDst(dstOpnd); memopInstr->SetSrc1(src1); memopInstr->SetSrc2(sizeOpnd); insertBeforeInstr->InsertBefore(memopInstr); loop->memOpInfo->instr = memopInstr; #if DBG_DUMP if (DO_MEMOP_TRACE()) { char valueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; baseOpnd->GetValueType().ToString(valueTypeStr); const int loopCountBufSize = 16; char16 loopCountBuf[loopCountBufSize]; if (loopCount->LoopCountMinusOneSym()) { swprintf_s(loopCountBuf, _u("s%u"), loopCount->LoopCountMinusOneSym()->m_id); } else { swprintf_s(loopCountBuf, _u("%u"), loopCount->LoopCountMinusOneConstantValue() + 1); } if (isMemset) { const Loop::MemSetCandidate* candidate = emitData->candidate->AsMemSet(); const int constBufSize = 32; char16 constBuf[constBufSize]; if (candidate->srcSym) { swprintf_s(constBuf, _u("s%u"), candidate->srcSym->m_id); } else { switch (candidate->constant.type) { case TyInt8: case TyInt16: case TyInt32: case TyInt64: swprintf_s(constBuf, sizeof(IntConstType) == 8 ? _u("%lld") : _u("%d"), candidate->constant.u.intConst.value); break; case TyFloat32: case TyFloat64: swprintf_s(constBuf, _u("%.4f"), candidate->constant.u.floatConst.value); break; case TyVar: swprintf_s(constBuf, sizeof(Js::Var) == 8 ? _u("0x%.16llX") : _u("0x%.8X"), candidate->constant.u.varConst.value); break; default: AssertMsg(false, "Unsupported constant type"); swprintf_s(constBuf, _u("Unknown")); break; } } TRACE_MEMOP_PHASE(MemSet, loop, emitData->stElemInstr, _u("ValueType: %S, Base: s%u, Index: s%u, Constant: %s, LoopCount: %s, IsIndexChangedBeforeUse: %d"), valueTypeStr, candidate->base, candidate->index, constBuf, loopCountBuf, bIndexAlreadyChanged); } else { const Loop::MemCopyCandidate* candidate = emitData->candidate->AsMemCopy(); TRACE_MEMOP_PHASE(MemCopy, loop, emitData->stElemInstr, _u("ValueType: %S, StBase: s%u, Index: s%u, LdBase: s%u, LoopCount: %s, IsIndexChangedBeforeUse: %d"), valueTypeStr, candidate->base, candidate->index, candidate->ldBase, loopCountBuf, bIndexAlreadyChanged); } } #endif Assert(noImplicitCallUsesToInsert->Count() == 0); bool isLikelyJsArray; if (emitData->stElemInstr->GetDst()->IsIndirOpnd()) { baseOpnd = emitData->stElemInstr->GetDst()->AsIndirOpnd()->GetBaseOpnd(); isLikelyJsArray = baseOpnd->GetValueType().IsLikelyArrayOrObjectWithArray(); ProcessNoImplicitCallArrayUses(baseOpnd, baseOpnd->IsArrayRegOpnd() ? baseOpnd->AsArrayRegOpnd() : nullptr, emitData->stElemInstr, isLikelyJsArray, true); } RemoveMemOpSrcInstr(memopInstr, emitData->stElemInstr, emitData->block); if (!isMemset) { IR::Instr* ldElemInstr = ((MemCopyEmitData*)emitData)->ldElemInstr; if (ldElemInstr->GetSrc1()->IsIndirOpnd()) { baseOpnd = ldElemInstr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd(); isLikelyJsArray = baseOpnd->GetValueType().IsLikelyArrayOrObjectWithArray(); ProcessNoImplicitCallArrayUses(baseOpnd, baseOpnd->IsArrayRegOpnd() ? baseOpnd->AsArrayRegOpnd() : nullptr, ldElemInstr, isLikelyJsArray, true); } RemoveMemOpSrcInstr(memopInstr, ldElemInstr, emitData->block); } InsertNoImplicitCallUses(memopInstr); noImplicitCallUsesToInsert->Clear(); } bool GlobOpt::InspectInstrForMemSetCandidate(Loop* loop, IR::Instr* instr, MemSetEmitData* emitData, bool& errorInInstr) { Assert(emitData && emitData->candidate && emitData->candidate->IsMemSet()); Loop::MemSetCandidate* candidate = (Loop::MemSetCandidate*)emitData->candidate; if (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict) { if (instr->GetDst()->IsIndirOpnd() && (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->base) && (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index) ) { Assert(instr->IsProfiledInstr()); emitData->stElemInstr = instr; emitData->bailOutKind = instr->GetBailOutKind(); return true; } TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Orphan StElemI_A detected")); errorInInstr = true; } else if (instr->m_opcode == Js::OpCode::LdElemI_A) { TRACE_MEMOP_PHASE_VERBOSE(MemSet, loop, instr, _u("Orphan LdElemI_A detected")); errorInInstr = true; } return false; } bool GlobOpt::InspectInstrForMemCopyCandidate(Loop* loop, IR::Instr* instr, MemCopyEmitData* emitData, bool& errorInInstr) { Assert(emitData && emitData->candidate && emitData->candidate->IsMemCopy()); Loop::MemCopyCandidate* candidate = (Loop::MemCopyCandidate*)emitData->candidate; if (instr->m_opcode == Js::OpCode::StElemI_A || instr->m_opcode == Js::OpCode::StElemI_A_Strict) { if ( instr->GetDst()->IsIndirOpnd() && (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->base) && (GetVarSymID(instr->GetDst()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index) ) { Assert(instr->IsProfiledInstr()); emitData->stElemInstr = instr; emitData->bailOutKind = instr->GetBailOutKind(); // Still need to find the LdElem return false; } TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Orphan StElemI_A detected")); errorInInstr = true; } else if (instr->m_opcode == Js::OpCode::LdElemI_A) { if ( emitData->stElemInstr && instr->GetSrc1()->IsIndirOpnd() && (GetVarSymID(instr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetStackSym()) == candidate->ldBase) && (GetVarSymID(instr->GetSrc1()->AsIndirOpnd()->GetIndexOpnd()->GetStackSym()) == candidate->index) ) { Assert(instr->IsProfiledInstr()); emitData->ldElemInstr = instr; ValueType stValueType = emitData->stElemInstr->GetDst()->AsIndirOpnd()->GetBaseOpnd()->GetValueType(); ValueType ldValueType = emitData->ldElemInstr->GetSrc1()->AsIndirOpnd()->GetBaseOpnd()->GetValueType(); if (stValueType != ldValueType) { #if DBG_DUMP char16 stValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; stValueType.ToString(stValueTypeStr); char16 ldValueTypeStr[VALUE_TYPE_MAX_STRING_SIZE]; ldValueType.ToString(ldValueTypeStr); TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("for mismatch in Load(%s) and Store(%s) value type"), ldValueTypeStr, stValueTypeStr); #endif errorInInstr = true; return false; } // We found both instruction for this candidate return true; } TRACE_MEMOP_PHASE_VERBOSE(MemCopy, loop, instr, _u("Orphan LdElemI_A detected")); errorInInstr = true; } return false; } // The caller is responsible to free the memory allocated between inOrderEmitData[iEmitData -> end] bool GlobOpt::ValidateMemOpCandidates(Loop * loop, _Out_writes_(iEmitData) MemOpEmitData** inOrderEmitData, int& iEmitData) { AnalysisAssert(iEmitData == (int)loop->memOpInfo->candidates->Count()); // We iterate over the second block of the loop only. MemOp Works only if the loop has exactly 2 blocks Assert(loop->blockList.HasTwo()); Loop::MemOpList::Iterator iter(loop->memOpInfo->candidates); BasicBlock* bblock = loop->blockList.Head()->next; Loop::MemOpCandidate* candidate = nullptr; MemOpEmitData* emitData = nullptr; // Iterate backward because the list of candidate is reversed FOREACH_INSTR_BACKWARD_IN_BLOCK(instr, bblock) { if (!candidate) { // Time to check next candidate if (!iter.Next()) { // We have been through the whole list of candidates, finish break; } candidate = iter.Data(); if (!candidate) { continue; } // Common check for memset and memcopy Loop::InductionVariableChangeInfo inductionVariableChangeInfo = { 0, 0 }; // Get the inductionVariable changeInfo if (!loop->memOpInfo->inductionVariableChangeInfoMap->TryGetValue(candidate->index, &inductionVariableChangeInfo)) { TRACE_MEMOP_VERBOSE(loop, nullptr, _u("MemOp skipped (s%d): no induction variable"), candidate->base); return false; } if (inductionVariableChangeInfo.unroll != candidate->count) { TRACE_MEMOP_VERBOSE(loop, nullptr, _u("MemOp skipped (s%d): not matching unroll count"), candidate->base); return false; } if (candidate->IsMemSet()) { Assert(!PHASE_OFF(Js::MemSetPhase, this->func)); emitData = JitAnew(this->alloc, MemSetEmitData); } else { Assert(!PHASE_OFF(Js::MemCopyPhase, this->func)); // Specific check for memcopy Assert(candidate->IsMemCopy()); Loop::MemCopyCandidate* memcopyCandidate = candidate->AsMemCopy(); if (memcopyCandidate->base == Js::Constants::InvalidSymID || memcopyCandidate->ldBase == Js::Constants::InvalidSymID || (memcopyCandidate->ldCount != memcopyCandidate->count)) { TRACE_MEMOP_PHASE(MemCopy, loop, nullptr, _u("(s%d): not matching ldElem and stElem"), candidate->base); return false; } emitData = JitAnew(this->alloc, MemCopyEmitData); } Assert(emitData); emitData->block = bblock; emitData->inductionVar = inductionVariableChangeInfo; emitData->candidate = candidate; } bool errorInInstr = false; bool candidateFound = candidate->IsMemSet() ? InspectInstrForMemSetCandidate(loop, instr, (MemSetEmitData*)emitData, errorInInstr) : InspectInstrForMemCopyCandidate(loop, instr, (MemCopyEmitData*)emitData, errorInInstr); if (errorInInstr) { JitAdelete(this->alloc, emitData); return false; } if (candidateFound) { AnalysisAssert(iEmitData > 0); if (iEmitData == 0) { // Explicit for OACR break; } inOrderEmitData[--iEmitData] = emitData; candidate = nullptr; emitData = nullptr; } } NEXT_INSTR_BACKWARD_IN_BLOCK; if (iter.IsValid()) { TRACE_MEMOP(loop, nullptr, _u("Candidates not found in loop while validating")); return false; } return true; } void GlobOpt::ProcessMemOp() { FOREACH_LOOP_IN_FUNC_EDITING(loop, this->func) { if (HasMemOp(loop)) { const int candidateCount = loop->memOpInfo->candidates->Count(); Assert(candidateCount > 0); LoopCount * loopCount = GetOrGenerateLoopCountForMemOp(loop); // If loopCount is not available we can not continue with memop if (!loopCount || !(loopCount->LoopCountMinusOneSym() || loopCount->LoopCountMinusOneConstantValue())) { TRACE_MEMOP(loop, nullptr, _u("MemOp skipped for no loop count")); loop->doMemOp = false; loop->memOpInfo->candidates->Clear(); continue; } // The list is reversed, check them and place them in order in the following array MemOpEmitData** inOrderCandidates = JitAnewArray(this->alloc, MemOpEmitData*, candidateCount); int i = candidateCount; if (ValidateMemOpCandidates(loop, inOrderCandidates, i)) { Assert(i == 0); // Process the valid MemOp candidate in order. for (; i < candidateCount; ++i) { // Emit EmitMemop(loop, loopCount, inOrderCandidates[i]); JitAdelete(this->alloc, inOrderCandidates[i]); } } else { Assert(i != 0); for (; i < candidateCount; ++i) { JitAdelete(this->alloc, inOrderCandidates[i]); } // One of the memop candidates did not validate. Do not emit for this loop. loop->doMemOp = false; loop->memOpInfo->candidates->Clear(); } // Free memory JitAdeleteArray(this->alloc, candidateCount, inOrderCandidates); } } NEXT_LOOP_EDITING; } void GlobOpt::PRE::FieldPRE(Loop *loop) { JitArenaAllocator *alloc = this->globOpt->tempAlloc; this->FindPossiblePRECandidates(loop, alloc); this->PreloadPRECandidates(loop); this->RemoveOverlyOptimisticInitialValues(loop); } bool GlobOpt::PRE::InsertSymDefinitionInLandingPad(StackSym * sym, Loop * loop, Sym ** objPtrCopyPropSym) { Assert(sym->IsSingleDef()); IR::Instr * symDefInstr = sym->GetInstrDef(); if (!GlobOpt::IsPREInstrSequenceCandidateLoad(symDefInstr->m_opcode)) { return false; } IR::Opnd * symDefInstrSrc1 = symDefInstr->GetSrc1(); if (symDefInstrSrc1->IsSymOpnd()) { Assert(symDefInstrSrc1->AsSymOpnd()->m_sym->IsPropertySym()); // $L1 // T1 = o.x (v1|T3) // T2 = T1.y (v2|T4) <-- T1 is not live in the loop landing pad // jmp $L1 // Trying to make T1 live in the landing pad // o.x PropertySym* propSym = symDefInstrSrc1->AsSymOpnd()->m_sym->AsPropertySym(); if (candidates->candidatesBv->Test(propSym->m_id)) { // If propsym is a PRE candidate, then it must have had the same value on all back edges. // So, just look up the value on one of the back edges. BasicBlock* loopTail = loop->GetAnyTailBlock(); Value * valueOnBackEdge = loopTail->globOptData.FindValue(propSym); // If o.x is not invariant in the loop, we can't use the preloaded value of o.x.y in the landing pad Value * valueInLandingPad = loop->landingPad->globOptData.FindValue(propSym); if (valueOnBackEdge->GetValueNumber() != valueInLandingPad->GetValueNumber()) { return false; } *objPtrCopyPropSym = valueOnBackEdge->GetValueInfo()->GetSymStore(); if (candidates->candidatesToProcess->Test(propSym->m_id)) { GlobHashBucket bucket; bucket.element = valueOnBackEdge; bucket.value = propSym; if (!PreloadPRECandidate(loop, &bucket)) { return false; } Assert(!candidates->candidatesToProcess->Test(propSym->m_id)); Assert(loop->landingPad->globOptData.IsLive(valueOnBackEdge->GetValueInfo()->GetSymStore())); // Inserted T3 = o.x // Now, we want to // 1. Insert T1 = o.x // 2. Insert T4 = T1.y // 3. Indentify T3 as the objptr copy prop sym for T1, and make T3.y live on the back-edges // #1 is done next. #2 and #3 are done as part of preloading T1.y // Insert T1 = o.x if (!InsertPropertySymPreloadInLandingPad(symDefInstr->Copy(), loop, propSym)) { return false; } return true; } else { // o.x was already processed as a PRE candidate. If we were successful in preloading o.x, // we can now insert T1 = o.x if (loop->landingPad->globOptData.IsLive(*objPtrCopyPropSym)) { // insert T1 = o.x if (!InsertPropertySymPreloadInLandingPad(symDefInstr->Copy(), loop, propSym)) { return false; } return true; } else { return false; } } } else { return false; } } else if (symDefInstrSrc1->IsRegOpnd()) { // T2 = T1 // T3 = T2.y // trying to insert def of T2 // T1 StackSym * symDefInstrSrc1Sym = symDefInstrSrc1->AsRegOpnd()->GetStackSym(); if (!loop->landingPad->globOptData.IsLive(symDefInstrSrc1Sym)) { if (symDefInstrSrc1Sym->IsSingleDef()) { if (!InsertSymDefinitionInLandingPad(symDefInstrSrc1Sym, loop, objPtrCopyPropSym)) { return false; } } } else { *objPtrCopyPropSym = symDefInstrSrc1Sym; } if (!(OpCodeAttr::TempNumberTransfer(symDefInstr->m_opcode) && OpCodeAttr::TempObjectTransfer(symDefInstr->m_opcode))) { *objPtrCopyPropSym = sym; } IR::Instr * instr = symDefInstr->Copy(); if (instr->m_opcode == Js::OpCode::BytecodeArgOutCapture) { instr->m_opcode = Js::OpCode::Ld_A; } InsertInstrInLandingPad(instr, loop); return true; } else { return false; } } void GlobOpt::PRE::InsertInstrInLandingPad(IR::Instr * instr, Loop * loop) { instr->GetSrc1()->SetIsJITOptimizedReg(true); if (instr->GetDst()) { instr->GetDst()->SetIsJITOptimizedReg(true); loop->landingPad->globOptData.liveVarSyms->Set(instr->GetDst()->GetStackSym()->m_id); } if (instr->HasAnyImplicitCalls()) { IR::Instr * bailInstr = globOpt->EnsureDisableImplicitCallRegion(loop); bailInstr->InsertBefore(instr); } else if (loop->endDisableImplicitCall) { loop->endDisableImplicitCall->InsertBefore(instr); } else { loop->landingPad->InsertAfter(instr); } instr->ClearByteCodeOffset(); instr->SetByteCodeOffset(loop->landingPad->GetFirstInstr()); } IR::Instr * GlobOpt::PRE::InsertPropertySymPreloadInLandingPad(IR::Instr * ldInstr, Loop * loop, PropertySym * propertySym) { IR::SymOpnd *ldSrc = ldInstr->GetSrc1()->AsSymOpnd(); if (ldSrc->m_sym != propertySym) { // It's possible that the property syms are different but have equivalent objPtrs. Verify their values. Value *val1 = globOpt->CurrentBlockData()->FindValue(ldSrc->m_sym->AsPropertySym()->m_stackSym); Value *val2 = globOpt->CurrentBlockData()->FindValue(propertySym->m_stackSym); if (!val1 || !val2 || val1->GetValueNumber() != val2->GetValueNumber()) { return nullptr; } } // Consider: Shouldn't be necessary once we have copy-prop in prepass... ldInstr->GetSrc1()->AsSymOpnd()->m_sym = propertySym; ldSrc = ldInstr->GetSrc1()->AsSymOpnd(); if (ldSrc->IsPropertySymOpnd()) { IR::PropertySymOpnd *propSymOpnd = ldSrc->AsPropertySymOpnd(); IR::PropertySymOpnd *newPropSymOpnd; newPropSymOpnd = propSymOpnd->AsPropertySymOpnd()->CopyWithoutFlowSensitiveInfo(this->globOpt->func); ldInstr->ReplaceSrc1(newPropSymOpnd); } if (ldInstr->GetDst()) { loop->landingPad->globOptData.liveVarSyms->Set(ldInstr->GetDst()->GetStackSym()->m_id); } InsertInstrInLandingPad(ldInstr, loop); return ldInstr; } void GlobOpt::PRE::MakePropertySymLiveOnBackEdges(PropertySym * propertySym, Loop * loop, Value * valueToAdd) { BasicBlock * loopHeader = loop->GetHeadBlock(); FOREACH_PREDECESSOR_BLOCK(blockPred, loopHeader) { if (!loop->IsDescendentOrSelf(blockPred->loop)) { // Not a loop back-edge continue; } // Insert it in the value table blockPred->globOptData.SetValue(valueToAdd, propertySym); // Make it a live field blockPred->globOptData.liveFields->Set(propertySym->m_id); } NEXT_PREDECESSOR_BLOCK; } void GlobOpt::PRE::RemoveOverlyOptimisticInitialValues(Loop * loop) { BasicBlock * landingPad = loop->landingPad; // For a property sym whose obj ptr sym wasn't live in the landing pad, we can optimistically (if the obj ptr sym was // single def) insert an initial value in the landing pad, with the hope that PRE could make the obj ptr sym live. // But, if PRE couldn't make the obj ptr sym live, we need to clear the value for the property sym from the landing pad for (auto it = loop->initialValueFieldMap.GetIteratorWithRemovalSupport(); it.IsValid(); it.MoveNext()) { PropertySym * propertySym = it.CurrentKey(); StackSym * objPtrSym = propertySym->m_stackSym; if (!landingPad->globOptData.IsLive(objPtrSym)) { Value * landingPadPropSymValue = landingPad->globOptData.FindValue(propertySym); Assert(landingPadPropSymValue); Assert(landingPadPropSymValue->GetValueNumber() == it.CurrentValue()->GetValueNumber()); Assert(landingPadPropSymValue->GetValueInfo()->GetSymStore() == propertySym); landingPad->globOptData.ClearSymValue(propertySym); it.RemoveCurrent(); } } } #if DBG_DUMP void GlobOpt::PRE::TraceFailedPreloadInLandingPad(const Loop *const loop, PropertySym * propertySym, const char16* reason) const { if (PHASE_TRACE(Js::FieldPREPhase, this->globOpt->func)) { int32 propertyId = propertySym->m_propertyId; SymID objectSymId = propertySym->m_stackSym->m_id; char16 propSymStr[32]; switch (propertySym->m_fieldKind) { case PropertyKindData: if (JITManager::GetJITManager()->IsOOPJITEnabled()) { swprintf_s(propSymStr, _u("s%d->#%d"), objectSymId, propertyId); } else { Js::PropertyRecord const* fieldName = propertySym->m_func->GetInProcThreadContext()->GetPropertyRecord(propertyId); swprintf_s(propSymStr, _u("s%d->%s"), objectSymId, fieldName->GetBuffer()); } break; case PropertyKindSlots: case PropertyKindSlotArray: swprintf_s(propSymStr, _u("s%d[%d]"), objectSymId, propertyId); break; case PropertyKindLocalSlots: swprintf_s(propSymStr, _u("s%dl[%d]"), objectSymId, propertyId); break; default: AssertMsg(0, "Unknown field kind"); break; } Output::Print(_u("** TRACE: Field PRE: ")); this->globOpt->func->DumpFullFunctionName(); Output::Print(_u(": Failed to pre-load (%s) in landing pad of loop #%d. Reason: %s "), propSymStr, loop->GetLoopNumber(), reason); Output::Print(_u("\n")); } } #endif