Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JIT: Extend escape analysis to account for arrays with non-gcref elements #104906

Open
wants to merge 78 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
78 commits
Select commit Hold shift + click to select a range
1b0e3d3
initial prototype
hez2010 Jul 15, 2024
57b7e42
Morph ARR_LENGTH and INDEX_ADDR
hez2010 Jul 15, 2024
1b5b25e
Fix incorrect array length storage
hez2010 Jul 15, 2024
395b735
Use offset and correct type
hez2010 Jul 15, 2024
17de70b
handle reassignment
hez2010 Jul 15, 2024
5443c42
range check
hez2010 Jul 15, 2024
b2d07da
throw range check failure
hez2010 Jul 15, 2024
b5ae9e7
update comments
hez2010 Jul 15, 2024
87b29de
add metrics
hez2010 Jul 15, 2024
eeb681d
minor cleanup
hez2010 Jul 15, 2024
dee9f38
Introduce new temp and implement local address morphing
hez2010 Jul 16, 2024
94c103b
handle index out-of-range
hez2010 Jul 16, 2024
12b297b
Refactor to remove duplicates
hez2010 Jul 16, 2024
e0fa91e
Remove invalid asserts
hez2010 Jul 16, 2024
9e0a04f
make compiler happy
hez2010 Jul 16, 2024
ae822f8
Address review feedbacks
hez2010 Jul 16, 2024
a4588bb
Fix INDEX_ADDR and add Sub
hez2010 Jul 16, 2024
32b9e26
Support IsAddressLessThan and its friends
hez2010 Jul 16, 2024
39d1ad9
Fix assertions
hez2010 Jul 16, 2024
0df0d58
Merge remote-tracking branch 'origin/main' into value-array-stack-alloc
hez2010 Jul 16, 2024
9f408b2
Use new overload
hez2010 Jul 16, 2024
418a62b
JIT: Remove GTF_IND_INVARIANT and GTF_IND_NONFAULTING flags checking
jakobbotsch Jul 16, 2024
4572408
Remove old comment
jakobbotsch Jul 16, 2024
9255762
Expose jitconfig
hez2010 Jul 16, 2024
1af84b9
Remove another assert
jakobbotsch Jul 16, 2024
629c793
Count
jakobbotsch Jul 16, 2024
b578203
Try 2 at counting
jakobbotsch Jul 16, 2024
b4445f6
Introduce BBF_HAS_NEWARR
hez2010 Jul 16, 2024
af9c40e
Early exit on debug as well
hez2010 Jul 16, 2024
8b54f5a
Update computed flags
hez2010 Jul 16, 2024
cba4d2c
Merge remote-tracking branch 'jakobbotsch/indir-flags' into value-arr…
hez2010 Jul 17, 2024
b497fae
Merge remote-tracking branch 'origin/main' into value-array-stack-alloc
hez2010 Jul 17, 2024
6eca58d
Partially revert 39d1ad9
hez2010 Jul 17, 2024
49d8509
Reuse existing comma node
hez2010 Jul 17, 2024
4c6e359
Respect IsBoundsChecked
hez2010 Jul 17, 2024
4d84379
Check lowerbound too
hez2010 Jul 17, 2024
c0cad85
Fix assertion take 2
hez2010 Jul 18, 2024
d28553a
Remove redundant jit-ee calls
hez2010 Jul 18, 2024
f42e78f
Merge branch 'main' into value-array-stack-alloc
hez2010 Jul 18, 2024
c21c4f7
Fix assertion again
hez2010 Jul 18, 2024
18ec558
Check array length
hez2010 Jul 18, 2024
eadb4ad
Fix assertion in another way
hez2010 Jul 18, 2024
9d4021c
Unset the flag to avoid unnecessary assert
hez2010 Jul 18, 2024
1cba8c0
Merge branch 'main' into value-array-stack-alloc
hez2010 Jul 19, 2024
1fff53e
Add tests
hez2010 Jul 19, 2024
d521a94
sigh
hez2010 Jul 19, 2024
97ee2be
Support R2R/NativeAOT
hez2010 Jul 19, 2024
c66bdb8
Merge branch 'main' into value-array-stack-alloc
hez2010 Jul 19, 2024
5bcb786
Fix building
hez2010 Jul 19, 2024
a01562e
cleanup
hez2010 Jul 19, 2024
e728d4f
remove invalid assert
hez2010 Jul 19, 2024
d73c5c5
double align on 32bit platform
hez2010 Jul 19, 2024
c9fea23
Use correct alignment for align8
hez2010 Jul 19, 2024
772bee6
Fix intrinsic expansion
hez2010 Jul 20, 2024
499ac0b
Merge branch 'main' into value-array-stack-alloc
hez2010 Jul 20, 2024
ffcbd85
Merge remote-tracking branch 'origin/main' into value-array-stack-alloc
hez2010 Jul 22, 2024
4789090
Merge branch 'main' into value-array-stack-alloc
hez2010 Aug 24, 2024
1b33170
Merge branch 'main' into value-array-stack-alloc
hez2010 Sep 20, 2024
9c81c04
Address some review feedback
hez2010 Sep 21, 2024
ed6c83c
Merge branch 'main' into value-array-stack-alloc
hez2010 Oct 4, 2024
6a7e6cb
Revert workaround to fgMorphSmpOp
hez2010 Oct 4, 2024
e4d91ed
Merge branch 'main' into value-array-stack-alloc
hez2010 Nov 26, 2024
aeeb9e3
Fix build failure
hez2010 Nov 26, 2024
0e2a01d
Merge branch 'main' into value-array-stack-alloc
hez2010 Dec 6, 2024
1914e80
Try fixing remorph issue
hez2010 Dec 6, 2024
f671085
Minimize asmdiff for tier-0
hez2010 Dec 6, 2024
d4fd6ae
Check CI
hez2010 Dec 6, 2024
b3ff72d
Take INDEX as non-escape
hez2010 Dec 7, 2024
49c39b9
Early exit when optimization is disabled
hez2010 Dec 7, 2024
a696f87
Format
hez2010 Dec 7, 2024
ce2de99
Merge branch 'main' into value-array-stack-alloc
hez2010 Dec 14, 2024
cc88979
Revert "Try fixing remorph issue"
hez2010 Dec 14, 2024
8d38e0c
Merge branch 'main' into value-array-stack-alloc
AndyAyersMS Dec 18, 2024
52b32f2
Introduce a flag for stack allocated arrays
hez2010 Dec 20, 2024
1f342bd
Handle in global morph
hez2010 Dec 21, 2024
5b51919
Some special cases
hez2010 Dec 21, 2024
ce41a8d
Avoid extensive remorph
hez2010 Dec 21, 2024
47114c6
Oops
hez2010 Dec 21, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions src/coreclr/jit/assertionprop.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -170,16 +170,22 @@ bool IntegralRange::Contains(int64_t value) const
{
GenTree* const addr = node->AsIndir()->Addr();

if (node->TypeIs(TYP_INT) && addr->OperIs(GT_ADD) && addr->gtGetOp1()->OperIs(GT_LCL_VAR) &&
addr->gtGetOp2()->IsIntegralConst(OFFSETOF__CORINFO_Span__length))
if (node->TypeIs(TYP_INT) && addr->OperIs(GT_ADD) && addr->gtGetOp1()->OperIs(GT_LCL_VAR))
{
GenTreeLclVar* const lclVar = addr->gtGetOp1()->AsLclVar();

if (compiler->lvaGetDesc(lclVar->GetLclNum())->IsSpan())
if (addr->gtGetOp2()->IsIntegralConst(OFFSETOF__CORINFO_Span__length) &&
compiler->lvaGetDesc(lclVar->GetLclNum())->IsSpan())
{
assert(compiler->lvaIsImplicitByRefLocal(lclVar->GetLclNum()));
return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)};
}

if (addr->gtGetOp2()->IsIntegralConst(OFFSETOF__CORINFO_Array__length) &&
compiler->lvaGetDesc(lclVar->GetLclNum())->lvStackAllocatedArray == 1)
{
return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::ArrayLenMax};
}
}
break;
}
Expand All @@ -189,9 +195,17 @@ bool IntegralRange::Contains(int64_t value) const
GenTreeLclFld* const lclFld = node->AsLclFld();
LclVarDsc* const varDsc = compiler->lvaGetDesc(lclFld);

if (node->TypeIs(TYP_INT) && varDsc->IsSpan() && lclFld->GetLclOffs() == OFFSETOF__CORINFO_Span__length)
if (node->TypeIs(TYP_INT))
{
return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)};
if (varDsc->IsSpan() && lclFld->GetLclOffs() == OFFSETOF__CORINFO_Span__length)
{
return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)};
}

if (varDsc->lvStackAllocatedArray == 1 && lclFld->GetLclOffs() == OFFSETOF__CORINFO_Array__length)
{
return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::ArrayLenMax};
}
}

break;
Expand Down
5 changes: 3 additions & 2 deletions src/coreclr/jit/compiler.h
Original file line number Diff line number Diff line change
Expand Up @@ -690,8 +690,9 @@ class LclVarDsc
unsigned char lvSingleDefDisqualifyReason = 'H';
#endif

unsigned char lvAllDefsAreNoGc : 1; // For pinned locals: true if all defs of this local are no-gc
unsigned char lvStackAllocatedBox : 1; // Local is a stack allocated box
unsigned char lvAllDefsAreNoGc : 1; // For pinned locals: true if all defs of this local are no-gc
unsigned char lvStackAllocatedBox : 1; // Local is a stack allocated box
unsigned char lvStackAllocatedArray : 1; // Local is a stack allocated array

#if FEATURE_MULTIREG_ARGS
regNumber lvRegNumForSlot(unsigned slotNum)
Expand Down
4 changes: 4 additions & 0 deletions src/coreclr/jit/fgdiagnostic.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -955,6 +955,10 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos)
{
fprintf(fgxFile, "\n hot=\"true\"");
}
if (block->HasFlag(BBF_HAS_NEWARR))
{
fprintf(fgxFile, "\n callsNewArr=\"true\"");
}
if (block->HasFlag(BBF_HAS_NEWOBJ))
{
fprintf(fgxFile, "\n callsNew=\"true\"");
Expand Down
4 changes: 2 additions & 2 deletions src/coreclr/jit/gentree.h
Original file line number Diff line number Diff line change
Expand Up @@ -7586,12 +7586,12 @@ struct GenTreeArrAddr : GenTreeUnOp

public:
GenTreeArrAddr(GenTree* addr, var_types elemType, CORINFO_CLASS_HANDLE elemClassHandle, uint8_t firstElemOffset)
: GenTreeUnOp(GT_ARR_ADDR, TYP_BYREF, addr DEBUGARG(/* largeNode */ false))
: GenTreeUnOp(GT_ARR_ADDR, addr->TypeGet(), addr DEBUGARG(/* largeNode */ false))
, m_elemClassHandle(elemClassHandle)
, m_elemType(elemType)
, m_firstElemOffset(firstElemOffset)
{
assert(addr->TypeIs(TYP_BYREF));
assert(addr->TypeIs(TYP_BYREF, TYP_I_IMPL));
assert(((elemType == TYP_STRUCT) && (elemClassHandle != NO_CLASS_HANDLE)) ||
hez2010 marked this conversation as resolved.
Show resolved Hide resolved
(elemClassHandle == NO_CLASS_HANDLE));
}
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/jit/jitconfigvalues.h
Original file line number Diff line number Diff line change
Expand Up @@ -668,6 +668,7 @@ CONFIG_STRING(JitObjectStackAllocationRange, "JitObjectStackAllocationRange")
RELEASE_CONFIG_INTEGER(JitObjectStackAllocation, "JitObjectStackAllocation", 1)
RELEASE_CONFIG_INTEGER(JitObjectStackAllocationRefClass, "JitObjectStackAllocationRefClass", 1)
RELEASE_CONFIG_INTEGER(JitObjectStackAllocationBoxedValueClass, "JitObjectStackAllocationBoxedValueClass", 1)
RELEASE_CONFIG_INTEGER(JitObjectStackAllocationArray, "JitObjectStackAllocationArray", 1)

RELEASE_CONFIG_INTEGER(JitEECallTimingInfo, "JitEECallTimingInfo", 0)

Expand Down
2 changes: 2 additions & 0 deletions src/coreclr/jit/jitmetadatalist.h
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ JITMETADATAMETRIC(NewRefClassHelperCalls, int, 0)
JITMETADATAMETRIC(StackAllocatedRefClasses, int, 0)
JITMETADATAMETRIC(NewBoxedValueClassHelperCalls, int, 0)
JITMETADATAMETRIC(StackAllocatedBoxedValueClasses, int, 0)
JITMETADATAMETRIC(NewArrayHelperCalls, int, 0)
JITMETADATAMETRIC(StackAllocatedArrays, int, 0)
JITMETADATAMETRIC(LocalAssertionCount, int, 0)
JITMETADATAMETRIC(LocalAssertionOverflow, int, 0)
JITMETADATAMETRIC(MorphTrackedLocals, int, 0)
Expand Down
71 changes: 71 additions & 0 deletions src/coreclr/jit/lclmorph.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1353,6 +1353,77 @@ class LocalAddressVisitor final : public GenTreeVisitor<LocalAddressVisitor>
break;
}

case GT_INDEX_ADDR:
{
assert(TopValue(2).Node() == node);
assert(TopValue(1).Node() == node->gtGetOp1());
assert(TopValue(0).Node() == node->gtGetOp2());

if (node->gtGetOp2()->IsCnsIntOrI() && TopValue(1).IsAddress() &&
m_compiler->lvaGetDesc(TopValue(1).LclNum())->lvStackAllocatedArray == 1)
{
ssize_t offset = node->AsIndexAddr()->gtElemOffset +
node->gtGetOp2()->AsIntCon()->IconValue() * node->AsIndexAddr()->gtElemSize;

// For stack allocated arrays the local size is the size of the entire array
if (!node->AsIndexAddr()->IsBoundsChecked() ||
(static_cast<ssize_t>(node->AsIndexAddr()->gtElemOffset) <= offset &&
offset < static_cast<ssize_t>(m_compiler->lvaLclExactSize(TopValue(1).LclNum()))))
hez2010 marked this conversation as resolved.
Show resolved Hide resolved
{
if (FitsIn<unsigned>(offset) &&
TopValue(2).AddOffset(TopValue(1), static_cast<unsigned>(offset)))
{
INDEBUG(TopValue(0).Consume());
PopValue();
PopValue();
break;
}
}
else
{
*use = m_compiler->gtNewOperNode(GT_COMMA, node->TypeGet(),
hez2010 marked this conversation as resolved.
Show resolved Hide resolved
m_compiler->gtNewHelperCallNode(CORINFO_HELP_RNGCHKFAIL,
TYP_VOID),
m_compiler->gtNewIconNode(0, TYP_BYREF));
m_stmtModified = true;
INDEBUG(TopValue(0).Consume());
PopValue();
INDEBUG(TopValue(0).Consume());
PopValue();
break;
}
}

EscapeValue(TopValue(0), node);
PopValue();
EscapeValue(TopValue(0), node);
PopValue();
break;
}

case GT_ARR_LENGTH:
{
assert(TopValue(1).Node() == node);
assert(TopValue(0).Node() == node->gtGetOp1());

if (TopValue(0).IsAddress() && m_compiler->lvaGetDesc(TopValue(0).LclNum())->lvStackAllocatedArray == 1)
{
GenTree* gtLclFld =
m_compiler->gtNewLclFldNode(TopValue(0).LclNum(), TYP_INT,
TopValue(0).Offset() + OFFSETOF__CORINFO_Array__length);
SequenceLocal(gtLclFld->AsLclVarCommon());
*use = gtLclFld;
m_stmtModified = true;
INDEBUG(TopValue(0).Consume());
PopValue();
break;
}

EscapeValue(TopValue(0), node);
PopValue();
break;
}

default:
while (TopValue(0).Node() != node)
{
Expand Down
5 changes: 5 additions & 0 deletions src/coreclr/jit/lclvars.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2820,6 +2820,11 @@ void Compiler::StructPromotionHelper::PromoteStructVar(unsigned lclNum)
fieldVarDsc->SetIsNeverNegative(true);
}

if (varDsc->lvStackAllocatedArray == 1 && fieldVarDsc->lvFldOffset == OFFSETOF__CORINFO_Array__length)
{
fieldVarDsc->SetIsNeverNegative(true);
}

// This new local may be the first time we've seen a long typed local.
if (fieldVarDsc->lvType == TYP_LONG)
{
Expand Down
35 changes: 34 additions & 1 deletion src/coreclr/jit/morph.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3571,6 +3571,7 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr)
{
indexAddr->Arr() = fgMorphTree(indexAddr->Arr());
indexAddr->Index() = fgMorphTree(indexAddr->Index());

indexAddr->AddAllEffectsFlags(indexAddr->Arr(), indexAddr->Index());

// Mark the indirection node as needing a range check if necessary.
Expand Down Expand Up @@ -3602,6 +3603,32 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr)
GenTree* indexDefn = nullptr; // non-NULL if we need to allocate a temp for the index expression
GenTreeBoundsChk* boundsCheck = nullptr;

if (arrRef->IsAnyLocal() && lvaGetDesc(arrRef->AsLclVarCommon()->GetLclNum())->lvStackAllocatedArray == 1)
{
index = fgMorphTree(index);
if (index->IsCnsIntOrI())
{
ssize_t offset = static_cast<ssize_t>(elemOffs) + index->AsIntCon()->IconValue() * elemSize;

// For stack allocated arrays the local size is the size of the entire array
if (!indexAddr->IsBoundsChecked() ||
(static_cast<ssize_t>(elemOffs) <= offset &&
offset < static_cast<ssize_t>(lvaLclExactSize(arrRef->AsLclVarCommon()->GetLclNum()))))
{
if (FitsIn<unsigned>(offset))
{
return fgMorphTree(gtNewOperNode(GT_ADD, indexAddr->TypeGet(), arrRef, gtNewIconNode(offset)));
}
}
else
{
return fgMorphTree(gtNewOperNode(GT_COMMA, indexAddr->TypeGet(),
gtNewHelperCallNode(CORINFO_HELP_RNGCHKFAIL, TYP_VOID),
gtNewIconNode(0, TYP_BYREF)));
}
}
}

// If we're doing range checking, introduce a GT_BOUNDS_CHECK node for the address.
if (indexAddr->IsBoundsChecked())
{
Expand Down Expand Up @@ -3633,7 +3660,7 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr)
}

if (((index->gtFlags & (GTF_ASG | GTF_CALL | GTF_GLOB_REF)) != 0) ||
gtComplexityExceeds(index, MAX_ARR_COMPLEXITY) || index->OperIs(GT_LCL_FLD) ||
gtComplexityExceeds(index, MAX_INDEX_COMPLEXITY) || index->OperIs(GT_LCL_FLD) ||
(index->OperIs(GT_LCL_VAR) && lvaIsLocalImplicitlyAccessedByRef(index->AsLclVar()->GetLclNum())))
{
unsigned indexTmpNum = lvaGrabTemp(true DEBUGARG("index expr"));
Expand Down Expand Up @@ -7755,6 +7782,12 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optA
return iconNode;
}
}
if (op1->IsAnyLocal() && lvaGetDesc(op1->AsLclVarCommon())->lvStackAllocatedArray == 1)
{
return fgMorphTree(
gtNewLclFldNode(op1->AsLclVarCommon()->GetLclNum(), tree->TypeGet(),
op1->AsLclVarCommon()->GetLclOffs() + OFFSETOF__CORINFO_Array__length));
}
break;

case GT_IND:
Expand Down
Loading
Loading