Skip to content

Change VMPTR IPC events to ensure endianness consistency #128058

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rcj1 wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Change VMPTR IPC events to ensure endianness consistency #128058

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5286823
change vmptr portable
rcj1 May 11, 2026
c999fe6
remove static assert
rcj1 May 12, 2026
0e7490a
missed a few of these
rcj1 May 12, 2026
3d40896
missed another
rcj1 May 12, 2026
c082e90
reverting framepointer
rcj1 May 13, 2026
c89aae6
copilolt comments
rcj1 May 13, 2026
05762c6
code review
rcj1 May 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/coreclr/debug/di/breakpoint.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -522,11 +522,11 @@ HRESULT CordbStepper::StepRange(BOOL fStepIn,

if (m_frame == NULL)
{
pEvent->StepData.frameToken = LEAF_MOST_FRAME;
pEvent->StepData.frameToken = (CORDB_ADDRESS)0;
}
else
{
pEvent->StepData.frameToken = m_frame->GetFramePointer();
pEvent->StepData.frameToken = PTR_TO_CORDB_ADDRESS(m_frame->GetFramePointer().GetSPValue());
}

pEvent->StepData.stepIn = (fStepIn != 0);
Expand Down Expand Up @@ -688,11 +688,11 @@ HRESULT CordbStepper::StepOut()

if (m_frame == NULL)
{
pEvent->StepData.frameToken = LEAF_MOST_FRAME;
pEvent->StepData.frameToken = (CORDB_ADDRESS)0;
}
else
{
pEvent->StepData.frameToken = m_frame->GetFramePointer();
pEvent->StepData.frameToken = PTR_TO_CORDB_ADDRESS(m_frame->GetFramePointer().GetSPValue());
}

pEvent->StepData.totalRangeCount = 0;
Expand Down
4 changes: 2 additions & 2 deletions src/coreclr/debug/di/process.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5580,7 +5580,7 @@ void CordbProcess::RawDispatchEvent(
{
STRESS_LOG4(LF_CORDB, LL_INFO100,
"RCET::DRCE: Exception2 0x%p 0x%X 0x%X 0x%X\n",
pEvent->ExceptionCallback2.framePointer.GetSPValue(),
CORDB_ADDRESS_TO_PTR(pEvent->ExceptionCallback2.framePointer),
pEvent->ExceptionCallback2.nOffset,
pEvent->ExceptionCallback2.eventType,
pEvent->ExceptionCallback2.dwFlags
Expand All @@ -5602,7 +5602,7 @@ void CordbProcess::RawDispatchEvent(
//
RSSmartPtr<CordbFrame> pFrame;

FramePointer fp = pEvent->ExceptionCallback2.framePointer;
FramePointer fp = FramePointer::MakeFramePointer(CORDB_ADDRESS_TO_PTR(pEvent->ExceptionCallback2.framePointer));
if (fp != LEAF_MOST_FRAME)
{
// The interface forces us to pass a FramePointer via an ICorDebugFrame.
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/debug/di/rsthread.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2184,7 +2184,7 @@ HRESULT CordbThread::InterceptCurrentException(ICorDebugFrame * pFrame)
GetProcess()->InitIPCEvent(&event, DB_IPCE_INTERCEPT_EXCEPTION, true, VMPTR_AppDomain::NullPtr());

event.InterceptException.vmThreadToken = m_vmThreadToken;
event.InterceptException.frameToken = pRealFrame->GetFramePointer();
event.InterceptException.frameToken = PTR_TO_CORDB_ADDRESS(pRealFrame->GetFramePointer().GetSPValue());

hr = GetProcess()->m_cordb->SendIPCEvent(GetProcess(), &event, sizeof(DebuggerIPCEvent));

Expand Down
18 changes: 9 additions & 9 deletions src/coreclr/debug/ee/debugger.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7158,7 +7158,7 @@ HRESULT Debugger::SendExceptionHelperAndBlock(
//
InitIPCEvent(ipce, DB_IPCE_EXCEPTION_CALLBACK2, pThread);

ipce->ExceptionCallback2.framePointer = framePointer;
ipce->ExceptionCallback2.framePointer = PTR_TO_CORDB_ADDRESS(framePointer.GetSPValue());
ipce->ExceptionCallback2.eventType = eventType;
ipce->ExceptionCallback2.nOffset = (UINT)nOffset;
ipce->ExceptionCallback2.dwFlags = dwFlags;
Expand Down Expand Up @@ -7311,7 +7311,7 @@ void Debugger::SendExceptionEventsWorker(

InitIPCEvent(ipce, DB_IPCE_EXCEPTION_CALLBACK2, pThread);

ipce->ExceptionCallback2.framePointer = framePointer;
ipce->ExceptionCallback2.framePointer = PTR_TO_CORDB_ADDRESS(framePointer.GetSPValue());
ipce->ExceptionCallback2.eventType = DEBUG_EXCEPTION_USER_FIRST_CHANCE;
ipce->ExceptionCallback2.nOffset = (UINT)nOffset;
ipce->ExceptionCallback2.dwFlags = fIsInterceptable ? DEBUG_EXCEPTION_CAN_BE_INTERCEPTED : 0;
Expand Down Expand Up @@ -7970,7 +7970,7 @@ void Debugger::SendCatchHandlerFound(
//
InitIPCEvent(ipce, DB_IPCE_EXCEPTION_CALLBACK2, pThread);

ipce->ExceptionCallback2.framePointer = fp;
ipce->ExceptionCallback2.framePointer = PTR_TO_CORDB_ADDRESS(fp.GetSPValue());
ipce->ExceptionCallback2.eventType = DEBUG_EXCEPTION_CATCH_HANDLER_FOUND;
ipce->ExceptionCallback2.nOffset = (UINT)nOffset;
ipce->ExceptionCallback2.dwFlags = dwFlags;
Expand Down Expand Up @@ -10292,7 +10292,7 @@ bool Debugger::HandleIPCEvent(DebuggerIPCEvent * pEvent)
LOG((LF_CORDB,LL_INFO10000, "D::HIPCE: frame SP:%p "
"StepIn:%s RangeIL:%s RangeCount:%u MapStop:0x%x "
"InterceptStop:0x%x AppD:%p\n",
pEvent->StepData.frameToken.GetSPValue(),
CORDB_ADDRESS_TO_PTR(pEvent->StepData.frameToken),
(pEvent->StepData.stepIn ? "true" : "false"),
(pEvent->StepData.rangeIL ? "true" : "false"),
pEvent->StepData.rangeCount,
Expand Down Expand Up @@ -10348,7 +10348,7 @@ bool Debugger::HandleIPCEvent(DebuggerIPCEvent * pEvent)

_ASSERTE(cRanges == 0 || ((cRanges > 0) && (cRanges == pEvent->StepData.rangeCount)));

if (!pStepper->Step(pEvent->StepData.frameToken,
if (!pStepper->Step(FramePointer::MakeFramePointer(CORDB_ADDRESS_TO_PTR(pEvent->StepData.frameToken)),
pEvent->StepData.stepIn,
&(pEvent->StepData.range),
cRanges,
Expand Down Expand Up @@ -10422,7 +10422,7 @@ bool Debugger::HandleIPCEvent(DebuggerIPCEvent * pEvent)
// Safe to stack trace b/c we're stopped.
StackTraceTicket ticket(pThread);

pStepper->StepOut(pEvent->StepData.frameToken, ticket);
pStepper->StepOut(FramePointer::MakeFramePointer(CORDB_ADDRESS_TO_PTR(pEvent->StepData.frameToken)), ticket);

pIPCResult->StepData.stepperToken.Set(pStepper);
}
Expand Down Expand Up @@ -11107,7 +11107,7 @@ HRESULT Debugger::GetAndSendInterceptCommand(DebuggerIPCEvent *event)
//
// Now start processing the parameters from the event.
//
FramePointer targetFramePointer = event->InterceptException.frameToken;
FramePointer targetFramePointer = FramePointer::MakeFramePointer(CORDB_ADDRESS_TO_PTR(event->InterceptException.frameToken));

ControllerStackInfo csi;

Expand Down Expand Up @@ -11437,7 +11437,7 @@ void Debugger::TypeHandleToBasicTypeInfo(AppDomain *pAppDomain, TypeHandle th, D
case ELEMENT_TYPE_BYREF:
res->vmTypeHandle = WrapTypeHandle(th);
res->metadataToken = mdTokenNil;
res->vmAssembly.SetRawPtr(NULL);
res->vmAssembly = VMPTR_Assembly::NullPtr();
break;

case ELEMENT_TYPE_CLASS:
Expand All @@ -11455,7 +11455,7 @@ void Debugger::TypeHandleToBasicTypeInfo(AppDomain *pAppDomain, TypeHandle th, D
default:
res->vmTypeHandle = VMPTR_TypeHandle::NullPtr();
res->metadataToken = mdTokenNil;
res->vmAssembly.SetRawPtr(NULL);
res->vmAssembly = VMPTR_Assembly::NullPtr();
break;
}
return;
Expand Down
Loading
Loading