Skip to content

Add APIs to allow calling conventions to provide requirements for being considered during heuristic calling convention detection #7861

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
D0ntPanic wants to merge 1 commit into
base: dev
Choose a base branch
from
+250 −0
Open

Add APIs to allow calling conventions to provide requirements for being considered during heuristic calling convention detection #7861

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions arch/x86/arch_x86.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3814,6 +3814,11 @@ class X86ThiscallCallingConvention: public X86BaseCallingConvention
return vector<uint32_t>{ XED_REG_ECX };
}

virtual vector<uint32_t> GetRequiredArgumentRegisters() override
{
return vector<uint32_t>{ XED_REG_ECX };
}

virtual bool IsStackAdjustedOnReturn() override
{
return true;
Expand Down
19 changes: 19 additions & 0 deletions binaryninjaapi.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17223,6 +17223,8 @@ namespace BinaryNinja {
static uint32_t* GetCalleeSavedRegistersCallback(void* ctxt, size_t* count);
static uint32_t* GetIntegerArgumentRegistersCallback(void* ctxt, size_t* count);
static uint32_t* GetFloatArgumentRegistersCallback(void* ctxt, size_t* count);
static uint32_t* GetRequiredArgumentRegistersCallback(void* ctxt, size_t* count);
static uint32_t* GetRequiredClobberedRegistersCallback(void* ctxt, size_t* count);
static void FreeRegisterListCallback(void* ctxt, uint32_t* regs, size_t len);

static bool AreArgumentRegistersSharedIndexCallback(void* ctxt);
Expand Down Expand Up @@ -17255,6 +17257,21 @@ namespace BinaryNinja {

virtual std::vector<uint32_t> GetIntegerArgumentRegisters();
virtual std::vector<uint32_t> GetFloatArgumentRegisters();

/*! Gets the set of registers that must be arguments for heuristic calling convention
detection to consider this calling convention as a valid option.

\return The set of registers that must be arguments
*/
virtual std::vector<uint32_t> GetRequiredArgumentRegisters();

/*! Gets the set of registers that must be clobbered for heuristic calling convention
detection to consider this calling convention as a valid option.

\return The set of registers that must be clobbered
*/
virtual std::vector<uint32_t> GetRequiredClobberedRegisters();

virtual bool AreArgumentRegistersSharedIndex();
virtual bool AreArgumentRegistersUsedForVarArgs();
virtual bool IsStackReservedForArgumentRegisters();
Expand Down Expand Up @@ -17287,6 +17304,8 @@ namespace BinaryNinja {

virtual std::vector<uint32_t> GetIntegerArgumentRegisters() override;
virtual std::vector<uint32_t> GetFloatArgumentRegisters() override;
virtual std::vector<uint32_t> GetRequiredArgumentRegisters() override;
virtual std::vector<uint32_t> GetRequiredClobberedRegisters() override;
virtual bool AreArgumentRegistersSharedIndex() override;
virtual bool AreArgumentRegistersUsedForVarArgs() override;
virtual bool IsStackReservedForArgumentRegisters() override;
Expand Down
4 changes: 4 additions & 0 deletions binaryninjacore.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2770,6 +2770,8 @@ extern "C"
uint32_t* (*getCalleeSavedRegisters)(void* ctxt, size_t* count);
uint32_t* (*getIntegerArgumentRegisters)(void* ctxt, size_t* count);
uint32_t* (*getFloatArgumentRegisters)(void* ctxt, size_t* count);
uint32_t* (*getRequiredArgumentRegisters)(void* ctxt, size_t* count);
uint32_t* (*getRequiredClobberedRegisters)(void* ctxt, size_t* count);
void (*freeRegisterList)(void* ctxt, uint32_t* regs, size_t len);

bool (*areArgumentRegistersSharedIndex)(void* ctxt);
Expand Down Expand Up @@ -7412,6 +7414,8 @@ extern "C"

BINARYNINJACOREAPI uint32_t* BNGetIntegerArgumentRegisters(BNCallingConvention* cc, size_t* count);
BINARYNINJACOREAPI uint32_t* BNGetFloatArgumentRegisters(BNCallingConvention* cc, size_t* count);
BINARYNINJACOREAPI uint32_t* BNGetRequiredArgumentRegisters(BNCallingConvention* cc, size_t* count);
BINARYNINJACOREAPI uint32_t* BNGetRequiredClobberedRegisters(BNCallingConvention* cc, size_t* count);
BINARYNINJACOREAPI bool BNAreArgumentRegistersSharedIndex(BNCallingConvention* cc);
BINARYNINJACOREAPI bool BNAreArgumentRegistersUsedForVarArgs(BNCallingConvention* cc);
BINARYNINJACOREAPI bool BNIsStackReservedForArgumentRegisters(BNCallingConvention* cc);
Expand Down
62 changes: 62 additions & 0 deletions callingconvention.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ CallingConvention::CallingConvention(Architecture* arch, const string& name)
cc.getCalleeSavedRegisters = GetCalleeSavedRegistersCallback;
cc.getIntegerArgumentRegisters = GetIntegerArgumentRegistersCallback;
cc.getFloatArgumentRegisters = GetFloatArgumentRegistersCallback;
cc.getRequiredArgumentRegisters = GetRequiredArgumentRegistersCallback;
cc.getRequiredClobberedRegisters = GetRequiredClobberedRegistersCallback;
cc.freeRegisterList = FreeRegisterListCallback;
cc.areArgumentRegistersSharedIndex = AreArgumentRegistersSharedIndexCallback;
cc.areArgumentRegistersUsedForVarArgs = AreArgumentRegistersUsedForVarArgsCallback;
Expand Down Expand Up @@ -119,6 +121,32 @@ uint32_t* CallingConvention::GetFloatArgumentRegistersCallback(void* ctxt, size_
}


uint32_t* CallingConvention::GetRequiredArgumentRegistersCallback(void* ctxt, size_t* count)
{
CallbackRef<CallingConvention> cc(ctxt);
vector<uint32_t> regs = cc->GetRequiredArgumentRegisters();
*count = regs.size();

uint32_t* result = new uint32_t[regs.size()];
for (size_t i = 0; i < regs.size(); i++)
result[i] = regs[i];
return result;
}


uint32_t* CallingConvention::GetRequiredClobberedRegistersCallback(void* ctxt, size_t* count)
{
CallbackRef<CallingConvention> cc(ctxt);
vector<uint32_t> regs = cc->GetRequiredClobberedRegisters();
*count = regs.size();

uint32_t* result = new uint32_t[regs.size()];
for (size_t i = 0; i < regs.size(); i++)
result[i] = regs[i];
return result;
}


void CallingConvention::FreeRegisterListCallback(void*, uint32_t* regs, size_t)
{
delete[] regs;
Expand Down Expand Up @@ -284,6 +312,18 @@ vector<uint32_t> CallingConvention::GetFloatArgumentRegisters()
}


vector<uint32_t> CallingConvention::GetRequiredArgumentRegisters()
{
return vector<uint32_t>();
}


vector<uint32_t> CallingConvention::GetRequiredClobberedRegisters()
{
return vector<uint32_t>();
}


bool CallingConvention::AreArgumentRegistersSharedIndex()
{
return false;
Expand Down Expand Up @@ -417,6 +457,28 @@ vector<uint32_t> CoreCallingConvention::GetFloatArgumentRegisters()
}


vector<uint32_t> CoreCallingConvention::GetRequiredArgumentRegisters()
{
size_t count;
uint32_t* regs = BNGetRequiredArgumentRegisters(m_object, &count);
vector<uint32_t> result;
result.insert(result.end(), regs, &regs[count]);
BNFreeRegisterList(regs);
return result;
}


vector<uint32_t> CoreCallingConvention::GetRequiredClobberedRegisters()
{
size_t count;
uint32_t* regs = BNGetRequiredClobberedRegisters(m_object, &count);
vector<uint32_t> result;
result.insert(result.end(), regs, &regs[count]);
BNFreeRegisterList(regs);
return result;
}


bool CoreCallingConvention::AreArgumentRegistersSharedIndex()
{
return BNAreArgumentRegistersSharedIndex(m_object);
Expand Down
54 changes: 54 additions & 0 deletions python/callingconvention.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ class CallingConvention:
callee_saved_regs = []
int_arg_regs = []
float_arg_regs = []
required_arg_regs = []
required_clobbered_regs = []
arg_regs_share_index = False
arg_regs_for_varargs = True
stack_reserved_for_arg_regs = False
Expand Down Expand Up @@ -70,6 +72,8 @@ def __init__(
self._get_int_arg_regs
)
self._cb.getFloatArgumentRegisters = self._cb.getFloatArgumentRegisters.__class__(self._get_float_arg_regs)
self._cb.getRequiredArgumentRegisters = self._cb.getRequiredArgumentRegisters.__class__(self._get_required_arg_regs)
self._cb.getRequiredClobberedRegisters = self._cb.getRequiredClobberedRegisters.__class__(self._get_required_clobbered_regs)
self._cb.freeRegisterList = self._cb.freeRegisterList.__class__(self._free_register_list)
self._cb.areArgumentRegistersSharedIndex = self._cb.areArgumentRegistersSharedIndex.__class__(
self._arg_regs_share_index
Expand Down Expand Up @@ -161,6 +165,26 @@ def __init__(
core.BNFreeRegisterList(regs)
self.__dict__["float_arg_regs"] = result

count = ctypes.c_ulonglong()
regs = core.BNGetRequiredArgumentRegisters(_handle, count)
assert regs is not None, "core.BNGetRequiredArgumentRegisters returned None"
result = []
arch = self.arch
for i in range(0, count.value):
result.append(arch.get_reg_name(regs[i]))
core.BNFreeRegisterList(regs)
self.__dict__["required_arg_regs"] = result

count = ctypes.c_ulonglong()
regs = core.BNGetRequiredClobberedRegisters(_handle, count)
assert regs is not None, "core.BNGetRequiredClobberedRegisters returned None"
result = []
arch = self.arch
for i in range(0, count.value):
result.append(arch.get_reg_name(regs[i]))
core.BNFreeRegisterList(regs)
self.__dict__["required_clobbered_regs"] = result

reg = core.BNGetIntegerReturnValueRegister(_handle)
if reg == 0xffffffff:
self.__dict__["int_return_reg"] = None
Expand Down Expand Up @@ -281,6 +305,36 @@ def _get_float_arg_regs(self, ctxt, count):
count[0] = 0
return None

def _get_required_arg_regs(self, ctxt, count):
try:
regs = self.__class__.required_arg_regs
count[0] = len(regs)
reg_buf = (ctypes.c_uint * len(regs))()
for i in range(0, len(regs)):
reg_buf[i] = self.arch.regs[regs[i]].index
result = ctypes.cast(reg_buf, ctypes.c_void_p)
self._pending_reg_lists[result.value] = (result, reg_buf)
return result.value
except:
log_error_for_exception("Unhandled Python exception in CallingConvention._get_required_arg_regs")
count[0] = 0
return None

def _get_required_clobbered_regs(self, ctxt, count):
try:
regs = self.__class__.required_clobbered_regs
count[0] = len(regs)
reg_buf = (ctypes.c_uint * len(regs))()
for i in range(0, len(regs)):
reg_buf[i] = self.arch.regs[regs[i]].index
result = ctypes.cast(reg_buf, ctypes.c_void_p)
self._pending_reg_lists[result.value] = (result, reg_buf)
return result.value
except:
log_error_for_exception("Unhandled Python exception in CallingConvention._get_required_clobbered_regs")
count[0] = 0
return None

def _free_register_list(self, ctxt, regs, count):
try:
buf = ctypes.cast(regs, ctypes.c_void_p)
Expand Down
Loading
Loading