Load module symbols from the debugger backend on demand (#210)

api/debuggerapi.h

@@ -698,6 +698,22 @@ namespace BinaryNinjaDebuggerAPI {
 
 		std::vector<DebugModule> GetModules();
 		std::vector<DebugMemoryRegion> GetMemoryMap();
+
+		// Read the symbols the debugger backend knows about for the named module and add them to the
+		// BinaryView as auto symbols. Returns the number of symbols added.
+		size_t LoadSymbolsForModule(const std::string& module);
+		// Load the backend symbols for every currently-loaded module. Returns the total number added.
+		size_t LoadSymbolsForAllModules();
+		// Remove the backend symbols previously added for the named module. Returns the number removed.
+		size_t RemoveSymbolsForModule(const std::string& module);
+		// Remove every backend symbol the debugger has added. Returns the number removed.
+		size_t RemoveAllLoadedSymbols();
+		// The base names of the modules for which backend symbols have been loaded.
+		std::vector<std::string> GetModulesWithLoadedSymbols();
+		// The number of backend symbols currently loaded for the named module (0 if none). The module may
+		// be given as either its base name or its full path.
+		size_t GetLoadedSymbolCountForModule(const std::string& module);
+
 		std::vector<DebugRegister> GetRegisters();
 		intx::uint512 GetRegisterValue(const std::string& name);
 		bool SetRegisterValue(const std::string& name, const intx::uint512& value);

api/debuggercontroller.cpp

@@ -300,6 +300,51 @@ std::vector<DebugMemoryRegion> DebuggerController::GetMemoryMap()
 }
 
 
+size_t DebuggerController::LoadSymbolsForModule(const std::string& module)
+{
+	return BNDebuggerLoadSymbolsForModule(m_object, module.c_str());
+}
+
+
+size_t DebuggerController::LoadSymbolsForAllModules()
+{
+	return BNDebuggerLoadSymbolsForAllModules(m_object);
+}
+
+
+size_t DebuggerController::RemoveSymbolsForModule(const std::string& module)
+{
+	return BNDebuggerRemoveSymbolsForModule(m_object, module.c_str());
+}
+
+
+size_t DebuggerController::RemoveAllLoadedSymbols()
+{
+	return BNDebuggerRemoveAllLoadedSymbols(m_object);
+}
+
+
+std::vector<std::string> DebuggerController::GetModulesWithLoadedSymbols()
+{
+	size_t count;
+	char** modules = BNDebuggerGetModulesWithLoadedSymbols(m_object, &count);
+
+	std::vector<std::string> result;
+	result.reserve(count);
+	for (size_t i = 0; i < count; i++)
+		result.emplace_back(modules[i]);
+
+	BNDebuggerFreeStringList(modules, count);
+	return result;
+}
+
+
+size_t DebuggerController::GetLoadedSymbolCountForModule(const std::string& module)
+{
+	return BNDebuggerGetLoadedSymbolCountForModule(m_object, module.c_str());
+}
+
+
 std::vector<DebugRegister> DebuggerController::GetRegisters()
 {
 	size_t count;

api/ffi.h

@@ -533,6 +533,23 @@ extern "C"
 	DEBUGGER_FFI_API BNDebugMemoryRegion* BNDebuggerGetMemoryMap(BNDebuggerController* controller, size_t* count);
 	DEBUGGER_FFI_API void BNDebuggerFreeMemoryRegions(BNDebugMemoryRegion* regions, size_t count);
 
+	// Read the symbols the debugger backend knows about for the named module and add them to the
+	// BinaryView as auto symbols. Returns the number of symbols added.
+	DEBUGGER_FFI_API size_t BNDebuggerLoadSymbolsForModule(BNDebuggerController* controller, const char* module);
+	// Load the backend symbols for every currently-loaded module. Returns the total number added.
+	DEBUGGER_FFI_API size_t BNDebuggerLoadSymbolsForAllModules(BNDebuggerController* controller);
+	// Remove the backend symbols previously added for the named module. Returns the number removed.
+	DEBUGGER_FFI_API size_t BNDebuggerRemoveSymbolsForModule(BNDebuggerController* controller, const char* module);
+	// Remove every backend symbol the debugger has added. Returns the number removed.
+	DEBUGGER_FFI_API size_t BNDebuggerRemoveAllLoadedSymbols(BNDebuggerController* controller);
+	// The base names of the modules for which backend symbols have been loaded. Free with
+	// BNDebuggerFreeStringList.
+	DEBUGGER_FFI_API char** BNDebuggerGetModulesWithLoadedSymbols(BNDebuggerController* controller, size_t* count);
+	// The number of backend symbols currently loaded for the named module (0 if none). The module may be
+	// given as either its base name or its full path.
+	DEBUGGER_FFI_API size_t BNDebuggerGetLoadedSymbolCountForModule(
+		BNDebuggerController* controller, const char* module);
+
 	DEBUGGER_FFI_API BNDebugRegister* BNDebuggerGetRegisters(BNDebuggerController* controller, size_t* count);
 	DEBUGGER_FFI_API void BNDebuggerFreeRegisters(BNDebugRegister* modules, size_t count);
 	DEBUGGER_FFI_API bool BNDebuggerSetRegisterValue(

api/python/debuggercontroller.py

@@ -1529,6 +1529,74 @@ def memory_map(self) -> List[DebugMemoryRegion]:
         dbgcore.BNDebuggerFreeMemoryRegions(regions, count.value)
         return result
 
+    def load_symbols_for_module(self, module: str) -> int:
+        """
+        Read the symbols that the debugger backend knows about for the given module and add them to
+        the BinaryView as auto symbols.
+
+        By default the debugger loads no symbols from the backend. Call this to load, on demand, all
+        symbols of a module (e.g., the exports of ``kernel32.dll``) so that the annotation process
+        becomes aware of them, e.g., when a register points to a Windows API function. The added
+        symbols are tracked internally and can be removed later with ``remove_symbols_for_module`` or
+        ``remove_all_loaded_symbols``.
+
+        Loading the same module again is idempotent: any symbols previously loaded for it are removed
+        first, so no duplicate symbols are created.
+
+        :param module: the module to load symbols for; either its short name or full path
+        :return: the number of symbols added
+        """
+        return dbgcore.BNDebuggerLoadSymbolsForModule(self.handle, module)
+
+    def load_symbols_for_all_modules(self) -> int:
+        """
+        Load the backend symbols for every currently-loaded module.
+
+        :return: the total number of symbols added
+        """
+        return dbgcore.BNDebuggerLoadSymbolsForAllModules(self.handle)
+
+    def remove_symbols_for_module(self, module: str) -> int:
+        """
+        Remove the backend symbols previously added for the given module.
+
+        :param module: the module to remove symbols for; either its short name or full path
+        :return: the number of symbols removed
+        """
+        return dbgcore.BNDebuggerRemoveSymbolsForModule(self.handle, module)
+
+    def remove_all_loaded_symbols(self) -> int:
+        """
+        Remove every backend symbol the debugger has added.
+
+        :return: the number of symbols removed
+        """
+        return dbgcore.BNDebuggerRemoveAllLoadedSymbols(self.handle)
+
+    @property
+    def modules_with_loaded_symbols(self) -> List[str]:
+        """
+        The base names of the modules for which backend symbols have been loaded.
+
+        :return: a list of module base names
+        """
+        count = ctypes.c_ulonglong()
+        modules = dbgcore.BNDebuggerGetModulesWithLoadedSymbols(self.handle, count)
+        result = []
+        for i in range(count.value):
+            result.append(modules[i].decode('utf-8'))
+        dbgcore.BNDebuggerFreeStringList(modules, count.value)
+        return result
+
+    def loaded_symbol_count_for_module(self, module: str) -> int:
+        """
+        The number of backend symbols currently loaded for the given module.
+
+        :param module: the module to query; either its short name or full path
+        :return: the number of loaded symbols, or 0 if none have been loaded for the module
+        """
+        return dbgcore.BNDebuggerGetLoadedSymbolCountForModule(self.handle, module)
+
     def rebase_to_remote_base(self) -> bool:
         """
         Rebase the input binary view to match the remote base address.

core/adapters/dbgengadapter.cpp

@@ -1819,6 +1819,53 @@ std::vector<DebugMemoryRegion> DbgEngAdapter::GetMemoryMap()
 }
 
 
+std::vector<DebugSymbol> DbgEngAdapter::GetSymbolsForModule(const DebugModule& module)
+{
+	std::vector<DebugSymbol> result;
+	if (!m_debugSymbols)
+		return result;
+
+	// Build the symbol-match pattern "<module>!*". dbgeng identifies modules in the bang syntax by their
+	// base name without extension.
+	std::string moduleName =
+		module.m_short_name.empty() ? DebugModule::GetPathBaseName(module.m_name) : module.m_short_name;
+	auto dot = moduleName.find_last_of('.');
+	if (dot != std::string::npos)
+		moduleName = moduleName.substr(0, dot);
+	if (moduleName.empty())
+		return result;
+
+	std::string pattern = moduleName + "!*";
+
+	uint64_t handle = 0;
+	if (m_debugSymbols->StartSymbolMatch(pattern.c_str(), &handle) != S_OK)
+		return result;
+
+	char nameBuffer[2048];
+	uint64_t offset = 0;
+	unsigned long matchSize = 0;
+	while (m_debugSymbols->GetNextSymbolMatch(handle, nameBuffer, sizeof(nameBuffer), &matchSize, &offset) == S_OK)
+	{
+		// GetNextSymbolMatch returns the fully-qualified "module!symbol" name.
+		std::string fullName = nameBuffer;
+		std::string shortName = fullName;
+		auto bang = fullName.find('!');
+		if (bang != std::string::npos)
+			shortName = fullName.substr(bang + 1);
+
+		if (shortName.empty() || (offset == 0))
+			continue;
+
+		// TODO: dbgeng's symbol match does not report whether a symbol is code or data; classify all as
+		// functions for now, which is correct for the common case of API exports.
+		result.emplace_back(shortName, fullName, fullName, offset, 0, true);
+	}
+
+	m_debugSymbols->EndSymbolMatch(handle);
+	return result;
+}
+
+
 bool DbgEngAdapter::BreakInto()
 {
 	if (ExecStatus() == DEBUG_STATUS_BREAK || ExecStatus() == DEBUG_STATUS_NO_DEBUGGEE)
@@ -2238,6 +2285,8 @@ bool DbgEngAdapter::SupportFeature(DebugAdapterCapacity feature)
 		return true;
 	case DebugAdapterSupportThreads:
 		return true;
+	case DebugAdapterSupportSymbols:
+		return true;
 	default:
 		return false;
 	}

core/adapters/dbgengadapter.h

@@ -225,6 +225,8 @@ namespace BinaryNinjaDebugger {
 
 		std::vector<DebugMemoryRegion> GetMemoryMap() override;
 
+		std::vector<DebugSymbol> GetSymbolsForModule(const DebugModule& module) override;
+
 		std::string GetTargetArchitecture() override;
 
 		DebugStopReason StopReason() override;

core/adapters/lldbadapter.cpp

@@ -1685,6 +1685,81 @@ std::vector<DebugMemoryRegion> LldbAdapter::GetMemoryMap()
 }
 
 
+std::vector<DebugSymbol> LldbAdapter::GetSymbolsForModule(const DebugModule& module)
+{
+	std::vector<DebugSymbol> result;
+
+	// Locate the SBModule that corresponds to the requested DebugModule. We match on the base file
+	// name so that host/guest path differences (see DebugModule::IsSameBaseModule) do not matter.
+	uint32_t numModules = m_target.GetNumModules();
+	for (uint32_t i = 0; i < numModules; i++)
+	{
+		SBModule sbModule = m_target.GetModuleAtIndex(i);
+		if (!sbModule.IsValid())
+			continue;
+
+		SBFileSpec fileSpec = sbModule.GetFileSpec();
+		char path[1024];
+		size_t len = fileSpec.GetPath(path, 1024);
+		std::string modulePath(path, len);
+		if (!module.IsSameBaseModule(modulePath))
+			continue;
+
+		size_t numSymbols = sbModule.GetNumSymbols();
+		result.reserve(numSymbols);
+		for (size_t j = 0; j < numSymbols; j++)
+		{
+			SBSymbol symbol = sbModule.GetSymbolAtIndex(j);
+			if (!symbol.IsValid())
+				continue;
+
+			SymbolType type = symbol.GetType();
+			bool isFunction;
+			switch (type)
+			{
+				case eSymbolTypeCode:
+				case eSymbolTypeResolver:
+					isFunction = true;
+					break;
+				case eSymbolTypeData:
+					isFunction = false;
+					break;
+				default:
+					// Skip everything else (e.g. compile units, line entries, trampolines), which do not
+					// correspond to a useful named address in the target.
+					continue;
+			}
+
+			SBAddress startAddress = symbol.GetStartAddress();
+			if (!startAddress.IsValid())
+				continue;
+
+			uint64_t address = startAddress.GetLoadAddress(m_target);
+			if ((address == 0) || (address == LLDB_INVALID_ADDRESS))
+				continue;
+
+			const char* name = symbol.GetName();
+			if ((name == nullptr) || (name[0] == '\0'))
+				continue;
+
+			std::string shortName = name;
+			std::string fullName = module.m_short_name.empty() ? shortName : module.m_short_name + "!" + shortName;
+			std::string rawName;
+			if (const char* mangled = symbol.GetMangledName())
+				rawName = mangled;
+			if (rawName.empty())
+				rawName = shortName;
+
+			result.emplace_back(shortName, fullName, rawName, address, symbol.GetSize(), isFunction);
+		}
+
+		break;
+	}
+
+	return result;
+}
+
+
 std::string LldbAdapter::GetTargetArchitecture()
 {
 	SBPlatform platform = m_target.GetPlatform();
@@ -2144,6 +2219,8 @@ bool LldbAdapter::SupportFeature(DebugAdapterCapacity feature)
 			return true;
 		case DebugAdapterSupportThreads:
 			return true;
+		case DebugAdapterSupportSymbols:
+			return true;
 		case DebugAdapterSupportTTD:
 			return false;
 		default:

core/adapters/lldbadapter.h

@@ -127,6 +127,8 @@ namespace BinaryNinjaDebugger {
 
 		std::vector<DebugMemoryRegion> GetMemoryMap() override;
 
+		std::vector<DebugSymbol> GetSymbolsForModule(const DebugModule& module) override;
+
 		std::string GetTargetArchitecture() override;
 
 		DebugStopReason StopReason() override;