Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SharedCache] Use m_exportInfos as an export list cache #6197

Open
wants to merge 7 commits into
base: dev
Choose a base branch
from
Open

[SharedCache] Use m_exportInfos as an export list cache #6197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b7ccf06
[SharedCache] Use `m_exportInfos` as an export list cache
Nov 25, 2024
64174ee
[SharedCache] Add parameter to `SharedCache::InitializeHeader` to che…
Nov 26, 2024
6edb250
[SharedCache] Improve the types for `m_exportInfos`
Nov 26, 2024
9e7d6e2
[SharedCache] Only setup undo actions and bulk modify in `SharedCache…
Nov 26, 2024
202a792
[SharedCache] Make `m_exportInfos` map's values a `shared_ptr`
Nov 26, 2024
c83ec00
[SharedCache] Remove commenting out of line from debugging
Nov 26, 2024
e0d623d
Merge branch 'dev' into use-cached-exports
Jan 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 119 additions & 87 deletions view/sharedcache/core/SharedCache.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,7 @@ int count_trailing_zeros(uint64_t value) {

struct SharedCache::State
{
std::unordered_map<uint64_t, std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>>>
exportInfos;
std::unordered_map<uint64_t, std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>>> exportInfos;
std::unordered_map<uint64_t, std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>>>
symbolInfos;

Expand Down Expand Up @@ -2705,48 +2704,51 @@ void SharedCache::InitializeHeader(

if (header.exportTriePresent && header.linkeditPresent && vm->AddressIsMapped(header.linkeditSegment.vmaddr))
{
auto symbols = SharedCache::ParseExportTrie(vm->MappingAtAddress(header.linkeditSegment.vmaddr).first.fileAccessor->lock(), header);
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> exportMapping;
for (const auto& symbol : symbols)
auto symbols = GetExportListForHeader(header, [&]() {
return vm->MappingAtAddress(header.linkeditSegment.vmaddr).first.fileAccessor->lock();
});
if (symbols)
{
exportMapping.push_back({symbol->GetAddress(), {symbol->GetType(), symbol->GetRawName()}});
if (typeLib)
for (const auto& [symbolAddress, symbol] : *symbols)
{
auto type = m_dscView->ImportTypeLibraryObject(typeLib, {symbol->GetFullName()});

if (type)
if (typeLib)
{
view->DefineAutoSymbolAndVariableOrFunction(view->GetDefaultPlatform(), symbol, type);
}
else
view->DefineAutoSymbol(symbol);
auto type = m_dscView->ImportTypeLibraryObject(typeLib, symbol->GetRawName());

if (view->GetAnalysisFunction(view->GetDefaultPlatform(), symbol->GetAddress()))
{
auto func = view->GetAnalysisFunction(view->GetDefaultPlatform(), symbol->GetAddress());
if (symbol->GetFullName() == "_objc_msgSend")
if (type)
{
func->SetHasVariableArguments(false);
view->DefineAutoSymbolAndVariableOrFunction(view->GetDefaultPlatform(), symbol, type);
}
else if (symbol->GetFullName().find("_objc_retain_x") != std::string::npos || symbol->GetFullName().find("_objc_release_x") != std::string::npos)
else
view->DefineAutoSymbol(symbol);

if (view->GetAnalysisFunction(view->GetDefaultPlatform(), symbolAddress))
{
auto x = symbol->GetFullName().rfind("x");
auto num = symbol->GetFullName().substr(x + 1);
auto func = view->GetAnalysisFunction(view->GetDefaultPlatform(), symbolAddress);
auto name = symbol->GetFullName();
if (name == "_objc_msgSend")
{
func->SetHasVariableArguments(false);
}
else if (name.find("_objc_retain_x") != std::string::npos || name.find("_objc_release_x") != std::string::npos)
{
auto x = name.rfind("x");
auto num = name.substr(x + 1);

std::vector<BinaryNinja::FunctionParameter> callTypeParams;
auto cc = m_dscView->GetDefaultArchitecture()->GetCallingConventionByName("apple-arm64-objc-fast-arc-" + num);
std::vector<BinaryNinja::FunctionParameter> callTypeParams;
auto cc = m_dscView->GetDefaultArchitecture()->GetCallingConventionByName("apple-arm64-objc-fast-arc-" + num);

callTypeParams.push_back({"obj", m_dscView->GetTypeByName({ "id" }), true, BinaryNinja::Variable()});
callTypeParams.push_back({"obj", m_dscView->GetTypeByName({ "id" }), true, BinaryNinja::Variable()});

auto funcType = BinaryNinja::Type::FunctionType(m_dscView->GetTypeByName({ "id" }), cc, callTypeParams);
func->SetUserType(funcType);
auto funcType = BinaryNinja::Type::FunctionType(m_dscView->GetTypeByName({ "id" }), cc, callTypeParams);
func->SetUserType(funcType);
}
}
}
else
view->DefineAutoSymbol(symbol);
}
else
view->DefineAutoSymbol(symbol);
}
MutableState().exportInfos[header.textBase] = std::move(exportMapping);
}
view->EndBulkModifySymbols();

Expand Down Expand Up @@ -2850,6 +2852,40 @@ std::vector<Ref<Symbol>> SharedCache::ParseExportTrie(std::shared_ptr<MMappedFil
return symbols;
}


std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>> SharedCache::GetExportListForHeader(SharedCacheMachOHeader header, std::function<std::shared_ptr<MMappedFileAccessor>()> provideLinkeditFile, bool* didModifyExportList)
{
if (auto it = State().exportInfos.find(header.textBase); it != State().exportInfos.end())
{
if (didModifyExportList)
*didModifyExportList = false;
return it->second;
}
else
{
std::shared_ptr<MMappedFileAccessor> linkeditFile = provideLinkeditFile();
if (!linkeditFile)
{
if (didModifyExportList)
*didModifyExportList = false;
return nullptr;
}

auto exportList = SharedCache::ParseExportTrie(linkeditFile, header);
auto exportMapping = std::make_shared<std::unordered_map<uint64_t, Ref<Symbol>>>(exportList.size());
for (const auto& sym : exportList)
{
exportMapping->insert_or_assign(sym->GetAddress(), sym);
}
WillMutateState();
MutableState().exportInfos.emplace(header.textBase, exportMapping);
if (didModifyExportList)
*didModifyExportList = true;
return MutableState().exportInfos[header.textBase];
}
}


std::vector<std::string> SharedCache::GetAvailableImages()
{
std::vector<std::string> installNames;
Expand All @@ -2863,34 +2899,35 @@ std::vector<std::string> SharedCache::GetAvailableImages()

std::vector<std::pair<std::string, Ref<Symbol>>> SharedCache::LoadAllSymbolsAndWait()
{
WillMutateState();

std::lock_guard initialLoadBlock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex);

bool doSave = false;
std::vector<std::pair<std::string, Ref<Symbol>>> symbols;
for (const auto& img : State().images)
{
auto header = HeaderForAddress(img.headerLocation);
std::shared_ptr<MMappedFileAccessor> mapping;
try {
mapping = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie %s for %s", header->exportTriePath.c_str(), header->installName.c_str());
auto exportList = GetExportListForHeader(*header, [&]() {
try {
auto mapping = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
return mapping;
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie %s for %s", header->exportTriePath.c_str(), header->installName.c_str());
return std::shared_ptr<MMappedFileAccessor>(nullptr);
}
}, &doSave);
if (!exportList)
continue;
}
auto exportList = SharedCache::ParseExportTrie(mapping, *header);
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> exportMapping;
for (const auto& sym : exportList)
for (const auto& [_, symbol] : *exportList)
{
exportMapping.push_back({sym->GetAddress(), {sym->GetType(), sym->GetRawName()}});
symbols.push_back({img.installName, sym});
symbols.push_back({img.installName, symbol});
}
MutableState().exportInfos[header->textBase] = std::move(exportMapping);
}

SaveToDSCView();
// Only save to DSC view if a header was actually loaded
if (doSave)
SaveToDSCView();

return symbols;
}
Expand Down Expand Up @@ -2970,58 +3007,55 @@ void SharedCache::FindSymbolAtAddrAndApplyToAddr(
auto header = HeaderForAddress(symbolLocation);
if (header)
{
std::shared_ptr<MMappedFileAccessor> mapping;
try {
mapping = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie for %s", header->installName.c_str());
return;
}
auto exportList = SharedCache::ParseExportTrie(mapping, *header);
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> exportMapping;
auto typeLib = TypeLibraryForImage(header->installName);
id = m_dscView->BeginUndoActions();
m_dscView->BeginBulkModifySymbols();
for (const auto& sym : exportList)

auto exportList = GetExportListForHeader(*header, [&]() {
try {
return MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie %s for %s", header->exportTriePath.c_str(), header->installName.c_str());
return std::shared_ptr<MMappedFileAccessor>(nullptr);
}
});

if (exportList)
{
exportMapping.push_back({sym->GetAddress(), {sym->GetType(), sym->GetRawName()}});
if (sym->GetAddress() == symbolLocation)
if (auto it = exportList->find(symbolLocation); it != exportList->end())
{
if (auto func = m_dscView->GetAnalysisFunction(m_dscView->GetDefaultPlatform(), targetLocation))
id = m_dscView->BeginUndoActions();
m_dscView->BeginBulkModifySymbols();

auto func = m_dscView->GetAnalysisFunction(m_dscView->GetDefaultPlatform(), targetLocation);
if (func)
{
m_dscView->DefineUserSymbol(
new Symbol(FunctionSymbol, prefix + sym->GetFullName(), targetLocation));
new Symbol(FunctionSymbol, prefix + it->second->GetFullName(), targetLocation));

if (typeLib)
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {sym->GetFullName()}))
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {it->second->GetFullName()}))
func->SetUserType(type);
}
else
{
m_dscView->DefineUserSymbol(
new Symbol(sym->GetType(), prefix + sym->GetFullName(), targetLocation));
new Symbol(it->second->GetType(), prefix + it->second->GetFullName(), targetLocation));

if (typeLib)
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {sym->GetFullName()}))
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {it->second->GetFullName()}))
m_dscView->DefineUserDataVariable(targetLocation, type);
}
if (triggerReanalysis)
{
auto func = m_dscView->GetAnalysisFunction(m_dscView->GetDefaultPlatform(), targetLocation);
if (func)
func->Reanalyze();
}
break;

m_dscView->EndBulkModifySymbols();
m_dscView->ForgetUndoActions(id);
}
}
{
std::lock_guard lock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex);
MutableState().exportInfos[header->textBase] = std::move(exportMapping);
}
m_dscView->EndBulkModifySymbols();
m_dscView->ForgetUndoActions(id);
}
}

Expand Down Expand Up @@ -3479,18 +3513,18 @@ void SharedCache::Store(SerializationContext& context) const

Serialize(context, "exportInfos");
context.writer.StartArray();
for (const auto& pair1 : State().exportInfos)
for (const auto& [headerLocation, symbolMap] : State().exportInfos)
{
context.writer.StartObject();
Serialize(context, "key", pair1.first);
Serialize(context, "key", headerLocation);
Serialize(context, "value");
context.writer.StartArray();
for (const auto& pair2 : pair1.second)
for (const auto& [symbolAddress, symbol] : *symbolMap)
{
context.writer.StartObject();
Serialize(context, "key", pair2.first);
Serialize(context, "val1", pair2.second.first);
Serialize(context, "val2", pair2.second.second);
Serialize(context, "key", symbolAddress);
Serialize(context, "val1", symbol->GetType());
Serialize(context, "val2", symbol->GetRawName());
context.writer.EndObject();
}
context.writer.EndArray();
Expand Down Expand Up @@ -3561,15 +3595,13 @@ void SharedCache::Load(DeserializationContext& context)

for (const auto& obj1 : context.doc["exportInfos"].GetArray())
{
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> innerVec;
std::unordered_map<uint64_t, Ref<Symbol>> innerMap;
for (const auto& obj2 : obj1["value"].GetArray())
{
std::pair<BNSymbolType, std::string> innerPair = {
(BNSymbolType)obj2["val1"].GetUint64(), obj2["val2"].GetString()};
innerVec.push_back({obj2["key"].GetUint64(), innerPair});
innerMap[obj2["key"].GetUint64()] = new Symbol((BNSymbolType)obj2["val1"].GetUint64(), obj2["val2"].GetString(), obj2["key"].GetUint64());
}

MutableState().exportInfos[obj1["key"].GetUint64()] = std::move(innerVec);
MutableState().exportInfos[obj1["key"].GetUint64()] = std::make_shared<std::unordered_map<uint64_t, Ref<Symbol>>>(std::move(innerMap));
}

for (auto& symbolInfo : context.doc["symbolInfos"].GetArray())
Expand Down
1 change: 1 addition & 0 deletions view/sharedcache/core/SharedCache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,7 @@ namespace SharedCacheCore {
uint64_t textBase, const std::string& currentText, size_t cursor, uint32_t endGuard);
std::vector<Ref<Symbol>> ParseExportTrie(
std::shared_ptr<MMappedFileAccessor> linkeditFile, SharedCacheMachOHeader header);
std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>> GetExportListForHeader(SharedCacheMachOHeader header, std::function<std::shared_ptr<MMappedFileAccessor>()> provideLinkeditFile, bool* didModifyExportList = nullptr);

Ref<TypeLibrary> TypeLibraryForImage(const std::string& installName);

Expand Down