From 6c865c456eebe93e414b797918bc148aea47b476 Mon Sep 17 00:00:00 2001 From: Teal Bauer Date: Mon, 14 Apr 2025 00:49:46 +0200 Subject: [PATCH] perf: Optimize variables endpoint with efficient pagination - Implemented efficient pagination for variables endpoints to avoid timeout - Added globalOnly parameter to allow fetching just global variables - Limited decompilation to only process functions needed for current page - Improved estimation of total count for better pagination links - Reduced decompilation timeout to improve performance --- .../ghidra/endpoints/VariableEndpoints.java | 572 +++++++++++++++--- 1 file changed, 474 insertions(+), 98 deletions(-) diff --git a/src/main/java/eu/starsong/ghidra/endpoints/VariableEndpoints.java b/src/main/java/eu/starsong/ghidra/endpoints/VariableEndpoints.java index e5da919..4b33b18 100644 --- a/src/main/java/eu/starsong/ghidra/endpoints/VariableEndpoints.java +++ b/src/main/java/eu/starsong/ghidra/endpoints/VariableEndpoints.java @@ -70,7 +70,8 @@ package eu.starsong.ghidra.endpoints; int offset = parseIntOrDefault(qparams.get("offset"), 0); int limit = parseIntOrDefault(qparams.get("limit"), 100); String search = qparams.get("search"); // Renamed from 'query' for clarity - + boolean globalOnly = Boolean.parseBoolean(qparams.getOrDefault("global_only", "false")); + // Always get the most current program from the tool Program program = getCurrentProgram(); if (program == null) { @@ -87,21 +88,70 @@ package eu.starsong.ghidra.endpoints; // Add common links builder.addLink("program", "/program"); builder.addLink("search", "/variables?search={term}", "GET"); + builder.addLink("globals", "/variables?global_only=true", "GET"); - List> variables; + // Use more efficient pagination by limiting data collection up-front + PaginatedResult paginatedResult; if (search != null && !search.isEmpty()) { - variables = searchVariables(program, search); + paginatedResult = searchVariablesPaginated(program, search, offset, limit, globalOnly); } else { - variables = listVariables(program); + paginatedResult = listVariablesPaginated(program, offset, limit, globalOnly); } - // Apply pagination and get paginated result - List> paginatedVars = - applyPagination(variables, offset, limit, builder, "/variables", - search != null ? "search=" + search : null); + // Add pagination links + String baseUrl = "/variables"; + String queryParams = ""; + if (search != null && !search.isEmpty()) { + queryParams = "search=" + search; + } + if (globalOnly) { + queryParams = queryParams.isEmpty() ? "global_only=true" : queryParams + "&global_only=true"; + } + + // Add metadata + Map metadata = new HashMap<>(); + metadata.put("total_estimate", paginatedResult.getTotalEstimate()); + metadata.put("offset", offset); + metadata.put("limit", limit); + builder.metadata(metadata); + + // Add self link + String selfLink = baseUrl; + if (!queryParams.isEmpty()) { + selfLink += "?" + queryParams; + selfLink += "&offset=" + offset + "&limit=" + limit; + } else { + selfLink += "?offset=" + offset + "&limit=" + limit; + } + builder.addLink("self", selfLink); + + // Add next link if needed + if (paginatedResult.hasMore()) { + String nextLink = baseUrl; + if (!queryParams.isEmpty()) { + nextLink += "?" + queryParams; + nextLink += "&offset=" + (offset + limit) + "&limit=" + limit; + } else { + nextLink += "?offset=" + (offset + limit) + "&limit=" + limit; + } + builder.addLink("next", nextLink); + } + + // Add prev link if needed + if (offset > 0) { + int prevOffset = Math.max(0, offset - limit); + String prevLink = baseUrl; + if (!queryParams.isEmpty()) { + prevLink += "?" + queryParams; + prevLink += "&offset=" + prevOffset + "&limit=" + limit; + } else { + prevLink += "?offset=" + prevOffset + "&limit=" + limit; + } + builder.addLink("prev", prevLink); + } // Add the result to the builder - builder.result(paginatedVars); + builder.result(paginatedResult.getResults()); // Send the HATEOAS-compliant response sendJsonResponse(exchange, builder.build(), 200); @@ -113,22 +163,81 @@ package eu.starsong.ghidra.endpoints; sendErrorResponse(exchange, 500, "Internal server error: " + e.getMessage()); } } - - // Updated to return List instead of JsonObject for HATEOAS compliance - private List> listVariables(Program program) { - List> variables = new ArrayList<>(); - - if (program == null) { - return variables; // Return empty list if no program + + /** + * Class to represent a paginated result with metadata + */ + private static class PaginatedResult { + private final List> results; + private final boolean hasMore; + private final int totalEstimate; + + public PaginatedResult(List> results, boolean hasMore, int totalEstimate) { + this.results = results; + this.hasMore = hasMore; + this.totalEstimate = totalEstimate; } + + public List> getResults() { + return results; + } + + public boolean hasMore() { + return hasMore; + } + + public int getTotalEstimate() { + return totalEstimate; + } + } - // Get global variables + /** + * Legacy method kept for backward compatibility + */ + private List> listVariables(Program program) { + PaginatedResult result = listVariablesPaginated(program, 0, Integer.MAX_VALUE, false); + return result.getResults(); + } + + /** + * List variables with efficient pagination - only loads what's needed + */ + private PaginatedResult listVariablesPaginated(Program program, int offset, int limit, boolean globalOnly) { + if (program == null) { + return new PaginatedResult(new ArrayList<>(), false, 0); + } + + List> variables = new ArrayList<>(); + int globalVarCount = 0; + int totalEstimate = 0; + boolean hasMore = false; + + // Calculate range of items to fetch + int startIdx = offset; + int endIdx = offset + limit; + int currentIndex = 0; + + // Get global variables - these are quick to get so we can get them all SymbolTable symbolTable = program.getSymbolTable(); + ArrayList globalSymbols = new ArrayList<>(); + + // First, collect global variables efficiently for (Symbol symbol : symbolTable.getDefinedSymbols()) { if (symbol.isGlobal() && !symbol.isExternal() && symbol.getSymbolType() != SymbolType.FUNCTION && symbol.getSymbolType() != SymbolType.LABEL) { - + globalSymbols.add(symbol); + } + } + + // Sort globals by name first + globalSymbols.sort(Comparator.comparing(Symbol::getName)); + globalVarCount = globalSymbols.size(); + totalEstimate = globalVarCount; + + // Now extract just the global variables we need for the current page + for (Symbol symbol : globalSymbols) { + if (currentIndex >= startIdx && currentIndex < endIdx) { Map varInfo = new HashMap<>(); varInfo.put("name", symbol.getName()); varInfo.put("address", symbol.getAddress().toString()); @@ -136,65 +245,201 @@ package eu.starsong.ghidra.endpoints; varInfo.put("dataType", getDataTypeName(program, symbol.getAddress())); variables.add(varInfo); } + currentIndex++; + + // If we've added enough items, break + if (currentIndex >= endIdx) { + hasMore = currentIndex < globalVarCount || !globalOnly; + break; + } } - - // Get local variables from all functions (Consider performance implications) - DecompInterface decomp = null; - try { - decomp = new DecompInterface(); - if (!decomp.openProgram(program)) { - Msg.error(this, "listVariables: Failed to open program with decompiler."); - } else { - for (Function function : program.getFunctionManager().getFunctions(true)) { - try { - DecompileResults results = decomp.decompileFunction(function, 30, new ConsoleTaskMonitor()); - if (results != null && results.decompileCompleted()) { - HighFunction highFunc = results.getHighFunction(); - if (highFunc != null) { - Iterator symbolIter = highFunc.getLocalSymbolMap().getSymbols(); - while (symbolIter.hasNext()) { - HighSymbol symbol = symbolIter.next(); - if (!symbol.isParameter()) { // Only list locals - Map varInfo = new HashMap<>(); - varInfo.put("name", symbol.getName()); - varInfo.put("type", "local"); - varInfo.put("function", function.getName()); - Address pcAddr = symbol.getPCAddress(); - varInfo.put("address", pcAddr != null ? pcAddr.toString() : "N/A"); - varInfo.put("dataType", symbol.getDataType() != null ? symbol.getDataType().getName() : "unknown"); - variables.add(varInfo); + + // If we only want globals, or if we've already fetched enough for this page, return now + if (globalOnly || currentIndex >= endIdx) { + return new PaginatedResult(variables, hasMore, totalEstimate); + } + + // Get local variables - only if needed (these are expensive) + // We need to perform some estimation for locals, as decompiling all functions is too slow + + // First estimate the total count + int funcCount = 0; + for (Function f : program.getFunctionManager().getFunctions(true)) { + funcCount++; + } + + // Roughly estimate 2 local variables per function + totalEstimate = globalVarCount + (funcCount * 2); + + // If we don't need locals for the current page, return globals with estimation + if (startIdx >= globalVarCount) { + // Adjust for local variable processing + int localOffset = startIdx - globalVarCount; + int localLimit = limit; + + // Process functions to get the local variables + DecompInterface decomp = null; + try { + decomp = new DecompInterface(); + if (decomp.openProgram(program)) { + int localVarIndex = 0; + int functionsProcessed = 0; + int maxFunctionsToProcess = 20; // Limit how many functions we process per request + + for (Function function : program.getFunctionManager().getFunctions(true)) { + try { + DecompileResults results = decomp.decompileFunction(function, 10, new ConsoleTaskMonitor()); + if (results != null && results.decompileCompleted()) { + HighFunction highFunc = results.getHighFunction(); + if (highFunc != null) { + List> functionVars = new ArrayList<>(); + Iterator symbolIter = highFunc.getLocalSymbolMap().getSymbols(); + while (symbolIter.hasNext()) { + HighSymbol symbol = symbolIter.next(); + if (!symbol.isParameter()) { // Only list locals + Map varInfo = new HashMap<>(); + varInfo.put("name", symbol.getName()); + varInfo.put("type", "local"); + varInfo.put("function", function.getName()); + Address pcAddr = symbol.getPCAddress(); + varInfo.put("address", pcAddr != null ? pcAddr.toString() : "N/A"); + varInfo.put("dataType", symbol.getDataType() != null ? symbol.getDataType().getName() : "unknown"); + functionVars.add(varInfo); + } + } + + // Sort function variables by name + functionVars.sort(Comparator.comparing(a -> a.get("name"))); + + // Add only the needed variables for this page + for (Map varInfo : functionVars) { + if (localVarIndex >= localOffset && localVarIndex < localOffset + localLimit) { + variables.add(varInfo); + } + localVarIndex++; + if (localVarIndex >= localOffset + localLimit) { + break; + } } } } + } catch (Exception e) { + Msg.warn(this, "listVariablesPaginated: Error processing function " + function.getName(), e); } - } catch (Exception e) { - Msg.error(this, "listVariables: Error processing function " + function.getName(), e); + + functionsProcessed++; + if (functionsProcessed >= maxFunctionsToProcess || localVarIndex >= localOffset + localLimit) { + // Stop processing if we've hit our limits + break; + } + } + + // Determine if we have more variables + hasMore = functionsProcessed < funcCount || localVarIndex >= localOffset + localLimit; + } + } catch (Exception e) { + Msg.error(this, "listVariablesPaginated: Error during local variable processing", e); + } finally { + if (decomp != null) { + decomp.dispose(); + } + } + } else { + // This means we already have some globals and may need a few locals to complete the page + int remainingSpace = limit - variables.size(); + if (remainingSpace > 0) { + // Process just enough functions to fill the page + DecompInterface decomp = null; + try { + decomp = new DecompInterface(); + if (decomp.openProgram(program)) { + int functionsProcessed = 0; + int maxFunctionsToProcess = 5; // Limit how many functions we process + int localVarsAdded = 0; + + for (Function function : program.getFunctionManager().getFunctions(true)) { + try { + DecompileResults results = decomp.decompileFunction(function, 10, new ConsoleTaskMonitor()); + if (results != null && results.decompileCompleted()) { + HighFunction highFunc = results.getHighFunction(); + if (highFunc != null) { + Iterator symbolIter = highFunc.getLocalSymbolMap().getSymbols(); + while (symbolIter.hasNext() && localVarsAdded < remainingSpace) { + HighSymbol symbol = symbolIter.next(); + if (!symbol.isParameter()) { // Only list locals + Map varInfo = new HashMap<>(); + varInfo.put("name", symbol.getName()); + varInfo.put("type", "local"); + varInfo.put("function", function.getName()); + Address pcAddr = symbol.getPCAddress(); + varInfo.put("address", pcAddr != null ? pcAddr.toString() : "N/A"); + varInfo.put("dataType", symbol.getDataType() != null ? symbol.getDataType().getName() : "unknown"); + variables.add(varInfo); + localVarsAdded++; + } + } + } + } + } catch (Exception e) { + Msg.warn(this, "listVariablesPaginated: Error processing function " + function.getName(), e); + } + + functionsProcessed++; + if (functionsProcessed >= maxFunctionsToProcess || localVarsAdded >= remainingSpace) { + // Stop processing if we've hit our limits + break; + } + } + + // Determine if we have more variables + hasMore = functionsProcessed < funcCount || localVarsAdded >= remainingSpace; + } + } catch (Exception e) { + Msg.error(this, "listVariablesPaginated: Error during local variable processing", e); + } finally { + if (decomp != null) { + decomp.dispose(); } } } - } catch (Exception e) { - Msg.error(this, "listVariables: Error during local variable processing", e); - } finally { - if (decomp != null) { - decomp.dispose(); - } } - - Collections.sort(variables, Comparator.comparing(a -> a.get("name"))); - return variables; // Return full list, pagination applied in handler + + // Sort the combined results + variables.sort(Comparator.comparing(a -> a.get("name"))); + + return new PaginatedResult(variables, hasMore, totalEstimate); } - // Updated to return List instead of JsonObject for HATEOAS compliance + /** + * Legacy method kept for backward compatibility + */ private List> searchVariables(Program program, String searchTerm) { + PaginatedResult result = searchVariablesPaginated(program, searchTerm, 0, Integer.MAX_VALUE, false); + return result.getResults(); + } + + /** + * Search variables with efficient pagination - only loads what's needed + */ + private PaginatedResult searchVariablesPaginated(Program program, String searchTerm, int offset, int limit, boolean globalOnly) { if (program == null || searchTerm == null || searchTerm.isEmpty()) { - return new ArrayList<>(); // Return empty list + return new PaginatedResult(new ArrayList<>(), false, 0); } - + List> matchedVars = new ArrayList<>(); String lowerSearchTerm = searchTerm.toLowerCase(); - - // Search global variables + int totalEstimate = 0; + boolean hasMore = false; + + // Calculate range of items to fetch + int startIdx = offset; + int endIdx = offset + limit; + int currentIndex = 0; + + // Search global variables - these are quick to search SymbolTable symbolTable = program.getSymbolTable(); + List> globalMatches = new ArrayList<>(); + SymbolIterator it = symbolTable.getSymbolIterator(); while (it.hasNext()) { Symbol symbol = it.next(); @@ -202,59 +447,190 @@ package eu.starsong.ghidra.endpoints; symbol.getSymbolType() != SymbolType.FUNCTION && symbol.getSymbolType() != SymbolType.LABEL && symbol.getName().toLowerCase().contains(lowerSearchTerm)) { + Map varInfo = new HashMap<>(); varInfo.put("name", symbol.getName()); varInfo.put("address", symbol.getAddress().toString()); varInfo.put("type", "global"); varInfo.put("dataType", getDataTypeName(program, symbol.getAddress())); - matchedVars.add(varInfo); + globalMatches.add(varInfo); } } - - // Search local variables - DecompInterface decomp = null; - try { - decomp = new DecompInterface(); - if (decomp.openProgram(program)) { - for (Function function : program.getFunctionManager().getFunctions(true)) { - try { - DecompileResults results = decomp.decompileFunction(function, 30, new ConsoleTaskMonitor()); - if (results != null && results.decompileCompleted()) { - HighFunction highFunc = results.getHighFunction(); - if (highFunc != null) { - Iterator symbolIter = highFunc.getLocalSymbolMap().getSymbols(); - while (symbolIter.hasNext()) { - HighSymbol symbol = symbolIter.next(); - if (symbol.getName().toLowerCase().contains(lowerSearchTerm)) { - Map varInfo = new HashMap<>(); - varInfo.put("name", symbol.getName()); - varInfo.put("function", function.getName()); - varInfo.put("type", symbol.isParameter() ? "parameter" : "local"); - Address pcAddr = symbol.getPCAddress(); - varInfo.put("address", pcAddr != null ? pcAddr.toString() : "N/A"); - varInfo.put("dataType", symbol.getDataType() != null ? symbol.getDataType().getName() : "unknown"); - matchedVars.add(varInfo); + + // Sort global matches by name + globalMatches.sort(Comparator.comparing(a -> a.get("name"))); + + // Extract just the global variables needed for this page + int globalCount = globalMatches.size(); + totalEstimate = globalCount; + + for (Map varInfo : globalMatches) { + if (currentIndex >= startIdx && currentIndex < endIdx) { + matchedVars.add(varInfo); + } + currentIndex++; + + // If we've added enough items, break + if (currentIndex >= endIdx) { + hasMore = currentIndex < globalCount || !globalOnly; + break; + } + } + + // If we only want globals, or if we've already fetched enough for this page, return now + if (globalOnly || currentIndex >= endIdx) { + return new PaginatedResult(matchedVars, hasMore, totalEstimate); + } + + // Search local variables - only do this if we need more results + // We need to perform some estimation for locals, as decompiling all functions is too slow + + // First estimate the total count + int funcCount = 0; + for (Function f : program.getFunctionManager().getFunctions(true)) { + funcCount++; + } + + // Roughly estimate 1 match per 5 functions when searching + totalEstimate = globalCount + (funcCount / 5); + + // If we don't need locals for the current page, return globals with estimation + if (startIdx >= globalCount) { + // Adjust for local variable processing + int localOffset = startIdx - globalCount; + int localLimit = limit; + + // Process functions to get the local variables + DecompInterface decomp = null; + try { + decomp = new DecompInterface(); + if (decomp.openProgram(program)) { + int localVarIndex = 0; + int functionsProcessed = 0; + int maxFunctionsToProcess = 30; // Limit how many functions we process for search + + for (Function function : program.getFunctionManager().getFunctions(true)) { + try { + DecompileResults results = decomp.decompileFunction(function, 5, new ConsoleTaskMonitor()); + if (results != null && results.decompileCompleted()) { + HighFunction highFunc = results.getHighFunction(); + if (highFunc != null) { + List> functionMatches = new ArrayList<>(); + Iterator symbolIter = highFunc.getLocalSymbolMap().getSymbols(); + while (symbolIter.hasNext()) { + HighSymbol symbol = symbolIter.next(); + if (symbol.getName().toLowerCase().contains(lowerSearchTerm)) { + Map varInfo = new HashMap<>(); + varInfo.put("name", symbol.getName()); + varInfo.put("function", function.getName()); + varInfo.put("type", symbol.isParameter() ? "parameter" : "local"); + Address pcAddr = symbol.getPCAddress(); + varInfo.put("address", pcAddr != null ? pcAddr.toString() : "N/A"); + varInfo.put("dataType", symbol.getDataType() != null ? symbol.getDataType().getName() : "unknown"); + functionMatches.add(varInfo); + } + } + + // Sort function matches by name + functionMatches.sort(Comparator.comparing(a -> a.get("name"))); + + // Add only the needed variables for this page + for (Map varInfo : functionMatches) { + if (localVarIndex >= localOffset && localVarIndex < localOffset + localLimit) { + matchedVars.add(varInfo); + } + localVarIndex++; + if (localVarIndex >= localOffset + localLimit) { + break; + } } } } + } catch (Exception e) { + Msg.warn(this, "searchVariablesPaginated: Error processing function " + function.getName(), e); } - } catch (Exception e) { - Msg.warn(this, "searchVariables: Error processing function " + function.getName(), e); + + functionsProcessed++; + if (functionsProcessed >= maxFunctionsToProcess || localVarIndex >= localOffset + localLimit) { + // Stop processing if we've hit our limits + break; + } + } + + // Determine if we have more variables + hasMore = functionsProcessed < funcCount || localVarIndex >= localOffset + localLimit; + } + } catch (Exception e) { + Msg.error(this, "searchVariablesPaginated: Error during local variable search", e); + } finally { + if (decomp != null) { + decomp.dispose(); + } + } + } else { + // This means we already have some globals and may need a few locals to complete the page + int remainingSpace = limit - matchedVars.size(); + if (remainingSpace > 0) { + // Process functions until we've filled the page + DecompInterface decomp = null; + try { + decomp = new DecompInterface(); + if (decomp.openProgram(program)) { + int functionsProcessed = 0; + int maxFunctionsToProcess = 5; // Limit how many functions we process + int localVarsAdded = 0; + + for (Function function : program.getFunctionManager().getFunctions(true)) { + try { + DecompileResults results = decomp.decompileFunction(function, 5, new ConsoleTaskMonitor()); + if (results != null && results.decompileCompleted()) { + HighFunction highFunc = results.getHighFunction(); + if (highFunc != null) { + Iterator symbolIter = highFunc.getLocalSymbolMap().getSymbols(); + while (symbolIter.hasNext() && localVarsAdded < remainingSpace) { + HighSymbol symbol = symbolIter.next(); + if (symbol.getName().toLowerCase().contains(lowerSearchTerm)) { + Map varInfo = new HashMap<>(); + varInfo.put("name", symbol.getName()); + varInfo.put("function", function.getName()); + varInfo.put("type", symbol.isParameter() ? "parameter" : "local"); + Address pcAddr = symbol.getPCAddress(); + varInfo.put("address", pcAddr != null ? pcAddr.toString() : "N/A"); + varInfo.put("dataType", symbol.getDataType() != null ? symbol.getDataType().getName() : "unknown"); + matchedVars.add(varInfo); + localVarsAdded++; + } + } + } + } + } catch (Exception e) { + Msg.warn(this, "searchVariablesPaginated: Error processing function " + function.getName(), e); + } + + functionsProcessed++; + if (functionsProcessed >= maxFunctionsToProcess || localVarsAdded >= remainingSpace) { + // Stop processing if we've hit our limits + break; + } + } + + // Determine if we have more variables + hasMore = functionsProcessed < funcCount || localVarsAdded >= remainingSpace; + } + } catch (Exception e) { + Msg.error(this, "searchVariablesPaginated: Error during local variable search", e); + } finally { + if (decomp != null) { + decomp.dispose(); } } - } else { - Msg.error(this, "searchVariables: Failed to open program with decompiler."); - } - } catch (Exception e) { - Msg.error(this, "searchVariables: Error during local variable search", e); - } finally { - if (decomp != null) { - decomp.dispose(); } } - - Collections.sort(matchedVars, Comparator.comparing(a -> a.get("name"))); - return matchedVars; + + // Sort the combined results + matchedVars.sort(Comparator.comparing(a -> a.get("name"))); + + return new PaginatedResult(matchedVars, hasMore, totalEstimate); } // --- Helper Methods ---