From 8475a229091ecd9fa443420d1c413f5e9eb91cd1 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 9 Feb 2024 14:43:12 +0100 Subject: [PATCH 01/29] Initial prototype for specifying function dfg-summaries --- .../aisec/cpg/TranslationConfiguration.kt | 15 ++ .../cpg/passes/inference/DFGSummaries.kt | 193 ++++++++++++++++++ .../cpg/enhancements/DFGSummariesTest.kt | 41 ++++ cpg-core/src/test/resources/function-dfg.json | 56 +++++ 4 files changed, 305 insertions(+) create mode 100644 cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGSummaries.kt create mode 100644 cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGSummariesTest.kt create mode 100644 cpg-core/src/test/resources/function-dfg.json diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index 74bfbea49c..41e347f3b3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.passes.* +import de.fraunhofer.aisec.cpg.passes.inference.DFGSummaries import de.fraunhofer.aisec.cpg.passes.order.* import java.io.File import java.nio.file.Path @@ -102,6 +103,8 @@ private constructor( */ val replacedPasses: Map>, KClass>>, KClass>>, + /** This list contains the files with function summaries which should be considered. */ + val functionSummaries: DFGSummaries, languages: List>, codeInNodes: Boolean, processAnnotations: Boolean, @@ -240,6 +243,7 @@ private constructor( private val passes = mutableListOf>>() private val replacedPasses = mutableMapOf>, KClass>>, KClass>>() + private val functionSummaries = mutableListOf() private var codeInNodes = true private var processAnnotations = false private var disableCleanup = false @@ -420,6 +424,16 @@ private constructor( return this } + fun registerFunctionSummaries(functionSummaries: List): Builder { + this.functionSummaries.addAll(functionSummaries) + return this + } + + fun registerFunctionSummary(functionSummary: File): Builder { + this.functionSummaries.add(functionSummary) + return this + } + /** Registers an additional [Language]. */ fun registerLanguage(language: Language<*>): Builder { languages.add(language) @@ -625,6 +639,7 @@ private constructor( includeBlocklist, orderPasses(), replacedPasses, + DFGSummaries.fromFiles(functionSummaries), languages, codeInNodes, processAnnotations, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGSummaries.kt new file mode 100644 index 0000000000..9e885edaf2 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGSummaries.kt @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.passes.inference + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration +import de.fraunhofer.aisec.cpg.graph.objectType +import de.fraunhofer.aisec.cpg.graph.parseName +import de.fraunhofer.aisec.cpg.graph.types.Type +import java.io.File + +class DFGSummaries { + private constructor() + + val functionToDFGEntryMap = mutableMapOf>() + + /** This function returns a list of [DataflowEntry] from the specified file. */ + private fun addEntriesFromFile(file: File): Map> { + val jsonStringFile = file.readText() + val mapper = ObjectMapper().registerKotlinModule() + val entries = mapper.readValue>(jsonStringFile) + for (entry in entries) { + functionToDFGEntryMap[entry.functionDeclaration] = entry.dataFlows + } + return functionToDFGEntryMap + } + + /** + * Adds the DFG edges to the [functionDeclaration] depending on the function summaries which are + * kept in this object. If no suitable entry was found, this method returns `false`. + */ + fun addFlowsToFunctionDeclaration(functionDeclaration: FunctionDeclaration): Boolean { + val dfgEntries = findFunctionDeclarationEntry(functionDeclaration) ?: return false + applyDfgEntryToFunctionDeclaration(functionDeclaration, dfgEntries) + return true + } + + private fun findFunctionDeclarationEntry(functionDecl: FunctionDeclaration): List? { + val language = functionDecl.language + val languageName = language?.javaClass?.name + val methodName = functionDecl.name + // The language and the method name have to match. If a signature is specified, it also has + // to match to the one of the FunctionDeclaration, null indicates that we accept everything. + val matchingEntries = + functionToDFGEntryMap.keys.filter { + it.language == languageName && + methodName.lastPartsMatch(it.methodName) && + (it.signature == null || + functionDecl.hasSignature( + it.signature.map { signatureType -> language.objectType(signatureType) } + )) + } + return if (matchingEntries.size == 1) { + // Only one entry => We take this one. + functionToDFGEntryMap[matchingEntries.single()] + } else if (matchingEntries.filter { it.signature != null }.size == 1) { + // Only one entry with a matching signature => We take this one. + functionToDFGEntryMap[matchingEntries.single { it.signature != null }] + } else { + /* There are multiple matching entries. We use the following routine: + * First, we filter for existing signatures. + * Second, we filter for the most precise class. + * If there are still multiple options, we take the longest signature and hope it's the most precise one. + */ + val typeEntryList = + matchingEntries + .filter { it.signature != null } + .map { + Pair( + language.parseName(it.methodName).parent?.let { it1 -> + language?.objectType(it1) + }, + it + ) + } + val mostPreciseClassEntries = mutableListOf() + var mostPreciseType = typeEntryList.first().first + var superTypes = getAllSupertypes(mostPreciseType) + for (typeEntry in typeEntryList) { + if (typeEntry.first == mostPreciseType) { + mostPreciseClassEntries.add(typeEntry.second) + } else if (typeEntry.first in superTypes) { + mostPreciseClassEntries.clear() + mostPreciseClassEntries.add(typeEntry.second) + mostPreciseType = typeEntry.first + superTypes = getAllSupertypes(typeEntry.first) + } + } + if (mostPreciseClassEntries.size > 1) { + mostPreciseClassEntries.sortByDescending { it.signature?.size ?: 0 } + } + functionToDFGEntryMap[matchingEntries.first()] + } + } + + private fun getAllSupertypes(type: Type?): Set { + if (type == null) return setOf() + val superTypes = type.superTypes + superTypes.addAll(type.superTypes.flatMap { getAllSupertypes(it) }) + return superTypes + } + + private fun applyDfgEntryToFunctionDeclaration( + functionDeclaration: FunctionDeclaration, + dfgEntries: List + ) { + for (entry in dfgEntries) { + val from = + if (entry.from.startsWith("param")) { + try { + val paramIndex = entry.from.removePrefix("param").toInt() + functionDeclaration.parameters[paramIndex] + } catch (e: NumberFormatException) { + null + } + } else if (entry.from == "base") { + (functionDeclaration as? MethodDeclaration)?.receiver + } else { + null + } + val to = + if (entry.to.startsWith("param")) { + try { + val paramIndex = entry.to.removePrefix("param").toInt() + functionDeclaration.parameters[paramIndex] + } catch (e: NumberFormatException) { + null + } + } else if (entry.to == "base") { + (functionDeclaration as? MethodDeclaration)?.receiver + } else if (entry.to == "return") { + functionDeclaration + } else if (entry.to.startsWith("return")) { + val returnIndex = entry.to.removePrefix("param").toInt() + // TODO: It would be nice if we could model the index. Not sure how this is done + functionDeclaration + } else { + null + } + // TODO: It would make sense to model properties here. Could be the index of a return + // value, full vs. partial flow or whatever comes to our minds in the future + to?.let { from?.addNextDFG(it) } + } + } + + private data class DataflowEntry( + val functionDeclaration: FunctionDeclarationEntry, + val dataFlows: List + ) + + data class FunctionDeclarationEntry( + val language: String, + val methodName: String, + val signature: List? = null + ) + + data class DFGEntry(val from: String, val to: String, val dfgType: String) + + companion object { + /** Generates a [DFGSummaries] object from the given [files]. */ + fun fromFiles(files: List): DFGSummaries { + val dfgSummaries = DFGSummaries() + files.forEach { dfgSummaries.addEntriesFromFile(it) } + return dfgSummaries + } + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGSummariesTest.kt new file mode 100644 index 0000000000..ad6a7e4ea5 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGSummariesTest.kt @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.enhancements + +import de.fraunhofer.aisec.cpg.passes.inference.DFGSummaries +import java.io.File +import kotlin.test.Test +import kotlin.test.assertTrue + +class DFGSummariesTest { + + @Test + fun testRecordInference() { + val summaries = DFGSummaries.fromFiles(listOf(File("src/test/resources/function-dfg.json"))) + + assertTrue(summaries.functionToDFGEntryMap.isNotEmpty()) + } +} diff --git a/cpg-core/src/test/resources/function-dfg.json b/cpg-core/src/test/resources/function-dfg.json new file mode 100644 index 0000000000..7c7c0f49f9 --- /dev/null +++ b/cpg-core/src/test/resources/function-dfg.json @@ -0,0 +1,56 @@ +[ + { + "functionDeclaration": { + "language": "de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage", + "methodName": "java.util.List.addAll", + "signature": ["int", "java.util.Object"] + }, + "dataFlows": [ + { + "from": "param2", + "to": "base", + "dfgType": "full" + } + ] + }, + { + "functionDeclaration": { + "language": "de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage", + "methodName": "java.util.List.addAll", + "signature": ["java.util.Object"] + }, + "dataFlows": [ + { + "from": "param1", + "to": "base", + "dfgType": "full" + } + ] + }, + { + "functionDeclaration": { + "language": "de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage", + "methodName": "java.util.List.add" + }, + "dataFlows": [ + { + "from": "param1", + "to": "base", + "dfgType": "full" + } + ] + }, + { + "functionDeclaration": { + "language": "de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage", + "methodName": "memcpy" + }, + "dataFlows": [ + { + "from": "param2", + "to": "param1", + "dfgType": "full" + } + ] + } +] \ No newline at end of file From dea03f95feb83c2852499f74774c9441ea155007 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 9 Feb 2024 15:22:43 +0100 Subject: [PATCH 02/29] Connect new summaries also to dfg of the declaration --- .../aisec/cpg/TranslationConfiguration.kt | 6 ++-- .../de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 36 +++++++++++++------ ...FGSummaries.kt => DFGFunctionSummaries.kt} | 12 +++---- ...iesTest.kt => DFGFunctionSummariesTest.kt} | 7 ++-- 4 files changed, 38 insertions(+), 23 deletions(-) rename cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/{DFGSummaries.kt => DFGFunctionSummaries.kt} (95%) rename cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/{DFGSummariesTest.kt => DFGFunctionSummariesTest.kt} (85%) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index 41e347f3b3..425b7f0907 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -34,7 +34,7 @@ import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.passes.* -import de.fraunhofer.aisec.cpg.passes.inference.DFGSummaries +import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries import de.fraunhofer.aisec.cpg.passes.order.* import java.io.File import java.nio.file.Path @@ -104,7 +104,7 @@ private constructor( val replacedPasses: Map>, KClass>>, KClass>>, /** This list contains the files with function summaries which should be considered. */ - val functionSummaries: DFGSummaries, + val functionSummaries: DFGFunctionSummaries, languages: List>, codeInNodes: Boolean, processAnnotations: Boolean, @@ -639,7 +639,7 @@ private constructor( includeBlocklist, orderPasses(), replacedPasses, - DFGSummaries.fromFiles(functionSummaries), + DFGFunctionSummaries.fromFiles(functionSummaries), languages, codeInNodes, processAnnotations, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index c175e00756..57eff44012 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -33,6 +33,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.IterativeGraphWalker import de.fraunhofer.aisec.cpg.helpers.Util +import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries import de.fraunhofer.aisec.cpg.passes.order.DependsOn /** Adds the DFG edges for various types of nodes. */ @@ -42,7 +43,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { val inferDfgForUnresolvedCalls = config.inferenceConfiguration.inferDfgForUnresolvedSymbols val walker = IterativeGraphWalker() walker.registerOnNodeVisit2 { node, parent -> - handle(node, parent, inferDfgForUnresolvedCalls) + handle(node, parent, inferDfgForUnresolvedCalls, config.functionSummaries) } for (tu in component.translationUnits) { walker.iterate(tu) @@ -53,7 +54,12 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { // Nothing to do } - protected fun handle(node: Node?, parent: Node?, inferDfgForUnresolvedSymbols: Boolean) { + protected fun handle( + node: Node?, + parent: Node?, + inferDfgForUnresolvedSymbols: Boolean, + functionSummaries: DFGFunctionSummaries + ) { when (node) { // Expressions is CallExpression -> handleCallExpression(node, inferDfgForUnresolvedSymbols) @@ -83,7 +89,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { is IfStatement -> handleIfStatement(node) // Declarations is FieldDeclaration -> handleFieldDeclaration(node) - is FunctionDeclaration -> handleFunctionDeclaration(node) + is FunctionDeclaration -> handleFunctionDeclaration(node, functionSummaries) is TupleDeclaration -> handleTupleDeclaration(node) is VariableDeclaration -> handleVariableDeclaration(node) } @@ -159,15 +165,23 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * Adds the DFG edge for a [FunctionDeclaration]. The data flows from the return statement(s) to * the function. */ - protected fun handleFunctionDeclaration(node: FunctionDeclaration) { + protected fun handleFunctionDeclaration( + node: FunctionDeclaration, + functionSummaries: DFGFunctionSummaries + ) { if (node.isInferred) { - // If the function is inferred, we connect all parameters to the function declaration. - // The condition should make sure that we don't add edges multiple times, i.e., we - // only handle the declaration exactly once. - node.addAllPrevDFG(node.parameters) - // If it's a method with a receiver, we connect that one too. - if (node is MethodDeclaration) { - node.receiver?.let { node.addPrevDFG(it) } + val summaryExists = functionSummaries.addFlowsToFunctionDeclaration(node) + + if (!summaryExists) { + // If the function is inferred, we connect all parameters to the function + // declaration. + // The condition should make sure that we don't add edges multiple times, i.e., we + // only handle the declaration exactly once. + node.addAllPrevDFG(node.parameters) + // If it's a method with a receiver, we connect that one too. + if (node is MethodDeclaration) { + node.receiver?.let { node.addPrevDFG(it) } + } } } else { node.allChildren().forEach { node.addPrevDFG(it) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt similarity index 95% rename from cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGSummaries.kt rename to cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 9e885edaf2..789a63d67f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -35,7 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.parseName import de.fraunhofer.aisec.cpg.graph.types.Type import java.io.File -class DFGSummaries { +class DFGFunctionSummaries { private constructor() val functionToDFGEntryMap = mutableMapOf>() @@ -183,11 +183,11 @@ class DFGSummaries { data class DFGEntry(val from: String, val to: String, val dfgType: String) companion object { - /** Generates a [DFGSummaries] object from the given [files]. */ - fun fromFiles(files: List): DFGSummaries { - val dfgSummaries = DFGSummaries() - files.forEach { dfgSummaries.addEntriesFromFile(it) } - return dfgSummaries + /** Generates a [DFGFunctionSummaries] object from the given [files]. */ + fun fromFiles(files: List): DFGFunctionSummaries { + val dfgFunctionSummaries = DFGFunctionSummaries() + files.forEach { dfgFunctionSummaries.addEntriesFromFile(it) } + return dfgFunctionSummaries } } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt similarity index 85% rename from cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGSummariesTest.kt rename to cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index ad6a7e4ea5..c94f3a6322 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -25,16 +25,17 @@ */ package de.fraunhofer.aisec.cpg.enhancements -import de.fraunhofer.aisec.cpg.passes.inference.DFGSummaries +import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries import java.io.File import kotlin.test.Test import kotlin.test.assertTrue -class DFGSummariesTest { +class DFGFunctionSummariesTest { @Test fun testRecordInference() { - val summaries = DFGSummaries.fromFiles(listOf(File("src/test/resources/function-dfg.json"))) + val summaries = + DFGFunctionSummaries.fromFiles(listOf(File("src/test/resources/function-dfg.json"))) assertTrue(summaries.functionToDFGEntryMap.isNotEmpty()) } From 4fffec7a0eca80d4cbaa5a07ee715a4bbfac10e6 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 9 Feb 2024 15:25:41 +0100 Subject: [PATCH 03/29] Handle non-existing list --- .../aisec/cpg/passes/inference/DFGFunctionSummaries.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 789a63d67f..8239733f6c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -62,6 +62,8 @@ class DFGFunctionSummaries { } private fun findFunctionDeclarationEntry(functionDecl: FunctionDeclaration): List? { + if (functionToDFGEntryMap.isEmpty()) return null + val language = functionDecl.language val languageName = language?.javaClass?.name val methodName = functionDecl.name @@ -82,7 +84,7 @@ class DFGFunctionSummaries { } else if (matchingEntries.filter { it.signature != null }.size == 1) { // Only one entry with a matching signature => We take this one. functionToDFGEntryMap[matchingEntries.single { it.signature != null }] - } else { + } else if (matchingEntries.isNotEmpty()) { /* There are multiple matching entries. We use the following routine: * First, we filter for existing signatures. * Second, we filter for the most precise class. @@ -116,6 +118,8 @@ class DFGFunctionSummaries { mostPreciseClassEntries.sortByDescending { it.signature?.size ?: 0 } } functionToDFGEntryMap[matchingEntries.first()] + } else { + null } } From af5fd2cd794c0fdc72edb1609b3e2abd582d65f4 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 9 Feb 2024 15:59:27 +0100 Subject: [PATCH 04/29] Document the file format --- cpg-core/src/test/resources/function-dfg.json | 10 +-- docs/docs/CPG/specs/dfg-function-summaries.md | 75 +++++++++++++++++++ docs/docs/CPG/specs/index.md | 1 + docs/docs/GettingStarted/library.md | 2 + docs/mkdocs.yaml | 1 + 5 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 docs/docs/CPG/specs/dfg-function-summaries.md diff --git a/cpg-core/src/test/resources/function-dfg.json b/cpg-core/src/test/resources/function-dfg.json index 7c7c0f49f9..df3cc5cda1 100644 --- a/cpg-core/src/test/resources/function-dfg.json +++ b/cpg-core/src/test/resources/function-dfg.json @@ -7,7 +7,7 @@ }, "dataFlows": [ { - "from": "param2", + "from": "param1", "to": "base", "dfgType": "full" } @@ -21,7 +21,7 @@ }, "dataFlows": [ { - "from": "param1", + "from": "param0", "to": "base", "dfgType": "full" } @@ -34,7 +34,7 @@ }, "dataFlows": [ { - "from": "param1", + "from": "param0", "to": "base", "dfgType": "full" } @@ -47,8 +47,8 @@ }, "dataFlows": [ { - "from": "param2", - "to": "param1", + "from": "param1", + "to": "param0", "dfgType": "full" } ] diff --git a/docs/docs/CPG/specs/dfg-function-summaries.md b/docs/docs/CPG/specs/dfg-function-summaries.md new file mode 100644 index 0000000000..19f078ce0c --- /dev/null +++ b/docs/docs/CPG/specs/dfg-function-summaries.md @@ -0,0 +1,75 @@ +# Specification: Data Flow Graph - Function Summaries + +For functions and methods which are part of the analyzed codebase, the CPG can track data flows interprocedurally to some extent. +However, for all functions and methods which cannot be analyzed, we have no information available. +For this case, we provide the user a way to specify custom summaries of the data flows through the function. +To do so, you need to fill a JSON file as follows: + +* The outer element is a list/array +* In this list, you add elements, each of which summarizes the flows for one function/method +* The element consists of two objects: The `functionDeclaration` and the `dataFlows` +* The `functionDeclaration` consists of: + * `language`: The FQN of the `Language` element which this function is relevant for. + * `methodName`: The FQN of the function or method. We use this one to identify the relevant function/method. Do not forget to add the class name and use the separators as specified by the `Language`. + * `signature` (*optional*): This optional element allows us to differentiate between overloaded functions (i.e., two functions have the same FQN but accept different arguments). If no `signature` is specified, it matches to any function/method with the name you specified. The `signature` is a list of FQNs of the types (as strings) +* The `dataFlows` element is a list of objects with the following elements: + * `from`: A description of the start-node of a DFG-edge. Valid options: + * `paramX`: where `X` is the offset (we start counting with 0) + * `base`: the receiver of the method (i.e., the object the method is called on) + * `to`: A description of the end-node of the DFG-edge. Valid options: + * `paramX` where `X` is the offset (we start counting with 0) + * `base` the receiver of the method (i.e., the object the method is called on) + * `return` the return value of the function + * `returnX` where `X` is a number and specifies the index of the return value (if multiple values are returned). + * `dfgType`: Here, you can give more information. Currently, this is unused but should later allow us to add the properties to the edge. + +An example of a file could look as follows: +```json +[ + { + "functionDeclaration": { + "language": "de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage", + "methodName": "java.util.List.addAll", + "signature": ["int", "java.util.Object"] + }, + "dataFlows": [ + { + "from": "param1", + "to": "base", + "dfgType": "full" + } + ] + }, + { + "functionDeclaration": { + "language": "de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage", + "methodName": "java.util.List.addAll", + "signature": ["java.util.Object"] + }, + "dataFlows": [ + { + "from": "param0", + "to": "base", + "dfgType": "full" + } + ] + }, + { + "functionDeclaration": { + "language": "de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage", + "methodName": "memcpy" + }, + "dataFlows": [ + { + "from": "param1", + "to": "param0", + "dfgType": "full" + } + ] + } +] +``` +This file configures the following edges: +* For a method declaration in Java `java.util.List.addAll(int, java.util.Object)`, the parameter 1 flows to the base (i.e., the list object) +* For a method declaration in Java `java.util.List.addAll(java.util.Object)`, the parameter 0 flows to the base (i.e., the list object) +* For a function declaration in C `memcpy` (and thus also CXX `std::memcpy`), the parameter 1 flows to parameter 0. \ No newline at end of file diff --git a/docs/docs/CPG/specs/index.md b/docs/docs/CPG/specs/index.md index 89971d3543..354ff321ed 100755 --- a/docs/docs/CPG/specs/index.md +++ b/docs/docs/CPG/specs/index.md @@ -16,4 +16,5 @@ links to the specifications of the following concepts: * Explore our [Graph Model](./graph) * [Data Flow Graph (DFG)](./dfg) +* [Data Flow Graph (DFG) Function Summaries](./dfg-function-summaries.md) * [Evaluation Order Graph (EOG)](./eog) diff --git a/docs/docs/GettingStarted/library.md b/docs/docs/GettingStarted/library.md index da8bfe840d..4f2b384943 100644 --- a/docs/docs/GettingStarted/library.md +++ b/docs/docs/GettingStarted/library.md @@ -59,6 +59,8 @@ val translationConfig = TranslationConfiguration For a complete list of available methods, please check the KDoc. +If you want/have to specify data flow summaries for some methods or functions, you add the method `registerFunctionSummary` when building the `TranslationCOnfiguration` and add a file with the format specified [here](../CPG/specs/dfg-function-summaries.md) + ## 3. Running the analysis Now it's time to get the CPG. All you have to do is to run the analysis with the diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index 7ab9c29691..1464c2678f 100755 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -161,6 +161,7 @@ nav: - CPG/specs/index.md - "Graph Schema": CPG/specs/graph.md - "Dataflow Graph (DFG)": CPG/specs/dfg.md + - "Dataflow Graph (DFG) Function Summaries": CPG/specs/dfg-function-summaries.md - "Evaluation Order Graph (EOG)": CPG/specs/eog.md - "Implementation": - CPG/impl/index.md From c1860a7d036197696684ba841db0d340cc2ab4cb Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Mon, 12 Feb 2024 08:38:44 +0100 Subject: [PATCH 05/29] Added yml support --- cpg-core/build.gradle.kts | 2 + .../passes/inference/DFGFunctionSummaries.kt | 13 +- .../enhancements/DFGFunctionSummariesTest.kt | 13 +- .../de/fraunhofer/aisec/cpg/passes/DFGTest.kt | 2 +- cpg-core/src/test/resources/function-dfg.yml | 36 +++++ docs/docs/CPG/specs/dfg-function-summaries.md | 125 ++++++++++++------ 6 files changed, 140 insertions(+), 51 deletions(-) create mode 100644 cpg-core/src/test/resources/function-dfg.yml diff --git a/cpg-core/build.gradle.kts b/cpg-core/build.gradle.kts index f34baef41a..00bde02dd2 100644 --- a/cpg-core/build.gradle.kts +++ b/cpg-core/build.gradle.kts @@ -65,6 +65,8 @@ dependencies { implementation(libs.bundles.log4j) implementation(libs.kotlin.reflect) + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.16.1") + testImplementation(libs.junit.params) testFixturesApi(libs.kotlin.test.junit5) // somehow just using testFixturesApi(kotlin("test")) does not work for testFixtures diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 8239733f6c..675d01b4b3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -25,7 +25,9 @@ */ package de.fraunhofer.aisec.cpg.passes.inference +import com.fasterxml.jackson.core.JsonFactory import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import com.fasterxml.jackson.module.kotlin.readValue import com.fasterxml.jackson.module.kotlin.registerKotlinModule import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration @@ -42,9 +44,14 @@ class DFGFunctionSummaries { /** This function returns a list of [DataflowEntry] from the specified file. */ private fun addEntriesFromFile(file: File): Map> { - val jsonStringFile = file.readText() - val mapper = ObjectMapper().registerKotlinModule() - val entries = mapper.readValue>(jsonStringFile) + val mapper = + if (file.extension.lowercase() in listOf("yaml", "yml")) { + ObjectMapper(YAMLFactory()) + } else { + ObjectMapper(JsonFactory()) + } + .registerKotlinModule() + val entries = mapper.readValue>(file) for (entry in entries) { functionToDFGEntryMap[entry.functionDeclaration] = entry.dataFlows } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index c94f3a6322..52b19a4a49 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -28,15 +28,22 @@ package de.fraunhofer.aisec.cpg.enhancements import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries import java.io.File import kotlin.test.Test +import kotlin.test.assertEquals import kotlin.test.assertTrue class DFGFunctionSummariesTest { @Test - fun testRecordInference() { - val summaries = + fun testParsingFile() { + val jsonSummaries = DFGFunctionSummaries.fromFiles(listOf(File("src/test/resources/function-dfg.json"))) - assertTrue(summaries.functionToDFGEntryMap.isNotEmpty()) + assertTrue(jsonSummaries.functionToDFGEntryMap.isNotEmpty()) + val yamlSummaries = + DFGFunctionSummaries.fromFiles(listOf(File("src/test/resources/function-dfg.yml"))) + + assertTrue(yamlSummaries.functionToDFGEntryMap.isNotEmpty()) + + assertEquals(jsonSummaries.functionToDFGEntryMap, yamlSummaries.functionToDFGEntryMap) } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt index 441ded7cbb..fa5bbee1db 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt @@ -489,7 +489,7 @@ class DFGTest { @Throws(Exception::class) fun testOutgoingDFGFromVariableDeclaration() { // TODO: IMHO this test is quite useless and can be merged into another one (e.g. - // testControlSensitiveDFGPassIfMerge). + // testControlSensitiveDFGPassIfMerge). val result = GraphExamples.getBasicSlice() val varA = TestUtils.findByUniqueName(result.variables, "a") diff --git a/cpg-core/src/test/resources/function-dfg.yml b/cpg-core/src/test/resources/function-dfg.yml new file mode 100644 index 0000000000..5964799dbc --- /dev/null +++ b/cpg-core/src/test/resources/function-dfg.yml @@ -0,0 +1,36 @@ +- functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage + methodName: java.util.List.addAll + signature: + - int + - java.util.Object + dataFlows: + - from: param1 + to: base + dfgType: full + +- functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage + methodName: java.util.List.addAll + signature: + - java.util.Object + dataFlows: + - from: param0 + to: base + dfgType: full + +- functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage + methodName: java.util.List.add + dataFlows: + - from: param0 + to: base + dfgType: full + +- functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage + methodName: memcpy + dataFlows: + - from: param1 + to: param0 + dfgType: full diff --git a/docs/docs/CPG/specs/dfg-function-summaries.md b/docs/docs/CPG/specs/dfg-function-summaries.md index 19f078ce0c..0e1bc6ff2e 100644 --- a/docs/docs/CPG/specs/dfg-function-summaries.md +++ b/docs/docs/CPG/specs/dfg-function-summaries.md @@ -3,7 +3,7 @@ For functions and methods which are part of the analyzed codebase, the CPG can track data flows interprocedurally to some extent. However, for all functions and methods which cannot be analyzed, we have no information available. For this case, we provide the user a way to specify custom summaries of the data flows through the function. -To do so, you need to fill a JSON file as follows: +To do so, you need to fill a JSON or YAML file as follows: * The outer element is a list/array * In this list, you add elements, each of which summarizes the flows for one function/method @@ -24,51 +24,88 @@ To do so, you need to fill a JSON file as follows: * `dfgType`: Here, you can give more information. Currently, this is unused but should later allow us to add the properties to the edge. An example of a file could look as follows: -```json -[ - { - "functionDeclaration": { - "language": "de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage", - "methodName": "java.util.List.addAll", - "signature": ["int", "java.util.Object"] - }, - "dataFlows": [ - { - "from": "param1", - "to": "base", - "dfgType": "full" - } - ] - }, - { - "functionDeclaration": { - "language": "de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage", - "methodName": "java.util.List.addAll", - "signature": ["java.util.Object"] + +=== "JSON" + + ```json + [ + { + "functionDeclaration": { + "language": "de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage", + "methodName": "java.util.List.addAll", + "signature": ["int", "java.util.Object"] + }, + "dataFlows": [ + { + "from": "param1", + "to": "base", + "dfgType": "full" + } + ] }, - "dataFlows": [ - { - "from": "param0", - "to": "base", - "dfgType": "full" - } - ] - }, - { - "functionDeclaration": { - "language": "de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage", - "methodName": "memcpy" + { + "functionDeclaration": { + "language": "de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage", + "methodName": "java.util.List.addAll", + "signature": ["java.util.Object"] + }, + "dataFlows": [ + { + "from": "param0", + "to": "base", + "dfgType": "full" + } + ] }, - "dataFlows": [ - { - "from": "param1", - "to": "param0", - "dfgType": "full" - } - ] - } -] -``` + { + "functionDeclaration": { + "language": "de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage", + "methodName": "memcpy" + }, + "dataFlows": [ + { + "from": "param1", + "to": "param0", + "dfgType": "full" + } + ] + } + ] + ``` + +=== "YAML" + + ```yml + - functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage + methodName: java.util.List.addAll + signature: + - int + - java.util.Object + dataFlows: + - from: param1 + to: base + dfgType: full + + - functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage + methodName: java.util.List.addAll + signature: + - java.util.Object + dataFlows: + - from: param0 + to: base + dfgType: full + + - functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage + methodName: memcpy + dataFlows: + - from: param1 + to: param0 + dfgType: full + ``` + This file configures the following edges: * For a method declaration in Java `java.util.List.addAll(int, java.util.Object)`, the parameter 1 flows to the base (i.e., the list object) * For a method declaration in Java `java.util.List.addAll(java.util.Object)`, the parameter 0 flows to the base (i.e., the list object) From 33de45cc03e12d0f73e77f68db1bf69d21db2d99 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Mon, 12 Feb 2024 08:43:11 +0100 Subject: [PATCH 06/29] Move dependency to toml file --- cpg-core/build.gradle.kts | 2 +- gradle/libs.versions.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cpg-core/build.gradle.kts b/cpg-core/build.gradle.kts index 00bde02dd2..be5d9c6b7e 100644 --- a/cpg-core/build.gradle.kts +++ b/cpg-core/build.gradle.kts @@ -65,7 +65,7 @@ dependencies { implementation(libs.bundles.log4j) implementation(libs.kotlin.reflect) - implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.16.1") + implementation(libs.jacksonyml) testImplementation(libs.junit.params) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b7bdc9832f..f8b39b9c06 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -22,6 +22,7 @@ neo4j-ogm-bolt-driver = { module = "org.neo4j:neo4j-ogm-bolt-driver", version.re javaparser = { module = "com.github.javaparser:javaparser-symbol-solver-core", version = "3.25.4"} jackson = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version = "2.16.0"} +jacksonyml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", version = "2.16.1"} eclipse-runtime = { module = "org.eclipse.platform:org.eclipse.core.runtime", version = "3.30.0"} osgi-service = { module = "org.osgi:org.osgi.service.prefs", version = "1.1.2"} icu4j = { module = "com.ibm.icu:icu4j", version = "74.2"} From fb44dffadfb058b767e10b6725f15598955a4019 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Mon, 12 Feb 2024 09:56:16 +0100 Subject: [PATCH 07/29] Document the class --- .../passes/inference/DFGFunctionSummaries.kt | 70 ++++++++++++++++--- 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 675d01b4b3..5ad9d059ff 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -30,6 +30,8 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import com.fasterxml.jackson.module.kotlin.readValue import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import de.fraunhofer.aisec.cpg.TranslationConfiguration.Builder +import de.fraunhofer.aisec.cpg.ancestors import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration import de.fraunhofer.aisec.cpg.graph.objectType @@ -37,9 +39,16 @@ import de.fraunhofer.aisec.cpg.graph.parseName import de.fraunhofer.aisec.cpg.graph.types.Type import java.io.File +/** + * If the user of the library registers one or multiple DFG-function summary files (via + * [Builder.registerFunctionSummaries] or [Builder.registerFunctionSummary]), this class is + * responsible for parsing the files, caching the result and adding the respective DFG summaries to + * the [FunctionDeclaration]. + */ class DFGFunctionSummaries { private constructor() + /** Caches a mapping of the [FunctionDeclarationEntry] to a list of its [DFGEntry]. */ val functionToDFGEntryMap = mutableMapOf>() /** This function returns a list of [DataflowEntry] from the specified file. */ @@ -68,6 +77,20 @@ class DFGFunctionSummaries { return true } + /** + * It identifies the "best match" of all [FunctionDeclarationEntry]s stored in the + * [functionToDFGEntryMap] for the given [functionDecl]. It therefore checks that + * 1) The languages match + * 2) The method/function names match + * 3) If there are multiple entries with different signatures, the signature has to match. If + * none of the entries with a signature matches, we take the "default" entry without a + * signature. + * 4) If it's a method (i.e., invoked on an object), we also consider which type of the + * receiver/base is the most precise one + * + * This method returns the list of [DFGEntry] for the "best match" or `null` if no entry + * matches. + */ private fun findFunctionDeclarationEntry(functionDecl: FunctionDeclaration): List? { if (functionToDFGEntryMap.isEmpty()) return null @@ -110,7 +133,7 @@ class DFGFunctionSummaries { } val mostPreciseClassEntries = mutableListOf() var mostPreciseType = typeEntryList.first().first - var superTypes = getAllSupertypes(mostPreciseType) + var superTypes = mostPreciseType?.ancestors?.map { it.type } ?: setOf() for (typeEntry in typeEntryList) { if (typeEntry.first == mostPreciseType) { mostPreciseClassEntries.add(typeEntry.second) @@ -118,7 +141,7 @@ class DFGFunctionSummaries { mostPreciseClassEntries.clear() mostPreciseClassEntries.add(typeEntry.second) mostPreciseType = typeEntry.first - superTypes = getAllSupertypes(typeEntry.first) + superTypes = mostPreciseType?.ancestors?.map { it.type } ?: setOf() } } if (mostPreciseClassEntries.size > 1) { @@ -130,13 +153,10 @@ class DFGFunctionSummaries { } } - private fun getAllSupertypes(type: Type?): Set { - if (type == null) return setOf() - val superTypes = type.superTypes - superTypes.addAll(type.superTypes.flatMap { getAllSupertypes(it) }) - return superTypes - } - + /** + * This method parses the [DFGEntry] entries in [dfgEntries] and adds the respective DFG edges + * between the parameters, receiver and potentially the [functionDeclaration] itself. + */ private fun applyDfgEntryToFunctionDeclaration( functionDeclaration: FunctionDeclaration, dfgEntries: List @@ -180,18 +200,48 @@ class DFGFunctionSummaries { } } + /** + * This class summarizes a data flow entry. Consists of the [functionDeclaration] for which it + * is relevant and a list [dataFlows] of data flow summaries. + */ private data class DataflowEntry( val functionDeclaration: FunctionDeclarationEntry, val dataFlows: List ) + /** + * This class is used to identify the [FunctionDeclaration] of interest for the specified flows. + */ data class FunctionDeclarationEntry( + /** The FQN of the [Language] for which this flow is relevant. */ val language: String, + /** The FQN of the [FunctionDeclaration] or [MethodDeclaration]. */ val methodName: String, + /** + * The signature of the [FunctionDeclaration]. We use a list of the FQN of the [Type]s of + * parameter. This is optional and if not specified, we perform the matching only based on + * the [methodName]. + */ val signature: List? = null ) - data class DFGEntry(val from: String, val to: String, val dfgType: String) + /** Represents a data flow entry. */ + data class DFGEntry( + /** + * The start of the DFG edge. Can be a parameter (`paramX`, where X is a number), or `base`. + */ + val from: String, + /** + * The end of the DFG edge. Can be a parameter (`paramX`, where X is a number), `base`, or + * the return value (`returnX`, where X is optional and a number indicating an index). + */ + val to: String, + /** + * A property which can give us more information. Currently, it's ignored, but it would make + * sense to add e.g. partial flows based on PR 1421. + */ + val dfgType: String + ) companion object { /** Generates a [DFGFunctionSummaries] object from the given [files]. */ From 9a9372d79a21dcabb883bdc253ead429e3d73c20 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Mon, 12 Feb 2024 17:51:05 +0100 Subject: [PATCH 08/29] Propagate DFG to calling expressions and arguments --- .../de/fraunhofer/aisec/cpg/helpers/Util.kt | 17 +++- .../de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 39 ++++++++ .../passes/inference/DFGFunctionSummaries.kt | 34 ++++++- .../enhancements/DFGFunctionSummariesTest.kt | 99 +++++++++++++++++++ cpg-core/src/test/resources/function-dfg.json | 13 +++ cpg-core/src/test/resources/function-dfg.yml | 8 ++ 6 files changed, 203 insertions(+), 7 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt index e50cb7348b..482908374c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt @@ -355,7 +355,12 @@ object Util { fun attachCallParameters(target: FunctionDeclaration, call: CallExpression) { // Add an incoming DFG edge from a member call's base to the method's receiver if (target is MethodDeclaration && call is MemberCallExpression && !call.isStatic) { - target.receiver?.let { receiver -> call.base?.addNextDFG(receiver) } + target.receiver?.let { receiver -> + call.base?.addNextDFG( + receiver, + mutableMapOf(Pair(Properties.CALLING_CONTEXT_IN, call)) + ) + } } // Connect the arguments to parameters @@ -370,12 +375,18 @@ object Util { if (param.isVariadic) { while (j < arguments.size) { // map all the following arguments to this variadic param - param.addPrevDFG(arguments[j]) + param.addPrevDFG( + arguments[j], + mutableMapOf(Pair(Properties.CALLING_CONTEXT_IN, call)) + ) j++ } break } else { - param.addPrevDFG(arguments[j]) + param.addPrevDFG( + arguments[j], + mutableMapOf(Pair(Properties.CALLING_CONTEXT_IN, call)) + ) } } j++ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index 57eff44012..404521b97c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -39,6 +39,8 @@ import de.fraunhofer.aisec.cpg.passes.order.DependsOn /** Adds the DFG edges for various types of nodes. */ @DependsOn(SymbolResolver::class) class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { + private val callsInferredFunctions = mutableListOf() + override fun accept(component: Component) { val inferDfgForUnresolvedCalls = config.inferenceConfiguration.inferDfgForUnresolvedSymbols val walker = IterativeGraphWalker() @@ -48,6 +50,40 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { for (tu in component.translationUnits) { walker.iterate(tu) } + + connectInferredCallArguments(config.functionSummaries) + } + + /** + * For inferred functions which have function summaries encoded, we connect the arguments to + * modified parameter to propagate the changes to the arguments out of the [FunctionDeclaration] + * again. + */ + private fun connectInferredCallArguments(functionSummaries: DFGFunctionSummaries) { + for (call in callsInferredFunctions) { + for (invoked in call.invokes.filter { it.isInferred }) { + val changedParams = + functionSummaries.functionToChangedParameters[invoked] ?: mapOf() + for ((param, _) in changedParams) { + if (param == (invoked as? MethodDeclaration)?.receiver) { + (call as? MemberCallExpression)?.base?.let { base -> + base.addPrevDFG( + param, + mutableMapOf(Pair(Properties.CALLING_CONTEXT_OUT, call)) + ) + // (base as? Reference)?.access = AccessValues.READWRITE + } + } else if (param is ParameterDeclaration) { + val arg = call.arguments[param.argumentIndex] + arg.addPrevDFG( + param, + mutableMapOf(Pair(Properties.CALLING_CONTEXT_OUT, call)) + ) + // (arg as? Reference)?.access = AccessValues.READWRITE + } + } + } + } } override fun cleanup() { @@ -430,6 +466,9 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { call.invokes.forEach { Util.attachCallParameters(it, call) call.addPrevDFG(it) + if (it.isInferred) { + callsInferredFunctions.add(call) + } } } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 5ad9d059ff..99807c11e0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -32,8 +32,8 @@ import com.fasterxml.jackson.module.kotlin.readValue import com.fasterxml.jackson.module.kotlin.registerKotlinModule import de.fraunhofer.aisec.cpg.TranslationConfiguration.Builder import de.fraunhofer.aisec.cpg.ancestors -import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.objectType import de.fraunhofer.aisec.cpg.graph.parseName import de.fraunhofer.aisec.cpg.graph.types.Type @@ -51,6 +51,16 @@ class DFGFunctionSummaries { /** Caches a mapping of the [FunctionDeclarationEntry] to a list of its [DFGEntry]. */ val functionToDFGEntryMap = mutableMapOf>() + /** + * Saves the information on which parameter(s) of a function are modified by the function. This + * is interesting since we need to add DFG edges between the modified parameter and the + * respective argument(s). For each [ParameterDeclaration] as well as the + * [MethodDeclaration.receiver] that has some incoming DFG-edge within this + * [FunctionDeclaration], we store all previous DFG nodes. + */ + val functionToChangedParameters = + mutableMapOf>>() + /** This function returns a list of [DataflowEntry] from the specified file. */ private fun addEntriesFromFile(file: File): Map> { val mapper = @@ -179,12 +189,28 @@ class DFGFunctionSummaries { if (entry.to.startsWith("param")) { try { val paramIndex = entry.to.removePrefix("param").toInt() - functionDeclaration.parameters[paramIndex] + val paramTo = functionDeclaration.parameters[paramIndex] + if (from != null) { + functionToChangedParameters + .computeIfAbsent(functionDeclaration) { mutableMapOf() } + .computeIfAbsent(paramTo) { mutableSetOf() } + .add(from) + } + paramTo } catch (e: NumberFormatException) { null } } else if (entry.to == "base") { - (functionDeclaration as? MethodDeclaration)?.receiver + val receiver = (functionDeclaration as? MethodDeclaration)?.receiver + if (from != null) { + if (receiver != null) { + functionToChangedParameters + .computeIfAbsent(functionDeclaration) { mutableMapOf() } + .computeIfAbsent(receiver, ::mutableSetOf) + .add(from) + } + } + receiver } else if (entry.to == "return") { functionDeclaration } else if (entry.to.startsWith("return")) { diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index 52b19a4a49..6372267785 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -25,10 +25,21 @@ */ package de.fraunhofer.aisec.cpg.enhancements +import de.fraunhofer.aisec.cpg.GraphExamples +import de.fraunhofer.aisec.cpg.InferenceConfiguration +import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.frontends.TestLanguage +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.builder.* +import de.fraunhofer.aisec.cpg.graph.edge.Properties +import de.fraunhofer.aisec.cpg.graph.functions +import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries import java.io.File import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertNotNull import kotlin.test.assertTrue class DFGFunctionSummariesTest { @@ -46,4 +57,92 @@ class DFGFunctionSummariesTest { assertEquals(jsonSummaries.functionToDFGEntryMap, yamlSummaries.functionToDFGEntryMap) } + + @Test + fun testPropagateArguments() { + val dfgTest = getDfgInferredCall() + assertNotNull(dfgTest) + + val main = dfgTest.functions["main"] + assertNotNull(main) + + val memcpy = dfgTest.functions["memcpy"] + assertNotNull(memcpy) + val param0 = memcpy.parameters[0] + val param1 = memcpy.parameters[1] + + val call = main.calls["memcpy"] + assertNotNull(call) + + val argA = call.arguments[0] + assertNotNull(argA) + + assertEquals(1, argA.nextDFG.size) + assertEquals(2, argA.prevDFG.size) + + val nextDfg = argA.nextDFGEdges.single() + assertEquals(call, nextDfg.getProperty(Properties.CALLING_CONTEXT_IN)) + assertEquals(param0, nextDfg.end) + + val prevDfgThroughFunction = + argA.prevDFGEdges.singleOrNull { + it.containsProperties(mapOf(Pair(Properties.CALLING_CONTEXT_OUT, call))) + } + assertNotNull(prevDfgThroughFunction) + assertEquals(param0, prevDfgThroughFunction.start) + + val variableDeclA = main.variables["a"] + assertNotNull(variableDeclA) + + val prevDfgNotThroughFunction = + argA.prevDFGEdges.singleOrNull { + !it.containsProperties(mapOf(Pair(Properties.CALLING_CONTEXT_OUT, call))) + } + assertNotNull(prevDfgNotThroughFunction) + assertEquals(variableDeclA, prevDfgNotThroughFunction.start) + + assertEquals(setOf(argA, param1), param0.prevDFG) + } + + companion object { + fun getDfgInferredCall(): TranslationResult { + val config = + TranslationConfiguration.builder() + .defaultPasses() + .registerLanguage(TestLanguage(".")) + .registerFunctionSummary(File("src/test/resources/function-dfg.yml")) + .inferenceConfiguration( + InferenceConfiguration.builder() + .inferDfgForUnresolvedCalls(true) + .inferFunctions(true) + .build() + ) + .build() + return GraphExamples.testFrontend(config).build { + translationResult { + translationUnit("DfgInferredCall.c") { + function("main", t("int")) { + body { + declare { + variable("a", t("char").pointer()) { literal(7, t("char")) } + } + + declare { + variable("b", t("char").pointer()) { literal(5, t("char")) } + } + + call("memcpy") { + ref("a") + ref("b") + literal(1, t("int")) + } + + returnStmt { ref("a") } + } + } + } + } + } + } + } } diff --git a/cpg-core/src/test/resources/function-dfg.json b/cpg-core/src/test/resources/function-dfg.json index df3cc5cda1..1a00d077f4 100644 --- a/cpg-core/src/test/resources/function-dfg.json +++ b/cpg-core/src/test/resources/function-dfg.json @@ -52,5 +52,18 @@ "dfgType": "full" } ] + }, + { + "functionDeclaration": { + "language": "de.fraunhofer.aisec.cpg.frontends.TestLanguage", + "methodName": "memcpy" + }, + "dataFlows": [ + { + "from": "param1", + "to": "param0", + "dfgType": "full" + } + ] } ] \ No newline at end of file diff --git a/cpg-core/src/test/resources/function-dfg.yml b/cpg-core/src/test/resources/function-dfg.yml index 5964799dbc..25489b6a3c 100644 --- a/cpg-core/src/test/resources/function-dfg.yml +++ b/cpg-core/src/test/resources/function-dfg.yml @@ -34,3 +34,11 @@ - from: param1 to: param0 dfgType: full + +- functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.TestLanguage + methodName: memcpy + dataFlows: + - from: param1 + to: param0 + dfgType: full \ No newline at end of file From e2e62364d7ab599c671d3149bfabe481426026d0 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Mon, 12 Feb 2024 19:42:49 +0100 Subject: [PATCH 09/29] Fix serialization --- .../aisec/cpg/graph/edge/PropertyEdgeConverterManager.kt | 4 ++++ .../de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdgeConverterManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdgeConverterManager.kt index 6c3f88862d..6a044bc58a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdgeConverterManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdgeConverterManager.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.graph.edge import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration.TemplateInitialization +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import java.util.function.Function /** @@ -49,6 +50,9 @@ class PropertyEdgeConverterManager private constructor() { addDeserializer("INSTANTIATION") { s: Any? -> if (s != null) TemplateInitialization.valueOf(s.toString()) else null } + addSerializer(CallExpression::class.java.name) { it.toString() } + addDeserializer("CALLING_CONTEXT_IN") { null } // TODO: Not supported yet + addDeserializer("CALLING_CONTEXT_OUT") { null } // TODO: Not supported yet } fun addSerializer(clazz: String, func: Function) { diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt index 412906fbfb..384fe34c76 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt @@ -41,12 +41,11 @@ import kotlin.reflect.jvm.javaField import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull -import org.junit.jupiter.api.Tag import org.neo4j.ogm.annotation.Relationship import org.neo4j.ogm.config.ObjectMapperFactory.objectMapper import picocli.CommandLine -@Tag("integration") +// @Tag("integration") class ApplicationTest { private fun createTranslationResult(): Pair { val topLevel = Paths.get("src").resolve("test").resolve("resources").toAbsolutePath() From 35dcf01350a6a1aa341b516f747ec584aefc5d7e Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 13 Feb 2024 09:16:51 +0100 Subject: [PATCH 10/29] integration test tag again --- .../kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt index 384fe34c76..df9f183096 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt @@ -45,7 +45,7 @@ import org.neo4j.ogm.annotation.Relationship import org.neo4j.ogm.config.ObjectMapperFactory.objectMapper import picocli.CommandLine -// @Tag("integration") +@Tag("integration") class ApplicationTest { private fun createTranslationResult(): Pair { val topLevel = Paths.get("src").resolve("test").resolve("resources").toAbsolutePath() From eed60091c3983a08c86ae535829cdd8cad384a22 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 13 Feb 2024 09:17:39 +0100 Subject: [PATCH 11/29] integration test tag again --- .../kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt index df9f183096..412906fbfb 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt @@ -41,6 +41,7 @@ import kotlin.reflect.jvm.javaField import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull +import org.junit.jupiter.api.Tag import org.neo4j.ogm.annotation.Relationship import org.neo4j.ogm.config.ObjectMapperFactory.objectMapper import picocli.CommandLine From d34e7abad0b0d3c9ebc3f65a2249c390cfa4809a Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 1 Mar 2024 15:10:54 +0100 Subject: [PATCH 12/29] New property edges --- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 40 +++++++++++++++++++ .../aisec/cpg/graph/edge/Dataflow.kt | 37 ++++++++++++++++- .../de/fraunhofer/aisec/cpg/helpers/Util.kt | 16 ++------ .../de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 18 +++------ .../enhancements/DFGFunctionSummariesTest.kt | 20 ++++++++-- 5 files changed, 101 insertions(+), 30 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index e1a1a857c0..93716bb483 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -257,6 +257,20 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider next.prevDFGEdges.add(edge) } + /** + * Adds a [Dataflow] edge from this node to [next], with the given [CallingContext] and + * [Granularity]. + */ + fun addNextDFGContext( + next: Node, + callingContext: CallingContext, + granularity: Granularity = default(), + ) { + val edge = ContextsensitiveDataflow(this, next, callingContext, granularity) + nextDFGEdges.add(edge) + next.prevDFGEdges.add(edge) + } + fun removeNextDFG(next: Node?) { if (next != null) { val thisRemove = @@ -279,6 +293,20 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider prev.nextDFGEdges.add(edge) } + /** + * Adds a [Dataflow] edge from [prev] node to this node, with the given [CallingContext] and + * [Granularity]. + */ + open fun addPrevDFGContext( + prev: Node, + callingContext: CallingContext, + granularity: Granularity = default(), + ) { + val edge = ContextsensitiveDataflow(prev, this, callingContext, granularity) + prevDFGEdges.add(edge) + prev.nextDFGEdges.add(edge) + } + fun addPrevCDG( prev: Node, properties: MutableMap = EnumMap(Properties::class.java) @@ -296,6 +324,18 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider prev.forEach { addPrevDFG(it, granularity) } } + /** + * Adds a [Dataflow] edge from all [prev] nodes to this node, with the given [CallingContext] + * and [Granularity]. + */ + fun addAllPrevDFGContext( + prev: Collection, + callingContext: CallingContext, + granularity: Granularity = full(), + ) { + prev.forEach { addPrevDFGContext(it, callingContext, granularity) } + } + fun addAllPrevPDG(prev: Collection, dependenceType: DependenceType) { addAllPrevPDGEdges(prev.map { PropertyEdge(it, this) }, dependenceType) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Dataflow.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Dataflow.kt index fb3dbf8199..498883b156 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Dataflow.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Dataflow.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TupleDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import org.neo4j.ogm.annotation.RelationshipEntity @@ -80,11 +81,43 @@ fun partial(target: Declaration?): PartialDataflowGranularity { * [granularity]. */ @RelationshipEntity -class Dataflow( +open class Dataflow( start: Node, end: Node, /** The granularity of this dataflow. */ - val granularity: Granularity = default(), + val granularity: Granularity = default() ) : PropertyEdge(start, end) { override val label: String = "DFG" } + +sealed interface CallingContext + +class CallingContextIn( + /** The call expression that affects this dataflow edge. */ + val callExpression: CallExpression +) : CallingContext + +class CallingContextOut( + /** The call expression that affects this dataflow edge. */ + val callExpression: CallExpression +) : CallingContext + +fun callIn(callExpression: CallExpression): CallingContext { + return CallingContextIn(callExpression) +} + +/** + * This edge class defines a flow of data between [start] and [end]. The flow must have a + * [callingContext] and can have a certain [granularity]. + */ +@RelationshipEntity +class ContextsensitiveDataflow( + start: Node, + end: Node, + /** The calling context affecting this dataflow. */ + val callingContext: CallingContext, + /** The granularity of this dataflow. */ + granularity: Granularity, +) : Dataflow(start, end, granularity) { + override val label: String = "DFG" +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt index 482908374c..613a450a60 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration +import de.fraunhofer.aisec.cpg.graph.edge.CallingContextIn import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation @@ -356,10 +357,7 @@ object Util { // Add an incoming DFG edge from a member call's base to the method's receiver if (target is MethodDeclaration && call is MemberCallExpression && !call.isStatic) { target.receiver?.let { receiver -> - call.base?.addNextDFG( - receiver, - mutableMapOf(Pair(Properties.CALLING_CONTEXT_IN, call)) - ) + call.base?.addNextDFGContext(receiver, CallingContextIn(call)) } } @@ -375,18 +373,12 @@ object Util { if (param.isVariadic) { while (j < arguments.size) { // map all the following arguments to this variadic param - param.addPrevDFG( - arguments[j], - mutableMapOf(Pair(Properties.CALLING_CONTEXT_IN, call)) - ) + param.addPrevDFGContext(arguments[j], CallingContextIn(call)) j++ } break } else { - param.addPrevDFG( - arguments[j], - mutableMapOf(Pair(Properties.CALLING_CONTEXT_IN, call)) - ) + param.addPrevDFGContext(arguments[j], CallingContextIn(call)) } } j++ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index 404521b97c..acae57be8d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.edge.CallingContextOut import de.fraunhofer.aisec.cpg.graph.edge.partial import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* @@ -66,19 +67,12 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { functionSummaries.functionToChangedParameters[invoked] ?: mapOf() for ((param, _) in changedParams) { if (param == (invoked as? MethodDeclaration)?.receiver) { - (call as? MemberCallExpression)?.base?.let { base -> - base.addPrevDFG( - param, - mutableMapOf(Pair(Properties.CALLING_CONTEXT_OUT, call)) - ) - // (base as? Reference)?.access = AccessValues.READWRITE - } + (call as? MemberCallExpression) + ?.base + ?.addPrevDFGContext(param, CallingContextOut(call)) } else if (param is ParameterDeclaration) { val arg = call.arguments[param.argumentIndex] - arg.addPrevDFG( - param, - mutableMapOf(Pair(Properties.CALLING_CONTEXT_OUT, call)) - ) + arg.addPrevDFGContext(param, CallingContextOut(call)) // (arg as? Reference)?.access = AccessValues.READWRITE } } @@ -465,7 +459,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { } else if (call.invokes.isNotEmpty()) { call.invokes.forEach { Util.attachCallParameters(it, call) - call.addPrevDFG(it) + call.addPrevDFGContext(it, CallingContextOut(call)) if (it.isInferred) { callsInferredFunctions.add(call) } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index 6372267785..e2ff16f9a2 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -32,7 +32,9 @@ import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.frontends.TestLanguage import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.builder.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties +import de.fraunhofer.aisec.cpg.graph.edge.CallingContextIn +import de.fraunhofer.aisec.cpg.graph.edge.CallingContextOut +import de.fraunhofer.aisec.cpg.graph.edge.ContextsensitiveDataflow import de.fraunhofer.aisec.cpg.graph.functions import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries @@ -79,14 +81,23 @@ class DFGFunctionSummariesTest { assertEquals(1, argA.nextDFG.size) assertEquals(2, argA.prevDFG.size) + /* + The flows should be as follows: + VariableDeclaration["a"] -> Reference["a" (argument of call)] -CallingContextIn-> ParameterDeclaration -CallingContextOut-> Reference["a" (return)] + */ val nextDfg = argA.nextDFGEdges.single() - assertEquals(call, nextDfg.getProperty(Properties.CALLING_CONTEXT_IN)) + assertEquals( + call, + ((nextDfg as? ContextsensitiveDataflow)?.callingContext as? CallingContextIn) + ?.callExpression + ) assertEquals(param0, nextDfg.end) val prevDfgThroughFunction = argA.prevDFGEdges.singleOrNull { - it.containsProperties(mapOf(Pair(Properties.CALLING_CONTEXT_OUT, call))) + ((it as? ContextsensitiveDataflow)?.callingContext as? CallingContextOut) + ?.callExpression != call } assertNotNull(prevDfgThroughFunction) assertEquals(param0, prevDfgThroughFunction.start) @@ -96,7 +107,8 @@ class DFGFunctionSummariesTest { val prevDfgNotThroughFunction = argA.prevDFGEdges.singleOrNull { - !it.containsProperties(mapOf(Pair(Properties.CALLING_CONTEXT_OUT, call))) + ((it as? ContextsensitiveDataflow)?.callingContext as? CallingContextOut) + ?.callExpression != call } assertNotNull(prevDfgNotThroughFunction) assertEquals(variableDeclA, prevDfgNotThroughFunction.start) From d72d9ccd6b017ad4a9c32092804c535e72a46f10 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 1 Mar 2024 15:49:58 +0100 Subject: [PATCH 13/29] Flows are now also control flow sensitive --- .../cpg/passes/ControlFlowSensitiveDFGPass.kt | 38 ++++++++++++++---- .../de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 2 +- .../enhancements/DFGFunctionSummariesTest.kt | 40 ++++++++++--------- 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt index 9a70abaee7..35588bbc74 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt @@ -28,9 +28,7 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.partial +import de.fraunhofer.aisec.cpg.graph.edge.* import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement @@ -122,11 +120,15 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass ) } } else { - key.addAllPrevDFG( - value.elements.filterNot { - (it is VariableDeclaration || it is ParameterDeclaration) && key == it + value.elements.forEach { + if ((it is VariableDeclaration || it is ParameterDeclaration) && key == it) { + // Nothing to do + } else if (it in edgePropertiesMap && edgePropertiesMap[it] is CallingContext) { + key.addPrevDFGContext(it, (edgePropertiesMap[it] as CallingContext)) + } else { + key.addPrevDFG(it) } - ) + } } } } @@ -388,6 +390,26 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass currentNode, PowersetLattice(identitySetOf(currentNode)) ) + } else if (currentNode is CallExpression) { + for (invoked in + currentNode.invokes.filter { + it in ctx.config.functionSummaries.functionToChangedParameters + }) { + val changedParams = + ctx.config.functionSummaries.functionToChangedParameters[invoked] ?: mapOf() + for ((param, _) in changedParams) { + if (param == (invoked as? MethodDeclaration)?.receiver) { + doubleState.declarationsState[ + ((currentNode as? MemberCallExpression)?.base as? Reference) + ?.refersTo] = PowersetLattice(identitySetOf(param)) + } else if (param is ParameterDeclaration) { + val arg = currentNode.arguments[param.argumentIndex] + doubleState.declarationsState[(arg as? Reference)?.refersTo] = + PowersetLattice(identitySetOf(param)) + } + edgePropertiesMap[param] = CallingContextOut(currentNode) + } + } } else { doubleState.declarationsState.push( currentNode, @@ -397,6 +419,8 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass return state } + val edgePropertiesMap = mutableMapOf() + /** * Checks if the node performs an operation and an assignment at the same time e.g. with the * operators +=, -=, *=, ... diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index acae57be8d..c771d94c4d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -52,7 +52,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { walker.iterate(tu) } - connectInferredCallArguments(config.functionSummaries) + // connectInferredCallArguments(config.functionSummaries) } /** diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index e2ff16f9a2..6aad4f8a73 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -37,6 +37,8 @@ import de.fraunhofer.aisec.cpg.graph.edge.CallingContextOut import de.fraunhofer.aisec.cpg.graph.edge.ContextsensitiveDataflow import de.fraunhofer.aisec.cpg.graph.functions import de.fraunhofer.aisec.cpg.graph.pointer +import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries import java.io.File import kotlin.test.Test @@ -78,14 +80,14 @@ class DFGFunctionSummariesTest { val argA = call.arguments[0] assertNotNull(argA) - - assertEquals(1, argA.nextDFG.size) - assertEquals(2, argA.prevDFG.size) /* The flows should be as follows: VariableDeclaration["a"] -> Reference["a" (argument of call)] -CallingContextIn-> ParameterDeclaration -CallingContextOut-> Reference["a" (return)] */ + assertEquals(1, argA.nextDFG.size) + assertEquals(1, argA.prevDFG.size) + val nextDfg = argA.nextDFGEdges.single() assertEquals( call, @@ -94,26 +96,26 @@ class DFGFunctionSummariesTest { ) assertEquals(param0, nextDfg.end) - val prevDfgThroughFunction = - argA.prevDFGEdges.singleOrNull { - ((it as? ContextsensitiveDataflow)?.callingContext as? CallingContextOut) - ?.callExpression != call - } - assertNotNull(prevDfgThroughFunction) - assertEquals(param0, prevDfgThroughFunction.start) + val variableA = main.variables["a"] + assertNotNull(variableA) + assertEquals(mutableSetOf(variableA), argA.prevDFG) + + val prevDfgOfParam0 = param0.prevDFGEdges.singleOrNull { it !is ContextsensitiveDataflow } + assertNotNull(prevDfgOfParam0) + assertEquals(param1, prevDfgOfParam0.start) - val variableDeclA = main.variables["a"] - assertNotNull(variableDeclA) + val returnA = main.allChildren().singleOrNull()?.returnValue as? Reference + assertNotNull(returnA) - val prevDfgNotThroughFunction = - argA.prevDFGEdges.singleOrNull { + assertEquals(mutableSetOf(returnA), param0.nextDFG) + + // Check that also the CallingContext property is set correctly + val nextDfgOfParam0 = + param0.nextDFGEdges.singleOrNull { ((it as? ContextsensitiveDataflow)?.callingContext as? CallingContextOut) - ?.callExpression != call + ?.callExpression == call } - assertNotNull(prevDfgNotThroughFunction) - assertEquals(variableDeclA, prevDfgNotThroughFunction.start) - - assertEquals(setOf(argA, param1), param0.prevDFG) + assertEquals(returnA, nextDfgOfParam0?.end) } companion object { From 2eeb5de1d40dd1fe773fb2a989f6ad419ed0351a Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 12 Mar 2024 16:48:02 +0100 Subject: [PATCH 14/29] Copy property edges --- .../aisec/cpg/graph/ExpressionBuilder.kt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt index d4e8209bdf..4969796286 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.frontends.HasShortCircuitOperators import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.Node.Companion.EMPTY_NAME import de.fraunhofer.aisec.cpg.graph.NodeBuilder.log +import de.fraunhofer.aisec.cpg.graph.edge.ContextsensitiveDataflow import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.types.ProblemType @@ -561,8 +562,17 @@ fun Literal.duplicate(implicit: Boolean): Literal { duplicate.comment = this.comment duplicate.file = this.file duplicate.name = this.name.clone() - duplicate.nextDFG = this.nextDFG - duplicate.prevDFG = this.prevDFG + for (next in this.nextDFGEdges) { + if (next is ContextsensitiveDataflow) + duplicate.addNextDFGContext(next.end, next.callingContext, next.granularity) + else duplicate.addNextDFG(next.end, next.granularity) + } + for (next in this.prevDFGEdges) { + if (next is ContextsensitiveDataflow) + duplicate.addPrevDFGContext(next.start, next.callingContext, next.granularity) + else duplicate.addNextDFG(next.start, next.granularity) + } + // TODO: This loses the properties of the edges. duplicate.nextEOG = this.nextEOG duplicate.prevEOG = this.prevEOG duplicate.isImplicit = implicit From 8e0d577e49d01302f0dc8ca35b652d3e99bfdd6d Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 12 Mar 2024 16:48:14 +0100 Subject: [PATCH 15/29] Remove unused method --- .../de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index c771d94c4d..47cf7ae892 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -51,33 +51,6 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { for (tu in component.translationUnits) { walker.iterate(tu) } - - // connectInferredCallArguments(config.functionSummaries) - } - - /** - * For inferred functions which have function summaries encoded, we connect the arguments to - * modified parameter to propagate the changes to the arguments out of the [FunctionDeclaration] - * again. - */ - private fun connectInferredCallArguments(functionSummaries: DFGFunctionSummaries) { - for (call in callsInferredFunctions) { - for (invoked in call.invokes.filter { it.isInferred }) { - val changedParams = - functionSummaries.functionToChangedParameters[invoked] ?: mapOf() - for ((param, _) in changedParams) { - if (param == (invoked as? MethodDeclaration)?.receiver) { - (call as? MemberCallExpression) - ?.base - ?.addPrevDFGContext(param, CallingContextOut(call)) - } else if (param is ParameterDeclaration) { - val arg = call.arguments[param.argumentIndex] - arg.addPrevDFGContext(param, CallingContextOut(call)) - // (arg as? Reference)?.access = AccessValues.READWRITE - } - } - } - } } override fun cleanup() { From a017f3975ca7bf754e592cad76e85c8794957aec Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 12 Mar 2024 16:52:34 +0100 Subject: [PATCH 16/29] Re-add connector method if ControlFlowSensitiveDFG is not executed --- .../de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index 47cf7ae892..f4d0ec0bf7 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -51,6 +51,34 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { for (tu in component.translationUnits) { walker.iterate(tu) } + if (config.registeredPasses.any { ControlFlowSensitiveDFGPass::class !in it }) { + connectInferredCallArguments(config.functionSummaries) + } + } + + /** + * For inferred functions which have function summaries encoded, we connect the arguments to + * modified parameter to propagate the changes to the arguments out of the [FunctionDeclaration] + * again. + */ + private fun connectInferredCallArguments(functionSummaries: DFGFunctionSummaries) { + for (call in callsInferredFunctions) { + for (invoked in call.invokes.filter { it.isInferred }) { + val changedParams = + functionSummaries.functionToChangedParameters[invoked] ?: mapOf() + for ((param, _) in changedParams) { + if (param == (invoked as? MethodDeclaration)?.receiver) { + (call as? MemberCallExpression) + ?.base + ?.addPrevDFGContext(param, CallingContextOut(call)) + } else if (param is ParameterDeclaration) { + val arg = call.arguments[param.argumentIndex] + arg.addPrevDFGContext(param, CallingContextOut(call)) + // (arg as? Reference)?.access = AccessValues.READWRITE + } + } + } + } } override fun cleanup() { From bbce365c5396f23e82f7c821709b1b46b2694265 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Thu, 14 Mar 2024 15:15:32 +0100 Subject: [PATCH 17/29] Fix --- .../src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index f4d0ec0bf7..c14625886f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -51,7 +51,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { for (tu in component.translationUnits) { walker.iterate(tu) } - if (config.registeredPasses.any { ControlFlowSensitiveDFGPass::class !in it }) { + if (config.registeredPasses.all { ControlFlowSensitiveDFGPass::class !in it }) { connectInferredCallArguments(config.functionSummaries) } } From 937734eeb256f084ad3d94bc31c6ffbdca0b7f26 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Thu, 14 Mar 2024 15:37:14 +0100 Subject: [PATCH 18/29] Add another test --- .../aisec/cpg/TranslationConfiguration.kt | 11 +++ .../de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 5 +- .../enhancements/DFGFunctionSummariesTest.kt | 76 ++++++++++++++++++- 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index 425b7f0907..556d0dc2aa 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -401,6 +401,17 @@ private constructor( return this } + inline fun > unregisterPass(): Builder { + unregisterPass(P::class) + return this + } + + /** Unregister a [Pass]. */ + fun unregisterPass(passType: KClass>): Builder { + passes.remove(passType) + return this + } + /** Register an additional [Pass]. */ fun registerPass(passType: KClass>): Builder { passes.add(passType) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index c14625886f..3a68fb04c1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -74,7 +74,10 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { } else if (param is ParameterDeclaration) { val arg = call.arguments[param.argumentIndex] arg.addPrevDFGContext(param, CallingContextOut(call)) - // (arg as? Reference)?.access = AccessValues.READWRITE + (arg as? Reference)?.let { + it.access = AccessValues.READWRITE + it.refersTo?.let { it1 -> it.addNextDFG(it1) } + } } } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index 6aad4f8a73..b8efe1f1b6 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -39,6 +39,7 @@ import de.fraunhofer.aisec.cpg.graph.functions import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import de.fraunhofer.aisec.cpg.passes.ControlFlowSensitiveDFGPass import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries import java.io.File import kotlin.test.Test @@ -118,8 +119,72 @@ class DFGFunctionSummariesTest { assertEquals(returnA, nextDfgOfParam0?.end) } + @Test + fun testPropagateArgumentsControlFlowInsensitive() { + val dfgTest = getDfgInferredCall { unregisterPass() } + assertNotNull(dfgTest) + + val main = dfgTest.functions["main"] + assertNotNull(main) + + val memcpy = dfgTest.functions["memcpy"] + assertNotNull(memcpy) + val param0 = memcpy.parameters[0] + val param1 = memcpy.parameters[1] + + val call = main.calls["memcpy"] + assertNotNull(call) + + val argA = call.arguments[0] + assertNotNull(argA) + /* + The flows should be as follows: + VariableDeclaration["a"] -> { Reference["a" (argument of call)], Reference["a" (return)] } + Reference["a" (argument of call)] -CallingContextIn-> ParameterDeclaration -CallingContextOut-> Reference["a" (argument of call)] -> VariableDeclaration["a"] + */ + + assertEquals(2, argA.nextDFG.size) + assertEquals(2, argA.prevDFG.size) + + val nextDfg = + argA.nextDFGEdges.singleOrNull { + ((it as? ContextsensitiveDataflow)?.callingContext as? CallingContextIn) + ?.callExpression == call + } + assertNotNull(nextDfg) + assertEquals(param0, nextDfg.end) + + val variableA = main.variables["a"] + assertNotNull(variableA) + assertEquals(mutableSetOf(variableA, param0), argA.prevDFG) + + val prevDfgOfParam0 = param0.prevDFGEdges.singleOrNull { it !is ContextsensitiveDataflow } + assertNotNull(prevDfgOfParam0) + assertEquals(param1, prevDfgOfParam0.start) + + val returnA = main.allChildren().singleOrNull()?.returnValue as? Reference + assertNotNull(returnA) + + assertEquals(mutableSetOf(argA), param0.nextDFG) + + assertEquals(mutableSetOf(returnA, argA), variableA.nextDFG) + + // Check that also the CallingContext property is set correctly + val nextDfgOfParam0 = + param0.nextDFGEdges.singleOrNull { + ((it as? ContextsensitiveDataflow)?.callingContext as? CallingContextOut) + ?.callExpression == call + } + assertEquals(argA, nextDfgOfParam0?.end) + } + companion object { - fun getDfgInferredCall(): TranslationResult { + fun getDfgInferredCall( + customConfig: TranslationConfiguration.Builder.() -> TranslationConfiguration.Builder = + { + this + } + ): TranslationResult { val config = TranslationConfiguration.builder() .defaultPasses() @@ -131,7 +196,16 @@ class DFGFunctionSummariesTest { .inferFunctions(true) .build() ) + .customConfig() .build() + /* + int main() { + char *a = 7; + char *b = 5; + memcpy(a, b, 1); + return a; + } + */ return GraphExamples.testFrontend(config).build { translationResult { translationUnit("DfgInferredCall.c") { From aac8bbc0aa0ab1bd663c21dd5993d193ed6cec2a Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Thu, 14 Mar 2024 15:57:24 +0100 Subject: [PATCH 19/29] Coverage++ --- .../aisec/cpg/graph/ExpressionBuilderTest.kt | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt new file mode 100644 index 0000000000..03298eb79f --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph + +import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration +import de.fraunhofer.aisec.cpg.graph.edge.CallingContextIn +import de.fraunhofer.aisec.cpg.graph.edge.ContextsensitiveDataflow +import de.fraunhofer.aisec.cpg.graph.edge.PartialDataflowGranularity +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class ExpressionBuilderTest { + @Test + fun testDuplicateWithDFGProperties() { + val node1 = Literal() + val node2 = Reference() + val granularity = PartialDataflowGranularity(FieldDeclaration()) + val callingContextIn = CallingContextIn(CallExpression()) + node1.addPrevDFGContext(node2, callingContextIn, granularity) + + val clone = node1.duplicate(false) + val clonedPrevDFG = clone.prevDFGEdges.single() + assertTrue(clonedPrevDFG is ContextsensitiveDataflow) + assertEquals(callingContextIn, clonedPrevDFG.callingContext) + assertEquals(granularity, clonedPrevDFG.granularity) + + assertEquals(setOf(node1, clone), node2.nextDFG) + } + + @Test + fun testDuplicateWithDFGProperties2() { + val node1 = Literal() + val node2 = Reference() + val granularity = PartialDataflowGranularity(FieldDeclaration()) + val callingContextIn = CallingContextIn(CallExpression()) + node1.addNextDFGContext(node2, callingContextIn, granularity) + + val clone = node1.duplicate(false) + val clonedPrevDFG = clone.nextDFGEdges.single() + assertTrue(clonedPrevDFG is ContextsensitiveDataflow) + assertEquals(callingContextIn, clonedPrevDFG.callingContext) + assertEquals(granularity, clonedPrevDFG.granularity) + + assertEquals(setOf(node1, clone), node2.prevDFG) + } +} From 28bc31e9779d3846e8f40dea93524c5af6b31e68 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Thu, 14 Mar 2024 16:38:25 +0100 Subject: [PATCH 20/29] Try more tests, problems with ContextProvider --- .../enhancements/DFGFunctionSummariesTest.kt | 45 +++++++++++++++++++ cpg-core/src/test/resources/function-dfg2.yml | 36 +++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 cpg-core/src/test/resources/function-dfg2.yml diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index b8efe1f1b6..db16e05afc 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -63,6 +63,51 @@ class DFGFunctionSummariesTest { assertEquals(jsonSummaries.functionToDFGEntryMap, yamlSummaries.functionToDFGEntryMap) } + @Test + fun testMatching() { + val code = + GraphExamples.testFrontend( + TranslationConfiguration.builder() + .defaultPasses() + .registerLanguage(TestLanguage(".")) + .registerFunctionSummary(File("src/test/resources/function-dfg2.yml")) + .inferenceConfiguration( + InferenceConfiguration.builder() + .inferDfgForUnresolvedCalls(true) + .inferFunctions(true) + .build() + ) + .build() + ) + .build { + translationResult { + translationUnit("DfgInferredCall.c") { + function("main", t("int")) { + body { + declare { + variable("a", t("test.List")) { construct("test.List") } + } + memberCall("addAll", ref("a", t("test.List"))) { + literal(1, t("int")) + construct("test.Object") + } + + returnStmt { ref("a") } + } + } + } + } + } + + val methodAddAllTwoArgs = code.methods["addAll"] + assertNotNull(methodAddAllTwoArgs) + assertEquals(2, methodAddAllTwoArgs.parameters.size) + assertEquals( + setOf(methodAddAllTwoArgs.receiver!!), + methodAddAllTwoArgs.parameters[1].nextDFG + ) + } + @Test fun testPropagateArguments() { val dfgTest = getDfgInferredCall() diff --git a/cpg-core/src/test/resources/function-dfg2.yml b/cpg-core/src/test/resources/function-dfg2.yml new file mode 100644 index 0000000000..1ba2d35878 --- /dev/null +++ b/cpg-core/src/test/resources/function-dfg2.yml @@ -0,0 +1,36 @@ +- functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.TestLanguage + methodName: test.List.addAll + signature: + - int + - test.Object + dataFlows: + - from: param1 + to: base + dfgType: full + +- functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.TestLanguage + methodName: test.List.addAll + signature: + - test.Object + dataFlows: + - from: param0 + to: base + dfgType: full + +- functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.TestLanguage + methodName: test.List.add + dataFlows: + - from: param0 + to: base + dfgType: full + +- functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.TestLanguage + methodName: memcpy + dataFlows: + - from: param1 + to: param0 + dfgType: full \ No newline at end of file From 548cffbbd0392f9da515702836ed0c1497b382e6 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 15 Mar 2024 07:54:21 +0100 Subject: [PATCH 21/29] Fix problem with types --- .../aisec/cpg/passes/inference/DFGFunctionSummaries.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 99807c11e0..74c6032b6a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -115,7 +115,9 @@ class DFGFunctionSummaries { methodName.lastPartsMatch(it.methodName) && (it.signature == null || functionDecl.hasSignature( - it.signature.map { signatureType -> language.objectType(signatureType) } + it.signature.map { signatureType -> + functionDecl.objectType(signatureType) + } )) } return if (matchingEntries.size == 1) { From 7501ac83ae3ed76cf40908247ae954872289554d Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 15 Mar 2024 11:27:44 +0100 Subject: [PATCH 22/29] Handle supertypes properly, extend test --- .../passes/inference/DFGFunctionSummaries.kt | 39 ++++++++--- .../enhancements/DFGFunctionSummariesTest.kt | 67 +++++++++++++++++-- 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 74c6032b6a..31d429dab7 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -37,6 +37,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.objectType import de.fraunhofer.aisec.cpg.graph.parseName import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.isDerivedFrom import java.io.File /** @@ -111,14 +112,36 @@ class DFGFunctionSummaries { // to match to the one of the FunctionDeclaration, null indicates that we accept everything. val matchingEntries = functionToDFGEntryMap.keys.filter { - it.language == languageName && - methodName.lastPartsMatch(it.methodName) && - (it.signature == null || - functionDecl.hasSignature( - it.signature.map { signatureType -> - functionDecl.objectType(signatureType) - } - )) + // The language has to match otherwise the remaining comparison is useless + if (it.language == languageName) { + // Split the name if we have a FQN + val entryMethodName = language.parseName(it.methodName) + val entryRecord = + entryMethodName.parent?.let { + functionDecl.objectType(entryMethodName.parent) + } + methodName.lastPartsMatch( + entryMethodName.localName + ) && // The local name has to match + // If it's a method, the record declaration has to be compatible with the + // type of the entry's record declaration. We take the type of the method + // name's parent and generate a type from it. We then check if this type is + // a supertype + (entryRecord == null || + (functionDecl as? MethodDeclaration) + ?.recordDeclaration + ?.toType() + ?.isDerivedFrom(entryRecord) == true) && + // The parameter types have to match + (it.signature == null || + functionDecl.hasSignature( + it.signature.map { signatureType -> + functionDecl.objectType(signatureType) + } + )) + } else { + false + } } return if (matchingEntries.size == 1) { // Only one entry => We take this one. diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index db16e05afc..647c5445ed 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -39,8 +39,10 @@ import de.fraunhofer.aisec.cpg.graph.functions import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import de.fraunhofer.aisec.cpg.graph.types.recordDeclaration import de.fraunhofer.aisec.cpg.passes.ControlFlowSensitiveDFGPass import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries +import de.fraunhofer.aisec.cpg.passes.inference.startInference import java.io.File import kotlin.test.Test import kotlin.test.assertEquals @@ -91,6 +93,30 @@ class DFGFunctionSummariesTest { literal(1, t("int")) construct("test.Object") } + val specialListType = t("test.SpecialList") + ctx?.let { + val recordDecl = + specialListType + .startInference(it) + ?.inferRecordDeclaration( + specialListType, + this@translationUnit + ) + specialListType.recordDeclaration = recordDecl + val listType = t("test.List") + recordDecl?.addSuperClass(listType) + specialListType.superTypes.add(listType) + } + + memberCall("addAll", ref("a", specialListType)) { + literal(1, t("int")) + construct("test.Object") + } + + memberCall("addAll", ref("a", t("random.Type"))) { + literal(1, t("int")) + construct("test.Object") + } returnStmt { ref("a") } } @@ -99,12 +125,43 @@ class DFGFunctionSummariesTest { } } - val methodAddAllTwoArgs = code.methods["addAll"] - assertNotNull(methodAddAllTwoArgs) - assertEquals(2, methodAddAllTwoArgs.parameters.size) + // Explicitly specified + val listAddAllTwoArgs = code.methods["test.List.addAll"] + assertNotNull(listAddAllTwoArgs) + assertEquals(2, listAddAllTwoArgs.parameters.size) + assertEquals( + setOf(listAddAllTwoArgs.receiver!!), + listAddAllTwoArgs.parameters[1].nextDFG + ) + // No flow from param0 or receiver specified => Should be empty and differ from default + // behavior + assertEquals(setOf(), listAddAllTwoArgs.parameters[0].nextDFG) + assertEquals(setOf(), listAddAllTwoArgs.prevDFG) + + // Specified by parent class' method List.addAll + val specialListAddAllTwoArgs = code.methods["test.SpecialList.addAll"] + assertNotNull(specialListAddAllTwoArgs) + assertEquals(2, specialListAddAllTwoArgs.parameters.size) + assertEquals( + setOf(specialListAddAllTwoArgs.receiver!!), + specialListAddAllTwoArgs.parameters[1].nextDFG + ) + // No flow from param0 or receiver specified => Should be empty and differ from default + // behavior + assertEquals(setOf(), specialListAddAllTwoArgs.parameters[0].nextDFG) + assertEquals(setOf(), specialListAddAllTwoArgs.prevDFG) + + // Not specified => Default behavior (param0 and param1 and receiver to method declaration). + val randomTypeAddAllTwoArgs = code.methods["random.Type.addAll"] + assertNotNull(randomTypeAddAllTwoArgs) + assertEquals(2, randomTypeAddAllTwoArgs.parameters.size) assertEquals( - setOf(methodAddAllTwoArgs.receiver!!), - methodAddAllTwoArgs.parameters[1].nextDFG + setOf( + randomTypeAddAllTwoArgs.parameters[1], + randomTypeAddAllTwoArgs.parameters[0], + randomTypeAddAllTwoArgs.receiver!! + ), + randomTypeAddAllTwoArgs.prevDFG ) } From f98479f2a44189bc7d6276b676b4add0e51d3347 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 15 Mar 2024 15:02:07 +0100 Subject: [PATCH 23/29] More fixes, more tests, more types --- .../passes/inference/DFGFunctionSummaries.kt | 41 +++++++- .../enhancements/DFGFunctionSummariesTest.kt | 95 ++++++++++++++++--- cpg-core/src/test/resources/function-dfg2.yml | 28 ++++++ 3 files changed, 147 insertions(+), 17 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 31d429dab7..431d78c82c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -153,7 +153,9 @@ class DFGFunctionSummaries { /* There are multiple matching entries. We use the following routine: * First, we filter for existing signatures. * Second, we filter for the most precise class. - * If there are still multiple options, we take the longest signature and hope it's the most precise one. + * If there are still multiple options, we take the longest signature. + * If this also didn't help to get a precise result, we iterate through the parameters and take the most precise one. We start with index 0 and count upwards, so if param0 leads to a single result, we're done and other entries won't be considered even if all the remaining parameters are more precise or whatever. + * If nothing helped to get a unique entry, we pick the first remaining entry and hope it's the most precise one. */ val typeEntryList = matchingEntries @@ -161,12 +163,12 @@ class DFGFunctionSummaries { .map { Pair( language.parseName(it.methodName).parent?.let { it1 -> - language?.objectType(it1) + functionDecl.objectType(it1) }, it ) } - val mostPreciseClassEntries = mutableListOf() + var mostPreciseClassEntries = mutableListOf() var mostPreciseType = typeEntryList.first().first var superTypes = mostPreciseType?.ancestors?.map { it.type } ?: setOf() for (typeEntry in typeEntryList) { @@ -179,10 +181,39 @@ class DFGFunctionSummaries { superTypes = mostPreciseType?.ancestors?.map { it.type } ?: setOf() } } + val maxSignature = mostPreciseClassEntries.mapNotNull { it.signature?.size }.max() if (mostPreciseClassEntries.size > 1) { - mostPreciseClassEntries.sortByDescending { it.signature?.size ?: 0 } + mostPreciseClassEntries = + mostPreciseClassEntries + .filter { it.signature?.size == maxSignature } + .toMutableList() } - functionToDFGEntryMap[matchingEntries.first()] + // Filter parameter types. We start with parameter 0 and continue. Let's hope we remove + // some entries here. + var argIndex = 0 + while (mostPreciseClassEntries.size > 1 && argIndex < maxSignature) { + mostPreciseType = + mostPreciseClassEntries.first().signature?.get(argIndex)?.let { + functionDecl.objectType(it) + } + superTypes = mostPreciseType?.ancestors?.map { it.type } ?: setOf() + val newMostPrecise = mutableListOf() + for (entry in mostPreciseClassEntries) { + val currentType = + entry.signature?.get(argIndex)?.let { functionDecl.objectType(it) } + if (currentType == mostPreciseType) { + newMostPrecise.add(entry) + } else if (currentType in superTypes) { + newMostPrecise.clear() + newMostPrecise.add(entry) + mostPreciseType = currentType + superTypes = mostPreciseType?.ancestors?.map { it.type } ?: setOf() + } + } + argIndex++ + mostPreciseClassEntries = newMostPrecise + } + functionToDFGEntryMap[mostPreciseClassEntries.first()] } else { null } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index 647c5445ed..b113f15269 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -86,13 +86,22 @@ class DFGFunctionSummariesTest { translationUnit("DfgInferredCall.c") { function("main", t("int")) { body { - declare { - variable("a", t("test.List")) { construct("test.List") } - } - memberCall("addAll", ref("a", t("test.List"))) { - literal(1, t("int")) - construct("test.Object") + // We need three types with a type hierarchy. + val objectType = t("test.Object") + val listType = t("test.List") + ctx?.let { + val recordDecl = + listType + .startInference(it) + ?.inferRecordDeclaration( + listType, + this@translationUnit + ) + listType.recordDeclaration = recordDecl + recordDecl?.addSuperClass(objectType) + listType.superTypes.add(objectType) } + val specialListType = t("test.SpecialList") ctx?.let { val recordDecl = @@ -103,22 +112,50 @@ class DFGFunctionSummariesTest { this@translationUnit ) specialListType.recordDeclaration = recordDecl - val listType = t("test.List") recordDecl?.addSuperClass(listType) specialListType.superTypes.add(listType) } - memberCall("addAll", ref("a", specialListType)) { + val verySpecialListType = t("test.VerySpecialList") + ctx?.let { + val recordDecl = + specialListType + .startInference(it) + ?.inferRecordDeclaration( + specialListType, + this@translationUnit + ) + specialListType.recordDeclaration = recordDecl + recordDecl?.addSuperClass(listType) + specialListType.superTypes.add(listType) + } + + memberCall("addAll", construct("test.VerySpecialList")) { literal(1, t("int")) construct("test.Object") } - memberCall("addAll", ref("a", t("random.Type"))) { + memberCall("addAll", construct("test.SpecialList")) { + literal(1, t("int")) + construct("test.List") + } + + memberCall("addAll", construct("test.SpecialList")) { + literal(1, t("int")) + construct("test.Object") + } + + memberCall("addAll", construct("test.List")) { + literal(1, t("int")) + construct("test.Object") + } + + memberCall("addAll", construct("random.Type")) { literal(1, t("int")) construct("test.Object") } - returnStmt { ref("a") } + returnStmt { literal(0, t("int")) } } } } @@ -138,8 +175,11 @@ class DFGFunctionSummariesTest { assertEquals(setOf(), listAddAllTwoArgs.parameters[0].nextDFG) assertEquals(setOf(), listAddAllTwoArgs.prevDFG) - // Specified by parent class' method List.addAll - val specialListAddAllTwoArgs = code.methods["test.SpecialList.addAll"] + // Specified by parent class' method List.addAll(int, Object) + val specialListAddAllTwoArgs = + code.methods("test.SpecialList.addAll").first { + it.parameters[1].type.name.lastPartsMatch("test.Object") + } assertNotNull(specialListAddAllTwoArgs) assertEquals(2, specialListAddAllTwoArgs.parameters.size) assertEquals( @@ -151,6 +191,37 @@ class DFGFunctionSummariesTest { assertEquals(setOf(), specialListAddAllTwoArgs.parameters[0].nextDFG) assertEquals(setOf(), specialListAddAllTwoArgs.prevDFG) + // Specified by parent class' method List.addAll(int, List) + val specialListAddAllSpecializedArgs = + code.methods("test.SpecialList.addAll").first { + it.parameters[1].type.name.lastPartsMatch("test.List") + } + assertNotNull(specialListAddAllSpecializedArgs) + assertEquals(2, specialListAddAllSpecializedArgs.parameters.size) + // Very weird data flow specified: receiver to param0 and param1 to return. + assertEquals( + setOf(specialListAddAllSpecializedArgs.parameters[0]), + specialListAddAllSpecializedArgs.receiver?.nextDFG ?: setOf() + ) + assertEquals( + setOf(specialListAddAllSpecializedArgs), + specialListAddAllSpecializedArgs.parameters[1].nextDFG + ) + + // Specified by parent class' method List.addAll(int, List) + val verySpecialListAddAllSpecializedArgs = code.methods["test.VerySpecialList.addAll"] + assertNotNull(verySpecialListAddAllSpecializedArgs) + assertEquals(2, verySpecialListAddAllSpecializedArgs.parameters.size) + // Very weird data flow specified: receiver to param0 and param1 to return. + assertEquals( + setOf(verySpecialListAddAllSpecializedArgs.parameters[0]), + verySpecialListAddAllSpecializedArgs.receiver?.nextDFG ?: setOf() + ) + assertEquals( + setOf(verySpecialListAddAllSpecializedArgs), + verySpecialListAddAllSpecializedArgs.parameters[1].nextDFG + ) + // Not specified => Default behavior (param0 and param1 and receiver to method declaration). val randomTypeAddAllTwoArgs = code.methods["random.Type.addAll"] assertNotNull(randomTypeAddAllTwoArgs) diff --git a/cpg-core/src/test/resources/function-dfg2.yml b/cpg-core/src/test/resources/function-dfg2.yml index 1ba2d35878..b13bdebae9 100644 --- a/cpg-core/src/test/resources/function-dfg2.yml +++ b/cpg-core/src/test/resources/function-dfg2.yml @@ -13,7 +13,35 @@ language: de.fraunhofer.aisec.cpg.frontends.TestLanguage methodName: test.List.addAll signature: + - int + - test.List + dataFlows: + - from: base + to: param0 + dfgType: full + - from: param1 + to: return + dfgType: full + +- functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.TestLanguage + methodName: test.VerySpecialList.addAll + signature: + - int - test.Object + dataFlows: + - from: base + to: param0 + dfgType: full + - from: param1 + to: return + dfgType: full + +- functionDeclaration: + language: de.fraunhofer.aisec.cpg.frontends.TestLanguage + methodName: test.List.addAll + signature: + - test.List dataFlows: - from: param0 to: base From 8d1b68687dca84e03dc5f9a42a6e8ac2648ec97f Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 15 Mar 2024 15:17:33 +0100 Subject: [PATCH 24/29] Documentation --- .../cpg/enhancements/DFGFunctionSummariesTest.kt | 13 +++++++++---- docs/docs/CPG/specs/dfg-function-summaries.md | 10 +++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index b113f15269..9afa3b3022 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -162,7 +162,8 @@ class DFGFunctionSummariesTest { } } - // Explicitly specified + // Explicitly specified. Easiest case. Base class, directly specified. Overloaded things + // don't match. Child entries don't match. val listAddAllTwoArgs = code.methods["test.List.addAll"] assertNotNull(listAddAllTwoArgs) assertEquals(2, listAddAllTwoArgs.parameters.size) @@ -175,7 +176,8 @@ class DFGFunctionSummariesTest { assertEquals(setOf(), listAddAllTwoArgs.parameters[0].nextDFG) assertEquals(setOf(), listAddAllTwoArgs.prevDFG) - // Specified by parent class' method List.addAll(int, Object) + // Specified by parent class' method List.addAll(int, Object). Test that parent of base is + // also taken into account. val specialListAddAllTwoArgs = code.methods("test.SpecialList.addAll").first { it.parameters[1].type.name.lastPartsMatch("test.Object") @@ -191,7 +193,8 @@ class DFGFunctionSummariesTest { assertEquals(setOf(), specialListAddAllTwoArgs.parameters[0].nextDFG) assertEquals(setOf(), specialListAddAllTwoArgs.prevDFG) - // Specified by parent class' method List.addAll(int, List) + // Specified by parent class' method List.addAll(int, List). Tests the most precise + // signature matching in case of function overloading. val specialListAddAllSpecializedArgs = code.methods("test.SpecialList.addAll").first { it.parameters[1].type.name.lastPartsMatch("test.List") @@ -208,7 +211,9 @@ class DFGFunctionSummariesTest { specialListAddAllSpecializedArgs.parameters[1].nextDFG ) - // Specified by parent class' method List.addAll(int, List) + // Specified by VerySpecialList.addAll(int, Object), overrides List.addAll(int, Object). + // Tests that we take the most precise base class. The entry of List.addAll(int, Object) is + // also applicable but isn't the most precise one (due to the base) val verySpecialListAddAllSpecializedArgs = code.methods["test.VerySpecialList.addAll"] assertNotNull(verySpecialListAddAllSpecializedArgs) assertEquals(2, verySpecialListAddAllSpecializedArgs.parameters.size) diff --git a/docs/docs/CPG/specs/dfg-function-summaries.md b/docs/docs/CPG/specs/dfg-function-summaries.md index 0e1bc6ff2e..8bf268b9dd 100644 --- a/docs/docs/CPG/specs/dfg-function-summaries.md +++ b/docs/docs/CPG/specs/dfg-function-summaries.md @@ -109,4 +109,12 @@ An example of a file could look as follows: This file configures the following edges: * For a method declaration in Java `java.util.List.addAll(int, java.util.Object)`, the parameter 1 flows to the base (i.e., the list object) * For a method declaration in Java `java.util.List.addAll(java.util.Object)`, the parameter 0 flows to the base (i.e., the list object) -* For a function declaration in C `memcpy` (and thus also CXX `std::memcpy`), the parameter 1 flows to parameter 0. \ No newline at end of file +* For a function declaration in C `memcpy` (and thus also CXX `std::memcpy`), the parameter 1 flows to parameter 0. + + +Note: If multiple function summaries match a method/function declaration (after the normal matching considering the language, local name of the function/method, signature if applicable and type hierarchy of the base object), we use the following routine to identify ideally a single entry: +1. We filter for existing signatures since it's more precisely specified than the generic "catch all" without a signature-element. +2. We filter for the most precise class of the base. +3. If there are still multiple options, we take the longest signature. +4. If this also didn't help to get a precise result, we iterate through the parameters and for index `i`, we pick the entry with the most precise matching type. We start with index 0 and count upwards, so if param0 leads to a single result, we're done and other entries won't be considered even if all the remaining parameters are more precise or whatever. +5. If nothing helped to get a unique entry, we pick the first remaining entry and hope it's the most precise one. \ No newline at end of file From 43ffd46305912c84a576096e7824a035ea816bca Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Mon, 18 Mar 2024 10:02:28 +0100 Subject: [PATCH 25/29] test coverage++ --- .../cpg/passes/ControlFlowSensitiveDFGPass.kt | 36 +++++++++++-------- .../enhancements/DFGFunctionSummariesTest.kt | 7 +++- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt index 35588bbc74..cf9ab92dc5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt @@ -391,24 +391,32 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass PowersetLattice(identitySetOf(currentNode)) ) } else if (currentNode is CallExpression) { - for (invoked in + val functionsWithSummaries = currentNode.invokes.filter { it in ctx.config.functionSummaries.functionToChangedParameters - }) { - val changedParams = - ctx.config.functionSummaries.functionToChangedParameters[invoked] ?: mapOf() - for ((param, _) in changedParams) { - if (param == (invoked as? MethodDeclaration)?.receiver) { - doubleState.declarationsState[ - ((currentNode as? MemberCallExpression)?.base as? Reference) - ?.refersTo] = PowersetLattice(identitySetOf(param)) - } else if (param is ParameterDeclaration) { - val arg = currentNode.arguments[param.argumentIndex] - doubleState.declarationsState[(arg as? Reference)?.refersTo] = - PowersetLattice(identitySetOf(param)) + } + if (functionsWithSummaries.isNotEmpty()) { + for (invoked in functionsWithSummaries) { + val changedParams = + ctx.config.functionSummaries.functionToChangedParameters[invoked] ?: mapOf() + for ((param, _) in changedParams) { + if (param == (invoked as? MethodDeclaration)?.receiver) { + doubleState.declarationsState[ + ((currentNode as? MemberCallExpression)?.base as? Reference) + ?.refersTo] = PowersetLattice(identitySetOf(param)) + } else if (param is ParameterDeclaration) { + val arg = currentNode.arguments[param.argumentIndex] + doubleState.declarationsState[(arg as? Reference)?.refersTo] = + PowersetLattice(identitySetOf(param)) + } + edgePropertiesMap[param] = CallingContextOut(currentNode) } - edgePropertiesMap[param] = CallingContextOut(currentNode) } + } else { + doubleState.declarationsState.push( + currentNode, + doubleState.declarationsState[currentEdge.start] + ) } } else { doubleState.declarationsState.push( diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index 9afa3b3022..911cf1f7e5 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -145,10 +145,15 @@ class DFGFunctionSummariesTest { construct("test.Object") } - memberCall("addAll", construct("test.List")) { + declare { + variable("a", t("test.List")) { construct("test.List") } + } + + memberCall("addAll", ref("a", t("test.List"))) { literal(1, t("int")) construct("test.Object") } + call("print") { ref("a", t("test.List")) } memberCall("addAll", construct("random.Type")) { literal(1, t("int")) From 66654b450fd39a5a02eefd9c29e355e2262fe8dc Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 19 Mar 2024 09:11:52 +0100 Subject: [PATCH 26/29] All the small comments --- .../aisec/cpg/TranslationConfiguration.kt | 20 +-------- .../aisec/cpg/graph/ExpressionBuilder.kt | 10 ++--- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 12 +++--- .../aisec/cpg/graph/edge/Dataflow.kt | 13 +++--- .../de/fraunhofer/aisec/cpg/helpers/Util.kt | 6 +-- .../cpg/passes/ControlFlowSensitiveDFGPass.kt | 2 +- .../de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 6 +-- .../passes/inference/DFGFunctionSummaries.kt | 5 +-- .../enhancements/DFGFunctionSummariesTest.kt | 43 +++++++++++-------- .../aisec/cpg/graph/ExpressionBuilderTest.kt | 10 ++--- 10 files changed, 58 insertions(+), 69 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index 556d0dc2aa..679628861e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -401,17 +401,6 @@ private constructor( return this } - inline fun > unregisterPass(): Builder { - unregisterPass(P::class) - return this - } - - /** Unregister a [Pass]. */ - fun unregisterPass(passType: KClass>): Builder { - passes.remove(passType) - return this - } - /** Register an additional [Pass]. */ fun registerPass(passType: KClass>): Builder { passes.add(passType) @@ -435,13 +424,8 @@ private constructor( return this } - fun registerFunctionSummaries(functionSummaries: List): Builder { - this.functionSummaries.addAll(functionSummaries) - return this - } - - fun registerFunctionSummary(functionSummary: File): Builder { - this.functionSummaries.add(functionSummary) + fun registerFunctionSummaries(vararg functionSummary: File): Builder { + this.functionSummaries.addAll(functionSummary) return this } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt index 4969796286..43a10114b8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt @@ -30,7 +30,7 @@ import de.fraunhofer.aisec.cpg.frontends.HasShortCircuitOperators import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.Node.Companion.EMPTY_NAME import de.fraunhofer.aisec.cpg.graph.NodeBuilder.log -import de.fraunhofer.aisec.cpg.graph.edge.ContextsensitiveDataflow +import de.fraunhofer.aisec.cpg.graph.edge.ContextSensitiveDataflow import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.types.ProblemType @@ -563,13 +563,13 @@ fun Literal.duplicate(implicit: Boolean): Literal { duplicate.file = this.file duplicate.name = this.name.clone() for (next in this.nextDFGEdges) { - if (next is ContextsensitiveDataflow) - duplicate.addNextDFGContext(next.end, next.callingContext, next.granularity) + if (next is ContextSensitiveDataflow) + duplicate.addNextDFGWithContext(next.end, next.callingContext, next.granularity) else duplicate.addNextDFG(next.end, next.granularity) } for (next in this.prevDFGEdges) { - if (next is ContextsensitiveDataflow) - duplicate.addPrevDFGContext(next.start, next.callingContext, next.granularity) + if (next is ContextSensitiveDataflow) + duplicate.addPrevDFGWithContext(next.start, next.callingContext, next.granularity) else duplicate.addNextDFG(next.start, next.granularity) } // TODO: This loses the properties of the edges. diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 93716bb483..523d97f5f3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -261,12 +261,12 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider * Adds a [Dataflow] edge from this node to [next], with the given [CallingContext] and * [Granularity]. */ - fun addNextDFGContext( + fun addNextDFGWithContext( next: Node, callingContext: CallingContext, granularity: Granularity = default(), ) { - val edge = ContextsensitiveDataflow(this, next, callingContext, granularity) + val edge = ContextSensitiveDataflow(this, next, callingContext, granularity) nextDFGEdges.add(edge) next.prevDFGEdges.add(edge) } @@ -297,12 +297,12 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider * Adds a [Dataflow] edge from [prev] node to this node, with the given [CallingContext] and * [Granularity]. */ - open fun addPrevDFGContext( + open fun addPrevDFGWithContext( prev: Node, callingContext: CallingContext, granularity: Granularity = default(), ) { - val edge = ContextsensitiveDataflow(prev, this, callingContext, granularity) + val edge = ContextSensitiveDataflow(prev, this, callingContext, granularity) prevDFGEdges.add(edge) prev.nextDFGEdges.add(edge) } @@ -328,12 +328,12 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider * Adds a [Dataflow] edge from all [prev] nodes to this node, with the given [CallingContext] * and [Granularity]. */ - fun addAllPrevDFGContext( + fun addAllPrevDFGWithContext( prev: Collection, callingContext: CallingContext, granularity: Granularity = full(), ) { - prev.forEach { addPrevDFGContext(it, callingContext, granularity) } + prev.forEach { addPrevDFGWithContext(it, callingContext, granularity) } } fun addAllPrevPDG(prev: Collection, dependenceType: DependenceType) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Dataflow.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Dataflow.kt index 498883b156..c8418b0db8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Dataflow.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Dataflow.kt @@ -94,24 +94,21 @@ sealed interface CallingContext class CallingContextIn( /** The call expression that affects this dataflow edge. */ - val callExpression: CallExpression + val call: CallExpression ) : CallingContext class CallingContextOut( /** The call expression that affects this dataflow edge. */ - val callExpression: CallExpression + val call: CallExpression ) : CallingContext -fun callIn(callExpression: CallExpression): CallingContext { - return CallingContextIn(callExpression) -} - /** * This edge class defines a flow of data between [start] and [end]. The flow must have a - * [callingContext] and can have a certain [granularity]. + * [callingContext] which allows for a context-sensitive dataflow analysis. This edge can also have + * a certain [granularity]. */ @RelationshipEntity -class ContextsensitiveDataflow( +class ContextSensitiveDataflow( start: Node, end: Node, /** The calling context affecting this dataflow. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt index 613a450a60..2d14475f89 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt @@ -357,7 +357,7 @@ object Util { // Add an incoming DFG edge from a member call's base to the method's receiver if (target is MethodDeclaration && call is MemberCallExpression && !call.isStatic) { target.receiver?.let { receiver -> - call.base?.addNextDFGContext(receiver, CallingContextIn(call)) + call.base?.addNextDFGWithContext(receiver, CallingContextIn(call)) } } @@ -373,12 +373,12 @@ object Util { if (param.isVariadic) { while (j < arguments.size) { // map all the following arguments to this variadic param - param.addPrevDFGContext(arguments[j], CallingContextIn(call)) + param.addPrevDFGWithContext(arguments[j], CallingContextIn(call)) j++ } break } else { - param.addPrevDFGContext(arguments[j], CallingContextIn(call)) + param.addPrevDFGWithContext(arguments[j], CallingContextIn(call)) } } j++ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt index cf9ab92dc5..dd125c1d1d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt @@ -124,7 +124,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass if ((it is VariableDeclaration || it is ParameterDeclaration) && key == it) { // Nothing to do } else if (it in edgePropertiesMap && edgePropertiesMap[it] is CallingContext) { - key.addPrevDFGContext(it, (edgePropertiesMap[it] as CallingContext)) + key.addPrevDFGWithContext(it, (edgePropertiesMap[it] as CallingContext)) } else { key.addPrevDFG(it) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index 3a68fb04c1..1f45e11a15 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -70,10 +70,10 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { if (param == (invoked as? MethodDeclaration)?.receiver) { (call as? MemberCallExpression) ?.base - ?.addPrevDFGContext(param, CallingContextOut(call)) + ?.addPrevDFGWithContext(param, CallingContextOut(call)) } else if (param is ParameterDeclaration) { val arg = call.arguments[param.argumentIndex] - arg.addPrevDFGContext(param, CallingContextOut(call)) + arg.addPrevDFGWithContext(param, CallingContextOut(call)) (arg as? Reference)?.let { it.access = AccessValues.READWRITE it.refersTo?.let { it1 -> it.addNextDFG(it1) } @@ -463,7 +463,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { } else if (call.invokes.isNotEmpty()) { call.invokes.forEach { Util.attachCallParameters(it, call) - call.addPrevDFGContext(it, CallingContextOut(call)) + call.addPrevDFGWithContext(it, CallingContextOut(call)) if (it.isInferred) { callsInferredFunctions.add(call) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 431d78c82c..a89e0d014b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -42,9 +42,8 @@ import java.io.File /** * If the user of the library registers one or multiple DFG-function summary files (via - * [Builder.registerFunctionSummaries] or [Builder.registerFunctionSummary]), this class is - * responsible for parsing the files, caching the result and adding the respective DFG summaries to - * the [FunctionDeclaration]. + * [Builder.registerFunctionSummaries]), this class is responsible for parsing the files, caching + * the result and adding the respective DFG summaries to the [FunctionDeclaration]. */ class DFGFunctionSummaries { private constructor() diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index 911cf1f7e5..108a1387e8 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -34,13 +34,13 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.builder.* import de.fraunhofer.aisec.cpg.graph.edge.CallingContextIn import de.fraunhofer.aisec.cpg.graph.edge.CallingContextOut -import de.fraunhofer.aisec.cpg.graph.edge.ContextsensitiveDataflow +import de.fraunhofer.aisec.cpg.graph.edge.ContextSensitiveDataflow import de.fraunhofer.aisec.cpg.graph.functions import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.recordDeclaration -import de.fraunhofer.aisec.cpg.passes.ControlFlowSensitiveDFGPass +import de.fraunhofer.aisec.cpg.passes.* import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries import de.fraunhofer.aisec.cpg.passes.inference.startInference import java.io.File @@ -72,7 +72,7 @@ class DFGFunctionSummariesTest { TranslationConfiguration.builder() .defaultPasses() .registerLanguage(TestLanguage(".")) - .registerFunctionSummary(File("src/test/resources/function-dfg2.yml")) + .registerFunctionSummaries(File("src/test/resources/function-dfg2.yml")) .inferenceConfiguration( InferenceConfiguration.builder() .inferDfgForUnresolvedCalls(true) @@ -248,7 +248,7 @@ class DFGFunctionSummariesTest { @Test fun testPropagateArguments() { - val dfgTest = getDfgInferredCall() + val dfgTest = getDfgInferredCall() { defaultPasses() } assertNotNull(dfgTest) val main = dfgTest.functions["main"] @@ -275,8 +275,7 @@ class DFGFunctionSummariesTest { val nextDfg = argA.nextDFGEdges.single() assertEquals( call, - ((nextDfg as? ContextsensitiveDataflow)?.callingContext as? CallingContextIn) - ?.callExpression + ((nextDfg as? ContextSensitiveDataflow)?.callingContext as? CallingContextIn)?.call ) assertEquals(param0, nextDfg.end) @@ -284,7 +283,7 @@ class DFGFunctionSummariesTest { assertNotNull(variableA) assertEquals(mutableSetOf(variableA), argA.prevDFG) - val prevDfgOfParam0 = param0.prevDFGEdges.singleOrNull { it !is ContextsensitiveDataflow } + val prevDfgOfParam0 = param0.prevDFGEdges.singleOrNull { it !is ContextSensitiveDataflow } assertNotNull(prevDfgOfParam0) assertEquals(param1, prevDfgOfParam0.start) @@ -296,15 +295,26 @@ class DFGFunctionSummariesTest { // Check that also the CallingContext property is set correctly val nextDfgOfParam0 = param0.nextDFGEdges.singleOrNull { - ((it as? ContextsensitiveDataflow)?.callingContext as? CallingContextOut) - ?.callExpression == call + ((it as? ContextSensitiveDataflow)?.callingContext as? CallingContextOut)?.call == + call } assertEquals(returnA, nextDfgOfParam0?.end) } @Test fun testPropagateArgumentsControlFlowInsensitive() { - val dfgTest = getDfgInferredCall { unregisterPass() } + // We don't use the ControlFlowSensitiveDFGPass here to check the method + // DFGPass.connectInferredCallArguments + val dfgTest = getDfgInferredCall { + this.registerPass() + registerPass() + registerPass() + registerPass() + registerPass() + registerPass() + registerPass() + registerPass() + } assertNotNull(dfgTest) val main = dfgTest.functions["main"] @@ -331,8 +341,8 @@ class DFGFunctionSummariesTest { val nextDfg = argA.nextDFGEdges.singleOrNull { - ((it as? ContextsensitiveDataflow)?.callingContext as? CallingContextIn) - ?.callExpression == call + ((it as? ContextSensitiveDataflow)?.callingContext as? CallingContextIn)?.call == + call } assertNotNull(nextDfg) assertEquals(param0, nextDfg.end) @@ -341,7 +351,7 @@ class DFGFunctionSummariesTest { assertNotNull(variableA) assertEquals(mutableSetOf(variableA, param0), argA.prevDFG) - val prevDfgOfParam0 = param0.prevDFGEdges.singleOrNull { it !is ContextsensitiveDataflow } + val prevDfgOfParam0 = param0.prevDFGEdges.singleOrNull { it !is ContextSensitiveDataflow } assertNotNull(prevDfgOfParam0) assertEquals(param1, prevDfgOfParam0.start) @@ -355,8 +365,8 @@ class DFGFunctionSummariesTest { // Check that also the CallingContext property is set correctly val nextDfgOfParam0 = param0.nextDFGEdges.singleOrNull { - ((it as? ContextsensitiveDataflow)?.callingContext as? CallingContextOut) - ?.callExpression == call + ((it as? ContextSensitiveDataflow)?.callingContext as? CallingContextOut)?.call == + call } assertEquals(argA, nextDfgOfParam0?.end) } @@ -370,9 +380,8 @@ class DFGFunctionSummariesTest { ): TranslationResult { val config = TranslationConfiguration.builder() - .defaultPasses() .registerLanguage(TestLanguage(".")) - .registerFunctionSummary(File("src/test/resources/function-dfg.yml")) + .registerFunctionSummaries(File("src/test/resources/function-dfg.yml")) .inferenceConfiguration( InferenceConfiguration.builder() .inferDfgForUnresolvedCalls(true) diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt index 03298eb79f..0719885c97 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt @@ -27,7 +27,7 @@ package de.fraunhofer.aisec.cpg.graph import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration import de.fraunhofer.aisec.cpg.graph.edge.CallingContextIn -import de.fraunhofer.aisec.cpg.graph.edge.ContextsensitiveDataflow +import de.fraunhofer.aisec.cpg.graph.edge.ContextSensitiveDataflow import de.fraunhofer.aisec.cpg.graph.edge.PartialDataflowGranularity import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal @@ -43,11 +43,11 @@ class ExpressionBuilderTest { val node2 = Reference() val granularity = PartialDataflowGranularity(FieldDeclaration()) val callingContextIn = CallingContextIn(CallExpression()) - node1.addPrevDFGContext(node2, callingContextIn, granularity) + node1.addPrevDFGWithContext(node2, callingContextIn, granularity) val clone = node1.duplicate(false) val clonedPrevDFG = clone.prevDFGEdges.single() - assertTrue(clonedPrevDFG is ContextsensitiveDataflow) + assertTrue(clonedPrevDFG is ContextSensitiveDataflow) assertEquals(callingContextIn, clonedPrevDFG.callingContext) assertEquals(granularity, clonedPrevDFG.granularity) @@ -60,11 +60,11 @@ class ExpressionBuilderTest { val node2 = Reference() val granularity = PartialDataflowGranularity(FieldDeclaration()) val callingContextIn = CallingContextIn(CallExpression()) - node1.addNextDFGContext(node2, callingContextIn, granularity) + node1.addNextDFGWithContext(node2, callingContextIn, granularity) val clone = node1.duplicate(false) val clonedPrevDFG = clone.nextDFGEdges.single() - assertTrue(clonedPrevDFG is ContextsensitiveDataflow) + assertTrue(clonedPrevDFG is ContextSensitiveDataflow) assertEquals(callingContextIn, clonedPrevDFG.callingContext) assertEquals(granularity, clonedPrevDFG.granularity) From 4b3152aeafbdd4217cd1cb7866d7119572d3b9de Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 19 Mar 2024 13:58:01 +0100 Subject: [PATCH 27/29] More comments, smaller fixes --- .../cpg/passes/ControlFlowSensitiveDFGPass.kt | 112 ++++++++++++++---- .../passes/inference/DFGFunctionSummaries.kt | 6 + 2 files changed, 98 insertions(+), 20 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt index dd125c1d1d..0d505e56c0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt @@ -123,8 +123,14 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass value.elements.forEach { if ((it is VariableDeclaration || it is ParameterDeclaration) && key == it) { // Nothing to do - } else if (it in edgePropertiesMap && edgePropertiesMap[it] is CallingContext) { - key.addPrevDFGWithContext(it, (edgePropertiesMap[it] as CallingContext)) + } else if ( + Pair(it, key) in edgePropertiesMap && + edgePropertiesMap[Pair(it, key)] is CallingContext + ) { + key.addPrevDFGWithContext( + it, + (edgePropertiesMap[Pair(it, key)] as CallingContext) + ) } else { key.addPrevDFG(it) } @@ -221,6 +227,16 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass // later for READ accesses. val declState = doubleState.declarationsState[currentNode.objectIdentifier()] if (declState != null) { + // We check if we have something relevant for this node (because there was an + // entry for the incoming edge) in the edgePropertiesMap and, if so, we generate + // a dedicated entry for the edge between declState and currentNode. + val relevantProperties = + edgePropertiesMap.filter { + it.key.first in declState.elements && it.key.second == null + } + relevantProperties.forEach { + edgePropertiesMap[Pair(it.key.first, currentNode)] = it.value + } state.push(currentNode, declState) } else { // If we do not have a stored state of our object+field, we can use the field @@ -241,6 +257,16 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass // later for READ accesses. val declState = doubleState.declarationsState[currentNode.objectIdentifier()] if (declState != null) { + // We check if we have something relevant for this node (because there was an + // entry for the incoming edge) in the edgePropertiesMap and, if so, we generate + // a dedicated entry for the edge between declState and currentNode. + val relevantProperties = + edgePropertiesMap.filter { + it.key.first in declState.elements && it.key.second == null + } + relevantProperties.forEach { + edgePropertiesMap[Pair(it.key.first, currentNode)] = it.value + } state.push(currentNode, declState) } else { // If we do not have a stored state of our object+field, we can use the field @@ -281,7 +307,18 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass writtenDeclaration = input.refersTo if (writtenDeclaration != null) { - state.push(input, doubleState.declarationsState[writtenDeclaration]) + val prev = doubleState.declarationsState[writtenDeclaration] + // We check if we have something relevant for this node (because there was an entry + // for the incoming edge) in the edgePropertiesMap and, if so, we generate a + // dedicated entry for the edge between declState and currentNode. + val relevantProperties = + edgePropertiesMap.filter { + it.key.first in (prev?.elements ?: setOf()) && it.key.second == null + } + relevantProperties.forEach { + edgePropertiesMap[Pair(it.key.first, currentNode)] = it.value + } + state.push(input, prev) doubleState.declarationsState[writtenDeclaration] = PowersetLattice(identitySetOf(input)) } @@ -294,8 +331,16 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass writtenDeclaration = (lhs as? Reference)?.refersTo if (writtenDeclaration != null && lhs != null) { + val prev = doubleState.declarationsState[writtenDeclaration] + val relevantProperties = + edgePropertiesMap.filter { + it.key.first in (prev?.elements ?: setOf()) && it.key.second == null + } + relevantProperties.forEach { + edgePropertiesMap[Pair(it.key.first, currentNode)] = it.value + } // Data flows from the last writes to the lhs variable to this node - state.push(lhs, doubleState.declarationsState[writtenDeclaration]) + state.push(lhs, prev) // The whole current node is the place of the last update, not (only) the lhs! doubleState.declarationsState[writtenDeclaration] = @@ -311,6 +356,16 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass doubleState.declarationsState[currentNode.refersTo]?.let { // We only read the variable => Get previous write which have been collected in // the other steps + // We check if we have something relevant for this node (because there was an entry + // for the incoming edge) in the edgePropertiesMap and, if so, we generate a + // dedicated entry for the edge between declState and currentNode. + val relevantProperties = + edgePropertiesMap.filter { it2 -> + it2.key.first in it.elements && it2.key.second == null + } + relevantProperties.forEach { + edgePropertiesMap[Pair(it.key.first, currentNode)] = it.value + } state.push(currentNode, it) } } else if ( @@ -391,28 +446,37 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass PowersetLattice(identitySetOf(currentNode)) ) } else if (currentNode is CallExpression) { + // If the CallExpression invokes a function for which we have a function summary, we use + // the summary to identify the last write to a parameter (or receiver) and match it to + // the respective argument or the base. + // Since this Reference r is manipulated inside the invoked function, the next + // read-access of a Reference r' with r'.refersTo == r.refersTo will be affected by the + // node that has been stored inside the function summary for this particular + // parameter/receiver, and we store this last write-access in the state. + // As the node is in another function, we also store the CallingContext of the call + // expression in the edgePropertiesMap. val functionsWithSummaries = - currentNode.invokes.filter { - it in ctx.config.functionSummaries.functionToChangedParameters - } + currentNode.invokes.filter { ctx.config.functionSummaries.hasSummary(it) } if (functionsWithSummaries.isNotEmpty()) { for (invoked in functionsWithSummaries) { - val changedParams = - ctx.config.functionSummaries.functionToChangedParameters[invoked] ?: mapOf() + val changedParams = ctx.config.functionSummaries.getLastWrites(invoked) for ((param, _) in changedParams) { - if (param == (invoked as? MethodDeclaration)?.receiver) { - doubleState.declarationsState[ - ((currentNode as? MemberCallExpression)?.base as? Reference) - ?.refersTo] = PowersetLattice(identitySetOf(param)) - } else if (param is ParameterDeclaration) { - val arg = currentNode.arguments[param.argumentIndex] - doubleState.declarationsState[(arg as? Reference)?.refersTo] = - PowersetLattice(identitySetOf(param)) - } - edgePropertiesMap[param] = CallingContextOut(currentNode) + val arg = + when (param) { + (invoked as? MethodDeclaration)?.receiver -> + (currentNode as? MemberCallExpression)?.base as? Reference + is ParameterDeclaration -> + currentNode.arguments[param.argumentIndex] as? Reference + else -> null + } + doubleState.declarationsState[arg?.refersTo] = + PowersetLattice(identitySetOf(param)) + // TODO: Map probably needs param and a second node! + edgePropertiesMap[Pair(param, null)] = CallingContextOut(currentNode) } } } else { + // The default behavior so we continue with the next EOG thing. doubleState.declarationsState.push( currentNode, doubleState.declarationsState[currentEdge.start] @@ -427,7 +491,15 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass return state } - val edgePropertiesMap = mutableMapOf() + /** + * We use this map to store additional information on the DFG edges which we cannot keep in the + * state. This is for example the case to identify if the resulting edge will receive a + * context-sensitivity label (i.e., if the node used as key is somehow inside the called + * function and the next usage happens inside the function under analysis right now). The key of + * an entry works as follows: The 1st item in the pair is the prevDFG of the 2nd item. If the + * 2nd item is null, it's obviously not relevant. Ultimately, it will be 2nd -prevDFG-> 1st. + */ + val edgePropertiesMap = mutableMapOf, Any>() /** * Checks if the node performs an operation and an assignment at the same time e.g. with the diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index a89e0d014b..b6b52b82bc 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -61,6 +61,12 @@ class DFGFunctionSummaries { val functionToChangedParameters = mutableMapOf>>() + fun hasSummary(functionDeclaration: FunctionDeclaration) = + functionDeclaration in functionToChangedParameters + + fun getLastWrites(functionDeclaration: FunctionDeclaration): Map> = + functionToChangedParameters[functionDeclaration] ?: mapOf() + /** This function returns a list of [DataflowEntry] from the specified file. */ private fun addEntriesFromFile(file: File): Map> { val mapper = From 47f07371f36365e08371a7b948e9c0edc3340185 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 19 Mar 2024 14:08:05 +0100 Subject: [PATCH 28/29] Merge methods together --- .../aisec/cpg/graph/ExpressionBuilder.kt | 16 +++-- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 65 +++++++------------ .../graph/statements/expressions/Reference.kt | 5 +- .../de/fraunhofer/aisec/cpg/helpers/Util.kt | 6 +- .../cpg/passes/ControlFlowSensitiveDFGPass.kt | 5 +- .../de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 6 +- .../aisec/cpg/graph/ExpressionBuilderTest.kt | 4 +- 7 files changed, 45 insertions(+), 62 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt index 43a10114b8..6a72ab6484 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt @@ -563,14 +563,18 @@ fun Literal.duplicate(implicit: Boolean): Literal { duplicate.file = this.file duplicate.name = this.name.clone() for (next in this.nextDFGEdges) { - if (next is ContextSensitiveDataflow) - duplicate.addNextDFGWithContext(next.end, next.callingContext, next.granularity) - else duplicate.addNextDFG(next.end, next.granularity) + duplicate.addNextDFG( + next.end, + next.granularity, + (next as? ContextSensitiveDataflow)?.callingContext + ) } for (next in this.prevDFGEdges) { - if (next is ContextSensitiveDataflow) - duplicate.addPrevDFGWithContext(next.start, next.callingContext, next.granularity) - else duplicate.addNextDFG(next.start, next.granularity) + duplicate.addPrevDFG( + next.start, + next.granularity, + (next as? ContextSensitiveDataflow)?.callingContext + ) } // TODO: This loses the properties of the edges. duplicate.nextEOG = this.nextEOG diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 523d97f5f3..c39731febf 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -251,22 +251,14 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider fun addNextDFG( next: Node, granularity: Granularity = default(), + callingContext: CallingContext? = null, ) { - val edge = Dataflow(this, next, granularity) - nextDFGEdges.add(edge) - next.prevDFGEdges.add(edge) - } - - /** - * Adds a [Dataflow] edge from this node to [next], with the given [CallingContext] and - * [Granularity]. - */ - fun addNextDFGWithContext( - next: Node, - callingContext: CallingContext, - granularity: Granularity = default(), - ) { - val edge = ContextSensitiveDataflow(this, next, callingContext, granularity) + val edge = + if (callingContext != null) { + ContextSensitiveDataflow(this, next, callingContext, granularity) + } else { + Dataflow(this, next, granularity) + } nextDFGEdges.add(edge) next.prevDFGEdges.add(edge) } @@ -283,26 +275,21 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider } } - /** Adds a [Dataflow] edge from [prev] node to this node, with the given [Granularity]. */ - open fun addPrevDFG( - prev: Node, - granularity: Granularity = default(), - ) { - val edge = Dataflow(prev, this, granularity) - prevDFGEdges.add(edge) - prev.nextDFGEdges.add(edge) - } - /** - * Adds a [Dataflow] edge from [prev] node to this node, with the given [CallingContext] and - * [Granularity]. + * Adds a [Dataflow] edge from [prev] node to this node, with the given [Granularity] and + * [CallingContext]. */ - open fun addPrevDFGWithContext( + open fun addPrevDFG( prev: Node, - callingContext: CallingContext, granularity: Granularity = default(), + callingContext: CallingContext? = null, ) { - val edge = ContextSensitiveDataflow(prev, this, callingContext, granularity) + val edge = + if (callingContext != null) { + ContextSensitiveDataflow(prev, this, callingContext, granularity) + } else { + Dataflow(prev, this, granularity) + } prevDFGEdges.add(edge) prev.nextDFGEdges.add(edge) } @@ -316,24 +303,16 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider prev.nextCDGEdges.add(edge) } - /** Adds a [Dataflow] edge from all [prev] nodes to this node, with the given [Granularity]. */ - fun addAllPrevDFG( - prev: Collection, - granularity: Granularity = full(), - ) { - prev.forEach { addPrevDFG(it, granularity) } - } - /** - * Adds a [Dataflow] edge from all [prev] nodes to this node, with the given [CallingContext] - * and [Granularity]. + * Adds a [Dataflow] edge from all [prev] nodes to this node, with the given [Granularity] and + * [CallingContext] if applicable. */ - fun addAllPrevDFGWithContext( + fun addAllPrevDFG( prev: Collection, - callingContext: CallingContext, granularity: Granularity = full(), + callingContext: CallingContext? = null, ) { - prev.forEach { addPrevDFGWithContext(it, callingContext, granularity) } + prev.forEach { addPrevDFG(it, granularity, callingContext) } } fun addAllPrevPDG(prev: Collection, dependenceType: DependenceType) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt index 69b856f013..11e4f24478 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt @@ -32,6 +32,7 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.edge.CallingContext import de.fraunhofer.aisec.cpg.graph.edge.Granularity import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.types.HasType @@ -147,8 +148,8 @@ open class Reference : Expression(), HasType.TypeObserver, HasAliases { return super.hashCode() } - override fun addPrevDFG(prev: Node, granularity: Granularity) { - super.addPrevDFG(prev, granularity) + override fun addPrevDFG(prev: Node, granularity: Granularity, callingContext: CallingContext?) { + super.addPrevDFG(prev, granularity, callingContext) // We want to propagate assigned types all through the previous DFG nodes. Therefore, we // override the DFG adding function here and add a type observer to the previous node (if it diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt index 2d14475f89..44171e159c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt @@ -357,7 +357,7 @@ object Util { // Add an incoming DFG edge from a member call's base to the method's receiver if (target is MethodDeclaration && call is MemberCallExpression && !call.isStatic) { target.receiver?.let { receiver -> - call.base?.addNextDFGWithContext(receiver, CallingContextIn(call)) + call.base?.addNextDFG(receiver, callingContext = CallingContextIn(call)) } } @@ -373,12 +373,12 @@ object Util { if (param.isVariadic) { while (j < arguments.size) { // map all the following arguments to this variadic param - param.addPrevDFGWithContext(arguments[j], CallingContextIn(call)) + param.addPrevDFG(arguments[j], callingContext = CallingContextIn(call)) j++ } break } else { - param.addPrevDFGWithContext(arguments[j], CallingContextIn(call)) + param.addPrevDFG(arguments[j], callingContext = CallingContextIn(call)) } } j++ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt index 0d505e56c0..e0fbbaa55a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt @@ -127,9 +127,9 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass Pair(it, key) in edgePropertiesMap && edgePropertiesMap[Pair(it, key)] is CallingContext ) { - key.addPrevDFGWithContext( + key.addPrevDFG( it, - (edgePropertiesMap[Pair(it, key)] as CallingContext) + callingContext = (edgePropertiesMap[Pair(it, key)] as? CallingContext) ) } else { key.addPrevDFG(it) @@ -471,7 +471,6 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass } doubleState.declarationsState[arg?.refersTo] = PowersetLattice(identitySetOf(param)) - // TODO: Map probably needs param and a second node! edgePropertiesMap[Pair(param, null)] = CallingContextOut(currentNode) } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index 1f45e11a15..e1680daa4d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -70,10 +70,10 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { if (param == (invoked as? MethodDeclaration)?.receiver) { (call as? MemberCallExpression) ?.base - ?.addPrevDFGWithContext(param, CallingContextOut(call)) + ?.addPrevDFG(param, callingContext = CallingContextOut(call)) } else if (param is ParameterDeclaration) { val arg = call.arguments[param.argumentIndex] - arg.addPrevDFGWithContext(param, CallingContextOut(call)) + arg.addPrevDFG(param, callingContext = CallingContextOut(call)) (arg as? Reference)?.let { it.access = AccessValues.READWRITE it.refersTo?.let { it1 -> it.addNextDFG(it1) } @@ -463,7 +463,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { } else if (call.invokes.isNotEmpty()) { call.invokes.forEach { Util.attachCallParameters(it, call) - call.addPrevDFGWithContext(it, CallingContextOut(call)) + call.addPrevDFG(it, callingContext = CallingContextOut(call)) if (it.isInferred) { callsInferredFunctions.add(call) } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt index 0719885c97..f5ce80f271 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt @@ -43,7 +43,7 @@ class ExpressionBuilderTest { val node2 = Reference() val granularity = PartialDataflowGranularity(FieldDeclaration()) val callingContextIn = CallingContextIn(CallExpression()) - node1.addPrevDFGWithContext(node2, callingContextIn, granularity) + node1.addPrevDFG(node2, granularity, callingContextIn) val clone = node1.duplicate(false) val clonedPrevDFG = clone.prevDFGEdges.single() @@ -60,7 +60,7 @@ class ExpressionBuilderTest { val node2 = Reference() val granularity = PartialDataflowGranularity(FieldDeclaration()) val callingContextIn = CallingContextIn(CallExpression()) - node1.addNextDFGWithContext(node2, callingContextIn, granularity) + node1.addNextDFG(node2, granularity, callingContextIn) val clone = node1.duplicate(false) val clonedPrevDFG = clone.nextDFGEdges.single() From e9ecdac226d46cf64d3f3e7c798579f6425e0d1b Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 19 Mar 2024 14:48:33 +0100 Subject: [PATCH 29/29] Less C&P --- .../cpg/passes/ControlFlowSensitiveDFGPass.kt | 50 ++++++------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt index cd628969aa..c99894be79 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt @@ -139,6 +139,16 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass } } + /** + * Checks if there's an entry in [edgePropertiesMap] with key `(x, null)` where `x` is in [from] + * and, if so, adds an entry with key `(x, to)` and the same value + */ + protected fun findAndSetProperties(from: Set, to: Node) { + edgePropertiesMap + .filter { it.key.first in from && it.key.second == null } + .forEach { edgePropertiesMap[Pair(it.key.first, to)] = it.value } + } + /** * Removes all the incoming and outgoing DFG edges for each variable declaration in the block of * code [node]. @@ -230,13 +240,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass // We check if we have something relevant for this node (because there was an // entry for the incoming edge) in the edgePropertiesMap and, if so, we generate // a dedicated entry for the edge between declState and currentNode. - val relevantProperties = - edgePropertiesMap.filter { - it.key.first in declState.elements && it.key.second == null - } - relevantProperties.forEach { - edgePropertiesMap[Pair(it.key.first, currentNode)] = it.value - } + findAndSetProperties(declState.elements, currentNode) state.push(currentNode, declState) } else { // If we do not have a stored state of our object+field, we can use the field @@ -260,13 +264,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass // We check if we have something relevant for this node (because there was an // entry for the incoming edge) in the edgePropertiesMap and, if so, we generate // a dedicated entry for the edge between declState and currentNode. - val relevantProperties = - edgePropertiesMap.filter { - it.key.first in declState.elements && it.key.second == null - } - relevantProperties.forEach { - edgePropertiesMap[Pair(it.key.first, currentNode)] = it.value - } + findAndSetProperties(declState.elements, currentNode) state.push(currentNode, declState) } else { // If we do not have a stored state of our object+field, we can use the field @@ -311,13 +309,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass // We check if we have something relevant for this node (because there was an entry // for the incoming edge) in the edgePropertiesMap and, if so, we generate a // dedicated entry for the edge between declState and currentNode. - val relevantProperties = - edgePropertiesMap.filter { - it.key.first in (prev?.elements ?: setOf()) && it.key.second == null - } - relevantProperties.forEach { - edgePropertiesMap[Pair(it.key.first, currentNode)] = it.value - } + findAndSetProperties(prev?.elements ?: setOf(), currentNode) state.push(input, prev) doubleState.declarationsState[writtenDeclaration] = PowersetLattice(identitySetOf(input)) @@ -332,13 +324,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass if (writtenDeclaration != null && lhs != null) { val prev = doubleState.declarationsState[writtenDeclaration] - val relevantProperties = - edgePropertiesMap.filter { - it.key.first in (prev?.elements ?: setOf()) && it.key.second == null - } - relevantProperties.forEach { - edgePropertiesMap[Pair(it.key.first, currentNode)] = it.value - } + findAndSetProperties(prev?.elements ?: setOf(), currentNode) // Data flows from the last writes to the lhs variable to this node state.push(lhs, prev) @@ -359,13 +345,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass // We check if we have something relevant for this node (because there was an entry // for the incoming edge) in the edgePropertiesMap and, if so, we generate a // dedicated entry for the edge between declState and currentNode. - val relevantProperties = - edgePropertiesMap.filter { it2 -> - it2.key.first in it.elements && it2.key.second == null - } - relevantProperties.forEach { - edgePropertiesMap[Pair(it.key.first, currentNode)] = it.value - } + findAndSetProperties(it.elements, currentNode) state.push(currentNode, it) } } else if (