diff --git a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt index 56fb4c93483..c211cd3082b 100644 --- a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt +++ b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt @@ -33,7 +33,6 @@ import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.ForStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.passes.EdgeCachePass -import de.fraunhofer.aisec.cpg.passes.astParent import org.slf4j.Logger import org.slf4j.LoggerFactory diff --git a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt index a5467d553cd..9d7f8528351 100644 --- a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt +++ b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt @@ -35,7 +35,6 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.ConstructExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.passes.astParent import org.slf4j.Logger import org.slf4j.LoggerFactory diff --git a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluatorTest.kt b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluatorTest.kt index 392465e7483..cae09478973 100644 --- a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluatorTest.kt +++ b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluatorTest.kt @@ -783,7 +783,12 @@ class ValueEvaluatorTest { comparison.lhs = aRef comparison.rhs = newLiteral(1) - var cond = newConditionalExpression(comparison, newLiteral(2), aRef) + var cond = + newConditionalExpression().withChildren { + it.condition = comparison + it.thenExpression = newLiteral(2) + it.elseExpression = aRef + } assertEquals(1, cond.evaluate()) // handle equals @@ -791,11 +796,21 @@ class ValueEvaluatorTest { comparison.lhs = aRef comparison.rhs = newLiteral(1) - cond = newConditionalExpression(comparison, newLiteral(2), aRef) + cond = + newConditionalExpression().withChildren { + it.condition = comparison + it.thenExpression = newLiteral(2) + it.elseExpression = aRef + } assertEquals(2, cond.evaluate()) // handle invalid - cond = newConditionalExpression(newProblemExpression(), newLiteral(2), aRef) + cond = + newConditionalExpression().withChildren { + it.condition = newProblemExpression() + it.thenExpression = newLiteral(2) + it.elseExpression = aRef + } assertEquals("{}", cond.evaluate()) } } diff --git a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/ValueEvaluationTests.kt b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/ValueEvaluationTests.kt index 62a578e9b7e..32e7968e05c 100644 --- a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/ValueEvaluationTests.kt +++ b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/ValueEvaluationTests.kt @@ -294,11 +294,11 @@ class ValueEvaluationTests { declare { variable("a", t("int")) { this.initializer = - newConditionalExpression( - ref("b") lt literal(2, t("int")), - literal(3, t("int")), - literal(5, t("int")).inc() - ) + newConditionalExpression().withChildren { + it.condition = ref("b") lt literal(2, t("int")) + it.thenExpression = literal(3, t("int")) + it.elseExpression = literal(5, t("int")).inc() + } } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt index 7d5b45a2a65..ba8e0871adc 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt @@ -54,7 +54,8 @@ abstract class Handler { + RawNodeTypeProvider, + AstStackProvider by frontend { protected val map = HashMap, HandlerInterface>() private val typeOfT: Class<*>? diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt index e6e97aa12db..fa67d3c43bb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt @@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import de.fraunhofer.aisec.cpg.sarif.Region import java.io.File import java.util.* +import kotlin.collections.ArrayDeque import org.slf4j.LoggerFactory /** @@ -61,13 +62,16 @@ abstract class LanguageFrontend( ScopeProvider, NamespaceProvider, ContextProvider, - RawNodeTypeProvider { + RawNodeTypeProvider, + AstStackProvider { val scopeManager: ScopeManager = ctx.scopeManager val typeManager: TypeManager = ctx.typeManager val config: TranslationConfiguration = ctx.config var currentTU: TranslationUnitDeclaration? = null + override val astStack: ArrayDeque = ArrayDeque() + @Throws(TranslationException::class) fun parseAll(): List { val units = ArrayList() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt index a1f23310211..b803b51146c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt @@ -48,8 +48,8 @@ fun MetadataProvider.newTranslationUnitDeclaration( ): TranslationUnitDeclaration { val node = TranslationUnitDeclaration() node.applyMetadata(this, name, rawNode, true) - log(node) + return node } @@ -59,16 +59,16 @@ fun MetadataProvider.newTranslationUnitDeclaration( * requires an appropriate [MetadataProvider], such as a [LanguageFrontend] as an additional * prepended argument. */ -@JvmOverloads fun MetadataProvider.newFunctionDeclaration( name: CharSequence?, localNameOnly: Boolean = false, - rawNode: Any? = null, + rawNode: Any? = null ): FunctionDeclaration { val node = FunctionDeclaration() node.applyMetadata(this@MetadataProvider, name, rawNode, localNameOnly) log(node) + return node } @@ -83,7 +83,7 @@ fun MetadataProvider.newMethodDeclaration( name: CharSequence?, isStatic: Boolean = false, recordDeclaration: RecordDeclaration? = null, - rawNode: Any? = null + rawNode: Any? = null, ): MethodDeclaration { val node = MethodDeclaration() node.applyMetadata(this, name, rawNode, defaultNamespace = recordDeclaration?.name) @@ -437,6 +437,7 @@ fun MetadataProvider.newNamespaceDeclaration( node.applyMetadata(this, name, rawNode) log(node) + return node } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt index c5f956a7578..8d4684d6b65 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt @@ -122,15 +122,11 @@ fun MetadataProvider.newUnaryOperator( @JvmOverloads fun MetadataProvider.newAssignExpression( operatorCode: String = "=", - lhs: List = listOf(), - rhs: List = listOf(), rawNode: Any? = null ): AssignExpression { val node = AssignExpression() node.applyMetadata(this, operatorCode, rawNode, true) node.operatorCode = operatorCode - node.lhs = lhs - node.rhs = rhs log(node) @@ -183,9 +179,6 @@ fun MetadataProvider.newConstructExpression( */ @JvmOverloads fun MetadataProvider.newConditionalExpression( - condition: Expression, - thenExpression: Expression? = null, - elseExpression: Expression? = null, type: Type = unknownType(), rawNode: Any? = null ): ConditionalExpression { @@ -193,9 +186,6 @@ fun MetadataProvider.newConditionalExpression( node.applyMetadata(this, EMPTY_NAME, rawNode, true) node.type = type - node.condition = condition - node.thenExpression = thenExpression - node.elseExpression = elseExpression log(node) return node diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt index 4200759a065..86bf2820f59 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt @@ -41,7 +41,6 @@ import de.fraunhofer.aisec.cpg.graph.statements.WhileStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker -import de.fraunhofer.aisec.cpg.passes.astParent import kotlin.math.absoluteValue /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index d9eca8716d6..7f412c00cc9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -158,6 +158,8 @@ open class Node : var astChildren: List = listOf() get() = SubgraphWalker.getAstChildren(this) + @Relationship("AST") var astParent: Node? = null + /** Virtual property for accessing [prevEOGEdges] without property edges. */ @PopulatedByPass(EvaluationOrderGraphPass::class) var prevEOG: List by PropertyEdgeDelegate(Node::prevEOGEdges, false) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt index cf7e12e01ee..22a0be60455 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt @@ -30,6 +30,8 @@ import de.fraunhofer.aisec.cpg.frontends.* import de.fraunhofer.aisec.cpg.graph.Node.Companion.EMPTY_NAME import de.fraunhofer.aisec.cpg.graph.NodeBuilder.LOGGER import de.fraunhofer.aisec.cpg.graph.NodeBuilder.log +import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.* @@ -39,6 +41,8 @@ import de.fraunhofer.aisec.cpg.passes.inference.IsInferredProvider import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import de.fraunhofer.aisec.cpg.sarif.Region import java.net.URI +import java.util.* +import kotlin.collections.ArrayDeque import org.slf4j.LoggerFactory object NodeBuilder { @@ -91,6 +95,10 @@ interface NamespaceProvider : MetadataProvider { val namespace: Name? } +interface AstStackProvider : MetadataProvider { + val astStack: ArrayDeque +} + /** * Applies various metadata on this [Node], based on the kind of provider in [provider]. This can * include: @@ -148,6 +156,10 @@ fun Node.applyMetadata( ) } + if (provider is AstStackProvider) { + provider.astStack.lastOrNull().let { this.astParent = it } + } + if (name != null) { val namespace = if (provider is NamespaceProvider) { @@ -217,14 +229,11 @@ fun MetadataProvider.newAnnotation(name: CharSequence?, rawNode: Any? = null): A @JvmOverloads fun MetadataProvider.newAnnotationMember( name: CharSequence?, - value: Expression?, rawNode: Any? = null ): AnnotationMember { val node = AnnotationMember() node.applyMetadata(this, name, rawNode, true) - node.value = value - log(node) return node } @@ -380,3 +389,69 @@ private fun Node.setCodeAndLocation( } this.location = provider.locationOf(rawNode) } + +context(AstStackProvider, ContextProvider) +fun T.withChildren( + hasScope: Boolean = false, + isGlobalScope: Boolean = false, + init: (T) -> Unit +): T { + val scopeManager = + this@ContextProvider.ctx?.scopeManager + ?: throw TranslationException( + "Trying to create node children without a ContextProvider. This will fail." + ) + + (this@AstStackProvider).astStack.addLast(this@withChildren) + + if (isGlobalScope && this is TranslationUnitDeclaration) { + scopeManager.resetToGlobal(this) + init(this) + } else if (hasScope) { + scopeManager.enterScope(this) + init(this) + scopeManager.leaveScope(this) + } else { + init(this) + } + + (this@AstStackProvider).astStack.removeLast() + + return this +} + +/** + * This function can be used to set the [Node.astParent] of this node to the current node on the + * [AstStackProvider]'s stack. This is particularly useful if the node was created outside of the + * [withChildren] lambda. This is a usual pattern if the node is optionally wrapped in something + * else. + * + * Example: + * ```kotlin + * val expr = newReference("p") + * + * return if (isDeref) { + * newUnaryOperator("*", prefix = true, postfix = false).withChildren { + * it.input = expr.withParent() + * } + * } else { + * expr + * } + * ``` + */ +context(AstStackProvider) +fun T.withParent(): T { + this.astParent = (this@AstStackProvider).astStack.lastOrNull() + return this +} + +context(ContextProvider) +fun T.declare(): T { + val scopeManager = + this@ContextProvider.ctx?.scopeManager + ?: throw TranslationException( + "Trying to create node children without a ContextProvider. This will fail." + ) + scopeManager.addDeclaration(this) + return this +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt index 78ce712c10f..0d8ec895061 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt @@ -1088,7 +1088,11 @@ operator fun Expression.plus(rhs: Expression): BinaryOperator { */ context(LanguageFrontend<*, *>, StatementHolder) operator fun Expression.plusAssign(rhs: Expression) { - val node = (this@LanguageFrontend).newAssignExpression("+=", listOf(this), listOf(rhs)) + val node = + (this@LanguageFrontend).newAssignExpression("+=").withChildren { + it.rhs = listOf(rhs) + it.lhs = listOf(this) + } (this@StatementHolder) += node } @@ -1292,8 +1296,10 @@ fun Expression.conditional( thenExpression: Expression, elseExpression: Expression ): ConditionalExpression { - val node = - (this@LanguageFrontend).newConditionalExpression(condition, thenExpression, elseExpression) + val node = (this@LanguageFrontend).newConditionalExpression() + node.condition = condition + node.thenExpression = thenExpression + node.elseExpression = elseExpression if (this@Holder is StatementHolder) { (this@Holder) += node @@ -1326,7 +1332,11 @@ infix fun Expression.assign(init: AssignExpression.() -> Expression): AssignExpr */ context(LanguageFrontend<*, *>, Holder) infix fun Expression.assign(rhs: Expression): AssignExpression { - val node = (this@LanguageFrontend).newAssignExpression("=", listOf(this), listOf(rhs)) + val node = + (this@LanguageFrontend).newAssignExpression("=").withChildren { + it.rhs = listOf(rhs) + it.lhs = listOf(this) + } if (this@Holder is StatementHolder) { this@Holder += node @@ -1341,7 +1351,11 @@ infix fun Expression.assign(rhs: Expression): AssignExpression { */ context(LanguageFrontend<*, *>, Holder) infix fun Expression.assignPlus(rhs: Expression): AssignExpression { - val node = (this@LanguageFrontend).newAssignExpression("+=", listOf(this), listOf(rhs)) + val node = + (this@LanguageFrontend).newAssignExpression("+=").withChildren { + it.rhs = listOf(rhs) + it.lhs = listOf(this) + } if (this@Holder is StatementHolder) { this@Holder += node @@ -1356,7 +1370,11 @@ infix fun Expression.assignPlus(rhs: Expression): AssignExpression { */ context(LanguageFrontend<*, *>, Holder) infix fun Expression.assignAsExpr(rhs: Expression): AssignExpression { - val node = (this@LanguageFrontend).newAssignExpression("=", listOf(this), listOf(rhs)) + val node = + (this@LanguageFrontend).newAssignExpression("=").withChildren { + it.rhs = listOf(rhs) + it.lhs = listOf(this) + } node.usedAsExpression = true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt index 5dc54d9509c..1d01395ee5b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt @@ -92,4 +92,8 @@ open class DeclarationStatement : Statement() { } override fun hashCode() = Objects.hash(super.hashCode(), declarations) + + operator fun plusAssign(declaration: Declaration) { + addToPropertyEdgeDeclaration(declaration) + } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/ASTParentTester.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/ASTParentTester.kt new file mode 100644 index 00000000000..faceb46d33e --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/ASTParentTester.kt @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.helpers + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.graph.Component +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression + +object ASTParentTester { + + /** + * This helper functions verifies that every "forward" AST edge has exactly one reverse edge to + * the AST "parent". + */ + fun checkASTParents(result: TranslationResult): List> { + val flatAST = SubgraphWalker.flattenAST(result) + val errorList = mutableListOf>() + for (parent in flatAST) { + when (parent) { + is TranslationResult, + is Component, + is NamespaceDeclaration, + is TranslationUnitDeclaration -> continue // ignore + else -> {} + } + if (parent.isInferred) continue // TODO fix & remove + for (child in parent.astChildren) { + if (child.isInferred) continue // TODO fix & remove + if (child is ProblemExpression) continue // TODO fix & remove + if (parent !== child.astParent) { + errorList.add(Pair(parent, child)) + } + } + } + return errorList + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EdgeCachePass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EdgeCachePass.kt index 3d97fca01d5..0e0c29ef19e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EdgeCachePass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EdgeCachePass.kt @@ -143,8 +143,3 @@ class EdgeCachePass(ctx: TranslationContext) : ComponentPass(ctx) { // nothing to do } } - -val Node.astParent: Node? - get() { - return Edges.to(this, EdgeType.AST).firstOrNull()?.source - } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ArgumentHolderTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ArgumentHolderTest.kt index 207ac245153..183b3e504f2 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ArgumentHolderTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ArgumentHolderTest.kt @@ -43,7 +43,7 @@ class ArgumentHolderTest { newCastExpression(), newIfStatement(), newReturnStatement(), - newConditionalExpression(newLiteral(true)), + newConditionalExpression().withChildren { it.condition = newLiteral(true) }, newDoStatement(), newInitializerListExpression(), newKeyValueExpression(), diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilderTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilderTest.kt new file mode 100644 index 00000000000..2652d371dcf --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilderTest.kt @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph + +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.UnaryOperator +import kotlin.test.Test +import kotlin.test.assertIs +import kotlin.test.assertNotNull + +class NodeBuilderTest { + + @Test + fun testNestedNodeBuilder() { + with(TestLanguageFrontend()) { + val tu = + newTranslationUnitDeclaration("my.file").withChildren(isGlobalScope = true) { + val func = + newFunctionDeclaration("main").withChildren(hasScope = true) { + val param = newParameterDeclaration("param1") + + scopeManager.addDeclaration(param) + } + + scopeManager.addDeclaration(func) + } + + tu.nodes + .filter { it != tu } + .forEach { assertNotNull(it.astParent, "${it.name} has no parent") } + } + } + + @Test + fun testWithParent() { + with(TestLanguageFrontend()) { + fun create(isDeref: Boolean): Expression { + return newBlock().withChildren { + val expr = newReference("p") + + it += + if (isDeref) { + newUnaryOperator("*", prefix = true, postfix = false).withChildren { + it.input = expr.withParent() + } + } else { + expr + } + } + } + + val node1 = create(false) + var ref = node1.refs.firstOrNull() + assertNotNull(ref) + assertIs(ref.astParent) + + val node2 = create(true) + ref = node2.refs.firstOrNull() + assertNotNull(ref) + assertIs(ref.astParent) + } + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt index 730a2c1eb89..3676cc4ca3b 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt @@ -42,7 +42,11 @@ class AssignExpressionTest { val refB = newReference("b") // Simple assignment from "b" to "a". Both types are unknown at this point - val stmt = newAssignExpression(lhs = listOf(refA), rhs = listOf(refB)) + val stmt = + newAssignExpression().withChildren { + it.lhs = listOf(refA) + it.rhs = listOf(refB) + } // Type listeners should be configured assertContains(refB.typeObservers, stmt) @@ -78,7 +82,10 @@ class AssignExpressionTest { // Assignment from "func()" to "a" and "err". val stmt = - newAssignExpression(lhs = listOf(refA, refErr), rhs = listOf(call)) + newAssignExpression().withChildren { + it.lhs = listOf(refA, refErr) + it.rhs = listOf(call) + } body = newBlock() body as Block += stmt diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/scopes/ScopeManagerTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/scopes/ScopeManagerTest.kt index 47eef6eb9d5..42c78c0c48d 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/scopes/ScopeManagerTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/scopes/ScopeManagerTest.kt @@ -26,10 +26,6 @@ package de.fraunhofer.aisec.cpg.passes.scopes import de.fraunhofer.aisec.cpg.* -import de.fraunhofer.aisec.cpg.frontends.TestLanguage -import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend -import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.scopes.NameScope import de.fraunhofer.aisec.cpg.test.* import kotlin.test.* @@ -40,7 +36,7 @@ internal class ScopeManagerTest : BaseTest() { fun setUp() { config = TranslationConfiguration.builder().defaultPasses().build() } - + /* @Test fun testMerge() { val tm = TypeManager() @@ -52,7 +48,7 @@ internal class ScopeManagerTest : BaseTest() { // build a namespace declaration in f1.cpp with the namespace A val namespaceA1 = frontend1.newNamespaceDeclaration("A") s1.enterScope(namespaceA1) - val func1 = frontend1.newFunctionDeclaration("func1") + val func1 = newFunctionDeclaration("func1") s1.addDeclaration(func1) s1.leaveScope(namespaceA1) @@ -61,12 +57,15 @@ internal class ScopeManagerTest : BaseTest() { TestLanguageFrontend("::", TestLanguage(), TranslationContext(config, s2, tm)) s2.resetToGlobal(frontend2.newTranslationUnitDeclaration("f1.cpp", null)) - // and do the same in the other file - val namespaceA2 = frontend2.newNamespaceDeclaration("A") - s2.enterScope(namespaceA2) - val func2 = frontend2.newFunctionDeclaration("func2") - s2.addDeclaration(func2) - s2.leaveScope(namespaceA2) + with(frontend2) { + // and do the same in the other file + val namespaceA2 = frontend2.newNamespaceDeclaration("A") + s2.enterScope(namespaceA2) + val func2 = newFunctionDeclaration("func2") + s2.addDeclaration(func2) + s2.leaveScope(namespaceA2) + + } // merge the two scopes. this replicates the behaviour of parseParallel val final = ScopeManager() @@ -135,5 +134,5 @@ internal class ScopeManagerTest : BaseTest() { val scope = s.lookupScope("A::B") assertNotNull(scope) - } + }*/ } diff --git a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt index 8be47cdcd3f..9a73a8e6c44 100644 --- a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt +++ b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt @@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.helpers.ASTParentTester import de.fraunhofer.aisec.cpg.test.TestUtils.ENFORCE_MEMBER_EXPRESSION import de.fraunhofer.aisec.cpg.test.TestUtils.ENFORCE_REFERENCES import java.io.File @@ -153,7 +154,10 @@ fun analyze( configModifier?.accept(builder) val config = builder.build() val analyzer = TranslationManager.builder().config(config).build() - return analyzer.analyze().get() + val result = analyzer.analyze().get() + var errorList = ASTParentTester.checkASTParents(result) + assertEquals(listOf(), errorList, "AST parents do not match") + return result } /** diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt index 18a6e92bf64..290091ed53e 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt @@ -63,12 +63,14 @@ class DeclarationHandler(frontend: GoLanguageFrontend) : // because the syntax is only there to ensure that this method is part // of the struct, but it is not modifying the receiver. if (recvField?.names?.isNotEmpty() == true) { - method.receiver = - newVariableDeclaration( - recvField.names[0].name, - recordType, - rawNode = recvField - ) + method.withChildren { + it.receiver = + newVariableDeclaration( + recvField.names[0].name, + recordType, + rawNode = recvField + ) + } } if (recordType !is UnknownType) { @@ -103,60 +105,58 @@ class DeclarationHandler(frontend: GoLanguageFrontend) : newFunctionDeclaration(funcDecl.name.name, localNameOnly, rawNode = funcDecl) } - frontend.scopeManager.enterScope(func) - - if (func is MethodDeclaration && func.receiver != null) { - // Add the receiver do the scope manager, so we can resolve the receiver value - frontend.scopeManager.addDeclaration(func.receiver) - } + func.withChildren(hasScope = true) { + if (func is MethodDeclaration && func.receiver != null) { + // Add the receiver do the scope manager, so we can resolve the receiver value + func.receiver?.declare() + } - val returnTypes = mutableListOf() - - // Build return types (and variables) - val results = funcDecl.type.results - if (results != null) { - for (returnVar in results.list) { - returnTypes += frontend.typeOf(returnVar.type) - - // If the function has named return variables, be sure to declare them as well - if (returnVar.names.isNotEmpty()) { - val param = - newVariableDeclaration( - returnVar.names[0].name, - frontend.typeOf(returnVar.type), - rawNode = returnVar - ) - - // Add parameter to scope - frontend.scopeManager.addDeclaration(param) + val returnTypes = mutableListOf() + + // Build return types (and variables) + val results = funcDecl.type.results + if (results != null) { + for (returnVar in results.list) { + returnTypes += frontend.typeOf(returnVar.type) + + // If the function has named return variables, be sure to declare them as well + if (returnVar.names.isNotEmpty()) { + val param = + newVariableDeclaration( + returnVar.names[0].name, + frontend.typeOf(returnVar.type), + rawNode = returnVar + ) + + // Add parameter to scope + param.declare() + } } } - } - func.type = frontend.typeOf(funcDecl.type) - func.returnTypes = returnTypes - - // Parse parameters - handleFuncParams(funcDecl.type.params) - - // Only parse function body in non-dependencies - if (!frontend.isDependency) { - // Check, if the last statement is a return statement, otherwise we insert an implicit - // one - val body = funcDecl.body?.let { frontend.statementHandler.handle(it) } - if (body is Block) { - val last = body.statements.lastOrNull() - if (last !is ReturnStatement) { - val ret = newReturnStatement() - ret.isImplicit = true - body += ret - } + func.type = frontend.typeOf(funcDecl.type) + func.returnTypes = returnTypes + + // Parse parameters + handleFuncParams(funcDecl.type.params) + + // Only parse function body in non-dependencies + if (!frontend.isDependency) { + // Check, if the last statement is a return statement, otherwise we insert an + // implicit one + func.body = + (funcDecl.body?.let { frontend.statementHandler.handle(it) } as? Block) + ?.withChildren { body -> + val last = body.statements.lastOrNull() + if (last !is ReturnStatement) { + val ret = newReturnStatement() + ret.isImplicit = true + body += ret + } + } } - func.body = body } - frontend.scopeManager.leaveScope(func) - // Leave scope of record, if applicable (func as? MethodDeclaration)?.recordDeclaration?.let { frontend.scopeManager.leaveScope(it) diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt index 8a389e1a589..6ce68390ecd 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt @@ -124,9 +124,11 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : } private fun handleBinaryExpr(binaryExpr: GoStandardLibrary.Ast.BinaryExpr): BinaryOperator { - val binOp = newBinaryOperator(binaryExpr.opString, rawNode = binaryExpr) - binOp.lhs = handle(binaryExpr.x) - binOp.rhs = handle(binaryExpr.y) + val binOp = + newBinaryOperator(binaryExpr.opString, rawNode = binaryExpr).withChildren { + it.lhs = handle(binaryExpr.x) + it.rhs = handle(binaryExpr.y) + } return binOp } @@ -165,9 +167,11 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : } private fun handleIndexExpr(indexExpr: GoStandardLibrary.Ast.IndexExpr): SubscriptExpression { - val ase = newSubscriptExpression(rawNode = indexExpr) - ase.arrayExpression = frontend.expressionHandler.handle(indexExpr.x) - ase.subscriptExpression = frontend.expressionHandler.handle(indexExpr.index) + val ase = + newSubscriptExpression(rawNode = indexExpr).withChildren { + it.arrayExpression = frontend.expressionHandler.handle(indexExpr.x) + it.subscriptExpression = frontend.expressionHandler.handle(indexExpr.index) + } return ase } @@ -182,14 +186,13 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : is GoStandardLibrary.Ast.InterfaceType, is GoStandardLibrary.Ast.StructType, is GoStandardLibrary.Ast.MapType, -> { - val cast = newCastExpression(rawNode = callExpr) - cast.castType = frontend.typeOf(unwrapped) + return newCastExpression(rawNode = callExpr).withChildren { + it.castType = frontend.typeOf(unwrapped) - if (callExpr.args.isNotEmpty()) { - cast.expression = frontend.expressionHandler.handle(callExpr.args[0]) + if (callExpr.args.isNotEmpty()) { + it.expression = frontend.expressionHandler.handle(callExpr.args[0]) + } } - - return cast } } @@ -222,29 +225,28 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : } // Differentiate between calls and member calls based on the callee - val call = - if (callee is MemberExpression) { + return if (callee is MemberExpression) { newMemberCallExpression(callee, rawNode = callExpr) } else { newCallExpression(callee, name, rawNode = callExpr) } - call.type = unknownType() - - // TODO(oxisto) Add type constraints - if (typeConstraints.isNotEmpty()) { - log.debug( - "Call {} has type constraints ({}), but we cannot add them to the call expression yet", - call.name, - typeConstraints.joinToString(", ") { it.name } - ) - } - - // Parse and add call arguments - for (arg in callExpr.args) { - call += handle(arg) - } + .withChildren { + it.type = unknownType() + + // TODO(oxisto) Add type constraints + if (typeConstraints.isNotEmpty()) { + log.debug( + "Call {} has type constraints ({}), but we cannot add them to the call expression yet", + it.name, + typeConstraints.joinToString(", ") { it.name } + ) + } - return call + // Parse and add call arguments + for (arg in callExpr.args) { + it += handle(arg) + } + } } /** @@ -274,21 +276,19 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : return newProblemExpression("could not create NewExpression with empty arguments") } - val n = newNewExpression(rawNode = callExpr) - - // First argument is type - val type = frontend.typeOf(callExpr.args[0]) - - // new is a pointer, so need to reference the type with a pointer - n.type = type.reference(PointerType.PointerOrigin.POINTER) + return newNewExpression(rawNode = callExpr).withChildren { + // First argument is type + val type = frontend.typeOf(callExpr.args[0]) - // a new expression also needs an initializer, which is usually a ConstructExpression - val construct = newConstructExpression(rawNode = callExpr) - construct.type = type + // new is a pointer, so need to reference the type with a pointer + it.type = type.reference(PointerType.PointerOrigin.POINTER) - n.initializer = construct + // a new expression also needs an initializer, which is usually a ConstructExpression + val construct = newConstructExpression(rawNode = callExpr) + construct.type = type - return n + it.initializer = construct + } } private fun handleMakeExpr(callExpr: GoStandardLibrary.Ast.CallExpr): Expression { @@ -301,24 +301,21 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : val expression = // Actually make() can make more than just arrays, i.e. channels and maps if (args[0] is GoStandardLibrary.Ast.ArrayType) { - val array = newNewArrayExpression(rawNode = callExpr) - - // second argument is a dimension (if this is an array), usually a literal - if (args.size > 1) { - array.addDimension(handle(args[1])) + newNewArrayExpression(rawNode = callExpr).withChildren { + // second argument is a dimension (if this is an array), usually a literal + if (args.size > 1) { + it.addDimension(handle(args[1])) + } } - array } else { // Create at least a generic construct expression for the given map or channel type // and provide the remaining arguments - val construct = newConstructExpression(rawNode = callExpr) - - // Pass the remaining arguments - for (expr in args.subList(1.coerceAtMost(args.size - 1), args.size - 1)) { - handle(expr).let { construct += it } + newConstructExpression(rawNode = callExpr).withChildren { construct -> + // Pass the remaining arguments + for (expr in args.subList(1.coerceAtMost(args.size - 1), args.size - 1)) { + handle(expr).let { construct += it } + } } - - construct } // First argument is always the type @@ -370,26 +367,23 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : * as its subscriptExpression to share some code between this and an index expression. */ private fun handleSliceExpr(sliceExpr: GoStandardLibrary.Ast.SliceExpr): SubscriptExpression { - val ase = newSubscriptExpression(rawNode = sliceExpr) - ase.arrayExpression = frontend.expressionHandler.handle(sliceExpr.x) - - // Build the slice expression - val range = newRangeExpression(rawNode = sliceExpr) - sliceExpr.low?.let { range.floor = frontend.expressionHandler.handle(it) } - sliceExpr.high?.let { range.ceiling = frontend.expressionHandler.handle(it) } - sliceExpr.max?.let { range.third = frontend.expressionHandler.handle(it) } - - ase.subscriptExpression = range - - return ase + return newSubscriptExpression(rawNode = sliceExpr).withChildren { ase -> + ase.arrayExpression = frontend.expressionHandler.handle(sliceExpr.x) + + // Build the slice expression + ase.subscriptExpression = + newRangeExpression(rawNode = sliceExpr).withChildren { range -> + sliceExpr.low?.let { range.floor = frontend.expressionHandler.handle(it) } + sliceExpr.high?.let { range.ceiling = frontend.expressionHandler.handle(it) } + sliceExpr.max?.let { range.third = frontend.expressionHandler.handle(it) } + } + } } - private fun handleStarExpr(starExpr: GoStandardLibrary.Ast.StarExpr): UnaryOperator { - val op = newUnaryOperator("*", postfix = false, prefix = false, rawNode = starExpr) - op.input = handle(starExpr.x) - - return op - } + private fun handleStarExpr(starExpr: GoStandardLibrary.Ast.StarExpr) = + newUnaryOperator("*", postfix = false, prefix = false, rawNode = starExpr).withChildren { + it.input = handle(starExpr.x) + } private fun handleTypeAssertExpr( typeAssertExpr: GoStandardLibrary.Ast.TypeAssertExpr @@ -398,39 +392,22 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : // "special" type assertion `.(type)`, which is used in a type switch to retrieve the type // of the variable. In this case we treat it as a special unary operator. return if (typeAssertExpr.type == null) { - val op = - newUnaryOperator( - ".(type)", - postfix = true, - prefix = false, - rawNode = typeAssertExpr - ) - op.input = handle(typeAssertExpr.x) - op + newUnaryOperator(".(type)", postfix = true, prefix = false, rawNode = typeAssertExpr) + .withChildren { it.input = handle(typeAssertExpr.x) } } else { - val cast = newCastExpression(rawNode = typeAssertExpr) - - // Parse the inner expression - cast.expression = handle(typeAssertExpr.x) + newCastExpression(rawNode = typeAssertExpr).withChildren { cast -> + // Parse the inner expression + cast.expression = handle(typeAssertExpr.x) - // The type can be null, but only in certain circumstances, i.e, a type switch - typeAssertExpr.type?.let { cast.castType = frontend.typeOf(it) } - cast + // The type can be null, but only in certain circumstances, i.e, a type switch + typeAssertExpr.type?.let { cast.castType = frontend.typeOf(it) } + } } } - private fun handleUnaryExpr(unaryExpr: GoStandardLibrary.Ast.UnaryExpr): UnaryOperator { - val op = - newUnaryOperator( - unaryExpr.opString, - postfix = false, - prefix = false, - rawNode = unaryExpr - ) - op.input = handle(unaryExpr.x) - - return op - } + private fun handleUnaryExpr(unaryExpr: GoStandardLibrary.Ast.UnaryExpr) = + newUnaryOperator(unaryExpr.opString, postfix = false, prefix = false, rawNode = unaryExpr) + .withChildren { op -> op.input = handle(unaryExpr.x) } /** * handleCompositeLit handles a composite literal, which we need to translate into a combination @@ -443,32 +420,29 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : // the "outer" one. See below val type = compositeLit.type?.let { frontend.typeOf(it) } ?: unknownType() - val list = newInitializerListExpression(type, rawNode = compositeLit) - list.type = type + return newInitializerListExpression(type, rawNode = compositeLit).withChildren { list -> + list.type = type - val expressions = mutableListOf() - for (elem in compositeLit.elts) { - val expression = handle(elem) - expressions += expression - } - - list.initializers = expressions + val expressions = mutableListOf() + for (elem in compositeLit.elts) { + val expression = handle(elem) + expressions += expression + } - return list + list.initializers = expressions + } } - /* - // handleFuncLit handles a function literal, which we need to translate into a combination of a - // LambdaExpression and a function declaration. - */ - fun handleFuncLit(funcLit: GoStandardLibrary.Ast.FuncLit): LambdaExpression { - val lambda = newLambdaExpression(rawNode = funcLit) - // Parse the expression as a function declaration with a little trick - lambda.function = - frontend.declarationHandler.handle(funcLit.toDecl()) as? FunctionDeclaration - - return lambda - } + /** + * handleFuncLit handles a function literal, which we need to translate into a combination of a + * LambdaExpression and a function declaration. + */ + fun handleFuncLit(funcLit: GoStandardLibrary.Ast.FuncLit) = + newLambdaExpression(rawNode = funcLit).withChildren { lambda -> + // Parse the expression as a function declaration with a little trick + lambda.function = + frontend.declarationHandler.handle(funcLit.toDecl()) as? FunctionDeclaration + } companion object { val builtins = diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt index 656894b58de..b9719cd3b62 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt @@ -59,6 +59,11 @@ abstract class GoHandler): List { + return list.mapNotNull(this::handle) + } + abstract fun handleNode(node: HandlerNode): ResultNode /** diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt index c0c6df6a107..204dfb16def 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt @@ -167,76 +167,81 @@ class GoLanguageFrontend(language: Language, ctx: Translatio currentFile = f currentFileSet = fset - val tu = newTranslationUnitDeclaration(file.absolutePath, rawNode = f) - scopeManager.resetToGlobal(tu) - currentTU = tu - - // We need to keep imports on a special file scope. We can simulate this by "entering" the - // translation unit - scopeManager.enterScope(tu) - - // We parse the imports specifically and not as part of the handler later - for (spec in f.imports) { - val import = specificationHandler.handle(spec) - scopeManager.addDeclaration(import) - } - - val p = newNamespaceDeclaration(f.name.name) - scopeManager.enterScope(p) - - try { - // we need to construct the package "path" (e.g. "encoding/json") out of the - // module path as well as the current directory in relation to the topLevel - var packagePath = file.parentFile.relativeTo(topLevel) - - // If we are in a module, we need to prepend the module path to it. There is an - // exception if we are in the "std" module, which represents the standard library - val modulePath = currentModule?.module?.mod?.path - if (modulePath != null && modulePath != "std") { - packagePath = File(modulePath).resolve(packagePath) + return newTranslationUnitDeclaration(file.absolutePath, rawNode = f).withChildren( + isGlobalScope = true + ) { tu -> + currentTU = tu + + // We need to keep imports on a special file scope. We can simulate this by "entering" + // the translation unit + scopeManager.enterScope(tu) + + // We parse the imports specifically and not as part of the handler later + for (spec in f.imports) { + val import = specificationHandler.handle(spec) + scopeManager.addDeclaration(import) } - p.path = packagePath.path - } catch (ex: IllegalArgumentException) { - log.error( - "Could not relativize package path to top level. Cannot set package path.", - ex - ) - } + var p = + newNamespaceDeclaration(f.name.name).withChildren(hasScope = true) { p -> + try { + // we need to construct the package "path" (e.g. "encoding/json") out of the + // module path as well as the current directory in relation to the topLevel + var packagePath = file.parentFile.relativeTo(topLevel) + + // If we are in a module, we need to prepend the module path to it. There is + // an exception if we are in the "std" module, which represents the standard + // library + val modulePath = currentModule?.module?.mod?.path + if (modulePath != null && modulePath != "std") { + packagePath = File(modulePath).resolve(packagePath) + } - for (decl in f.decls) { - // Retrieve all top level declarations. One "Decl" could potentially - // contain multiple CPG declarations. - val declaration = declarationHandler.handle(decl) - if (declaration is DeclarationSequence) { - declaration.declarations.forEach { scopeManager.addDeclaration(it) } - } else { - // We need to be careful with method declarations. We need to put them in the - // respective name scope of the record and NOT on the global scope / namespace scope - // TODO: this is broken if we see the declaration of the method before the class :( - if (declaration is MethodDeclaration) { - declaration.recordDeclaration?.let { - scopeManager.enterScope(it) - scopeManager.addDeclaration(declaration) - scopeManager.leaveScope(it) - // But still add it to the AST of the namespace so our AST walker can find - // it - p.declarations += declaration + p.path = packagePath.path + } catch (ex: IllegalArgumentException) { + log.error( + "Could not relativize package path to top level. Cannot set package path.", + ex + ) } - } else { - scopeManager.addDeclaration(declaration) - } - } - } - scopeManager.leaveScope(p) - scopeManager.leaveScope(tu) + for (decl in f.decls) { + // Retrieve all top level declarations. One "Decl" could potentially + // contain multiple CPG declarations. + val declaration = declarationHandler.handle(decl) + if (declaration is DeclarationSequence) { + declaration.declarations.forEach { it.declare() } + } else { + // We need to be careful with method declarations. We need to put them + // in the respective name scope of the record and NOT on the global + // scope / + // namespace scope + // TODO: this is broken if we see the declaration of the method before + // the class :( + if (declaration is MethodDeclaration) { + declaration.recordDeclaration?.let { + scopeManager.enterScope(it) + scopeManager.addDeclaration(declaration) + scopeManager.leaveScope(it) + // But still add it to the AST of the namespace so our AST + // walker can find + // it + p.declarations += declaration + } + } else { + scopeManager.addDeclaration(declaration) + } + } + } + } - scopeManager.resetToGlobal(tu) + scopeManager.leaveScope(tu) - scopeManager.addDeclaration(p) + scopeManager.resetToGlobal(tu) - return tu + // Declare our namespace + p.declare() + } } override fun typeOf(type: GoStandardLibrary.Ast.Expr): Type { @@ -318,8 +323,7 @@ class GoLanguageFrontend(language: Language, ctx: Translatio // Create an anonymous struct, this will add it to the scope manager. This is // somewhat duplicate, but the easiest for now. We need to create it in the - // global - // scope to avoid namespace issues + // global scope to avoid namespace issues var record = scopeManager.withScope(scopeManager.globalScope) { var record = specificationHandler.buildRecordDeclaration(type, name) @@ -331,14 +335,12 @@ class GoLanguageFrontend(language: Language, ctx: Translatio } is GoStandardLibrary.Ast.InterfaceType -> { // Go allows to use anonymous interface as type. This is something we cannot - // model - // properly in the CPG yet. In order to at least deal with this partially, we - // construct a ObjectType and put the methods and their types into the type. + // model properly in the CPG yet. In order to at least deal with this partially, + // we construct a ObjectType and put the methods and their types into the type. // In the easiest case this is the empty interface `interface{}`, which we then // consider to be the "any" type. `any` is actually a type alias for - // `interface{}`, - // but in modern Go `any` is preferred. + // `interface{}`, but in modern Go `any` is preferred. if (type.methods.list.isEmpty()) { return primitiveType("any") } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt index 21db017ade5..a0bbccaf64a 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.scopes.NameScope +import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.helpers.Util class SpecificationHandler(frontend: GoLanguageFrontend) : @@ -103,85 +104,73 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : structType: GoStandardLibrary.Ast.StructType, name: CharSequence, typeSpec: GoStandardLibrary.Ast.TypeSpec? = null, - ): RecordDeclaration { - val record = newRecordDeclaration(name, "struct", rawNode = typeSpec) - - frontend.scopeManager.enterScope(record) - - if (!structType.incomplete) { - for (field in structType.fields.list) { - val type = frontend.typeOf(field.type) - - // A field can also have no name, which means that it is embedded. In this case, it - // can be accessed by the local name of its type and therefore we name the field - // accordingly. We use the modifiers property to denote that this is an embedded - // field, so we can easily retrieve them later - val (fieldName, modifiers) = - if (field.names.isEmpty()) { - // Retrieve the root type local name - Pair(type.root.name.localName, listOf("embedded")) - } else { - Pair(field.names[0].name, listOf()) - } + ) = + newRecordDeclaration(name, "struct", rawNode = typeSpec).withChildren(hasScope = true) { + if (!structType.incomplete) { + for (field in structType.fields.list) { + val type = frontend.typeOf(field.type) + + // A field can also have no name, which means that it is embedded. In this case, + // it can be accessed by the local name of its type and therefore we name the + // field accordingly. We use the modifiers property to denote that this is an + // embedded field, so we can easily retrieve them later + val (fieldName, modifiers) = + if (field.names.isEmpty()) { + // Retrieve the root type local name + Pair(type.root.name.localName, listOf("embedded")) + } else { + Pair(field.names[0].name, listOf()) + } - val decl = newFieldDeclaration(fieldName, type, modifiers, rawNode = field) - frontend.scopeManager.addDeclaration(decl) + newFieldDeclaration(fieldName, type, modifiers, rawNode = field).declare() + } } } - frontend.scopeManager.leaveScope(record) - - return record - } - private fun handleInterfaceTypeSpec( typeSpec: GoStandardLibrary.Ast.TypeSpec, interfaceType: GoStandardLibrary.Ast.InterfaceType - ): Declaration { - val record = newRecordDeclaration(typeSpec.name.name, "interface", rawNode = typeSpec) - - // Make sure to register the type - frontend.typeManager.registerType(record.toType()) - - frontend.scopeManager.enterScope(record) - - if (!interfaceType.incomplete) { - for (field in interfaceType.methods.list) { - val type = frontend.typeOf(field.type) - - // Even though this list is called "Methods", it contains all kinds - // of things, so we need to proceed with caution. Only if the - // "method" actually has a name, we declare a new method - // declaration. - if (field.names.isNotEmpty()) { - val method = newMethodDeclaration(field.names[0].name, rawNode = field) - method.type = type - - frontend.scopeManager.enterScope(method) - - val params = (field.type as? GoStandardLibrary.Ast.FuncType)?.params - if (params != null) { - frontend.declarationHandler.handleFuncParams(params) + ) = + newRecordDeclaration(typeSpec.name.name, "interface", rawNode = typeSpec).withChildren( + hasScope = true + ) { record -> + // Make sure to register the type + frontend.typeManager.registerType(record.toType()) + + if (!interfaceType.incomplete) { + for (field in interfaceType.methods.list) { + val type = frontend.typeOf(field.type) + + // Even though this list is called "Methods", it contains all kinds + // of things, so we need to proceed with caution. Only if the + // "method" actually has a name, we declare a new method + // declaration. + if (field.names.isNotEmpty()) { + newMethodDeclaration(field.names[0].name, rawNode = field) + .withChildren(hasScope = true) { method -> + method.type = type + + val params = (field.type as? GoStandardLibrary.Ast.FuncType)?.params + if (params != null) { + frontend.declarationHandler.handleFuncParams(params) + } + } + .declare() + } else { + log.debug( + "Adding {} as super class of interface {}", + type.name, + record.name + ) + // Otherwise, it contains either types or interfaces. For now, we + // hope that it only has interfaces. We consider embedded + // interfaces as sort of super types for this interface. + record.addSuperClass(type) } - - frontend.scopeManager.leaveScope(method) - - frontend.scopeManager.addDeclaration(method) - } else { - log.debug("Adding {} as super class of interface {}", type.name, record.name) - // Otherwise, it contains either types or interfaces. For now, we - // hope that it only has interfaces. We consider embedded - // interfaces as sort of super types for this interface. - record.addSuperClass(type) } } } - frontend.scopeManager.leaveScope(record) - - return record - } - /** * // handleValueSpec handles parsing of an ast.ValueSpec, which is a variable declaration. * Since this can potentially declare multiple variables with one "spec", this returns a @@ -212,22 +201,22 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : } else { ident.name } - val decl = newVariableDeclaration(fqn, rawNode = valueSpec) - - if (valueSpec.type != null) { - decl.type = frontend.typeOf(valueSpec.type!!) - } else { - decl.type = autoType() - } - - if (valueSpec.values.isNotEmpty()) { - tuple.initializer = frontend.expressionHandler.handle(valueSpec.values[0]) - } - // We need to manually add the variables to the scope manager - frontend.scopeManager.addDeclaration(decl) - - tuple += decl + tuple += + newVariableDeclaration(fqn, rawNode = valueSpec) + .withChildren { decl -> + if (valueSpec.type != null) { + decl.type = frontend.typeOf(valueSpec.type!!) + } else { + decl.type = autoType() + } + + if (valueSpec.values.isNotEmpty()) { + tuple.initializer = + frontend.expressionHandler.handle(valueSpec.values[0]) + } + } + .declare() } return tuple } else { @@ -245,54 +234,59 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : } else { ident.name } - val decl = newVariableDeclaration(fqn, rawNode = valueSpec) - if (type != null) { - decl.type = type - } else { - decl.type = autoType() - } - - if (valueSpec.values.size > nameIdx) { - // the initializer is in the "Values" slice with the respective index - decl.initializer = frontend.expressionHandler.handle(valueSpec.values[nameIdx]) - } - - // If we are in a const declaration, we need to do something rather unusual. - // If we have an initializer, we need to set this as the current const initializer, - // because following specs will "inherit" the one from the previous line. - // - // Note: we cannot just take the already parsed initializer, but instead we need to - // reparse the raw AST expression, so that `iota` gets evaluated differently for - // each spec - if (frontend.declCtx.currentDecl?.tok == 64) { - var initializerExpr = valueSpec.values.getOrNull(nameIdx) - if (initializerExpr != null) { - // Set the current initializer - frontend.declCtx.constInitializers[nameIdx] = initializerExpr - - // Set the const type - frontend.declCtx.constType = type - } else { - // Fetch expr from existing initializers - initializerExpr = frontend.declCtx.constInitializers[nameIdx] - if (initializerExpr == null) { - Util.errorWithFileLocation( - decl, - log, - "Const declaration is missing its initializer" - ) + sequence += + newVariableDeclaration(fqn, rawNode = valueSpec).withChildren { decl -> + if (type != null) { + decl.type = type as Type } else { - decl.initializer = frontend.expressionHandler.handle(initializerExpr) + decl.type = autoType() } - } - type = frontend.declCtx.constType - if (type != null) { - decl.type = type - } - } + if (valueSpec.values.size > nameIdx) { + // the initializer is in the "Values" slice with the respective index + decl.initializer = + frontend.expressionHandler.handle(valueSpec.values[nameIdx]) + } - sequence += decl + // If we are in a const declaration, we need to do something rather unusual. + // If we have an initializer, we need to set this as the current const + // initializer, + // because following specs will "inherit" the one from the previous line. + // + // Note: we cannot just take the already parsed initializer, but instead we + // need to + // reparse the raw AST expression, so that `iota` gets evaluated differently + // for + // each spec + if (frontend.declCtx.currentDecl?.tok == 64) { + var initializerExpr = valueSpec.values.getOrNull(nameIdx) + if (initializerExpr != null) { + // Set the current initializer + frontend.declCtx.constInitializers[nameIdx] = initializerExpr + + // Set the const type + frontend.declCtx.constType = type + } else { + // Fetch expr from existing initializers + initializerExpr = frontend.declCtx.constInitializers[nameIdx] + if (initializerExpr == null) { + Util.errorWithFileLocation( + decl, + log, + "Const declaration is missing its initializer" + ) + } else { + decl.initializer = + frontend.expressionHandler.handle(initializerExpr) + } + } + + type = frontend.declCtx.constType + if (type != null) { + decl.type = type as Type + } + } + } } return sequence diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt index dcc0154fac6..198c192273b 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt @@ -60,64 +60,61 @@ class StatementHandler(frontend: GoLanguageFrontend) : } } - private fun handleAssignStmt(assignStmt: GoStandardLibrary.Ast.AssignStmt): AssignExpression { - val lhs = assignStmt.lhs.map { frontend.expressionHandler.handle(it) } - val rhs = assignStmt.rhs.map { frontend.expressionHandler.handle(it) } - - // We need to explicitly set the operator code on this assignment as - // something which potentially declares a variable, so we can resolve this - // in our extra pass. - val operatorCode = - if (assignStmt.tok == 47) { - ":=" - } else { - "=" + /** + * Handles an assignment statement. We need to explicitly set the operator code on this + * assignment as something which potentially declares a variable, so we can resolve this in our + * extra pass. + */ + private fun handleAssignStmt(assignStmt: GoStandardLibrary.Ast.AssignStmt) = + newAssignExpression( + operatorCode = + if (assignStmt.tok == 47) { + ":=" + } else { + "=" + }, + rawNode = assignStmt + ) + .withChildren { expr -> + expr.rhs = frontend.expressionHandler.handle(assignStmt.rhs) + expr.lhs = frontend.expressionHandler.handle(assignStmt.lhs) } - return newAssignExpression(operatorCode, lhs, rhs, rawNode = assignStmt) - } - - private fun handleBranchStmt(branchStmt: GoStandardLibrary.Ast.BranchStmt): Statement { + private fun handleBranchStmt(branchStmt: GoStandardLibrary.Ast.BranchStmt) = when (branchStmt.tokString) { "break" -> { - val stmt = newBreakStatement(rawNode = branchStmt) - branchStmt.label?.let { stmt.label = it.name } - return stmt + newBreakStatement(rawNode = branchStmt).withChildren { stmt -> + branchStmt.label?.let { stmt.label = it.name } + } } "continue" -> { - val stmt = newContinueStatement(rawNode = branchStmt) - branchStmt.label?.let { stmt.label = it.name } - return stmt + newContinueStatement(rawNode = branchStmt).withChildren { stmt -> + branchStmt.label?.let { stmt.label = it.name } + } } "goto" -> { - val stmt = newGotoStatement(rawNode = branchStmt) - branchStmt.label?.let { stmt.labelName = it.name } - return stmt + newGotoStatement(rawNode = branchStmt).withChildren { stmt -> + branchStmt.label?.let { stmt.labelName = it.name } + } } + else -> + newProblemExpression( + "unknown token \"${branchStmt.tokString}\" in branch statement" + ) } - return newProblemExpression("unknown token \"${branchStmt.tokString}\" in branch statement") - } - - private fun handleBlockStmt(blockStmt: GoStandardLibrary.Ast.BlockStmt): Statement { - val compound = newBlock(rawNode = blockStmt) - - frontend.scopeManager.enterScope(compound) - - for (stmt in blockStmt.list) { - val node = handle(stmt) - // Do not add case statements to the block because the already add themselves in - // handleCaseClause. Otherwise, the order of case's would be wrong - if (node !is CaseStatement) { - compound += node + private fun handleBlockStmt(blockStmt: GoStandardLibrary.Ast.BlockStmt) = + newBlock(rawNode = blockStmt).withChildren(hasScope = true) { + for (stmt in blockStmt.list) { + val node = handle(stmt) + // Do not add case statements to the block because the already add themselves in + // handleCaseClause. Otherwise, the order of case's would be wrong + if (node !is CaseStatement) { + it += node + } } } - frontend.scopeManager.leaveScope(compound) - - return compound - } - private fun handleCaseClause( caseClause: GoStandardLibrary.Ast.CaseClause, typeSwitchLhs: Node? = null, @@ -168,30 +165,36 @@ class StatementHandler(frontend: GoLanguageFrontend) : val stmt = newDeclarationStatement() stmt.isImplicit = true - val decl = newVariableDeclaration(typeSwitchLhs.name) - if (case is CaseStatement) { - decl.type = (case.caseExpression as? TypeExpression)?.type ?: unknownType() - } else { - // We need to work with type listeners here because they might not have their type - // yet - typeSwitchRhs.registerTypeObserver( - object : HasType.TypeObserver { - override fun typeChanged(newType: Type, src: HasType) { - decl.type = newType - } - - override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { - // Nothing to do - } + val decl = + newVariableDeclaration(typeSwitchLhs.name).withChildren { + if (case is CaseStatement) { + it.type = (case.caseExpression as? TypeExpression)?.type ?: unknownType() + } else { + // We need to work with type listeners here because they might not have + // their type + // yet + typeSwitchRhs.registerTypeObserver( + object : HasType.TypeObserver { + override fun typeChanged(newType: Type, src: HasType) { + it.type = newType + } + + override fun assignedTypeChanged( + assignedTypes: Set, + src: HasType + ) { + // Nothing to do + } + } + ) } - ) - } - decl.initializer = typeSwitchRhs + it.initializer = typeSwitchRhs + } // Add the variable to the declaration statement as well as to the current scope (aka // our block wrapper) stmt.addToPropertyEdgeDeclaration(decl) - frontend.scopeManager.addDeclaration(decl) + decl.declare() if (block != null) { block += stmt @@ -220,19 +223,18 @@ class StatementHandler(frontend: GoLanguageFrontend) : private fun handleDeclStmt(declStmt: GoStandardLibrary.Ast.DeclStmt): DeclarationStatement { // Let's create a variable declaration (wrapped with a declaration stmt) with // this, because we define the variable here - val stmt = newDeclarationStatement(rawNode = declStmt) - val sequence = frontend.declarationHandler.handle(declStmt.decl) - if (sequence is DeclarationSequence) { - for (declaration in sequence.declarations) { - frontend.scopeManager.addDeclaration(declaration) + return newDeclarationStatement(rawNode = declStmt).withChildren { stmt -> + val sequence = frontend.declarationHandler.handle(declStmt.decl) + if (sequence is DeclarationSequence) { + for (declaration in sequence.declarations) { + frontend.scopeManager.addDeclaration(declaration) + } + stmt.declarations = sequence.asList() + } else { + frontend.scopeManager.addDeclaration(sequence) + stmt.singleDeclaration = sequence } - stmt.declarations = sequence.asList() - } else { - frontend.scopeManager.addDeclaration(sequence) - stmt.singleDeclaration = sequence } - - return stmt } /** @@ -241,11 +243,9 @@ class StatementHandler(frontend: GoLanguageFrontend) : * this 1:1, so we basically we create a call expression to a built-in call. // We adjust the * EOG of the call later in an extra pass. */ - private fun handleDeferStmt(deferStmt: GoStandardLibrary.Ast.DeferStmt): UnaryOperator { - val op = newUnaryOperator("defer", postfix = false, prefix = true, rawNode = deferStmt) - op.input = frontend.expressionHandler.handle(deferStmt.call) - return op - } + private fun handleDeferStmt(deferStmt: GoStandardLibrary.Ast.DeferStmt) = + newUnaryOperator("defer", postfix = false, prefix = true, rawNode = deferStmt) + .withChildren { op -> op.input = frontend.expressionHandler.handle(deferStmt.call) } /** * This function handles the `go` statement, which is a special keyword in go that starts the @@ -253,182 +253,120 @@ class StatementHandler(frontend: GoLanguageFrontend) : * we create a call expression to a built-in call. */ private fun handleGoStmt(goStmt: GoStandardLibrary.Ast.GoStmt): CallExpression { + // TODO: this will not set the ast parent of the callee correctly val ref = newReference("go") - val call = newCallExpression(ref, "go", rawNode = goStmt) - call += frontend.expressionHandler.handle(goStmt.call) - - return call - } - - private fun handleForStmt(forStmt: GoStandardLibrary.Ast.ForStmt): ForStatement { - val stmt = newForStatement(rawNode = forStmt) - - frontend.scopeManager.enterScope(stmt) - - forStmt.init?.let { stmt.initializerStatement = handle(it) } - forStmt.cond?.let { stmt.condition = frontend.expressionHandler.handle(it) } - forStmt.post?.let { stmt.iterationStatement = handle(it) } - forStmt.body?.let { stmt.statement = handle(it) } - - frontend.scopeManager.leaveScope(stmt) - - return stmt - } - - private fun handleIncDecStmt(incDecStmt: GoStandardLibrary.Ast.IncDecStmt): UnaryOperator { - val op = - newUnaryOperator( - incDecStmt.tokString, - postfix = true, - prefix = false, - rawNode = incDecStmt - ) - op.input = frontend.expressionHandler.handle(incDecStmt.x) - - return op - } - - private fun handleIfStmt(ifStmt: GoStandardLibrary.Ast.IfStmt): IfStatement { - val stmt = newIfStatement(rawNode = ifStmt) - - frontend.scopeManager.enterScope(stmt) - - ifStmt.init?.let { stmt.initializerStatement = frontend.statementHandler.handle(it) } - - stmt.condition = frontend.expressionHandler.handle(ifStmt.cond) - stmt.thenStatement = frontend.statementHandler.handle(ifStmt.body) - - ifStmt.`else`?.let { stmt.elseStatement = frontend.statementHandler.handle(it) } - - frontend.scopeManager.leaveScope(stmt) - - return stmt - } - - private fun handleLabeledStmt(labeledStmt: GoStandardLibrary.Ast.LabeledStmt): LabelStatement { - val stmt = newLabelStatement(rawNode = labeledStmt) - stmt.subStatement = handle(labeledStmt.stmt) - stmt.label = labeledStmt.label.name - - return stmt - } - - private fun handleRangeStmt(rangeStmt: GoStandardLibrary.Ast.RangeStmt): ForEachStatement { - val forEach = newForEachStatement(rawNode = rangeStmt) - - frontend.scopeManager.enterScope(forEach) - - // TODO: Support other use cases that do not use DEFINE - if (rangeStmt.tokString == ":=") { - val stmt = newDeclarationStatement() - - // TODO: not really the best way to deal with this - // TODO: key type is always int. we could set this - rangeStmt.key?.let { - val ref = frontend.expressionHandler.handle(it) - if (ref is Reference) { - val key = newVariableDeclaration(ref.name, rawNode = it) - frontend.scopeManager.addDeclaration(key) - stmt.addToPropertyEdgeDeclaration(key) - } - } - - // TODO: not really the best way to deal with this - rangeStmt.value?.let { - val ref = frontend.expressionHandler.handle(it) - if (ref is Reference) { - val key = newVariableDeclaration(ref.name, rawNode = it) - frontend.scopeManager.addDeclaration(key) - stmt.addToPropertyEdgeDeclaration(key) - } - } - - forEach.variable = stmt + return newCallExpression(ref, "go", rawNode = goStmt).withChildren { + it += frontend.expressionHandler.handle(goStmt.call) } - - forEach.iterable = frontend.expressionHandler.handle(rangeStmt.x) - forEach.statement = frontend.statementHandler.handle(rangeStmt.body) - - frontend.scopeManager.leaveScope(forEach) - - return forEach } - private fun handleReturnStmt(returnStmt: GoStandardLibrary.Ast.ReturnStmt): ReturnStatement { - val `return` = newReturnStatement(rawNode = returnStmt) - - val results = returnStmt.results - if (results.isNotEmpty()) { - val expr = frontend.expressionHandler.handle(results[0]) - - // TODO: parse more than one result expression - `return`.returnValue = expr - } else { - // TODO: connect result statement to result variables + private fun handleForStmt(forStmt: GoStandardLibrary.Ast.ForStmt) = + newForStatement(rawNode = forStmt).withChildren(hasScope = true) { stmt -> + forStmt.init?.let { stmt.initializerStatement = handle(it) } + forStmt.cond?.let { stmt.condition = frontend.expressionHandler.handle(it) } + forStmt.post?.let { stmt.iterationStatement = handle(it) } + forStmt.body?.let { stmt.statement = handle(it) } } - return `return` - } - - private fun handleSendStmt(sendStmt: GoStandardLibrary.Ast.SendStmt): BinaryOperator { - val op = newBinaryOperator("<-", rawNode = sendStmt) - op.lhs = frontend.expressionHandler.handle(sendStmt.chan) - op.rhs = frontend.expressionHandler.handle(sendStmt.value) - - return op - } - - private fun handleSwitchStmt(switchStmt: GoStandardLibrary.Ast.SwitchStmt): Statement { - val switch = newSwitchStatement(rawNode = switchStmt) - - frontend.scopeManager.enterScope(switch) + private fun handleIncDecStmt(incDecStmt: GoStandardLibrary.Ast.IncDecStmt) = + newUnaryOperator(incDecStmt.tokString, postfix = true, prefix = false, rawNode = incDecStmt) + .withChildren { op -> op.input = frontend.expressionHandler.handle(incDecStmt.x) } - switchStmt.init?.let { switch.initializerStatement = handle(it) } - switchStmt.tag?.let { switch.selector = frontend.expressionHandler.handle(it) } + private fun handleIfStmt(ifStmt: GoStandardLibrary.Ast.IfStmt) = + newIfStatement(rawNode = ifStmt).withChildren(hasScope = true) { stmt -> + ifStmt.init?.let { stmt.initializerStatement = frontend.statementHandler.handle(it) } - val block = - handle(switchStmt.body) as? Block ?: return newProblemExpression("missing switch body") + stmt.condition = frontend.expressionHandler.handle(ifStmt.cond) + stmt.thenStatement = frontend.statementHandler.handle(ifStmt.body) - switch.statement = block + ifStmt.`else`?.let { stmt.elseStatement = frontend.statementHandler.handle(it) } + } - frontend.scopeManager.leaveScope(switch) + private fun handleLabeledStmt(labeledStmt: GoStandardLibrary.Ast.LabeledStmt) = + newLabelStatement(rawNode = labeledStmt).withChildren { stmt -> + stmt.subStatement = handle(labeledStmt.stmt) + stmt.label = labeledStmt.label.name + } - return switch - } + private fun handleRangeStmt(rangeStmt: GoStandardLibrary.Ast.RangeStmt) = + newForEachStatement(rawNode = rangeStmt).withChildren(hasScope = true) { forEach -> + // TODO: Support other use cases that do not use DEFINE + if (rangeStmt.tokString == ":=") { + forEach.variable = + newDeclarationStatement().withChildren { stmt -> + // TODO: not really the best way to deal with this + // TODO: key type is always int. we could set this + rangeStmt.key?.let { + val ref = frontend.expressionHandler.handle(it) + if (ref is Reference) { + stmt += newVariableDeclaration(ref.name, rawNode = it).declare() + } + } - private fun handleTypeSwitchStmt( - typeSwitchStmt: GoStandardLibrary.Ast.TypeSwitchStmt - ): SwitchStatement { - val switch = newSwitchStatement(rawNode = typeSwitchStmt) + // TODO: not really the best way to deal with this + rangeStmt.value?.let { + val ref = frontend.expressionHandler.handle(it) + if (ref is Reference) { + stmt += newVariableDeclaration(ref.name, rawNode = it).declare() + } + } + } + } - frontend.scopeManager.enterScope(switch) + forEach.iterable = frontend.expressionHandler.handle(rangeStmt.x) + forEach.statement = frontend.statementHandler.handle(rangeStmt.body) + } - typeSwitchStmt.init?.let { switch.initializerStatement = handle(it) } + private fun handleReturnStmt(returnStmt: GoStandardLibrary.Ast.ReturnStmt) = + newReturnStatement(rawNode = returnStmt).withChildren { `return` -> + val results = returnStmt.results + if (results.isNotEmpty()) { + val expr = frontend.expressionHandler.handle(results[0]) - val assign = frontend.statementHandler.handle(typeSwitchStmt.assign) - val (lhs, rhs) = - if (assign is AssignExpression) { - val rhs = assign.rhs.singleOrNull() - switch.selector = rhs - Pair(assign.lhs.singleOrNull(), (rhs as? UnaryOperator)?.input) + // TODO: parse more than one result expression + `return`.returnValue = expr } else { - Pair(null, null) + // TODO: connect result statement to result variables } + } - val body = newBlock(rawNode = typeSwitchStmt.body) + private fun handleSendStmt(sendStmt: GoStandardLibrary.Ast.SendStmt) = + newBinaryOperator("<-", rawNode = sendStmt).withChildren { op -> + op.lhs = frontend.expressionHandler.handle(sendStmt.chan) + op.rhs = frontend.expressionHandler.handle(sendStmt.value) + } - frontend.scopeManager.enterScope(body) + private fun handleSwitchStmt(switchStmt: GoStandardLibrary.Ast.SwitchStmt) = + newSwitchStatement(rawNode = switchStmt).withChildren(hasScope = true) { switch -> + switchStmt.init?.let { switch.initializerStatement = handle(it) } + switchStmt.tag?.let { switch.selector = frontend.expressionHandler.handle(it) } - for (c in typeSwitchStmt.body.list.filterIsInstance()) { - handleCaseClause(c, lhs, rhs) + switch.statement = + handle(switchStmt.body) as? Block ?: newProblemExpression("missing switch body") } - frontend.scopeManager.leaveScope(body) - - switch.statement = body + private fun handleTypeSwitchStmt(typeSwitchStmt: GoStandardLibrary.Ast.TypeSwitchStmt) = + newSwitchStatement(rawNode = typeSwitchStmt).withChildren(hasScope = true) { switch -> + typeSwitchStmt.init?.let { switch.initializerStatement = handle(it) } - frontend.scopeManager.leaveScope(switch) + val assign = frontend.statementHandler.handle(typeSwitchStmt.assign) + val (lhs, rhs) = + if (assign is AssignExpression) { + val rhs = assign.rhs.singleOrNull() + switch.selector = rhs + Pair(assign.lhs.singleOrNull(), (rhs as? UnaryOperator)?.input) + } else { + Pair(null, null) + } - return switch - } + switch.statement = + newBlock(rawNode = typeSwitchStmt.body).withChildren(hasScope = true) { + for (c in + typeSwitchStmt.body.list.filterIsInstance< + GoStandardLibrary.Ast.CaseClause + >()) { + handleCaseClause(c, lhs, rhs) + } + } + } } diff --git a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt index 194c5cda295..a0132951f71 100644 --- a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt +++ b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt @@ -551,7 +551,11 @@ class ExpressionHandler(lang: LLVMIRLanguageFrontend) : val value1 = frontend.getOperandValueAtIndex(instr, 1) val value2 = frontend.getOperandValueAtIndex(instr, 2) - return newConditionalExpression(cond, value1, value2, value1.type) + return newConditionalExpression(value1.type).withChildren { + it.condition = cond + it.thenExpression = value1 + it.elseExpression = value2 + } } /** diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 5c5cdf4cb92..c7acf2e2178 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -150,18 +150,18 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } private fun handleSlice(node: Python.ASTSlice): Expression { - val slice = newRangeExpression(rawNode = node) - slice.floor = node.lower?.let { handle(it) } - slice.ceiling = node.upper?.let { handle(it) } - slice.third = node.step?.let { handle(it) } - return slice + return newRangeExpression(rawNode = node).withChildren(hasScope = false) { slice -> + slice.floor = node.lower?.let { lower -> handle(lower) } + slice.ceiling = node.upper?.let { upper -> handle(upper) } + slice.third = node.step?.let { step -> handle(step) } + } } private fun handleSubscript(node: Python.ASTSubscript): Expression { - val subscriptExpression = newSubscriptExpression(rawNode = node) - subscriptExpression.arrayExpression = handle(node.value) - subscriptExpression.subscriptExpression = handle(node.slice) - return subscriptExpression + return newSubscriptExpression(rawNode = node).withChildren(hasScope = false) { sub -> + sub.arrayExpression = handle(node.value) + sub.subscriptExpression = handle(node.slice) + } } private fun handleBoolOp(node: Python.ASTBoolOp): Expression { @@ -170,75 +170,73 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : is Python.ASTAnd -> "and" is Python.ASTOr -> "or" } - val ret = newBinaryOperator(operatorCode = op, rawNode = node) if (node.values.size != 2) { return newProblemExpression( "Expected exactly two expressions but got " + node.values.size, rawNode = node ) } - ret.lhs = handle(node.values[0]) - ret.rhs = handle(node.values[1]) - return ret + return newBinaryOperator(operatorCode = op, rawNode = node).withChildren(hasScope = false) { + it.lhs = handle(node.values[0]) + it.rhs = handle(node.values[1]) + } } private fun handleList(node: Python.ASTList): Expression { - val lst = mutableListOf() - for (e in node.elts) { - lst += handle(e) + return newInitializerListExpression(rawNode = node).withChildren(hasScope = false) { + val lst = mutableListOf() + for (e in node.elts) { + lst += handle(e) + } + it.initializers = lst + it.type = frontend.objectType("list") } - val ile = newInitializerListExpression(rawNode = node) - ile.type = frontend.objectType("list") - ile.initializers = lst - return ile } private fun handleSet(node: Python.ASTSet): Expression { - val lst = mutableListOf() - for (e in node.elts) { - lst += handle(e) + return newInitializerListExpression(rawNode = node).withChildren(hasScope = false) { + val lst = mutableListOf() + for (e in node.elts) { + lst += handle(e) + } + it.initializers = lst + it.type = frontend.objectType("set") } - val ile = newInitializerListExpression(rawNode = node) - ile.type = frontend.objectType("set") - ile.initializers = lst - return ile } private fun handleTuple(node: Python.ASTTuple): Expression { - val lst = mutableListOf() - for (e in node.elts) { - lst += handle(e) + return newInitializerListExpression(rawNode = node).withChildren(hasScope = false) { + val lst = mutableListOf() + for (e in node.elts) { + lst += handle(e) + } + it.initializers = lst + it.type = frontend.objectType("tuple") } - val ile = newInitializerListExpression(rawNode = node) - ile.type = frontend.objectType("tuple") - ile.initializers = lst.toList() - return ile } private fun handleIfExp(node: Python.ASTIfExp): Expression { - return newConditionalExpression( - condition = handle(node.test), - thenExpression = handle(node.body), - elseExpression = handle(node.orelse), - rawNode = node - ) + return newConditionalExpression(rawNode = node).withChildren(hasScope = false) { + it.condition = handle(node.test) + it.thenExpression = handle(node.body) + it.elseExpression = handle(node.orelse) + } } private fun handleDict(node: Python.ASTDict): Expression { - val lst = mutableListOf() - for (i in node.values.indices) { // TODO: keys longer than values possible? - // Here we can not use node as raw node as it spans all keys and values - lst += - newKeyValueExpression( - key = node.keys[i]?.let { handle(it) }, - value = handle(node.values[i]), - ) - .codeAndLocationFromChildren(node) + return newInitializerListExpression(rawNode = node).withChildren(hasScope = false) { ile -> + val lst = mutableListOf() + for (i in node.values.indices) { // TODO: keys longer than values possible? + // Here we can not use node as raw node as it spans all keys and values + lst += + newKeyValueExpression().codeAndLocationFromChildren(node).withChildren { kve -> + kve.key = node.keys[i]?.let { key -> handle(key) } + kve.value = handle(node.values[i]) + } + } + ile.initializers = lst + ile.type = frontend.objectType("dict") } - val ile = newInitializerListExpression(rawNode = node) - ile.type = frontend.objectType("dict") - ile.initializers = lst.toList() - return ile } private fun handleCompare(node: Python.ASTCompare): Expression { @@ -258,26 +256,29 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : is Python.ASTIn -> "in" is Python.ASTNotIn -> "not in" } - val ret = newBinaryOperator(op, rawNode = node) - ret.lhs = handle(node.left) - ret.rhs = handle(node.comparators.first()) - return ret + return newBinaryOperator(operatorCode = op, rawNode = node).withChildren(hasScope = false) { + it.lhs = handle(node.left) + it.rhs = handle(node.comparators.first()) + } } private fun handleBinOp(node: Python.ASTBinOp): Expression { val op = frontend.operatorToString(node.op) - val ret = newBinaryOperator(operatorCode = op, rawNode = node) - ret.lhs = handle(node.left) - ret.rhs = handle(node.right) - return ret + return newBinaryOperator(operatorCode = op, rawNode = node).withChildren(hasScope = false) { + it.lhs = handle(node.left) + it.rhs = handle(node.right) + } } private fun handleUnaryOp(node: Python.ASTUnaryOp): Expression { val op = frontend.operatorUnaryToString(node.op) - val ret = - newUnaryOperator(operatorCode = op, postfix = false, prefix = false, rawNode = node) - ret.input = handle(node.operand) - return ret + return newUnaryOperator( + operatorCode = op, + postfix = false, + prefix = false, + rawNode = node + ) // TODO prefix? + .withChildren(hasScope = false) { it.input = handle(node.operand) } } private fun handleAttribute(node: Python.ASTAttribute): Expression { @@ -292,7 +293,11 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : // Yes, it's an import, so we need to construct a reference with an FQN newReference(base.name.fqn(node.attr), rawNode = node) } else { - newMemberExpression(name = node.attr, base = base, rawNode = node) + newMemberExpression(name = node.attr, base = base, rawNode = node).withChildren( + hasScope = false + ) { + it.base.withParent() + } } return ref @@ -347,7 +352,9 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : val ret = if (callee is MemberExpression) { - newMemberCallExpression(callee, rawNode = node) + newMemberCallExpression(callee, rawNode = node).withChildren(hasScope = false) { + it.callee.withParent() + } } else { // try to resolve -> [ConstructExpression] val currentScope = frontend.scopeManager.currentScope @@ -361,19 +368,21 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : constructExpr.type = record.toType() constructExpr } else { - newCallExpression(callee, rawNode = node) + newCallExpression(callee, rawNode = node).withChildren(hasScope = false) { + it.callee.withParent() + } } } - for (arg in node.args) { - ret.addArgument(handle(arg)) - } + return ret.withChildren(hasScope = false) { + for (arg in node.args) { + ret.addArgument(handle(arg)) + } - for (keyword in node.keywords) { - ret.addArgument(handle(keyword.value), keyword.arg) + for (keyword in node.keywords) { + ret.addArgument(handle(keyword.value), keyword.arg) + } } - - return ret } private fun isImport(name: Name): Boolean { @@ -406,16 +415,18 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return r } - private fun handleLambda(node: Python.ASTLambda): Expression { - val lambda = newLambdaExpression(rawNode = node) - val function = newFunctionDeclaration(name = "", rawNode = node) - frontend.scopeManager.enterScope(function) - for (arg in node.args.args) { - this.frontend.statementHandler.handleArgument(arg) + private fun handleLambda( + node: Python.ASTLambda + ): Expression { // TODO: scope for lambda / function or both? + return newLambdaExpression(rawNode = node).withChildren(hasScope = false) { lambda -> + lambda.function = + newFunctionDeclaration(name = "", rawNode = node).withChildren(hasScope = true) { + function -> + for (arg in node.args.args) { + this.frontend.statementHandler.handleArgument(arg) + } + function.body = handle(node.body) + } } - function.body = handle(node.body) - frontend.scopeManager.leaveScope(function) - lambda.function = function - return lambda } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt index 0aef6a52f56..eeb2c42173c 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt @@ -137,13 +137,10 @@ interface Python { */ sealed class ASTBASEstmt(pyObject: PyObject) : AST(pyObject), WithPythonLocation - /** - * ``` - * ast.FunctionDef = class FunctionDef(stmt) - * | FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) - * ``` + /* + FunctionDef and AsyncFunctionDef have the same fields. They differ according to the Python grammar, but making them inherent from a mutual class allows us to remove duplicate code. */ - class ASTFunctionDef(pyObject: PyObject) : ASTBASEstmt(pyObject) { + abstract class ASTFunctionOrAsyncFunctionDef(pyObject: PyObject) : ASTBASEstmt(pyObject) { val name: String by lazy { "name" of pyObject } val args: ASTarguments by lazy { "args" of pyObject } @@ -156,6 +153,13 @@ interface Python { val type_comment: String? by lazy { "type_comment" of pyObject } } + /** + * ``` + * ast.FunctionDef = class FunctionDef(stmt) + * | FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) + * ``` + */ + class ASTFunctionDef(pyObject: PyObject) : ASTFunctionOrAsyncFunctionDef(pyObject) /** * ``` @@ -163,19 +167,7 @@ interface Python { * | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) * ``` */ - class ASTAsyncFunctionDef(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val name: String by lazy { "name" of pyObject } - - val args: ASTarguments by lazy { "args" of pyObject } - - val body: List by lazy { "body" of pyObject } - - val decorator_list: List by lazy { "decorator_list" of pyObject } - - val returns: ASTBASEexpr? by lazy { "returns" of pyObject } - - val type_comment: String? by lazy { "type_comment" of pyObject } - } + class ASTAsyncFunctionDef(pyObject: PyObject) : ASTFunctionOrAsyncFunctionDef(pyObject) /** * ``` diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt index bc53dc070d6..291d7752802 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt @@ -260,21 +260,21 @@ class PythonLanguageFrontend(language: Language, ctx: Tr "Python ast of type ${fromPython(pyAST).javaClass} is not supported yet" ) // could be one of "ast.{Module,Interactive,Expression,FunctionType} - val tud = newTranslationUnitDeclaration(path, rawNode = pythonASTModule) - scopeManager.resetToGlobal(tud) - - val nsdName = Path(path).nameWithoutExtension - val nsd = newNamespaceDeclaration(nsdName, rawNode = pythonASTModule) - tud.addDeclaration(nsd) - - scopeManager.enterScope(nsd) - for (stmt in pythonASTModule.body) { - nsd.statements += statementHandler.handle(stmt) - } - scopeManager.leaveScope(nsd) - - scopeManager.addDeclaration(nsd) - + val tud = + newTranslationUnitDeclaration(name = path, rawNode = pythonASTModule).withChildren( + isGlobalScope = true + ) { + val nsdName = Path(path).nameWithoutExtension + val nsd = + newNamespaceDeclaration(name = nsdName, rawNode = pythonASTModule).withChildren( + hasScope = true + ) { + for (stmt in pythonASTModule.body) { + it.statements += statementHandler.handle(stmt) + } + } + scopeManager.addDeclaration(nsd) + } return tud } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 6959642678b..39e0ff86ea4 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -31,7 +31,6 @@ import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression import de.fraunhofer.aisec.cpg.graph.types.FunctionType @@ -41,8 +40,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : override fun handleNode(node: Python.ASTBASEstmt): Statement { return when (node) { is Python.ASTClassDef -> handleClassDef(node) - is Python.ASTFunctionDef -> handleFunctionDef(node) - is Python.ASTAsyncFunctionDef -> handleAsyncFunctionDef(node) + is Python.ASTFunctionOrAsyncFunctionDef -> handleFunctionDef(node) is Python.ASTPass -> return newEmptyStatement(rawNode = node) is Python.ASTImportFrom -> handleImportFrom(node) is Python.ASTAssign -> handleAssign(node) @@ -52,7 +50,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : is Python.ASTAnnAssign -> handleAnnAssign(node) is Python.ASTExpr -> handleExpressionStatement(node) is Python.ASTFor -> handleFor(node) - is Python.ASTAsyncFor -> handleAsyncFor(node) + is Python.ASTAsyncFor -> handleAsyncFor(node) // TODO async duplicate is Python.ASTWhile -> handleWhile(node) is Python.ASTImport -> handleImport(node) is Python.ASTBreak -> newBreakStatement(rawNode = node) @@ -66,7 +64,8 @@ class StatementHandler(frontend: PythonLanguageFrontend) : is Python.ASTTry, is Python.ASTTryStar, is Python.ASTWith, - is Python.ASTAsyncWith -> + is Python.ASTAsyncWith // TODO async duplicate + -> newProblemExpression( "The statement of class ${node.javaClass} is not supported yet", rawNode = node @@ -75,28 +74,27 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } private fun handleImport(node: Python.ASTImport): Statement { - val declStmt = newDeclarationStatement(rawNode = node) - for (imp in node.names) { - val alias = imp.asname - val decl = - if (alias != null) { - newImportDeclaration( - parseName(imp.name), - false, - parseName(alias), - rawNode = imp - ) - } else { - newImportDeclaration(parseName(imp.name), false, rawNode = imp) - } - frontend.scopeManager.addDeclaration(decl) - declStmt.addToPropertyEdgeDeclaration(decl) + return newDeclarationStatement(rawNode = node).withChildren(hasScope = false) { + for (imp in node.names) { + val alias = imp.asname + val decl = + if (alias != null) { + newImportDeclaration( + parseName(imp.name), + false, + parseName(alias), + rawNode = imp + ) + } else { + newImportDeclaration(parseName(imp.name), false, rawNode = imp) + } + frontend.scopeManager.addDeclaration(decl) + it.addToPropertyEdgeDeclaration(decl) + } } - return declStmt } private fun handleImportFrom(node: Python.ASTImportFrom): Statement { - val declStmt = newDeclarationStatement(rawNode = node) val level = node.level if (level == null || level > 0) { return newProblemExpression( @@ -104,58 +102,70 @@ class StatementHandler(frontend: PythonLanguageFrontend) : ) } - val module = parseName(node.module ?: "") - for (imp in node.names) { - // We need to differentiate between a wildcard import and an individual symbol. - // Wildcards luckily do not have aliases - val decl = - if (imp.name == "*") { - // In the wildcard case, our "import" is the module name and we set "wildcard" - // to true - newImportDeclaration(module, true, rawNode = imp) - } else { - // If we import an individual symbol, we need to FQN the symbol with our module - // name and import that. We also need to take care of any alias - val name = module.fqn(imp.name) - val alias = imp.asname - if (alias != null) { - newImportDeclaration(name, false, parseName(alias), rawNode = imp) + return newDeclarationStatement(rawNode = node).withChildren(hasScope = false) { + val module = parseName(node.module ?: "") + for (imp in node.names) { + // We need to differentiate between a wildcard import and an individual symbol. + // Wildcards luckily do not have aliases + val decl = + if (imp.name == "*") { + // In the wildcard case, our "import" is the module name and we set + // "wildcard" + // to true + newImportDeclaration(module, true, rawNode = imp) } else { - newImportDeclaration(name, false, rawNode = imp) + // If we import an individual symbol, we need to FQN the symbol with our + // module + // name and import that. We also need to take care of any alias + val name = module.fqn(imp.name) + val alias = imp.asname + if (alias != null) { + newImportDeclaration(name, false, parseName(alias), rawNode = imp) + } else { + newImportDeclaration(name, false, rawNode = imp) + } } - } - // Finally, add our declaration to the scope and the declaration statement - frontend.scopeManager.addDeclaration(decl) - declStmt.addToPropertyEdgeDeclaration(decl) + // Finally, add our declaration to the scope and the declaration statement + frontend.scopeManager.addDeclaration(decl) + it.addToPropertyEdgeDeclaration(decl) + } } - return declStmt } private fun handleWhile(node: Python.ASTWhile): Statement { - val ret = newWhileStatement(rawNode = node) - ret.condition = frontend.expressionHandler.handle(node.test) - ret.statement = makeBlock(node.body).codeAndLocationFromChildren(node) - node.orelse.firstOrNull()?.let { TODO("Not supported") } - return ret + if (node.orelse.isNotEmpty()) { + return newProblemExpression("orelse not supported for while statements", rawNode = node) + } + return newWhileStatement(rawNode = node).withChildren(hasScope = false) { + it.condition = frontend.expressionHandler.handle(node.test) + it.statement = makeBlock(node.body).codeAndLocationFromChildren(node) + } } private fun handleFor(node: Python.ASTFor): Statement { - val ret = newForEachStatement(rawNode = node) - ret.iterable = frontend.expressionHandler.handle(node.iter) - ret.variable = frontend.expressionHandler.handle(node.target) - ret.statement = makeBlock(node.body).codeAndLocationFromChildren(node) - node.orelse.firstOrNull()?.let { TODO("Not supported") } - return ret + if (node.orelse.isNotEmpty()) { + return newProblemExpression("orelse not supported for for statements", rawNode = node) + } + return newForEachStatement(rawNode = node).withChildren(hasScope = false) { + it.iterable = frontend.expressionHandler.handle(node.iter) + it.variable = frontend.expressionHandler.handle(node.target) + it.statement = makeBlock(node.body).codeAndLocationFromChildren(node) + } } private fun handleAsyncFor(node: Python.ASTAsyncFor): Statement { - val ret = newForEachStatement(rawNode = node) - ret.iterable = frontend.expressionHandler.handle(node.iter) - ret.variable = frontend.expressionHandler.handle(node.target) - ret.statement = makeBlock(node.body).codeAndLocationFromChildren(node) - node.orelse.firstOrNull()?.let { TODO("Not supported") } - return ret + if (node.orelse.isNotEmpty()) { + return newProblemExpression( + "orelse not supported for asyncfor statements", + rawNode = node + ) + } + return newForEachStatement(rawNode = node).withChildren(hasScope = false) { + it.iterable = frontend.expressionHandler.handle(node.iter) + it.variable = frontend.expressionHandler.handle(node.target) + it.statement = makeBlock(node.body).codeAndLocationFromChildren(node) + } } private fun handleExpressionStatement(node: Python.ASTExpr): Statement { @@ -164,89 +174,73 @@ class StatementHandler(frontend: PythonLanguageFrontend) : private fun handleAnnAssign(node: Python.ASTAnnAssign): Statement { // TODO: annotations - val lhs = frontend.expressionHandler.handle(node.target) - return if (node.value != null) { - newAssignExpression( - lhs = listOf(lhs), - rhs = listOf(frontend.expressionHandler.handle(node.value!!)), // TODO !! - rawNode = node - ) - } else { - lhs + return when (node.value) { + null -> frontend.expressionHandler.handle(node.target) + else -> { + newAssignExpression(rawNode = node).withChildren(hasScope = false) { + it.lhs = listOf(frontend.expressionHandler.handle(node.target)) + it.rhs = listOf(frontend.expressionHandler.handle(node.value!!)) // TODO !! + } + } } } private fun handleIf(node: Python.ASTIf): Statement { - val ret = newIfStatement(rawNode = node) - ret.condition = frontend.expressionHandler.handle(node.test) - ret.thenStatement = - if (node.body.isNotEmpty()) { - makeBlock(node.body).codeAndLocationFromChildren(node) - } else { - null - } - ret.elseStatement = - if (node.orelse.isNotEmpty()) { - makeBlock(node.orelse).codeAndLocationFromChildren(node) - } else { - null - } - return ret + return newIfStatement(rawNode = node).withChildren(hasScope = false) { + it.condition = frontend.expressionHandler.handle(node.test) + it.thenStatement = + if (node.body.isNotEmpty()) { + makeBlock(node.body).codeAndLocationFromChildren(node) + } else { + null + } + it.elseStatement = + if (node.orelse.isNotEmpty()) { + makeBlock(node.orelse).codeAndLocationFromChildren(node) + } else { + null + } + } } private fun handleReturn(node: Python.ASTReturn): Statement { - val ret = newReturnStatement(rawNode = node) - node.value?.let { ret.returnValue = frontend.expressionHandler.handle(it) } - return ret + return newReturnStatement(rawNode = node).withChildren(hasScope = false) { ret -> + node.value?.let { value -> ret.returnValue = frontend.expressionHandler.handle(value) } + } } private fun handleAssign(node: Python.ASTAssign): Statement { - val lhs = node.targets.map { frontend.expressionHandler.handle(it) } - val rhs = frontend.expressionHandler.handle(node.value) - if (rhs is List<*>) - newAssignExpression( - lhs = lhs, - rhs = - rhs.map { - (it as? Expression) - ?: newProblemExpression( - "There was an issue with an argument.", - rawNode = node - ) - }, - rawNode = node - ) - return newAssignExpression(lhs = lhs, rhs = listOf(rhs), rawNode = node) + return newAssignExpression(rawNode = node).withChildren(hasScope = false) { + it.lhs = node.targets.map { target -> frontend.expressionHandler.handle(target) } + it.rhs = listOf(frontend.expressionHandler.handle(node.value)) + } } private fun handleAugAssign(node: Python.ASTAugAssign): Statement { - val lhs = frontend.expressionHandler.handle(node.target) - val rhs = frontend.expressionHandler.handle(node.value) val op = frontend.operatorToString(node.op) + "=" - return newAssignExpression( - operatorCode = op, - lhs = listOf(lhs), - rhs = listOf(rhs), - rawNode = node - ) + return newAssignExpression(rawNode = node).withChildren(hasScope = false) { + it.operatorCode = op + it.lhs = listOf(frontend.expressionHandler.handle(node.target)) + it.rhs = listOf(frontend.expressionHandler.handle(node.value)) + } } private fun handleClassDef(stmt: Python.ASTClassDef): Statement { - val cls = newRecordDeclaration(stmt.name, "class", rawNode = stmt) - stmt.bases.map { cls.superClasses.add(frontend.typeOf(it)) } - - frontend.scopeManager.enterScope(cls) - - stmt.keywords.map { TODO() } - - for (s in stmt.body) { - when (s) { - is Python.ASTFunctionDef -> handleFunctionDef(s, cls) - else -> cls.addStatement(handleNode(s)) + val cls = + newRecordDeclaration(stmt.name, "class", rawNode = stmt).withChildren( + hasScope = true + ) { record -> + stmt.bases.map { base -> record.superClasses.add(frontend.typeOf(base)) } + stmt.keywords.map { TODO() } + + for (s in stmt.body) { + when (s) { + is Python.ASTFunctionDef -> handleFunctionDef(s, record) + else -> record.addStatement(handleNode(s)) + } + } } - } - frontend.scopeManager.leaveScope(cls) frontend.scopeManager.addDeclaration(cls) return wrapDeclarationToStatement(cls) @@ -266,7 +260,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * `receiver` (most often called `self`). */ private fun handleFunctionDef( - s: Python.ASTFunctionDef, + s: Python.ASTFunctionOrAsyncFunctionDef, recordDeclaration: RecordDeclaration? = null ): DeclarationStatement { val result = @@ -288,89 +282,27 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } else { newFunctionDeclaration(name = s.name, rawNode = s) } - frontend.scopeManager.enterScope(result) - - // Handle decorators (which are translated into CPG "annotations") - result.addAnnotations(handleAnnotations(s)) - - // Handle return type and calculate function type - if (result is ConstructorDeclaration) { - // Return type of the constructor is always its record declaration type - result.returnTypes = listOf(recordDeclaration?.toType() ?: unknownType()) - } else { - result.returnTypes = listOf(frontend.typeOf(s.returns)) - } - result.type = FunctionType.computeType(result) - - handleArguments(s.args, result, recordDeclaration) - - if (s.body.isNotEmpty()) { - result.body = makeBlock(s.body).codeAndLocationFromChildren(s) - } - frontend.scopeManager.leaveScope(result) - frontend.scopeManager.addDeclaration(result) + result.withChildren(hasScope = true) { + // Handle decorators (which are translated into CPG "annotations") + result.addAnnotations(handleAnnotations(s)) - return wrapDeclarationToStatement(result) - } - - /** - * We have to consider multiple things when matching Python's FunctionDef to the CPG: - * - A [Python.ASTFunctionDef] is a [Statement] from Python's point of view. The CPG sees it as - * a declaration -> we have to wrap the result in a [DeclarationStatement]. - * - A [Python.ASTFunctionDef] could be one of - * - a [ConstructorDeclaration] if it appears in a record and its [name] is `__init__` - * - a [MethodeDeclaration] if it appears in a record, and it isn't a - * [ConstructorDeclaration] - * - a [FunctionDeclaration] if neither of the above apply - * - * In case of a [ConstructorDeclaration] or[MethodDeclaration]: the first argument is the - * `receiver` (most often called `self`). - */ - private fun handleAsyncFunctionDef( - s: Python.ASTAsyncFunctionDef, - recordDeclaration: RecordDeclaration? = null - ): DeclarationStatement { - val result = - if (recordDeclaration != null) { - if (s.name == "__init__") { - newConstructorDeclaration( - name = s.name, - recordDeclaration = recordDeclaration, - rawNode = s - ) - } else { - newMethodDeclaration( - name = s.name, - recordDeclaration = recordDeclaration, - isStatic = false, - rawNode = s - ) - } + // Handle return type and calculate function type + if (result is ConstructorDeclaration) { + // Return type of the constructor is always its record declaration type + result.returnTypes = listOf(recordDeclaration?.toType() ?: unknownType()) } else { - newFunctionDeclaration(name = s.name, rawNode = s) + result.returnTypes = listOf(frontend.typeOf(s.returns)) } - frontend.scopeManager.enterScope(result) - - // Handle decorators (which are translated into CPG "annotations") - result.addAnnotations(handleAnnotations(s)) - - // Handle return type and calculate function type - if (result is ConstructorDeclaration) { - // Return type of the constructor is always its record declaration type - result.returnTypes = listOf(recordDeclaration?.toType() ?: unknownType()) - } else { - result.returnTypes = listOf(frontend.typeOf(s.returns)) - } - result.type = FunctionType.computeType(result) + result.type = FunctionType.computeType(result) - handleArguments(s.args, result, recordDeclaration) + handleArguments(s.args, result, recordDeclaration) - if (s.body.isNotEmpty()) { - result.body = makeBlock(s.body).codeAndLocationFromChildren(s) + if (s.body.isNotEmpty()) { + result.body = makeBlock(s.body).codeAndLocationFromChildren(s) + } } - frontend.scopeManager.leaveScope(result) frontend.scopeManager.addDeclaration(result) return wrapDeclarationToStatement(result) @@ -484,11 +416,9 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } } - private fun handleAnnotations(node: Python.ASTAsyncFunctionDef): Collection { - return handleDeclaratorList(node, node.decorator_list) - } - - private fun handleAnnotations(node: Python.ASTFunctionDef): Collection { + private fun handleAnnotations( + node: Python.ASTFunctionOrAsyncFunctionDef + ): Collection { return handleDeclaratorList(node, node.decorator_list) } @@ -515,31 +445,31 @@ class StatementHandler(frontend: PythonLanguageFrontend) : val annotation = newAnnotation( - name = - Name( - localName = decFuncParsed.name.localName, - parent = decFuncParsed.base.name - ), - rawNode = node - ) - for (arg in decorator.args) { - val argParsed = frontend.expressionHandler.handle(arg) - annotation.members += - newAnnotationMember( - "annotationArg" + decorator.args.indexOf(arg), // TODO - argParsed, - rawNode = arg + name = + Name( + localName = decFuncParsed.name.localName, + parent = decFuncParsed.base.name + ), + rawNode = node ) - } - for (keyword in decorator.keywords) { - annotation.members += - newAnnotationMember( - name = keyword.arg, - value = frontend.expressionHandler.handle(keyword.value), - rawNode = keyword - ) - } - + .withChildren(hasScope = false) { + for (arg in decorator.args) { + val argParsed = frontend.expressionHandler.handle(arg) + it.members += + newAnnotationMember( + "annotationArg" + decorator.args.indexOf(arg), // TODO + rawNode = arg + ) + .withChildren { it.value = argParsed } + } + for (keyword in decorator.keywords) { + it.members += + newAnnotationMember(name = keyword.arg, rawNode = keyword) + .withChildren { + it.value = frontend.expressionHandler.handle(keyword.value) + } + } + } annotations += annotation } @@ -547,11 +477,11 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } private fun makeBlock(stmts: List, rawNode: Python.AST? = null): Block { - val result = newBlock(rawNode = rawNode) - for (stmt in stmts) { - result.addStatement(handle(stmt)) + return newBlock(rawNode = rawNode).withChildren(hasScope = false) { + for (stmt in stmts) { + it.addStatement(handle(stmt)) + } } - return result } internal fun handleArgument(node: Python.ASTarg) { diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt index 28a6dec7441..3af515e9dc5 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt @@ -91,7 +91,9 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx) { val resolved = scopeManager.resolveReference(node) // Nothing to create - if (resolved != null) return null + if (resolved != null) { + return null + } val decl = if (scopeManager.isInRecord) { @@ -128,13 +130,17 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx) { decl.isImplicit = true if (decl is FieldDeclaration) { + decl.astParent = scopeManager.currentRecord scopeManager.currentRecord?.addField(decl) scopeManager.withScope(scopeManager.currentRecord?.scope) { scopeManager.addDeclaration(decl) } } else { + // copy the AST parent from the initial reference node + decl.astParent = node.astParent scopeManager.addDeclaration(decl) } + return decl } @@ -152,8 +158,11 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx) { .findValue(target) ?.registerTypeObserver(InitializerTypePropagation(handled)) - // Add it to our assign expression, so that we can find it in the AST - assignExpression.declarations += handled + // Add it to our assign expression, so that we can find it in the AST. + // [FieldDeclaration]s are stored at the [RecordDeclaration] level. + if (handled !is FieldDeclaration) { + assignExpression.declarations += handled + } } } } @@ -164,7 +173,7 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx) { when (node.variable) { is Reference -> { val handled = handleReference(node.variable as Reference) - if (handled is Declaration) { + if (handled is Declaration && node !is FieldDeclaration) { handled.let { node.addDeclaration(it) } } } diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 7a82e97a2ad..e4f130bbb1d 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -138,7 +138,7 @@ class PythonFrontendTest : BaseTest() { assertFullName("print", callExpression) - val literal = callExpression.arguments.first() as? Literal<*> + val literal = callExpression.arguments.firstOrNull() as? Literal<*> assertNotNull(literal) assertEquals("bar(s) here: ", literal.value) @@ -419,9 +419,8 @@ class PythonFrontendTest : BaseTest() { // assertEquals(tu.primitiveType("int"), i.type) // self.somevar = i - val someVarDeclaration = - ((bar.body as? Block)?.statements?.get(0) as? AssignExpression)?.declarations?.first() - as? FieldDeclaration + val someVarDeclaration = recordFoo.declarations["somevar"] as? FieldDeclaration + assertNotNull(someVarDeclaration) assertLocalName("somevar", someVarDeclaration) assertEquals(i, (someVarDeclaration.firstAssignment as? Reference)?.refersTo) @@ -540,7 +539,7 @@ class PythonFrontendTest : BaseTest() { val ifThen = (countStmt.thenStatement as? Block)?.statements?.get(0) as? CallExpression assertNotNull(ifThen) assertEquals(methCount, ifThen.invokes.first()) - assertEquals(countParam, (ifThen.arguments.first() as? Reference)?.refersTo) + assertEquals(countParam, (ifThen.arguments.firstOrNull() as? Reference)?.refersTo) assertNull(countStmt.elseStatement) // class c1(counter) @@ -914,7 +913,7 @@ class PythonFrontendTest : BaseTest() { val literals = commentedNodes.filterIsInstance>() assertEquals(1, literals.size) - assertEquals("# comment start", literals.first().comment) + assertEquals("# comment start", literals.firstOrNull()?.comment) val params = commentedNodes.filterIsInstance() assertEquals(2, params.size) @@ -923,12 +922,12 @@ class PythonFrontendTest : BaseTest() { val assignment = commentedNodes.filterIsInstance() assertEquals(2, assignment.size) - assertEquals("# A comment# a number", assignment.first().comment) - assertEquals("# comment end", assignment.last().comment) + assertEquals("# A comment# a number", assignment.firstOrNull()?.comment) + assertEquals("# comment end", assignment.lastOrNull()?.comment) val block = commentedNodes.filterIsInstance() assertEquals(1, block.size) - assertEquals("# foo", block.first().comment) + assertEquals("# foo", block.firstOrNull()?.comment) val kvs = commentedNodes.filterIsInstance() assertEquals(2, kvs.size) @@ -1007,10 +1006,10 @@ class PythonFrontendTest : BaseTest() { // dataflow from first loop to foo call val loopVar = firstLoop.variable as? Reference assertNotNull(loopVar) - assert(fooCall.arguments.first().prevDFG.contains(loopVar)) + assert(fooCall.arguments.firstOrNull()?.prevDFG?.contains(loopVar) == true) // dataflow from var declaration to foo call (in case for loop is not executed) - assert(fooCall.arguments.first().prevDFG.contains(varDefinedBeforeLoopRef)) + assert(fooCall.arguments.firstOrNull()?.prevDFG?.contains(varDefinedBeforeLoopRef) == true) // dataflow from range call to loop variable val secondLoopIterable = secondLoop.iterable as? CallExpression @@ -1022,7 +1021,7 @@ class PythonFrontendTest : BaseTest() { // dataflow from second loop var to bar call assertEquals( (secondLoop.variable as? Reference), - barCall.arguments.first().prevDFG.firstOrNull() + barCall.arguments.firstOrNull()?.prevDFG?.firstOrNull() ) }