diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 785475de5cc..31eb168719d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: go-version: 1.21 - name: Setup neo4j run: | - docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 neo4j || true + docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 || true - name: Determine Version run: | # determine version from tag diff --git a/README.md b/README.md index 3a9ef5911af..98c262f3b9b 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,14 @@ Instead of manually generating or editing the `gradle.properties` file, you can ### For Visualization Purposes -In order to get familiar with the graph itself, you can use the subproject [cpg-neo4j](https://github.com/Fraunhofer-AISEC/cpg/tree/master/cpg-neo4j). It uses this library to generate the CPG for a set of user-provided code files. The graph is then persisted to a [Neo4j](https://neo4j.com/) graph database. The advantage this has for the user, is that Neo4j's visualization software [Neo4j Browser](https://neo4j.com/developer/neo4j-browser/) can be used to graphically look at the CPG nodes and edges, instead of their Java representations. +In order to get familiar with the graph itself, you can use the subproject [cpg-neo4j](./cpg-neo4j). It uses this library to generate the CPG for a set of user-provided code files. The graph is then persisted to a [Neo4j](https://neo4j.com/) graph database. The advantage this has for the user, is that Neo4j's visualization software [Neo4j Browser](https://neo4j.com/developer/neo4j-browser/) can be used to graphically look at the CPG nodes and edges, instead of their Java representations. + +Please make sure, that the [APOC](https://neo4j.com/labs/apoc/) plugin is enabled on your neo4j server. It is used in mass-creating nodes and relationships. + +For example using docker: +``` +docker run -p 7474:7474 -p 7687:7687 -d -e NEO4J_AUTH=neo4j/password -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 +``` ### As Library diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index 3c4b1906f37..bb13e63375e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -36,6 +36,7 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.passes.* import de.fraunhofer.aisec.cpg.passes.configuration.* import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.io.File import java.nio.file.Path import java.util.* @@ -51,6 +52,7 @@ import org.slf4j.LoggerFactory * The configuration for the [TranslationManager] holds all information that is used during the * translation. */ +@DoNotPersist class TranslationConfiguration private constructor( /** Definition of additional symbols, mostly useful for C++. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt index 3d7636ee127..1524fe7050e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt @@ -26,11 +26,13 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.graph.Component +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist /** * The translation context holds all necessary managers and configurations needed during the * translation process. */ +@DoNotPersist class TranslationContext( /** The configuration for this translation. */ val config: TranslationConfiguration, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt index 05ace1a4dc8..7b4b3ddfb6a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt @@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder import de.fraunhofer.aisec.cpg.passes.Pass +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.* import java.util.concurrent.ConcurrentHashMap import org.neo4j.ogm.annotation.Relationship @@ -92,6 +93,7 @@ class TranslationResult( * @return the list of all translation units. */ @Deprecated(message = "translation units of individual components should be accessed instead") + @DoNotPersist val translationUnits: List get() { if (components.size == 1) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt index d9a32dbe22c..d20a0c82e58 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt @@ -45,4 +45,6 @@ class Assignment( /** The holder of this assignment */ @JsonIgnore val holder: AssignmentHolder -) : Edge(value, target as Node) +) : Edge(value, target as Node) { + override var labels = setOf("ASSIGMENT") +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt index 6f04d5dd1b1..65ff85b03b6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt @@ -109,7 +109,7 @@ fun > Node.printGraph( private fun Edge.label(): String { val builder = StringBuilder() builder.append("\"") - builder.append(this.label) + builder.append(this.labels.joinToString(",")) if (this is Dataflow) { var granularity = this.granularity diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 8d4ee551c88..de06b9d2be0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -45,9 +45,11 @@ import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.helpers.neo4j.LocationConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter import de.fraunhofer.aisec.cpg.passes.* +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import de.fraunhofer.aisec.cpg.processing.IVisitable import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import java.util.* +import kotlin.uuid.Uuid import org.apache.commons.lang3.builder.ToStringBuilder import org.apache.commons.lang3.builder.ToStringStyle import org.neo4j.ogm.annotation.* @@ -166,7 +168,7 @@ abstract class Node : var astChildren: List = listOf() get() = SubgraphWalker.getAstChildren(this) - @Transient var astParent: Node? = null + @DoNotPersist @Transient var astParent: Node? = null /** Virtual property for accessing [prevEOGEdges] without property edges. */ @PopulatedByPass(EvaluationOrderGraphPass::class) var prevEOG by unwrapping(Node::prevEOGEdges) @@ -189,6 +191,7 @@ abstract class Node : * Virtual property for accessing [nextDFGEdges] that have a * [de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity]. */ + @DoNotPersist @PopulatedByPass(DFGPass::class, ControlFlowSensitiveDFGPass::class) val prevFullDFG: List get() { @@ -212,6 +215,7 @@ abstract class Node : * Virtual property for accessing [nextDFGEdges] that have a * [de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity]. */ + @DoNotPersist @PopulatedByPass(DFGPass::class, ControlFlowSensitiveDFGPass::class) val nextFullDFG: List get() { @@ -252,7 +256,10 @@ abstract class Node : var isImplicit = false /** Required field for object graph mapping. It contains the node id. */ - @Id @GeneratedValue var id: Long? = null + @DoNotPersist @Id @GeneratedValue var legacyId: Long? = null + + /** Will replace [legacyId] */ + var id: Uuid = Uuid.random() /** Index of the argument if this node is used in a function call or parameter list. */ var argumentIndex = 0 diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt index 01e0b33c363..604bbb71966 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt @@ -25,4 +25,5 @@ */ package de.fraunhofer.aisec.cpg.graph +/** This interface represents all objects that can be persisted in a graph database. */ interface Persistable diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt index fc37bbca33f..76d597190e2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt @@ -27,6 +27,7 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.scopes.Symbol +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import org.neo4j.ogm.annotation.NodeEntity /** @@ -40,6 +41,7 @@ import org.neo4j.ogm.annotation.NodeEntity */ @NodeEntity abstract class Declaration : Node() { + @DoNotPersist val symbol: Symbol get() { return this.name.localName diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt index 6986ec21e20..bcfbd1a9ceb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt @@ -36,7 +36,7 @@ import org.neo4j.ogm.annotation.Relationship */ class FieldDeclaration : VariableDeclaration() { /** Specifies, whether this field declaration is also a definition, i.e. has an initializer. */ - private var isDefinition = false + var isDefinition = false /** If this is only a declaration, this provides a link to the definition of the field. */ @Relationship(value = "DEFINES") diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt index f20968b641f..d489bd4c8a5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt @@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -41,7 +42,7 @@ import org.neo4j.ogm.annotation.Relationship /** Represents the declaration or definition of a function. */ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStarterHolder { @Relationship("BODY") var bodyEdge = astOptionalEdgeOf() - /** The function body. Usually a [Block]. */ + /** The function body. Usualfly a [Block]. */ var body by unwrapping(FunctionDeclaration::bodyEdge) /** The list of function parameters. */ @@ -134,16 +135,6 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart return parameters.map { it.default } } - val defaultParameterSignature: List // TODO: What's this property? - get() = - parameters.map { - if (it.default != null) { - it.type - } else { - unknownType() - } - } - val signatureTypes: List get() = parameters.map { it.type } @@ -154,6 +145,7 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart .toString() } + @DoNotPersist override val eogStarters: List get() = listOfNotNull(this) @@ -179,11 +171,11 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart } } + @DoNotPersist override val declarations: List get() { val list = ArrayList() list.addAll(parameters) - list.addAll(records) return list } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt index c2e9927eb5e..20bebae19d6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.Objects import org.neo4j.ogm.annotation.Relationship @@ -75,6 +76,7 @@ class NamespaceDeclaration : Declaration(), DeclarationHolder, StatementHolder, override var statements by unwrapping(NamespaceDeclaration::statementEdges) + @DoNotPersist override val eogStarters: List get() { val list = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt index 86a9ceef33f..23440b37c2c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt @@ -33,6 +33,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.types.DeclaresType import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship import org.neo4j.ogm.annotation.Transient @@ -120,6 +121,7 @@ open class RecordDeclaration : templateEdges.removeIf { it.end == templateDeclaration } } + @DoNotPersist override val declarations: List get() { val list = ArrayList() @@ -162,6 +164,7 @@ open class RecordDeclaration : .toString() } + @DoNotPersist override val eogStarters: List get() { val list = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt index 2b53dc4efa9..f49c43fda32 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import org.neo4j.ogm.annotation.NodeEntity import org.neo4j.ogm.annotation.Relationship @@ -81,6 +82,7 @@ abstract class TemplateDeclaration : Declaration(), DeclarationHolder { return defaults } + @DoNotPersist override val declarations: List get() { val list = ArrayList() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt index 99b6f5ea162..4b479bcd367 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -74,6 +75,7 @@ class TranslationUnitDeclaration : .toString() } + @DoNotPersist override val eogStarters: List get() { val list = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt index acd7fe230f6..2755ad7a836 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt @@ -64,10 +64,9 @@ import de.fraunhofer.aisec.cpg.graph.types.TupleType class TupleDeclaration : VariableDeclaration() { /** The list of elements in this tuple. */ var elementEdges = - astEdgesOf( - onAdd = { registerTypeObserver(it.end) }, - onRemove = { unregisterTypeObserver(it.end) } - ) + astEdgesOf(onAdd = { registerTypeObserver(it.end) }) { + unregisterTypeObserver(it.end) + } var elements by unwrapping(TupleDeclaration::elementEdges) override var name: Name diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt index 83fecda3e0d..ebf498fc1b8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt @@ -75,7 +75,7 @@ open class VariableDeclaration : ValueDeclaration(), HasInitializer, HasType.Typ if (value is Reference) { value.resolutionHelper = this } - } + }, ) /** The (optional) initializer of the declaration. */ override var initializer by unwrapping(VariableDeclaration::initializerEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt index 1bb365dd633..064b0caa29d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt @@ -71,7 +71,7 @@ abstract class Edge : Persistable, Cloneable { end = edge.end } - @Transient open val label: String = "EDGE" + abstract var labels: Set /** * The index of this node, if it is stored in an diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt index ebdc6b69e56..e8bb941de20 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt @@ -37,6 +37,8 @@ open class AstEdge(start: Node, end: T) : Edge(start, end) { init { end.astParent = start } + + override var labels: Set = setOf("AST") } /** Creates an [AstEdges] container starting from this node. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt index b67f7711eaf..9e4518158aa 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt @@ -38,4 +38,7 @@ class TemplateArgument( /** A container for [TemplateArgument] edges. */ class TemplateArguments(thisRef: Node) : - AstEdges>(thisRef, init = ::TemplateArgument) + AstEdges>( + thisRef, + init = { start, end -> TemplateArgument(start, end) } + ) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt index e374adfcefd..5f3badedad5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt @@ -49,6 +49,8 @@ class ControlDependence( dependence = DependenceType.CONTROL } + override var labels = setOf("CDG") + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is ControlDependence) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt index 900eedc09b0..07ba1dc1047 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt @@ -93,7 +93,7 @@ open class Dataflow( @JsonIgnore var granularity: Granularity = default() ) : Edge(start, end) { - override val label: String = "DFG" + override var labels = setOf("DFG") override fun equals(other: Any?): Boolean { if (this === other) return true @@ -134,7 +134,7 @@ class ContextSensitiveDataflow( val callingContext: CallingContext ) : Dataflow(start, end, granularity) { - override val label: String = "DFG" + override var labels = setOf("DFG") override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt index af99e4dcaf7..7ac7126b3bf 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt @@ -68,6 +68,8 @@ class EvaluationOrder( result = 31 * result + branch.hashCode() return result } + + override var labels = setOf("EOG") } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt index 261b4957e4f..9bbaffad821 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt @@ -43,6 +43,8 @@ class Invoke( */ var dynamicInvoke: Boolean = false, ) : Edge(start, end) { + override var labels = setOf("INVOKES") + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is Invoke) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt index f5c7a863d75..0178456adfb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt @@ -52,6 +52,8 @@ class Usage( result = 31 * result + access.hashCode() return result } + + override var labels = setOf("USAGE") } /** A container for [Usage] edges. [NodeType] is necessary because of the Neo4J OGM. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt index 4280e774b99..b70f98b05da 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.graph.scopes +import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node /** @@ -37,6 +38,6 @@ sealed class NameScope(node: Node?) : StructureDeclarationScope(node) { init { astNode = node // Set the name so that we can use it as a namespace later - name = node?.name + name = node?.name ?: Name(EMPTY_NAME) } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt index cf322f7ff19..d393e137a5d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt @@ -27,7 +27,6 @@ package de.fraunhofer.aisec.cpg.graph.scopes import com.fasterxml.jackson.annotation.JsonBackReference import de.fraunhofer.aisec.cpg.frontends.Language -import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Node.Companion.TO_STRING_STYLE import de.fraunhofer.aisec.cpg.graph.declarations.Declaration @@ -35,13 +34,9 @@ import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.statements.LabelStatement import de.fraunhofer.aisec.cpg.graph.statements.LookupScopeStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter import org.apache.commons.lang3.builder.ToStringBuilder -import org.neo4j.ogm.annotation.GeneratedValue -import org.neo4j.ogm.annotation.Id import org.neo4j.ogm.annotation.NodeEntity import org.neo4j.ogm.annotation.Relationship -import org.neo4j.ogm.annotation.typeconversion.Convert /** * A symbol is a simple, local name. It is valid within the scope that declares it and all of its @@ -60,17 +55,11 @@ sealed class Scope( @Relationship(value = "SCOPE", direction = Relationship.Direction.INCOMING) @JsonBackReference open var astNode: Node? -) { - - /** Required field for object graph mapping. It contains the scope id. */ - @Id @GeneratedValue var id: Long? = null +) : Node() { /** FQN Name currently valid */ var scopedName: String? = null - /** The real new name */ - @Convert(NameConverter::class) var name: Name? = null - /** * Scopes are nested and therefore have a parent child relationship, this two members will help * navigate through the scopes,e.g. when looking up variables. diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt index 5f317eff4c0..7e93010f707 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt @@ -32,6 +32,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.Objects import org.neo4j.ogm.annotation.Relationship @@ -43,9 +44,11 @@ class CatchClause : Statement(), BranchingNode, EOGStarterHolder { @Relationship(value = "BODY") var bodyEdge = astOptionalEdgeOf() var body by unwrapping(CatchClause::bodyEdge) + @DoNotPersist override val branchedBy: Node? get() = parameter + @DoNotPersist override val eogStarters: List get() = listOf(this) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt index 029aef766f8..ed47af14a64 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt @@ -69,10 +69,10 @@ class ForEachStatement : LoopStatement(), BranchingNode, StatementHolder { override var statementEdges: AstEdges> get() { val statements = astEdgesOf() - variable?.let { statements.add(AstEdge(this, it)) } - iterable?.let { statements.add(AstEdge(this, it)) } - statement?.let { statements.add(AstEdge(this, it)) } - elseStatement?.let { statements.add(AstEdge(this, it)) } + statements += variableEdge + statements += iterableEdge + statements += statementEdge + statements += elseStatementEdge return statements } set(_) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt index 966a9135457..89ae115c2ae 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt @@ -81,13 +81,11 @@ class AssignExpression : var lhs by unwrapping(AssignExpression::lhsEdges) @Relationship("RHS") - /** The expressions on the right-hand side. */ var rhsEdges = - astEdgesOf( - onAdd = { it.end.registerTypeObserver(this) }, - onRemove = { it.end.unregisterTypeObserver(this) }, - ) + astEdgesOf(onAdd = { it.end.registerTypeObserver(this) }) { + it.end.unregisterTypeObserver(this) + } var rhs by unwrapping(AssignExpression::rhsEdges) /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt index e51e306f3d9..2e99cb4deae 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt @@ -60,7 +60,7 @@ class CollectionComprehension : Expression(), ArgumentHolder { @Relationship("STATEMENT") var statementEdge = astEdgeOf( - ProblemExpression("No statement provided but is required in ${this::class}") + ProblemExpression("No statement provided but is required in ${this::class}"), ) /** * This field contains the statement which is applied to each element of the input for which the diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt index 62fa90ca8ea..2f70d1026f1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt @@ -49,8 +49,9 @@ class InitializerListExpression : Expression(), ArgumentHolder, HasType.TypeObse var initializerEdges = astEdgesOf( onAdd = { it.end.registerTypeObserver(this) }, - onRemove = { it.end.unregisterTypeObserver(this) }, - ) + ) { + it.end.unregisterTypeObserver(this) + } /** Virtual property to access [initializerEdges] without property edges. */ var initializers by unwrapping(InitializerListExpression::initializerEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt index f16b284cdc8..2072ccae8f9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.helpers import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.io.File import java.nio.file.Path import java.time.Duration @@ -36,6 +37,7 @@ import java.util.* import org.slf4j.Logger import org.slf4j.LoggerFactory +@DoNotPersist class BenchmarkResults(val entries: List>) { val json: String @@ -175,6 +177,7 @@ constructor( } } +@DoNotPersist /** Represents some kind of measurements, e.g., on the performance or problems. */ open class MeasurementHolder @JvmOverloads diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt new file mode 100644 index 00000000000..27a25541ae6 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.persistence + +import de.fraunhofer.aisec.cpg.graph.Name +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.Persistable +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList +import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter +import java.math.BigInteger +import kotlin.reflect.KClass +import kotlin.reflect.KProperty1 +import kotlin.reflect.KTypeProjection +import kotlin.reflect.KVariance +import kotlin.reflect.KVisibility +import kotlin.reflect.full.createInstance +import kotlin.reflect.full.createType +import kotlin.reflect.full.hasAnnotation +import kotlin.reflect.full.isSubtypeOf +import kotlin.reflect.full.memberProperties +import kotlin.reflect.full.superclasses +import kotlin.reflect.full.withNullability +import kotlin.reflect.jvm.javaField +import kotlin.reflect.jvm.javaType +import kotlin.uuid.Uuid +import org.neo4j.ogm.annotation.Relationship +import org.neo4j.ogm.annotation.Relationship.Direction.INCOMING +import org.neo4j.ogm.annotation.typeconversion.Convert +import org.neo4j.ogm.typeconversion.AttributeConverter +import org.neo4j.ogm.typeconversion.CompositeAttributeConverter + +/** + * A cache used to store and retrieve sets of labels associated with specific Kotlin class types. + * + * This mutable map uses a Kotlin class type as the key and a set of strings representing associated + * labels as the value. The [labelCache] provides efficient lookup and prevents redundant + * re-computation of labels for the same class type. + */ +val labelCache: MutableMap, Set> = mutableMapOf() + +/** + * A cache mapping classes of type [Persistable] to their respective properties. + * + * This mutable map stores metadata about [Persistable] objects. For each specific class that + * implements the [Persistable] interface, it caches a mapping between property names and their + * corresponding [KProperty1] references. This allows efficient reflection-based access to class + * properties without repeatedly inspecting the class at runtime. + * + * The key in the map is the [KClass] of a subclass of [Persistable]. The value is a [Map] where the + * keys are strings representing the property names, and the values are [KProperty1] references + * pointing to those properties. This can be used for dynamic property access or serialization + * processes. + */ +val schemaPropertiesCache: + MutableMap, Map>> = + mutableMapOf() + +/** A cache mapping classes of type [Persistable] to their respective properties. */ +val schemaRelationshipCache: + MutableMap, Map>> = + mutableMapOf() + +/** + * Returns the [Persistable]'s properties. This DOES NOT include relationships, but only properties + * directly attached to the node/edge. + */ +fun Persistable.properties(): Map { + val properties = mutableMapOf() + for (entry in this::class.schemaProperties) { + val value = entry.value.call(this) + + if (value == null) { + continue + } + + value.convert(entry, properties) + } + + return properties +} + +/** + * Runs any conversions that are necessary by [CompositeAttributeConverter] and + * [AttributeConverter]. Since both of these classes are Neo4J OGM classes, we need to find new base + * types at some point. + */ +fun Any.convert( + entry: Map.Entry>, + properties: MutableMap +) { + val originalKey = entry.key + + val annotation = entry.value.javaField?.getAnnotation(Convert::class.java) + @Suppress("UNCHECKED_CAST") + if (annotation != null) { + val converter = annotation.value.createInstance() + if (converter is CompositeAttributeConverter<*>) { + properties += (converter as CompositeAttributeConverter).toGraphProperties(this) + } else if (converter is AttributeConverter<*, *>) { + properties.put( + originalKey, + (converter as AttributeConverter).toGraphProperty(this) + ) + } + } else if (this is Name && originalKey == "name") { + // needs to be extra because of the way annotations work, this will be re-designed once OGM + // is completely gone + properties += NameConverter().toGraphProperties(this) + } else if (this is Enum<*>) { + properties.put(originalKey, this.name) + } else if (this is Uuid) { + properties.put(originalKey, this.toString()) + } else if (this is BigInteger) { + properties.put(originalKey, this.toString()) + } else { + properties.put(originalKey, this) + } +} + +/** + * Represents a computed property for obtaining a set of labels associated with a Kotlin class. + * + * Recursively collects labels from the class hierarchy, including superclass labels, and adds the + * simple name of the current class to the set of labels. + * + * Interfaces and the Kotlin base class `Any` are excluded from the labels. The results are cached + * to improve performance. + */ +val KClass<*>.labels: Set + get() { + // Ignore interfaces and the Kotlin base class + if (this.java.isInterface || this == Any::class) { + return setOf() + } + + val cacheKey = this + + // Note: we cannot use computeIfAbsent here, because we are calling our function + // recursively and this would result in a ConcurrentModificationException + if (labelCache.containsKey(cacheKey)) { + return labelCache[cacheKey] ?: setOf() + } + + val labels = mutableSetOf() + labels.addAll(this.superclasses.flatMap { it.labels }) + this.simpleName?.let { labels.add(it) } + + // update the cache + labelCache[cacheKey] = labels + return labels + } + +internal val kClassType = KClass::class.createType(listOf(KTypeProjection.STAR)) +internal val nodeType = Node::class.createType() +internal val collectionType = Collection::class.createType(listOf(KTypeProjection.STAR)) +internal val collectionOfNodeType = + Collection::class.createType(listOf(KTypeProjection(variance = KVariance.OUT, type = nodeType))) +internal val edgeCollectionType = + EdgeCollection::class.createType(listOf(KTypeProjection.STAR, KTypeProjection.STAR)) +internal val mapType = Map::class.createType(listOf(KTypeProjection.STAR, KTypeProjection.STAR)) + +/** + * Retrieves a map of schema properties (not relationships!) for the given class implementing + * [Persistable]. + * + * This property computes a map that associates property names (as strings) to their corresponding + * [KProperty1] objects, which represent the properties defined in the class. Only properties for + * which [isSimpleProperty] returns true, are included. + * + * The computed map is cached to optimize subsequent lookups for properties of the same class. + */ +val KClass.schemaProperties: Map> + get() { + // Check, if we already computed the properties for this node's class + return schemaPropertiesCache.computeIfAbsent(this) { + val schema = mutableMapOf>() + val properties = it.memberProperties + for (property in properties) { + if (isSimpleProperty(property)) { + schema.put(property.name, property) + } + } + schema + } + } + +/** + * Provides a property that computes and returns a map of "relationships" for a given class + * implementing the [Persistable] interface. + * + * The relationships are represented as a `Map` where: + * - The key is the name of the relationship as a `String`. + * - The value is a reference to the property representing the relationship, encapsulated as a + * [KProperty1]. + * + * A "relationship" is determined based on a specific set of criteria defined by the + * [isRelationship] function. These criteria evaluate properties that are associated with other + * nodes in a graph model, excluding fields explicitly marked to skip persistence. + * + * The computed relationships are cached for performance optimization to ensure that repeated + * lookups do not re-evaluate the relationships for the same class. + * + * This property enhances schema introspection, allowing retrieval of relational data connections + * within classes modeled as entities in a graph database. + */ +val KClass.schemaRelationships: Map> + get() { + // Check, if we already computed the relationship for this node's class + return schemaRelationshipCache.computeIfAbsent(this) { + val schema = mutableMapOf>() + val properties = it.memberProperties + for (property in properties) { + if (isRelationship(property)) { + val name = property.relationshipName + schema.put(name, property) + } + } + schema + } + } + +/** + * Evaluates whether a given property qualifies as a "simple" property based on its characteristics. + * + * This evaluates to true, when + * - The property is not a list (see [collectionType]) + * - The property is not a map (see [mapType]) + * - The property is not a [KClass] + * - The property is not referring to a [Node] + * - The property is not an interface + * - The property does not have the annotation [DoNotPersist] and its return type does not have the + * annotation [DoNotPersist] + * + * @param property the property to be evaluated, belonging to a class implementing the Persistable + * interface + * @return true if the property satisfies the conditions of being a "simple" property, false + * otherwise + */ +private fun isSimpleProperty(property: KProperty1): Boolean { + val returnType = property.returnType.withNullability(false) + + return when { + property.visibility == KVisibility.PRIVATE -> false + property.hasAnnotation() -> false + returnType.hasAnnotation() -> false + (returnType.javaType as? Class<*>)?.getAnnotation(DoNotPersist::class.java) != null -> false + returnType.isSubtypeOf(kClassType) -> false + returnType.isSubtypeOf(collectionType) -> false + returnType.isSubtypeOf(mapType) -> false + returnType.isSubtypeOf(nodeType) -> false + (returnType.javaType as? Class<*>)?.isInterface == true -> false + else -> true + } +} + +/** + * Evaluates whether a given property qualifies as a "relationship" based on its characteristics. + * + * This evaluates to true, when + * - The property is not a delegate (Note: this might change in the future, once we re-design node + * properties if Neo4J OGM is completely removed) + * - The property is an [EdgeList] + * - The property is referring to a [Collection] of [Node] objects + * - The property is referring to a [Node] + * - The property does not have the annotation [DoNotPersist] + * - The property does not have the annotation [org.neo4j.ogm.annotation.Relationship] with an + * incoming direction (Note: We will replace this with our own annotation at some point) + * + * @param property the property to be evaluated, belonging to a class implementing the Persistable + * interface + * @return true if the property satisfies the conditions of being a "relationship", false otherwise + */ +private fun isRelationship(property: KProperty1): Boolean { + val returnType = property.returnType.withNullability(false) + + return when { + property.hasAnnotation() -> false + property.javaField?.type?.simpleName?.contains("Delegate") == true -> false + property.javaField?.getAnnotation(Relationship::class.java)?.direction == INCOMING -> false + property.visibility == KVisibility.PRIVATE -> false + returnType.isSubtypeOf(edgeCollectionType) -> true + returnType.isSubtypeOf(collectionOfNodeType) -> true + returnType.isSubtypeOf(nodeType) -> true + else -> false + } +} + +/** + * Retrieves the relational name associated with a property in the context of the raph schema. + * + * The `relationshipName` is determined based on the following rules: + * - If the property is annotated with the `@Relationship` annotation, the value of the annotation + * is used as the relationship name, provided it is non-null and not an empty string. + * - If the property name ends with "Edge", this suffix is removed. This adjustment is made to + * account for cases where two variables represent an edge, one named with "Edge" and another as + * the delegate without the suffix. The desired name is the one without "Edge". + * - The resulting name is converted to UPPER_SNAKE_CASE for standardization. + */ +val KProperty1.relationshipName: String + get() { + // If we have a (legacy) Neo4J annotation for our relationship, we take this one + // Note: We will replace this with something else in the future + val value = this.javaField?.getAnnotation(Relationship::class.java)?.value + if (value != null && value != "") { + return value + } + + // If the name ends with "Edge", we cut that of, since we always have two + // variables for real edges, one called "exampleEdge" and one just called + // "example" (which is only a delegate), but the name we want is "example". + // + // Replace camel case with UPPER_CASE + return this.name.substringBeforeLast("Edge").toUpperSnakeCase() + } + +/** + * Converts the current string to UPPER_SNAKE_CASE format. + * + * Each word boundary in camelCase or PascalCase naming convention is replaced with an underscore, + * and all characters are converted to uppercase. This is commonly used for representing constants + * or environment-style variable names. + * + * @return A string converted to UPPER_SNAKE_CASE. + */ +fun String.toUpperSnakeCase(): String { + val pattern = "(?<=.)[A-Z]".toRegex() + return this.replace(pattern, "_$0").uppercase() +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/DoNotPersist.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/DoNotPersist.kt new file mode 100644 index 00000000000..3843d71344d --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/DoNotPersist.kt @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.persistence + +/** This annotation is used to denote that this property or class should not be persisted */ +@Target(AnnotationTarget.PROPERTY, AnnotationTarget.CLASS) annotation class DoNotPersist() diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt new file mode 100644 index 00000000000..8d68203300b --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.persistence + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import kotlin.test.Test +import kotlin.test.assertEquals + +class TestCommon { + @Test + fun testSchemaProperties() { + val properties = FunctionDeclaration::class.schemaProperties + assertEquals( + setOf( + "complexity", + "isDefinition", + "signature", + "argumentIndex", + "code", + "comment", + "file", + "id", + "isImplicit", + "isInferred", + "location", + "name" + ), + properties.keys + ) + } + + @Test + fun testSchemaRelationships() { + var relationships = FunctionDeclaration::class.schemaRelationships + assertEquals( + listOf( + "ANNOTATIONS", + "ASSIGNED_TYPES", + "AST", + "BODY", + "CDG", + "DEFINES", + "DFG", + "EOG", + "LANGUAGE", + "OVERRIDES", + "PARAMETERS", + "PDG", + "RETURN_TYPES", + "SCOPE", + "SIGNATURE_TYPES", + "THROWS_TYPES", + "TYPE", + "USAGE", + ), + relationships.keys.sorted() + ) + + relationships = TranslationResult::class.schemaRelationships + assertEquals( + listOf( + "ADDITIONAL_NODES", + "ANNOTATIONS", + "AST", + "CDG", + "COMPONENTS", + "DFG", + "EOG", + "LANGUAGE", + "PDG", + "SCOPE", + ), + relationships.keys.sorted() + ) + } +} diff --git a/cpg-neo4j/README.md b/cpg-neo4j/README.md index b379d89493e..5ef4c510325 100644 --- a/cpg-neo4j/README.md +++ b/cpg-neo4j/README.md @@ -6,6 +6,13 @@ A simple tool to export a *code property graph* to a neo4j database. The application requires Java 17 or higher. +Please make sure, that the [APOC](https://neo4j.com/labs/apoc/) plugin is enabled on your neo4j server. It is used in mass-creating nodes and relationships. + +For example using docker: +``` +docker run -p 7474:7474 -p 7687:7687 -d -e NEO4J_AUTH=neo4j/password -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 +``` + ## Build Build (and install) a distribution using Gradle @@ -19,16 +26,19 @@ Please remember to adjust the `gradle.properties` before building the project. ## Usage ``` -./build/install/cpg-neo4j/bin/cpg-neo4j [--infer-nodes] [--load-includes] [--no-default-passes] +./build/install/cpg-neo4j/bin/cpg-neo4j [--infer-nodes] [--load-includes] [--no-default-passes] [--no-neo4j] [--no-purge-db] [--print-benchmark] - [--use-unity-build] [--benchmark-json=] + [--schema-json] [--schema-markdown] [--use-unity-build] + [--benchmark-json=] [--custom-pass-list=] [--export-json=] [--host=] [--includes-file=] + [--max-complexity-cf-dfg=] [--password=] [--port=] [--save-depth=] [--top-level=] - [--user=] ([...] | -S= - [-S=]... | + [--user=] + [--exclusion-patterns=]... ([...] + | -S= [-S=]... | --json-compilation-database= | --list-passes) [...] The paths to analyze. If module support is @@ -37,11 +47,14 @@ Please remember to adjust the `gradle.properties` before building the project. --benchmark-json= Save benchmark results to json file --custom-pass-list= - Add custom list of passes (includes - --no-default-passes) which is passed as a - comma-separated list; give either pass name if - pass is in list, or its FQDN (e.g. + Add custom list of passes (might be used + additional to --no-default-passes) which is + passed as a comma-separated list; give either + pass name if pass is in list, or its FQDN (e.g. --custom-pass-list=DFGPass,CallResolver) + --exclusion-patterns= + Configures an exclusion pattern for files or + directories that should not be parsed --export-json= Export cpg as json --host= Set the host of the neo4j Database (default: @@ -53,6 +66,11 @@ Please remember to adjust the `gradle.properties` before building the project. The path to an optional a JSON compilation database --list-passes Prints the list available passes --load-includes Enable TranslationConfiguration option loadIncludes + --max-complexity-cf-dfg= + Performance optimisation: Limit the + ControlFlowSensitiveDFGPass to functions with a + complexity less than what is specified here. -1 + (default) means no limit is used. --no-default-passes Do not register default passes [used for debugging] --no-neo4j Do not push cpg into neo4j [used for debugging] --no-purge-db Do no purge neo4j database before pushing the cpg @@ -69,6 +87,8 @@ Please remember to adjust the `gradle.properties` before building the project. --save-depth= Performance optimisation: Limit recursion depth form neo4j OGM when leaving the AST. -1 (default) means no limit is used. + --schema-json Print the CPGs nodes and edges that they can have. + --schema-markdown Print the CPGs nodes and edges that they can have. --top-level= Set top level directory of project structure. Default: Largest common path of all source files --use-unity-build Enable unity build mode for C++ (requires @@ -96,13 +116,4 @@ $ build/install/cpg-neo4j/bin/cpg-neo4j --export-json cpg-export.json --no-neo4j ``` To export the cpg from a neo4j database, you can use the neo4j `apoc` plugin. -There it's also possible to export only parts of the graph. - -## Known issues: - -- While importing sufficiently large projects with the parameter --save-depth=-1 - a java.lang.StackOverflowError may occur. - - This error could be solved by increasing the stack size with the JavaVM option: -Xss4m - - Otherwise the depth must be limited (e.g. 3 or 5) - -- While pushing a constant value larger than 2^63 - 1 a java.lang.IllegalArgumentException occurs. +There it's also possible to export only parts of the graph. \ No newline at end of file diff --git a/cpg-neo4j/build.gradle.kts b/cpg-neo4j/build.gradle.kts index 496f37df4e1..8a626188037 100644 --- a/cpg-neo4j/build.gradle.kts +++ b/cpg-neo4j/build.gradle.kts @@ -50,6 +50,7 @@ publishing { dependencies { // neo4j implementation(libs.bundles.neo4j) + implementation(libs.neo4j.driver) // Command line interface support implementation(libs.picocli) diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt new file mode 100644 index 00000000000..2e25a93e9f4 --- /dev/null +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.persistence + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.Persistable +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection +import de.fraunhofer.aisec.cpg.graph.nodes +import de.fraunhofer.aisec.cpg.helpers.Benchmark +import de.fraunhofer.aisec.cpg.helpers.IdentitySet +import de.fraunhofer.aisec.cpg.helpers.identitySetOf +import org.neo4j.driver.Session +import org.slf4j.LoggerFactory + +/** + * Defines the number of edges to be processed in a single batch operation during persistence. + * + * This constant is used for chunking collections of edges into smaller groups to optimize write + * performance and reduce memory usage when interacting with the Neo4j database. Specifically, it + * determines the maximum size of each chunk of edges to be persisted in one batch operation. + */ +const val edgeChunkSize = 10000 + +/** + * Specifies the maximum number of nodes to be processed in a single chunk during persistence + * operations. + * + * This constant is used to control the size of batches when persisting a list of nodes to the + * database. Breaking the list into chunks of this size helps improve performance and memory + * efficiency during database writes. Each chunk is handled individually, ensuring that operations + * remain manageable even for large data sets. + */ +const val nodeChunkSize = 10000 + +internal val log = LoggerFactory.getLogger("Persistence") + +internal typealias Relationship = Map + +/** + * Persists the current [TranslationResult] into a graph database. + * + * This method performs the following actions: + * - Logs information about the number and categories of nodes (e.g., AST nodes, scopes, types, + * languages) and edges that are being persisted. + * - Collects nodes that include AST nodes, scopes, types, and languages, as well as all associated + * edges. + * - Persists the collected nodes and edges. + * - Persists additional relationships between nodes, such as those related to types, scopes, and + * languages. + * - Utilizes a benchmarking mechanism to measure and log the time taken to complete the persistence + * operation. + * + * This method relies on the following context and properties: + * - The [TranslationResult.finalCtx] property for accessing the scope manager, type manager, and + * configuration. + * - A [Session] context to perform persistence actions. + */ +context(Session) +fun TranslationResult.persist() { + val b = Benchmark(Persistable::class.java, "Persisting translation result") + + val astNodes = this@persist.nodes + val connected = astNodes.flatMap { it.connectedNodes }.toSet() + val nodes = (astNodes + connected).distinct() + + log.info( + "Persisting {} nodes: AST nodes ({}), other nodes ({})", + nodes.size, + astNodes.size, + connected.size + ) + nodes.persist() + + val relationships = nodes.collectRelationships() + + log.info("Persisting {} relationships", relationships.size) + relationships.persist() + + b.stop() +} + +/** + * Persists a list of nodes into a database in chunks for efficient processing. + * + * This function utilizes the surrounding [Session] context to execute the database write + * operations. Nodes are processed in chunks of size determined by [nodeChunkSize], and each chunk + * is persisted using Cypher queries. The process is benchmarked using the [Benchmark] utility. + * + * The function generates a list of properties for the nodes, which includes their labels and other + * properties. These properties are used to construct Cypher queries that create nodes in the + * database with the given labels and properties. + * + * The function uses the APOC library for creating nodes in the database. For each node in the list, + * it extracts the labels and properties and executes the Cypher query to persist the node. + */ +context(Session) +private fun List.persist() { + this.chunked(nodeChunkSize).map { chunk -> + val b = Benchmark(Persistable::class.java, "Persisting chunk of ${chunk.size} nodes") + val params = + mapOf("props" to chunk.map { mapOf("labels" to it::class.labels) + it.properties() }) + this@Session.executeWrite { tx -> + tx.run( + """ + UNWIND ${"$"}props AS map + WITH map, apoc.map.removeKeys(map, ['labels']) AS properties + CALL apoc.create.node(map.labels, properties) YIELD node + RETURN node + """, + params + ) + .consume() + } + b.stop() + } +} + +/** + * Persists a collection of edges into a Neo4j graph database within the context of a [Session]. + * + * This method ensures that the required index for node IDs is created before proceeding with + * relationship creation. The edges are subdivided into chunks, and for each chunk, the + * relationships are created in the database. Neo4j does not support multiple labels on edges, so + * each edge is duplicated for each assigned label. The created relationships are associated with + * their respective nodes and additional properties derived from the edges. + * + * Constraints: + * - The session context is required to execute write transactions. + * - Edges should define their labels and properties for appropriate persistence. + * + * Mechanisms: + * - An index for [Node] IDs is created (if not already existing) to optimize matching operations. + * - Edges are chunked to avoid overloading transactional operations. + * - Relationship properties and labels are mapped before using database utilities for creation. + */ +context(Session) +private fun Collection.persist() { + // Create an index for the "id" field of node, because we are "MATCH"ing on it in the edge + // creation. We need to wait for this to be finished + this@Session.executeWrite { tx -> + tx.run("CREATE INDEX IF NOT EXISTS FOR (n:Node) ON (n.id)").consume() + } + + this.chunked(edgeChunkSize).map { chunk -> createRelationships(chunk) } +} + +context(Session) +private fun Collection>.persistEdgesOld() { + // Create an index for the "id" field of node, because we are "MATCH"ing on it in the edge + // creation. We need to wait for this to be finished + this@Session.executeWrite { tx -> + tx.run("CREATE INDEX IF NOT EXISTS FOR (n:Node) ON (n.id)").consume() + } + + this.chunked(edgeChunkSize).map { chunk -> + createRelationships( + chunk.flatMap { edge -> + // Since Neo4J does not support multiple labels on edges, but we do internally, we + // duplicate the edge for each label + edge.labels.map { label -> + mapOf( + "startId" to edge.start.id.toString(), + "endId" to edge.end.id.toString(), + "type" to label + ) + edge.properties() + } + } + ) + } +} + +/** + * Creates relationships in a graph database based on provided properties. + * + * @param props A list of maps, where each map represents properties of a relationship including + * keys such as `startId`, `endId`, and `type`. The `startId` and `endId` identify the nodes to + * connect, while `type` defines the type of the relationship. Additional properties for the + * relationship can also be included in the map. + */ +private fun Session.createRelationships( + props: List, +) { + val b = Benchmark(Persistable::class.java, "Persisting chunk of ${props.size} relationships") + val params = mapOf("props" to props) + executeWrite { tx -> + tx.run( + """ + UNWIND ${'$'}props AS map + MATCH (s:Node {id: map.startId}) + MATCH (e:Node {id: map.endId}) + WITH s, e, map, apoc.map.removeKeys(map, ['startId', 'endId', 'type']) AS properties + CALL apoc.create.relationship(s, map.type, properties, e) YIELD rel + RETURN rel + """ + .trimIndent(), + params + ) + .consume() + } + b.stop() +} + +/** + * Returns all [Node] objects that are connected with this node with some kind of relationship + * defined in [schemaRelationships]. + */ +val Persistable.connectedNodes: IdentitySet + get() { + val nodes = identitySetOf() + + for (entry in this::class.schemaRelationships) { + val value = entry.value.call(this) + if (value is EdgeCollection<*, *>) { + nodes += value.toNodeCollection() + } else if (value is List<*>) { + nodes += value.filterIsInstance() + } else if (value is Node) { + nodes += value + } + } + + return nodes + } + +private fun List.collectRelationships(): List { + val relationships = mutableListOf() + + for (node in this) { + for (entry in node::class.schemaRelationships) { + val value = entry.value.call(node) + if (value is EdgeCollection<*, *>) { + relationships += + value.map { edge -> + mapOf( + "startId" to edge.start.id.toString(), + "endId" to edge.end.id.toString(), + "type" to entry.key + ) + edge.properties() + } + } else if (value is List<*>) { + relationships += + value.filterIsInstance().map { end -> + mapOf( + "startId" to node.id.toString(), + "endId" to end.id.toString(), + "type" to entry.key + ) + } + } else if (value is Node) { + relationships += + mapOf( + "startId" to node.id.toString(), + "endId" to value.id.toString(), + "type" to entry.key + ) + } + } + } + + // Since Neo4J does not support multiple labels on edges, but we do internally, we + // duplicate the edge for each label + /*edge.labels.map { label -> + mapOf( + "startId" to edge.start.id.toString(), + "endId" to edge.end.id.toString(), + "type" to label + ) + edge.properties() + }*/ + return relationships +} diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index ea2b2257515..54fab47fb7f 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -30,37 +30,30 @@ import de.fraunhofer.aisec.cpg.* import de.fraunhofer.aisec.cpg.frontends.CompilationDatabase.Companion.fromFile import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.passes.* +import de.fraunhofer.aisec.cpg.persistence.persist import java.io.File import java.net.ConnectException import java.nio.file.Paths import java.util.concurrent.Callable import kotlin.reflect.KClass import kotlin.system.exitProcess -import org.neo4j.driver.exceptions.AuthenticationException -import org.neo4j.ogm.config.Configuration +import org.neo4j.driver.GraphDatabase import org.neo4j.ogm.context.EntityGraphMapper import org.neo4j.ogm.context.MappingContext import org.neo4j.ogm.cypher.compiler.MultiStatementCypherCompiler import org.neo4j.ogm.cypher.compiler.builders.node.DefaultNodeBuilder import org.neo4j.ogm.cypher.compiler.builders.node.DefaultRelationshipBuilder -import org.neo4j.ogm.exception.ConnectionException import org.neo4j.ogm.metadata.MetaData -import org.neo4j.ogm.session.Session -import org.neo4j.ogm.session.SessionFactory import org.slf4j.Logger import org.slf4j.LoggerFactory import picocli.CommandLine import picocli.CommandLine.ArgGroup private const val S_TO_MS_FACTOR = 1000 -private const val TIME_BETWEEN_CONNECTION_TRIES: Long = 2000 -private const val MAX_COUNT_OF_FAILS = 10 private const val EXIT_SUCCESS = 0 private const val EXIT_FAILURE = 1 -private const val VERIFY_CONNECTION = true private const val DEBUG_PARSER = true -private const val AUTO_INDEX = "none" -private const val PROTOCOL = "bolt://" +private const val PROTOCOL = "neo4j://" private const val DEFAULT_HOST = "localhost" private const val DEFAULT_PORT = 7687 @@ -84,6 +77,14 @@ data class JsonGraph(val nodes: List, val edges: List) /** * An application to export the cpg to a neo4j database. + * + * Please make sure, that the [APOC](https://neo4j.com/labs/apoc/) plugin is enabled on your neo4j + * server. It is used in mass-creating nodes and relationships. + * + * For example using docker: + * ``` + * docker run -p 7474:7474 -p 7687:7687 -d -e NEO4J_AUTH=neo4j/password -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 + * ``` */ class Application : Callable { @@ -251,6 +252,13 @@ class Application : Callable { ) private var topLevel: File? = null + @CommandLine.Option( + names = ["--exclusion-patterns"], + description = + ["Configures an exclusion pattern for files or directories that should not be parsed"] + ) + private var exclusionPatterns: List = listOf() + @CommandLine.Option( names = ["--benchmark-json"], description = ["Save benchmark results to json file"] @@ -381,33 +389,14 @@ class Application : Callable { * Pushes the whole translationResult to the neo4j db. * * @param translationResult, not null - * @throws InterruptedException, if the thread is interrupted while it try´s to connect to the - * neo4j db. - * @throws ConnectException, if there is no connection to bolt://localhost:7687 possible */ - @Throws(InterruptedException::class, ConnectException::class) fun pushToNeo4j(translationResult: TranslationResult) { - val bench = Benchmark(this.javaClass, "Push cpg to neo4j", false, translationResult) - log.info("Using import depth: $depth") - log.info( - "Count base nodes to save: " + - translationResult.components.size + - translationResult.additionalNodes.size - ) - - val sessionAndSessionFactoryPair = connect() - - val session = sessionAndSessionFactoryPair.first - session.beginTransaction().use { transaction -> - if (!noPurgeDb) session.purgeDatabase() - session.save(translationResult.components, depth) - session.save(translationResult.additionalNodes, depth) - transaction.commit() + val session = connect() + with(session) { + if (!noPurgeDb) executeWrite { tx -> tx.run("MATCH (n) DETACH DELETE n").consume() } + translationResult.persist() } - - session.clear() - sessionAndSessionFactoryPair.second.close() - bench.addMeasurement() + session.close() } /** @@ -420,41 +409,14 @@ class Application : Callable { * @throws ConnectException, if there is no connection to bolt://localhost:7687 possible */ @Throws(InterruptedException::class, ConnectException::class) - fun connect(): Pair { - var fails = 0 - var sessionFactory: SessionFactory? = null - var session: Session? = null - while (session == null && fails < MAX_COUNT_OF_FAILS) { - try { - val configuration = - Configuration.Builder() - .uri("$PROTOCOL$host:$port") - .credentials(neo4jUsername, neo4jPassword) - .verifyConnection(VERIFY_CONNECTION) - .build() - sessionFactory = SessionFactory(configuration, *packages) - - session = sessionFactory.openSession() - } catch (ex: ConnectionException) { - sessionFactory = null - fails++ - log.error( - "Unable to connect to localhost:7687, " + - "ensure the database is running and that " + - "there is a working network connection to it." - ) - Thread.sleep(TIME_BETWEEN_CONNECTION_TRIES) - } catch (ex: AuthenticationException) { - log.error("Unable to connect to localhost:7687, wrong username/password!") - exitProcess(EXIT_FAILURE) - } - } - if (session == null || sessionFactory == null) { - log.error("Unable to connect to localhost:7687") - exitProcess(EXIT_FAILURE) - } - assert(fails <= MAX_COUNT_OF_FAILS) - return Pair(session, sessionFactory) + fun connect(): org.neo4j.driver.Session { + val driver = + GraphDatabase.driver( + "$PROTOCOL$host:$port", + org.neo4j.driver.AuthTokens.basic(neo4jUsername, neo4jPassword) + ) + driver.verifyConnectivity() + return driver.session() } /** @@ -495,6 +457,7 @@ class Application : Callable { .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage") .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage") .loadIncludes(loadIncludes) + .exclusionPatterns(*exclusionPatterns.toTypedArray()) .addIncludesToGraph(loadIncludes) .debugParser(DEBUG_PARSER) .useUnityBuild(useUnityBuild) diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt index bf40cdb2a86..e121a98116d 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt @@ -43,9 +43,9 @@ import kotlin.test.assertNotNull import org.neo4j.ogm.annotation.Relationship import picocli.CommandLine -fun createTranslationResult(): Pair { +fun createTranslationResult(file: String = "client.cpp"): Pair { val topLevel = Paths.get("src").resolve("test").resolve("resources").toAbsolutePath() - val path = topLevel.resolve("client.cpp").toAbsolutePath() + val path = topLevel.resolve(file).toAbsolutePath() val cmd = CommandLine(Application::class.java) cmd.parseArgs(path.toString()) diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt index 96f477d58b6..91d2d937d38 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt @@ -25,20 +25,17 @@ */ package de.fraunhofer.aisec.cpg_vis_neo4j -import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend -import de.fraunhofer.aisec.cpg.graph.Name -import de.fraunhofer.aisec.cpg.graph.builder.translationResult -import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration -import de.fraunhofer.aisec.cpg.graph.functions +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal +import java.math.BigInteger import kotlin.test.Test import kotlin.test.assertEquals -import kotlin.test.assertNotNull +import kotlin.test.assertIs import org.junit.jupiter.api.Tag @Tag("integration") class Neo4JTest { @Test - @Throws(InterruptedException::class) fun testPush() { val (application, translationResult) = createTranslationResult() @@ -49,32 +46,15 @@ class Neo4JTest { } @Test - fun testSimpleNameConverter() { - val result = - with(TestLanguageFrontend()) { - translationResult { - val import = ImportDeclaration() - import.name = Name("myname") - import.alias = Name("myname", Name("myparent"), "::") - additionalNodes += import - } - } + fun testPushVeryLong() { + val (application, translationResult) = createTranslationResult("very_long.cpp") - val app = Application() - app.pushToNeo4j(result) + assertEquals(1, translationResult.variables.size) - val sessionAndSessionFactoryPair = app.connect() + val lit = translationResult.variables["l"]?.initializer + assertIs>(lit) + assertEquals(BigInteger("10958011617037158669"), lit.value) - val session = sessionAndSessionFactoryPair.first - session.beginTransaction().use { transaction -> - val imports = session.loadAll(ImportDeclaration::class.java) - assertNotNull(imports) - - var loadedImport = imports.singleOrNull() - assertNotNull(loadedImport) - assertEquals("myname", loadedImport.alias?.localName) - - transaction.commit() - } + application.pushToNeo4j(translationResult) } } diff --git a/cpg-neo4j/src/test/resources/very_long.cpp b/cpg-neo4j/src/test/resources/very_long.cpp new file mode 100644 index 00000000000..406c77bf2a8 --- /dev/null +++ b/cpg-neo4j/src/test/resources/very_long.cpp @@ -0,0 +1 @@ +unsigned long long l = 10958011617037158669ull; \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4a99128145e..883aaadbcd6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,6 +2,7 @@ kotlin = "2.0.20" kotlin19 = "1.9.10" neo4j = "4.0.10" +neo4j5 = "5.27.0" log4j = "2.24.0" spotless = "6.25.0" nexus-publish = "2.0.0" @@ -28,6 +29,7 @@ slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j"} apache-commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.17.0"} neo4j-ogm-core = { module = "org.neo4j:neo4j-ogm-core", version.ref = "neo4j"} neo4j-ogm-bolt-driver = { module = "org.neo4j:neo4j-ogm-bolt-driver", version.ref = "neo4j"} +neo4j-driver = { module = "org.neo4j.driver:neo4j-java-driver", version.ref = "neo4j5"} javaparser = { module = "com.github.javaparser:javaparser-symbol-solver-core", version = "3.26.0"} jackson = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version = "2.18.0"} @@ -55,7 +57,7 @@ mockito = { module = "org.mockito:mockito-core", version = "5.14.0"} kotlin-gradle = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" } dokka-gradle = { module = "org.jetbrains.dokka:dokka-gradle-plugin", version = "1.9.0" } # the dokka plugin is slightly behind the main Kotlin release cycle dokka-versioning = { module = "org.jetbrains.dokka:versioning-plugin", version = "1.9.0"} -kover-gradle = { module = "org.jetbrains.kotlinx:kover-gradle-plugin", version = "0.8.0" } +kover-gradle = { module = "org.jetbrains.kotlinx:kover-gradle-plugin", version = "0.9.0" } spotless-gradle = { module = "com.diffplug.spotless:spotless-plugin-gradle", version.ref = "spotless" } nexus-publish-gradle = { module = "io.github.gradle-nexus:publish-plugin", version.ref = "nexus-publish" }