From 47865caab0e2b162846daf9770365d943cba7eb6 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 24 Jan 2025 22:36:07 +0100 Subject: [PATCH 1/5] Do not enter local scopes in python --- .../aisec/cpg/frontends/cxx/CLanguage.kt | 1 + .../cpg/frontends/python/StatementHandler.kt | 18 +----------------- .../cpg/frontends/python/PythonFrontendTest.kt | 5 +++++ .../resources/python/variable_inference.py | 6 +++++- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt index 4a1c1498611..12eea68b325 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt @@ -114,6 +114,7 @@ open class CLanguage : hint: HasType?, targetHint: HasType?, ): CastResult { + return DirectMatch val match = super.tryCast(type, targetType, hint, targetHint) if (match != CastNotPossible) { return match diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index cda76a4066d..8810ea7836a 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -891,11 +891,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : handleArguments(s.args, result, recordDeclaration) if (s.body.isNotEmpty()) { - // Make sure we open a new (block) scope for the function body. This is not a 1:1 - // mapping to python scopes, since python only has a "function scope", but in the CPG - // the function scope only comprises the function arguments, and we need a block scope - // to hold all local variables within the function body. - result.body = makeBlock(s.body, parentNode = s, enterScope = true) + result.body = makeBlock(s.body, parentNode = s) } frontend.scopeManager.leaveScope(result) @@ -1154,29 +1150,17 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * This function "wraps" a list of [Python.AST.BaseStmt] nodes into a [Block]. Since the list * itself does not have a code/location, we need to employ [codeAndLocationFromChildren] on the * [parentNode]. - * - * Optionally, a new scope will be opened when [enterScope] is specified. This should be done - * VERY carefully, as Python has a very limited set of scopes and is most likely only to be used - * by [handleFunctionDef]. */ private fun makeBlock( stmts: List, parentNode: Python.AST.WithLocation, - enterScope: Boolean = false, ): Block { val result = newBlock() - if (enterScope) { - frontend.scopeManager.enterScope(result) - } for (stmt in stmts) { result.statements += handle(stmt) } - if (enterScope) { - frontend.scopeManager.leaveScope(result) - } - // Try to retrieve the code and location from the parent node, if it is a base stmt val ast = parentNode as? Python.AST.AST if (ast != null) { diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 79eda034dc8..c1c5fd1b930 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -1750,5 +1750,10 @@ class PythonFrontendTest : BaseTest() { // There is no field called "b" in the result. assertNull(tu.fields["b"]) + + val foo = tu.functions["foo"] + assertNotNull(foo) + val refersTo = foo.refs("fooA").map { it.refersTo } + refersTo.forEach { refersTo -> assertIs(refersTo) } } } diff --git a/cpg-language-python/src/test/resources/python/variable_inference.py b/cpg-language-python/src/test/resources/python/variable_inference.py index 954012e2216..9c647d10362 100644 --- a/cpg-language-python/src/test/resources/python/variable_inference.py +++ b/cpg-language-python/src/test/resources/python/variable_inference.py @@ -3,4 +3,8 @@ class SomeClass: def static_method(a): x = a b = x - return b \ No newline at end of file + return b + +def foo(fooA, b): + fooA = bar(fooA) + return fooA \ No newline at end of file From 9af87919a5dcd3c536f7870dfae7fe4fff279ea3 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 24 Jan 2025 22:39:40 +0100 Subject: [PATCH 2/5] Revert wronge change --- .../kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt index 12eea68b325..4a1c1498611 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt @@ -114,7 +114,6 @@ open class CLanguage : hint: HasType?, targetHint: HasType?, ): CastResult { - return DirectMatch val match = super.tryCast(type, targetType, hint, targetHint) if (match != CastNotPossible) { return match From 8aafc9f77a751aae7d151894842ea1f4f08ab134 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Fri, 24 Jan 2025 23:16:59 +0100 Subject: [PATCH 3/5] Fixed testGlobal --- .../python/statementHandler/StatementHandlerTest.kt | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index 2276641904e..653f965344b 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -251,16 +251,10 @@ class StatementHandlerTest : BaseTest() { var globalC = cVariables.firstOrNull { it.scope == pythonGlobalScope } assertNotNull(globalC) - var localC1 = - cVariables.firstOrNull { - it.scope?.astNode?.astParent?.name?.localName == "local_write" - } + var localC1 = cVariables.firstOrNull { it.scope?.astNode?.name?.localName == "local_write" } assertNotNull(localC1) - var localC2 = - cVariables.firstOrNull { - it.scope?.astNode?.astParent?.name?.localName == "error_write" - } + var localC2 = cVariables.firstOrNull { it.scope?.astNode?.name?.localName == "error_write" } assertNotNull(localC2) // In global_write, all references should point to global c From 83e0eb43afc351dafd019a498768e7c935007910 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Fri, 24 Jan 2025 23:18:55 +0100 Subject: [PATCH 4/5] Fixed testVarsAndFields --- .../cpg/frontends/python/PythonFrontendTest.kt | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index e1fd6e314f9..736896030e7 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -742,13 +742,22 @@ class PythonFrontendTest : BaseTest() { assertNotNull(classFieldDeclaredInFunction) assertNull(classFieldNoInitializer.initializer) - val localClassFieldNoInitializer = methBar.variables["classFieldNoInitializer"] + val localClassFieldNoInitializer = + methBar.variables[ + { it.name.localName == "classFieldNoInitializer" && it !is FieldDeclaration }] assertNotNull(localClassFieldNoInitializer) - val localClassFieldWithInit = methBar.variables["classFieldWithInit"] + val localClassFieldWithInit = + methBar.variables[ + { it.name.localName == "classFieldWithInit" && it !is FieldDeclaration }] assertNotNull(localClassFieldNoInitializer) - val localClassFieldDeclaredInFunction = methBar.variables["classFieldDeclaredInFunction"] + val localClassFieldDeclaredInFunction = + methBar.variables[ + { + it.name.localName == "classFieldDeclaredInFunction" && + it !is FieldDeclaration + }] assertNotNull(localClassFieldNoInitializer) // classFieldNoInitializer = classFieldWithInit From b497f5b0ec17b7a198bbf6b98c8f446dcf6e6b91 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Fri, 24 Jan 2025 23:20:18 +0100 Subject: [PATCH 5/5] Fixed testNonLocal --- .../aisec/cpg/frontends/python/StatementHandler.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 8810ea7836a..a98e484491a 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -33,7 +33,7 @@ import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage.Companion.MODIFIE import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.Annotation import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.scopes.LocalScope +import de.fraunhofer.aisec.cpg.graph.scopes.FunctionScope import de.fraunhofer.aisec.cpg.graph.scopes.NameScope import de.fraunhofer.aisec.cpg.graph.scopes.NamespaceScope import de.fraunhofer.aisec.cpg.graph.statements.* @@ -923,11 +923,10 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * into a [LookupScopeStatement]. */ private fun handleNonLocal(global: Python.AST.Nonlocal): LookupScopeStatement { - // We need to find the first outer function scope, or rather the block scope belonging to - // the function + // We need to find the first outer function scope var outerFunctionScope = frontend.scopeManager.firstScopeOrNull { - it is LocalScope && it != frontend.scopeManager.currentScope + it is FunctionScope && it != frontend.scopeManager.currentScope } return newLookupScopeStatement(