Feat/python dependency integration

analysis/build.gradle.kts

@@ -20,7 +20,8 @@ repositories {
 }
 
 dependencies {
-    implementation("com.github.MaibornWolff:TreeSitterExcavationSite:0.8.0")
+    // TEMPORARY: composite-build coordinate. REVERT to "com.github.MaibornWolff:TreeSitterExcavationSite:<tag>" before the DC PR.
+    implementation("de.maibornwolff.treesitter.excavationsite:treesitter-excavationsite")
     implementation("io.github.bonede:tree-sitter:0.26.3")
     implementation("io.github.bonede:tree-sitter-typescript:0.23.2")
     implementation("io.github.bonede:tree-sitter-php:0.23.11")

analysis/settings.gradle.kts

@@ -2,3 +2,7 @@ plugins {
     id("org.gradle.toolchains.foojay-resolver-convention") version "1.0.0"
 }
 rootProject.name = "dependacharta-analysis"
+
+// TEMPORARY: composite-build wiring to local TSE for the Python migration iteration phase.
+// REVERT before opening the DC PR — see plans/add-python-dependency-support.md Task 7.
+includeBuild("../../TreeSitterExcavationSite")

analysis/src/main/kotlin/de/maibornwolff/dependacharta/pipeline/analysis/analyzers/TseMappings.kt

@@ -25,6 +25,8 @@ fun DeclarationType.toNodeType(): NodeType {
         DeclarationType.INTERFACE -> NodeType.INTERFACE
         DeclarationType.ENUM -> NodeType.ENUM
         DeclarationType.ANNOTATION -> NodeType.ANNOTATION
+        DeclarationType.FUNCTION -> NodeType.FUNCTION
+        DeclarationType.VARIABLE -> NodeType.VARIABLE
         DeclarationType.UNKNOWN -> NodeType.UNKNOWN
     }
 }

analysis/src/main/kotlin/de/maibornwolff/dependacharta/pipeline/analysis/analyzers/python/PythonAnalyzer.kt

@@ -1,193 +1,150 @@
 package de.maibornwolff.dependacharta.pipeline.analysis.analyzers.python
 
 import de.maibornwolff.dependacharta.pipeline.analysis.analyzers.LanguageAnalyzer
-import de.maibornwolff.dependacharta.pipeline.analysis.analyzers.common.utils.nodeAsString
 import de.maibornwolff.dependacharta.pipeline.analysis.analyzers.common.utils.withoutFileSuffix
-import de.maibornwolff.dependacharta.pipeline.analysis.analyzers.python.queries.PythonDefinitionsQuery
-import de.maibornwolff.dependacharta.pipeline.analysis.analyzers.python.queries.PythonImportQuery
-import de.maibornwolff.dependacharta.pipeline.analysis.analyzers.python.queries.PythonTypeAttributeQuery
-import de.maibornwolff.dependacharta.pipeline.analysis.analyzers.python.queries.PythonTypeIdentifierQuery
-import de.maibornwolff.dependacharta.pipeline.analysis.model.*
+import de.maibornwolff.dependacharta.pipeline.analysis.analyzers.toNodeType
+import de.maibornwolff.dependacharta.pipeline.analysis.model.Dependency
+import de.maibornwolff.dependacharta.pipeline.analysis.model.FileInfo
+import de.maibornwolff.dependacharta.pipeline.analysis.model.FileReport
+import de.maibornwolff.dependacharta.pipeline.analysis.model.Node
+import de.maibornwolff.dependacharta.pipeline.analysis.model.NodeType
+import de.maibornwolff.dependacharta.pipeline.analysis.model.Path
+import de.maibornwolff.dependacharta.pipeline.analysis.model.Type
 import de.maibornwolff.dependacharta.pipeline.shared.SupportedLanguage
-import org.treesitter.TSNode
-import org.treesitter.TSParser
-import org.treesitter.TreeSitterPython
+import de.maibornwolff.treesitter.excavationsite.api.Declaration
+import de.maibornwolff.treesitter.excavationsite.api.DeclarationType
+import de.maibornwolff.treesitter.excavationsite.api.ImportDeclaration
+import de.maibornwolff.treesitter.excavationsite.api.ImportKind
+import de.maibornwolff.treesitter.excavationsite.api.Language
+import de.maibornwolff.treesitter.excavationsite.api.TreeSitterDependencies
+import de.maibornwolff.treesitter.excavationsite.api.UsedType
 
 class PythonAnalyzer(
     private val fileInfo: FileInfo
 ) : LanguageAnalyzer {
-    private val python = TreeSitterPython()
-    private val definitionsQuery = PythonDefinitionsQuery(python)
-    private val importQuery = PythonImportQuery(python)
-    private val identifierQuery = PythonTypeIdentifierQuery(python)
-    private val attributeQuery = PythonTypeAttributeQuery(python)
+    private companion object {
+        const val INIT = "__init__"
+        const val PY_SUFFIX = "py"
+        const val SEPARATOR = "."
+    }
 
     override fun analyze(): FileReport {
-        val rootNode = parseCode(fileInfo.content)
-        val filePathWithSlashes = fileInfo.physicalPathAsPath()
-        val pathWithoutFileEnding = filePathWithSlashes.withoutFileSuffix("py")
-        val modulePath = pathWithoutFileEnding.parts
-
-        val definitions = definitionsQuery.execute(rootNode)
+        val result = TreeSitterDependencies.analyze(fileInfo.content, Language.PYTHON)
+        val modulePath = fileInfo.physicalPathAsPath().withoutFileSuffix(PY_SUFFIX).parts
 
-        val importsFrom = importQuery.executeImportsFrom(rootNode, fileInfo.content, modulePath)
-        val wildcardImportsFrom = importQuery.executeWildcardImportsFrom(rootNode, fileInfo.content, modulePath)
-        val aliasedImportsFrom = importQuery.executeAliasedImportsFrom(rootNode, fileInfo.content, modulePath)
+        val resolvedImports = result.imports.map { it.resolveRelative(modulePath) }
+        val regularFromImports = resolvedImports.filter { it.kind == ImportKind.IMPORT_FROM && !it.isAliased && !it.isWildcard }
+        val wildcardFromImports = resolvedImports.filter { it.kind == ImportKind.IMPORT_FROM && it.isWildcard }
+        val aliasedFromImports = resolvedImports.filter { it.kind == ImportKind.IMPORT_FROM && it.isAliased }
+        val standardImports = resolvedImports.filter { it.kind == ImportKind.STANDARD }
 
-        val imports = importQuery.executeImports(rootNode, fileInfo.content)
-        val aliasedImports = importQuery.executeAliasedImports(rootNode, fileInfo.content, modulePath)
+        val regularFromTwins = regularFromImports.flatMap { it.twinPair() }
+        val wildcardFromTwins = wildcardFromImports.flatMap { it.twinPair() }
+        val basePreloadedDeps = regularFromTwins + wildcardFromTwins
 
         val nodes = mutableListOf<Node>()
-
-        if (modulePath.last() == "__init__") {
-            nodes.addAll(
-                importsFrom.filter { dependency -> "__init__" !in dependency.path.parts }.map { dependency ->
-                    Node(
-                        pathWithName = Path(modulePath + dependency.path.parts.last()),
-                        physicalPath = fileInfo.physicalPath,
-                        language = SupportedLanguage.PYTHON,
-                        nodeType = NodeType.UNKNOWN,
-                        dependencies = importsFrom.toSet(),
-                        usedTypes = setOf(Type.simple(dependency.path.parts.last()))
-                    )
-                }
-            )
+        if (modulePath.lastOrNull() == INIT) {
+            nodes.addAll(initFileReexportNodes(regularFromImports, regularFromTwins.toSet(), modulePath))
         }
-
-        if (definitions.isNotEmpty()) {
-            nodes.addAll(
-                definitions.map {
-                    extractNodeFromDefinition(
-                        modulePath,
-                        it,
-                        importsFrom + wildcardImportsFrom,
-                        aliasedImportsFrom,
-                        imports,
-                        aliasedImports,
-                    )
-                }
-            )
+        result.declarations.forEach { declaration ->
+            nodes.add(buildDeclarationNode(declaration, modulePath, basePreloadedDeps, aliasedFromImports, standardImports))
         }
-
         return FileReport(nodes)
     }
 
-    private fun parseCode(pythonCode: String): TSNode {
-        val parser = TSParser()
-        parser.language = python
-        val tree = parser.parseString(null, pythonCode)
-        return tree.rootNode
-    }
+    private fun initFileReexportNodes(
+        regularFromImports: List<ImportDeclaration>,
+        regularFromTwins: Set<Dependency>,
+        modulePath: List<String>
+    ): List<Node> =
+        regularFromImports.map { import ->
+            val name = import.path.last()
+            Node(
+                pathWithName = Path(modulePath + name),
+                physicalPath = fileInfo.physicalPath,
+                language = SupportedLanguage.PYTHON,
+                nodeType = NodeType.UNKNOWN,
+                dependencies = regularFromTwins,
+                usedTypes = setOf(Type.simple(name))
+            )
+        }
 
-    private fun extractNodeFromDefinition(
+    private fun buildDeclarationNode(
+        declaration: Declaration,
         modulePath: List<String>,
-        definition: TSNode,
-        importFromDependencies: List<Dependency>,
-        aliasedImportsFrom: Map<String, String>,
-        imports: List<String>,
-        aliasedImports: Map<String, String>,
+        basePreloadedDeps: List<Dependency>,
+        aliasedFromImports: List<ImportDeclaration>,
+        standardImports: List<ImportDeclaration>
     ): Node {
-        val mutableImportFromDependencies = importFromDependencies.toMutableList()
-        val nodeBody = nodeAsString(definition, fileInfo.content)
-        val definitionNode = parseCode(nodeBody).getChild(0)
+        val pathWithName = Path(modulePath + declaration.name)
+        val nodeType = declaration.type.toNodeType()
 
-        if (definition.type == "identifier") {
+        if (declaration.type == DeclarationType.VARIABLE) {
             return Node(
-                pathWithName = Path(modulePath + nodeBody),
+                pathWithName = pathWithName,
                 physicalPath = fileInfo.physicalPath,
                 language = SupportedLanguage.PYTHON,
-                nodeType = NodeType.VARIABLE,
-                dependencies = setOf(),
-                usedTypes = setOf()
+                nodeType = nodeType,
+                dependencies = emptySet(),
+                usedTypes = emptySet()
             )
         }
 
-        var classOrFunctionDefinitionNode = definitionNode
-        if (definition.type == "decorated_definition") {
-            classOrFunctionDefinitionNode = definitionNode.getChildByFieldName("definition")
-        }
+        val (identifierStream, attributeStream) = declaration.usedTypes.partition { it.namespacePrefix.isEmpty() }
+        val identifierNames = identifierStream.map { it.name }.toSet()
 
-        val definitionName = nodeAsString(classOrFunctionDefinitionNode.getChildByFieldName("name"), nodeBody)
-
-        val importFromTypes = buildImportFromTypes(
-            identifierQuery.execute(definitionNode, nodeBody),
-            aliasedImportsFrom,
-            mutableImportFromDependencies
-        )
-
-        val importDependencies = buildImportDependencies(imports, aliasedImports, definitionNode, nodeBody)
+        val deps = basePreloadedDeps.toMutableSet()
+        aliasedFromImports.forEach { import ->
+            if (import.path.last() in identifierNames) {
+                deps.addAll(import.twinPair())
+            }
+        }
+        attributeStream.forEach { ut ->
+            deps.addAll(matchStandardImportAttribute(ut, standardImports))
+        }
 
+        val usedTypes = identifierStream.map { Type.simple(it.name) }.toSet()
         return Node(
-            pathWithName = Path(modulePath + definitionName),
+            pathWithName = pathWithName,
             physicalPath = fileInfo.physicalPath,
             language = SupportedLanguage.PYTHON,
-            nodeType = nodeType(classOrFunctionDefinitionNode),
-            dependencies = mutableImportFromDependencies.toSet() + importDependencies,
-            usedTypes = importFromTypes
+            nodeType = nodeType,
+            dependencies = deps,
+            usedTypes = usedTypes
         )
     }
 
-    private fun nodeType(declaration: TSNode) =
-        when (declaration.type) {
-            "class_definition" -> NodeType.CLASS
-            "function_definition" -> NodeType.FUNCTION
-            else -> NodeType.UNKNOWN
-        }
-
-    private fun buildImportFromTypes(
-        types: List<String>,
-        aliasedImportsFrom: Map<String, String>,
-        mutableDependencies: MutableList<Dependency>
-    ): Set<Type> =
-        types
-            .map {
-                val aliasedType = checkAliasImportFrom(it, aliasedImportsFrom, mutableDependencies)
-                Type.simple(aliasedType)
-            }.toSet()
-
-    private fun checkAliasImportFrom(
-        type: String,
-        aliasedImportsFrom: Map<String, String>,
-        mutableDependencies: MutableList<Dependency>
-    ): String {
-        if (type in aliasedImportsFrom) {
-            val aliasedImport = aliasedImportsFrom[type]!!.split(".")
-            val aliasedType = aliasedImport.last()
-            mutableDependencies.add(Dependency(Path(aliasedImport)))
-            mutableDependencies.add(Dependency(Path(aliasedImport.dropLast(1) + listOf("__init__", aliasedType))))
-            return aliasedType
+    private fun matchStandardImportAttribute(
+        usedType: UsedType,
+        standardImports: List<ImportDeclaration>
+    ): List<Dependency> {
+        val joinedPrefix = usedType.namespacePrefix.joinToString(SEPARATOR)
+        return standardImports.flatMap { import ->
+            if (import.path.joinToString(SEPARATOR) == joinedPrefix) {
+                listOf(
+                    Dependency(Path(import.path + usedType.name)),
+                    Dependency(Path(import.path + INIT + usedType.name))
+                )
+            } else {
+                emptyList()
+            }
         }
+    }
 
-        return type
+    private fun ImportDeclaration.resolveRelative(modulePath: List<String>): ImportDeclaration {
+        val first = path.firstOrNull() ?: return this
+        if (first.isEmpty() || !first.all { it == '.' }) return this
+        val resolvedPath = modulePath.dropLast(first.length) + path.drop(1)
+        return copy(path = resolvedPath)
     }
 
-    private fun buildImportDependencies(
-        imports: List<String>,
-        aliasedImports: Map<String, String>,
-        definitionNode: TSNode,
-        nodeBody: String
-    ): Set<Dependency> {
-        val dependencies = mutableSetOf<Dependency>()
-        if (imports.isNotEmpty() || aliasedImports.isNotEmpty()) {
-            val attributes = attributeQuery.execute(definitionNode, nodeBody)
-            attributes.forEach { attribute ->
-                val attributeList = attribute.split(".").toMutableList()
-                if (attributeList.size >= 2) {
-                    val attributeType = attributeList.removeLast()
-                    val attributeIdentifier = attributeList.joinToString(".")
-                    if (attributeIdentifier in imports) {
-                        dependencies.add(Dependency(Path(attributeList + listOf(attributeType)), false))
-                        dependencies.add(Dependency(Path(attributeList + listOf("__init__", attributeType)), false))
-                    } else if (attributeIdentifier in aliasedImports) {
-                        val aliasedPath = Path(aliasedImports[attributeIdentifier]!!.split(".") + listOf(attributeType))
-                        dependencies.add(Dependency(aliasedPath, false))
-                        val aliasedInitPath =
-                            Path(aliasedImports[attributeIdentifier]!!.split(".") + listOf("__init__", attributeType))
-                        dependencies.add(Dependency(aliasedInitPath, false))
-                    }
-                }
-            }
+    private fun ImportDeclaration.twinPair(): List<Dependency> {
+        val canonical = Dependency(Path(path), isWildcard)
+        val initPath = if (isWildcard) {
+            path + INIT
+        } else {
+            path.dropLast(1) + INIT + path.last()
         }
-        return dependencies
+        return listOf(canonical, Dependency(Path(initPath), isWildcard))
     }
 }