Ramon/parsing improvements

.gitignore

@@ -3,4 +3,5 @@
 .qodana
 build
 **/lineage-panel-dist/**
-.kotlin
+.kotlin
+**/.kotlin/**

src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/models/Edge.kt

@@ -0,0 +1,15 @@
+package com.github.ramonvermeulen.dbtToolkit.models
+
+import com.google.gson.JsonObject
+
+data class Edge(
+    val parent: String,
+    val child: String,
+)
+
+fun Edge.toJson(): JsonObject {
+    val json = JsonObject()
+    json.addProperty("parent", parent)
+    json.addProperty("child", child)
+    return json
+}

src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/models/LineageInfo.kt

@@ -0,0 +1,34 @@
+package com.github.ramonvermeulen.dbtToolkit.models
+
+import com.google.gson.JsonArray
+import com.google.gson.JsonObject
+
+data class LineageInfo(
+    val nodes: MutableSet<Node>,
+    val edges: MutableSet<Edge>,
+) {
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as LineageInfo
+
+        if (nodes != other.nodes) return false
+        if (edges != other.edges) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = nodes.hashCode()
+        result = 31 * result + edges.hashCode()
+        return result
+    }
+}
+
+fun LineageInfo.toJson(): JsonObject {
+    val json = JsonObject() // t.b.d.
+    json.add("nodes", JsonArray().apply { nodes.forEach { add(it.toJson()) } })
+    json.add("edges", JsonArray().apply { edges.forEach { add(it.toJson()) } })
+    return json
+}

src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/models/Node.kt

@@ -0,0 +1,42 @@
+package com.github.ramonvermeulen.dbtToolkit.models
+
+import com.google.gson.JsonArray
+import com.google.gson.JsonObject
+
+data class Node(
+    val id: String,
+    val tests: List<String> = listOf(),
+    val isSelected: Boolean = false,
+    val relativePath: String,
+) {
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as Node
+
+        // isSelected is not considered in the equality check
+        // to prevent unnecessary re-renders of the UI lineage graph
+        if (id != other.id) return false
+        if (tests != other.tests) return false
+        if (relativePath != other.relativePath) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = id.hashCode()
+        result = 31 * result + tests.hashCode()
+        result = 31 * result + relativePath.hashCode()
+        return result
+    }
+}
+
+fun Node.toJson(): JsonObject {
+    val json = JsonObject()
+    json.addProperty("id", id)
+    json.addProperty("isSelected", isSelected)
+    json.add("tests", JsonArray().apply { tests.forEach { add(it) } })
+    json.addProperty("relativePath", relativePath)
+    return json
+}

src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/LineageInfoService.kt

@@ -0,0 +1,27 @@
+package com.github.ramonvermeulen.dbtToolkit.services
+
+import com.github.ramonvermeulen.dbtToolkit.models.LineageInfo
+import com.intellij.openapi.components.Service
+import com.intellij.openapi.project.DumbAware
+import com.intellij.openapi.project.DumbService
+import com.intellij.openapi.project.Project
+import com.intellij.util.messages.Topic
+
+interface LineageInfoListener {
+    fun lineageInfoChanged(newLineageInfo: LineageInfo)
+}
+
+@Service(Service.Level.PROJECT)
+class LineageInfoService(private val project: Project) : DumbAware {
+    private var publisher: LineageInfoListener = project.messageBus.syncPublisher(TOPIC)
+
+    fun setLineageInfo(lineageInfo: LineageInfo) {
+        DumbService.getInstance(project).runWhenSmart {
+            publisher.lineageInfoChanged(lineageInfo)
+        }
+    }
+
+    companion object {
+        val TOPIC = Topic.create("LineageInfoTopic", LineageInfoListener::class.java)
+    }
+}

src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/services/ManifestService.kt

@@ -1,6 +1,8 @@
 package com.github.ramonvermeulen.dbtToolkit.services
 
-import com.google.gson.JsonArray
+import com.github.ramonvermeulen.dbtToolkit.models.Edge
+import com.github.ramonvermeulen.dbtToolkit.models.LineageInfo
+import com.github.ramonvermeulen.dbtToolkit.models.Node
 import com.google.gson.JsonObject
 import com.google.gson.JsonParser
 import com.intellij.openapi.components.Service
@@ -10,164 +12,88 @@
 import java.util.concurrent.locks.ReentrantLock
 import kotlin.concurrent.withLock
 
-data class Node(
-    val id: String,
-    val tests: MutableList<String> = mutableListOf(),
-    val isSelected: Boolean = false,
-    val relativePath: String,
-) {
-    override fun equals(other: Any?): Boolean {
-        if (this === other) return true
-        if (javaClass != other?.javaClass) return false
-
-        other as Node
-
-        // isSelected is not considered in the equality check
-        // to prevent unnecessary re-renders of the UI lineage graph
-        if (id != other.id) return false
-        if (tests != other.tests) return false
-        if (relativePath != other.relativePath) return false
-
-        return true
-    }
-
-    override fun hashCode(): Int {
-        var result = id.hashCode()
-        result = 31 * result + tests.hashCode()
-        result = 31 * result + relativePath.hashCode()
-        return result
-    }
-}
-
-fun Node.toJson(): JsonObject {
-    val json = JsonObject()
-    json.addProperty("id", id)
-    json.addProperty("isSelected", isSelected)
-    json.add("tests", JsonArray().apply { tests.forEach { add(it) } })
-    json.addProperty("relativePath", relativePath)
-    return json
-}
-
-data class Edge(
-    val parent: String,
-    val child: String,
-)
-
-fun Edge.toJson(): JsonObject {
-    val json = JsonObject()
-    json.addProperty("parent", parent)
-    json.addProperty("child", child)
-    return json
-}
-
-data class LineageInfo(
-    val nodes: MutableSet<Node>,
-    val edges: MutableSet<Edge>,
-) {
-    override fun equals(other: Any?): Boolean {
-        if (this === other) return true
-        if (javaClass != other?.javaClass) return false
-
-        other as LineageInfo
-
-        if (nodes != other.nodes) return false
-        if (edges != other.edges) return false
-
-        return true
-    }
-
-    override fun hashCode(): Int {
-        var result = nodes.hashCode()
-        result = 31 * result + edges.hashCode()
-        return result
-    }
-}
-
-fun LineageInfo.toJson(): JsonObject {
-    val json = JsonObject() // t.b.d.
-    json.add("nodes", JsonArray().apply { nodes.forEach { add(it.toJson()) } })
-    json.add("edges", JsonArray().apply { edges.forEach { add(it.toJson()) } })
-    return json
-}
-
 @Service(Service.Level.PROJECT)
 class ManifestService(private var project: Project) {
-    private var settings = project.service<DbtToolkitSettingsService>()
+    private val lineageInfoService = project.service<LineageInfoService>()
+    private var settingsService = project.service<DbtToolkitSettingsService>()
     private val dbtCommandExecutorService = project.service<DbtCommandExecutorService>()
     private var manifest: JsonObject? = null
     private val manifestLock = ReentrantLock()
 
     @Synchronized
     private fun parseManifest() {
-        dbtCommandExecutorService.executeCommand(listOf("parse", "--no-partial-parse"))
-        val file = File("${settings.state.settingsDbtTargetDir}/manifest.json")
+        // todo(ramon) maybe remove --no-partial-parse flag since it is dbt 1.6 >
+        dbtCommandExecutorService.executeCommand(listOf("parse"))
+        val file = File("${settingsService.state.settingsDbtTargetDir}/manifest.json")
         if (file.exists()) {
             manifest = JsonParser.parseString(file.readText()).asJsonObject
         } else {
             println("Manifest file does not exist")
-            println("looked in the following path: ${settings.state.settingsDbtTargetDir}/manifest.json")
+            println("looked in the following path: ${settingsService.state.settingsDbtTargetDir}/manifest.json")
         }
     }
 
-    private fun getCompleteLineageForNode(node: String): LineageInfo {
+    // todo(ramon) reconsider this implementation, might be improved
+    private fun getCompleteLineageForNode(node: String) {
         val visited = mutableSetOf<String>()
         val nodes = mutableSetOf<Node>()
         val relationships = mutableSetOf<Edge>()
-        val manifestNodes = manifest!!.getAsJsonObject("nodes")
-        val manifestSources = manifest!!.getAsJsonObject("sources")
 
         fun depthFirstSearch(
             node: String,
             initialNode: Boolean = false,
         ) {
             if (node in visited) return
-            val tests = mutableListOf<String>()
             visited.add(node)
-            var nodePath = ""
-
-            val children = manifest!!.getAsJsonObject("child_map").getAsJsonArray(node)
-            val parents = manifest!!.getAsJsonObject("parent_map").getAsJsonArray(node)
-
-            if (manifestNodes.has(node)) {
-                nodePath = manifestNodes.getAsJsonObject(node).get("original_file_path").asString
-            }
-
-            if ("source" in node) {
-                nodePath = manifestSources.getAsJsonObject(node).get("original_file_path").asString
-            }
 
-            children?.forEach { child ->
-                if (child.asString.startsWith("test.")) {
-                    tests.add(child.asString)
-                    return@forEach
+            manifest?.run {
+                val manifestNodes = getAsJsonObject("nodes")
+                val manifestSources = getAsJsonObject("sources")
+                val children = getAsJsonObject("child_map").getAsJsonArray(node)
+                val parents = getAsJsonObject("parent_map").getAsJsonArray(node)
+
+                val nodePath: String =
+                    when {
+                        manifestNodes.has(node) -> manifestNodes.getAsJsonObject(node).get("original_file_path").asString
+                        "source" in node -> manifestSources.getAsJsonObject(node).get("original_file_path").asString
+                        else -> ""
+                    }
+
+                val tests =
+                    children?.mapNotNull { child ->
+                        child.asString.takeIf { it.startsWith("test.") }
+                    } ?: emptyList()
+
+                children?.forEach { child ->
+                    if (!child.asString.startsWith("test.")) {
+                        depthFirstSearch(child.asString)
+                        relationships.add(Edge(node, child.asString))
+                    }
+                }
+                parents?.forEach { parent ->
+                    depthFirstSearch(parent.asString)
+                    relationships.add(Edge(parent.asString, node))
                 }
-                depthFirstSearch(child.asString)
-                relationships.add(Edge(node, child.asString))
-            }
-            parents?.forEach { parent ->
-                depthFirstSearch(parent.asString)
-                relationships.add(Edge(parent.asString, node))
-            }
 
-            nodes.add(Node(node, isSelected = initialNode, tests = tests, relativePath = nodePath))
+                nodes.add(Node(node, isSelected = initialNode, tests = tests, relativePath = nodePath))
+            }
         }
 
         depthFirstSearch(node, true)
-
-        return LineageInfo(nodes, relationships)
+        lineageInfoService.setLineageInfo(lineageInfo = LineageInfo(nodes = nodes, edges = relationships))
     }
 
-    fun getLineageInfoForNode(
+    fun refreshLineageInfoForNode(
         node: String,
         enforceReparse: Boolean,
-    ): LineageInfo {
+    ) {
         manifestLock.withLock {
             if (enforceReparse || manifest == null ||
                 !(manifest!!.getAsJsonObject("nodes").has(node) || manifest!!.getAsJsonObject("sources").has(node))
             ) {
                 parseManifest()
             }
-            return getCompleteLineageForNode(node)
+            getCompleteLineageForNode(node)
         }
     }
 }

src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/DbtToolkitMainToolWindow.kt

@@ -31,9 +31,9 @@
 
         val panelCreators =
             mapOf(
-                "dbt lineage" to PanelInfo(Supplier { LineagePanel(project, toolWindow) }, false),
-                "dbt docs" to PanelInfo(Supplier { DocsPanel(project, toolWindow) }, true),
+                "dbt lineage" to PanelInfo(Supplier { LineagePanel(project) }, false),
+                "dbt docs" to PanelInfo(Supplier { DocsPanel(project) }, true),
                 "console (read-only)" to PanelInfo(Supplier { ConsoleOutputPanel(project, toolWindow) }, false),
             )
 
         val panels = mutableMapOf<String, IdeaPanel>()

src/main/kotlin/com/github/ramonvermeulen/dbtToolkit/ui/docs/DocsPanel.kt

@@ -13,7 +13,6 @@
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.components.service
 import com.intellij.openapi.project.Project
-import com.intellij.openapi.wm.ToolWindow
 import com.intellij.ui.jcef.JBCefApp
 import com.intellij.ui.jcef.JBCefBrowser
 import com.intellij.ui.jcef.JBCefBrowserBuilder
@@ -30,7 +29,7 @@
 import javax.swing.JPanel
 import javax.swing.SwingUtilities
 
-class DocsPanel(private var project: Project, private var toolWindow: ToolWindow) : IdeaPanel, Disposable {
+class DocsPanel(private var project: Project) : IdeaPanel, Disposable {
     private val docsService = project.service<DocsService>()
     private val settings = project.service<DbtToolkitSettingsService>()
     private val ourCefClient = JBCefApp.getInstance().createClient()
@@ -70,7 +69,7 @@
         val executor = Executors.newSingleThreadExecutor()
         val future =
             CompletableFuture.runAsync(
                 Runnable {
                     initiateCefRequestHandler()
                 },
                 executor,