Jn/gcp discovery improvements (#579)

* Resolves PROD-8041, bucket name fix * Fixes for PROD-8046. BigTable, BigQuery, CloudSQL assetId and resourceId updates
openraven · Oct 3, 2023 · 91b0bc5 · 91b0bc5
1 parent 26e6c66
commit 91b0bc5
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 4 deletions.
diff --git a/magpie-gcp/src/main/java/io/openraven/magpie/plugins/gcp/discovery/GCPUtils.java b/magpie-gcp/src/main/java/io/openraven/magpie/plugins/gcp/discovery/GCPUtils.java
@@ -62,4 +62,14 @@ public static Instant protobufTimestampToInstant(com.google.protobuf.Timestamp t
     }
     return Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos());
   }
+
+  public static String selfLinkToAssetId(String selfLink) {
+    return selfLink
+      .replaceFirst("https:", "")
+      .replaceFirst("/v1", "");
+  }
+
+  public static String assetNameToAssetId(String service, String projectId, String assetType, String assetName) {
+    return String.format("//%s.googleapis.com/projects/%s/%s/%s", service, projectId, assetType, assetName);
+  }
 }
diff --git a/...p/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/BigQueryDiscovery.java b/...p/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/BigQueryDiscovery.java
@@ -58,8 +58,12 @@ public void discover(ObjectMapper mapper, String projectId, Session session, Emi
     final String RESOURCE_TYPE = BigQueryDataset.RESOURCE_TYPE;
     bigQuery.listDatasets(projectId).iterateAll()
       .forEach(datasetProxy -> {
+        // Big Query Self links are null for some reason?  We've got to manually construct the assetId
+        final var assetId = GCPUtils.assetNameToAssetId("bigquery", projectId, "datasets", datasetProxy.getDatasetId().getDataset());
         Dataset datasetModel = bigQuery.getDataset(datasetProxy.getDatasetId());
-        var data = new MagpieGcpResource.MagpieGcpResourceBuilder(mapper, datasetProxy.getGeneratedId())
+        var data = new MagpieGcpResource.MagpieGcpResourceBuilder(mapper, assetId)
+          .withResourceName(datasetProxy.getDatasetId().getDataset())
+          .withResourceId(assetId)
           .withProjectId(projectId)
           .withResourceType(RESOURCE_TYPE)
           .withConfiguration(GCPUtils.asJsonNode(datasetModel))

diff --git a/...p/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/BigTableDiscovery.java b/...p/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/BigTableDiscovery.java
@@ -38,6 +38,8 @@
 import java.util.List;
 import java.util.Optional;
 
+import static io.openraven.magpie.plugins.gcp.discovery.GCPUtils.assetNameToAssetId;
+
 public class BigTableDiscovery implements GCPDiscovery {
   private static final String SERVICE = "bigTable";
 
@@ -62,12 +64,14 @@ public void discover(ObjectMapper mapper, String projectId, Session session, Emi
       }
 
       instances.listIterator().forEachRemaining(instance -> {
-        var data = new MagpieGcpResource.MagpieGcpResourceBuilder(mapper, instance.getId())
+        final var assetId = assetNameToAssetId("bigtable", projectId, "instances", instance.getId());
+        var data = new MagpieGcpResource.MagpieGcpResourceBuilder(mapper, assetId)
+          .withResourceName(instance.getDisplayName())
+          .withResourceId(assetId)
           .withProjectId(projectId)
           .withResourceType(RESOURCE_TYPE)
           .withConfiguration(GCPUtils.asJsonNode(instance))
           .build();
-
         discoverClusters(client, instance, data);
 
         emitter.emit(VersionedMagpieEnvelopeProvider.create(session, List.of(fullService() + ":instance"), data.toJsonNode()));

diff --git a/...ie-gcp/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/SqlDiscovery.java b/...ie-gcp/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/SqlDiscovery.java
@@ -46,7 +46,10 @@ public void discover(ObjectMapper mapper, String projectId, Session session, Emi
           continue;
         }
         for (var sqlInstance : response.getItems()) {
-          var data = new MagpieGcpResource.MagpieGcpResourceBuilder(mapper, sqlInstance.getName())
+          final var assetId = GCPUtils.selfLinkToAssetId(sqlInstance.getSelfLink());
+          var data = new MagpieGcpResource.MagpieGcpResourceBuilder(mapper, assetId)
+            .withResourceName(sqlInstance.getName())
+            .withResourceId(assetId)
             .withProjectId(projectId)
             .withResourceType(RESOURCE_TYPE)
             .withRegion(sqlInstance.getRegion())

diff --git a/...cp/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/StorageDiscovery.java b/...cp/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/StorageDiscovery.java
@@ -92,6 +92,7 @@ public void discover(ObjectMapper mapper, String projectId, Session session, Emi
       ))
         .withProjectId(projectId)
         .withResourceId(bucket.getName())
+        .withResourceName(bucket.getName())
         .withResourceType(RESOURCE_TYPE)
         .withRegion(bucket.getLocation().toLowerCase())
         .withCreatedIso(bucket.getCreateTimeOffsetDateTime().toInstant())