Discovery Updates (#577)

* Fix for PROD-7671 * Proposed fix for PROD-7668 (IAM Client Update) * Filter out roles that do not have the specific project as a parent. * Updated ComputeEngine discovery, using the optional credentialsProvider * Updated NetworkDiscovery, adding the missing configuration for the ZonesClient. Resolves PROD-7670 * Bumping from 0.7.4-SNAPSHOT to 0.8.0-SNAPSHOT due to behavorial and large number of dependency bumps
openraven · Sep 11, 2023 · f5d5c4a · f5d5c4a
1 parent ed6dbc5
commit f5d5c4a
Show file tree

Hide file tree

Showing 17 changed files with 119 additions and 122 deletions.
diff --git a/magpie-api/pom.xml b/magpie-api/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>magpie-parent</artifactId>
     <groupId>io.openraven.magpie</groupId>
-    <version>0.7.4-SNAPSHOT</version>
+    <version>0.8.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/magpie-aws/pom.xml b/magpie-aws/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>magpie-parent</artifactId>
     <groupId>io.openraven.magpie</groupId>
-    <version>0.7.4-SNAPSHOT</version>
+    <version>0.8.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/magpie-cli/pom.xml b/magpie-cli/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>magpie-parent</artifactId>
     <groupId>io.openraven.magpie</groupId>
-    <version>0.7.4-SNAPSHOT</version>
+    <version>0.8.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/magpie-core/pom.xml b/magpie-core/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>magpie-parent</artifactId>
     <groupId>io.openraven.magpie</groupId>
-    <version>0.7.4-SNAPSHOT</version>
+    <version>0.8.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
@@ -54,7 +54,7 @@
     <dependency>
       <groupId>io.openraven.magpie</groupId>
       <artifactId>magpie-persist</artifactId>
-      <version>0.7.4-SNAPSHOT</version>
+      <version>0.8.0-SNAPSHOT</version>
       <scope>compile</scope>
     </dependency>
 

diff --git a/magpie-data/pom.xml b/magpie-data/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>magpie-parent</artifactId>
     <groupId>io.openraven.magpie</groupId>
-    <version>0.7.4-SNAPSHOT</version>
+    <version>0.8.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/magpie-gcp/pom.xml b/magpie-gcp/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>magpie-parent</artifactId>
     <groupId>io.openraven.magpie</groupId>
-    <version>0.7.4-SNAPSHOT</version>
+    <version>0.8.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
@@ -245,9 +245,8 @@
       <version>0.16.1</version>
     </dependency>
     <dependency>
-      <groupId>com.google.apis</groupId>
-      <artifactId>google-api-services-iam</artifactId>
-      <version>v1-rev20210519-1.31.5</version>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-iam-admin</artifactId>
     </dependency>
     <dependency>
       <groupId>com.google.apis</groupId>

diff --git a/...rc/main/java/io/openraven/magpie/plugins/gcp/discovery/exception/DiscoveryExceptions.java b/...rc/main/java/io/openraven/magpie/plugins/gcp/discovery/exception/DiscoveryExceptions.java
@@ -1,5 +1,21 @@
+/*
+ * Copyright 2023 Open Raven Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package io.openraven.magpie.plugins.gcp.discovery.exception;
 
+import io.grpc.StatusRuntimeException;
 import io.sentry.Sentry;
 import io.sentry.SentryEvent;
 import io.sentry.SentryLevel;
@@ -13,6 +29,16 @@ public class DiscoveryExceptions {
 
   static public void onDiscoveryException(String resourceType, Exception ex) {
     logger.error("{} - Exception , with error {}", resourceType, ex.getMessage());
+
+    //
+    // We do not want to sent Sentry events for exceptions caused by StatusRuntimeExceptions indicating the root
+    // cause is a disabled API.
+    // See PROD-7671
+    //
+    if (ex instanceof StatusRuntimeException && ex.getMessage() != null && ex.getMessage().contains("API has not been used")) {
+      return;
+    }
+
     final var event = new SentryEvent();
     event.setLevel(SentryLevel.WARNING);
     final var message = new Message();

diff --git a/.../main/java/io/openraven/magpie/plugins/gcp/discovery/services/ComputeEngineDiscovery.java b/.../main/java/io/openraven/magpie/plugins/gcp/discovery/services/ComputeEngineDiscovery.java
@@ -43,22 +43,31 @@ public String service() {
   }
 
   public void discover(ObjectMapper mapper, String projectId, Session session, Emitter emitter, Logger logger, Optional<CredentialsProvider> maybeCredentialsProvider) {
-      try (var diskClient = DisksClient.create();
-           var instancesClient = InstancesClient.create();
-           var zoneClient = ZonesClient.create()) {
-          try {
-              discoverInstances(mapper, projectId, session, emitter, instancesClient, zoneClient);
-          }catch (IOException e) {
-              DiscoveryExceptions.onDiscoveryException("GCP::ComputeEngine::Instances", e);
-          }
-          try {
-              discoverDisks(mapper, projectId, session, emitter, diskClient, zoneClient);
-          }catch (IOException e) {
-              DiscoveryExceptions.onDiscoveryException("GCP::ComputeEngine::Disk", e);
-          }
+
+    final var diskSettings = DisksSettings.newBuilder();
+    final var instancesSettings = InstancesSettings.newBuilder();
+    final var zonesSettings = ZonesSettings.newBuilder();
+
+    maybeCredentialsProvider.ifPresent(diskSettings::setCredentialsProvider);
+    maybeCredentialsProvider.ifPresent(instancesSettings::setCredentialsProvider);
+    maybeCredentialsProvider.ifPresent(zonesSettings::setCredentialsProvider);
+
+    try (var diskClient = DisksClient.create(diskSettings.build());
+         var instancesClient = InstancesClient.create(instancesSettings.build());
+         var zoneClient = ZonesClient.create(zonesSettings.build())) {
+      try {
+        discoverInstances(mapper, projectId, session, emitter, instancesClient, zoneClient);
       } catch (IOException e) {
-          DiscoveryExceptions.onDiscoveryException("GCP::ComputeEngine::ClientAllocation", e);
+        DiscoveryExceptions.onDiscoveryException("GCP::ComputeEngine::Instances", e);
       }
+      try {
+        discoverDisks(mapper, projectId, session, emitter, diskClient, zoneClient);
+      } catch (IOException e) {
+        DiscoveryExceptions.onDiscoveryException("GCP::ComputeEngine::Disk", e);
+      }
+    } catch (IOException e) {
+      DiscoveryExceptions.onDiscoveryException("GCP::ComputeEngine::ClientAllocation", e);
+    }
   }
 
   private void discoverInstances(

diff --git a/...ie-gcp/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/IamDiscovery.java b/...ie-gcp/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/IamDiscovery.java
@@ -17,18 +17,12 @@
 package io.openraven.magpie.plugins.gcp.discovery.services;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.api.gax.core.CredentialsProvider;
-import com.google.api.services.iam.v1.Iam;
-import com.google.api.services.iam.v1.IamScopes;
-import com.google.api.services.iam.v1.model.ListRolesResponse;
-import com.google.api.services.iam.v1.model.ListServiceAccountsResponse;
-import com.google.api.services.iam.v1.model.ServiceAccount;
 import com.google.appengine.repackaged.com.google.common.base.Pair;
-import com.google.auth.http.HttpCredentialsAdapter;
-import com.google.auth.oauth2.GoogleCredentials;
-import com.google.cloud.resourcemanager.v3.ProjectName;
+import com.google.cloud.iam.admin.v1.IAMClient;
+import com.google.cloud.iam.admin.v1.IAMSettings;
+import com.google.iam.admin.v1.ListRolesRequest;
+import com.google.iam.admin.v1.ServiceAccount;
 import io.openraven.magpie.api.Emitter;
 import io.openraven.magpie.api.MagpieGcpResource;
 import io.openraven.magpie.api.Session;
@@ -40,7 +34,6 @@
 
 import java.io.IOException;
 import java.security.GeneralSecurityException;
-import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 
@@ -54,96 +47,63 @@ public String service() {
 
   public void discover(ObjectMapper mapper, String projectId, Session session, Emitter emitter, Logger logger, Optional<CredentialsProvider> maybeCredentialsProvider) {
     try {
-      Iam iamService = initService(maybeCredentialsProvider);
+      final var builder = IAMSettings.newBuilder();
+      maybeCredentialsProvider.ifPresent(builder::setCredentialsProvider);
 
-      discoverServiceAccounts(iamService, mapper, projectId, session, emitter);
-      discoverRoles(iamService, mapper, projectId, session, emitter);
+      try (final var client = IAMClient.create(builder.build())) {
+        discoverServiceAccounts(client, mapper, projectId, session, emitter, logger);
+        discoverRoles(client, mapper, projectId, session, emitter, logger);
+      }
     } catch (GeneralSecurityException | IOException e) {
       logger.error("Unable to finish IAM discovery, due to:", e);
     }
   }
 
-  public void discoverServiceAccounts(Iam iamService, ObjectMapper mapper, String projectId, Session session, Emitter emitter) throws GeneralSecurityException, IOException {
+  public void discoverServiceAccounts(IAMClient client, ObjectMapper mapper, String projectId, Session session, Emitter emitter, Logger logger) throws GeneralSecurityException, IOException {
     final String RESOURCE_TYPE = GcpIamServiceAccount.RESOURCE_TYPE;
-
-    var request = iamService.projects().serviceAccounts().list(ProjectName.of(projectId).toString());
-
-    ListServiceAccountsResponse response;
-    do {
-      response = request.execute();
-      if (response.getAccounts() == null) {
-        continue;
+    client.listServiceAccounts(com.google.iam.admin.v1.ProjectName.of(projectId)).iterateAll().forEach(sa -> {
+      var data = new MagpieGcpResource.MagpieGcpResourceBuilder(mapper, sa.getName())
+        .withProjectId(projectId)
+        .withResourceType(RESOURCE_TYPE)
+        .withResourceName(sa.getName())
+        .withConfiguration(GCPUtils.asJsonNode(sa.toBuilder()))
+        .build();
+
+      try {
+        discoverServiceAccountIamPolicy(client, sa, data);
+        discoverServiceAccountKeys(client, sa, data);
+      } catch (IOException ex) {
+        logger.warn("Couldn't discover ServiceAccount policy", ex);
       }
-      for (var serviceAccount : response.getAccounts()) {
-        var data = new MagpieGcpResource.MagpieGcpResourceBuilder(mapper, serviceAccount.getName())
-          .withProjectId(projectId)
-          .withResourceType(RESOURCE_TYPE)
-          .withConfiguration(GCPUtils.asJsonNode(serviceAccount))
-          .build();
-
-        discoverServiceAccountIamPolicy(iamService, serviceAccount, data);
-        discoverServiceAccountKeys(iamService, serviceAccount, data);
 
-        emitter.emit(VersionedMagpieEnvelopeProvider.create(session, List.of(fullService() + ":serviceAccount"), data.toJsonNode()));
-      }
-      request.setPageToken(response.getNextPageToken());
-    } while (response.getNextPageToken() != null);
+      emitter.emit(VersionedMagpieEnvelopeProvider.create(session, List.of(fullService() + ":serviceAccount"), data.toJsonNode()));
+    });
   }
 
-  private void discoverServiceAccountIamPolicy(Iam iamService, ServiceAccount serviceAccount, MagpieGcpResource data) throws IOException {
-    Iam.Projects.ServiceAccounts.GetIamPolicy getIamPolicyRequest =
-      iamService.projects().serviceAccounts().getIamPolicy(serviceAccount.getName());
-
+  private void discoverServiceAccountIamPolicy(IAMClient client, ServiceAccount serviceAccount, MagpieGcpResource data) throws IOException {
     final String fieldName = "iamPolicy";
+    final var policy = client.getIamPolicy(serviceAccount.getName());
 
-    GCPUtils.update(data.supplementaryConfiguration, Pair.of(fieldName, getIamPolicyRequest.execute()));
+    GCPUtils.update(data.supplementaryConfiguration, Pair.of(fieldName, policy.toBuilder()));
   }
-  private void discoverServiceAccountKeys(Iam iamService, ServiceAccount serviceAccount, MagpieGcpResource data) throws IOException {
+  private void discoverServiceAccountKeys(IAMClient client, ServiceAccount serviceAccount, MagpieGcpResource data) throws IOException {
     final String fieldName = "keys";
+    final var keys = client.listServiceAccountKeys(serviceAccount.getName(), List.of());  // Empty list requests all key types
 
-    var request = iamService.projects().serviceAccounts().keys().list(serviceAccount.getName());
-    GCPUtils.update(data.supplementaryConfiguration, Pair.of(fieldName, request.execute()));
+    GCPUtils.update(data.supplementaryConfiguration, Pair.of(fieldName, keys.toBuilder()));
   }
 
-  public void discoverRoles(Iam iamService, ObjectMapper mapper, String projectId, Session session, Emitter emitter) throws GeneralSecurityException, IOException {
+  public void discoverRoles(IAMClient client, ObjectMapper mapper, String projectId, Session session, Emitter emitter, Logger logger) throws GeneralSecurityException, IOException {
     final String RESOURCE_TYPE = GcpIamRole.RESOURCE_TYPE;
 
-    var request = iamService.roles().list();
+    client.listRoles(ListRolesRequest.newBuilder().setParent("projects/" + projectId).build()).iterateAll().forEach(role -> {
+      var data = new MagpieGcpResource.MagpieGcpResourceBuilder(mapper, role.getName())
+        .withProjectId(projectId)
+        .withResourceType(RESOURCE_TYPE)
+        .withConfiguration(GCPUtils.asJsonNode(role.toBuilder()))
+        .build();
 
-    ListRolesResponse response;
-    do {
-      response = request.execute();
-      if (response.getRoles() == null) {
-        continue;
-      }
-      for (var serviceAccount : response.getRoles()) {
-        var data = new MagpieGcpResource.MagpieGcpResourceBuilder(mapper, serviceAccount.getName())
-          .withProjectId(projectId)
-          .withResourceType(RESOURCE_TYPE)
-          .withConfiguration(GCPUtils.asJsonNode(serviceAccount))
-          .build();
-
-
-        emitter.emit(VersionedMagpieEnvelopeProvider.create(session, List.of(fullService() + ":role"), data.toJsonNode()));
-      }
-      request.setPageToken(response.getNextPageToken());
-    } while (response.getNextPageToken() != null);
-  }
-
-  private static Iam initService(Optional<CredentialsProvider> maybeCredentialsProvider) throws GeneralSecurityException, IOException {
-    GoogleCredentials credential;
-    if(maybeCredentialsProvider.isPresent()){
-        credential = (GoogleCredentials) maybeCredentialsProvider.get().getCredentials();
-    } else {
-        credential = GoogleCredentials.getApplicationDefault();
-    }
-    credential.createScoped(Collections.singleton(IamScopes.CLOUD_PLATFORM));
-
-    return new Iam.Builder(
-      GoogleNetHttpTransport.newTrustedTransport(),
-      JacksonFactory.getDefaultInstance(),
-      new HttpCredentialsAdapter(credential))
-      .setApplicationName("service-accounts")
-      .build();
+      emitter.emit(VersionedMagpieEnvelopeProvider.create(session, List.of(fullService() + ":role"), data.toJsonNode()));
+    });
   }
 }
diff --git a/...src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/MonitoringDiscovery.java b/...src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/MonitoringDiscovery.java
@@ -18,7 +18,6 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.api.gax.core.CredentialsProvider;
-import com.google.api.gax.core.CredentialsProvider;
 import com.google.cloud.monitoring.v3.AlertPolicyServiceClient;
 import com.google.cloud.monitoring.v3.AlertPolicyServiceSettings;
 import com.google.cloud.monitoring.v3.GroupServiceClient;
@@ -34,9 +33,9 @@
 import io.openraven.magpie.data.gcp.monitoring.MonitoringAlertPolicy;
 import io.openraven.magpie.data.gcp.monitoring.MonitoringGroup;
 import io.openraven.magpie.data.gcp.monitoring.MonitoringService;
-import io.openraven.magpie.plugins.gcp.discovery.exception.DiscoveryExceptions;
 import io.openraven.magpie.plugins.gcp.discovery.GCPUtils;
 import io.openraven.magpie.plugins.gcp.discovery.VersionedMagpieEnvelopeProvider;
+import io.openraven.magpie.plugins.gcp.discovery.exception.DiscoveryExceptions;
 import org.slf4j.Logger;
 
 import java.io.IOException;

diff --git a/...cp/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/NetworkDiscovery.java b/...cp/src/main/java/io/openraven/magpie/plugins/gcp/discovery/services/NetworkDiscovery.java
@@ -26,6 +26,7 @@
 import com.google.cloud.compute.v1.SubnetworksClient;
 import com.google.cloud.compute.v1.SubnetworksSettings;
 import com.google.cloud.compute.v1.ZonesClient;
+import com.google.cloud.compute.v1.ZonesSettings;
 import io.openraven.magpie.api.Emitter;
 import io.openraven.magpie.api.MagpieGcpResource;
 import io.openraven.magpie.api.Session;
@@ -50,15 +51,18 @@ public String service() {
   @Override
   public void discover(ObjectMapper mapper, String projectId, Session session, Emitter emitter, Logger logger, Optional<CredentialsProvider> maybeCredentialsProvider) {
     final String RESOURCE_TYPE = io.openraven.magpie.data.gcp.vpc.Network.RESOURCE_TYPE;
-    var networkSettingsBuilder = NetworksSettings.newBuilder();
-    var subnetworkSettingsBuilder = SubnetworksSettings.newBuilder();
-    maybeCredentialsProvider.ifPresent(provider -> {
-        networkSettingsBuilder.setCredentialsProvider(provider);
-        subnetworkSettingsBuilder.setCredentialsProvider(provider);
-    });
-    try (NetworksClient networkClient = NetworksClient.create(networkSettingsBuilder.build());
-         SubnetworksClient subnetworkClient = SubnetworksClient.create(subnetworkSettingsBuilder.build());
-         var zoneClient = ZonesClient.create()
+
+    final var networkSettings = NetworksSettings.newBuilder();
+    final var subnetworkSettings = SubnetworksSettings.newBuilder();
+    final var zonesSettings = ZonesSettings.newBuilder();
+
+    maybeCredentialsProvider.ifPresent(networkSettings::setCredentialsProvider);
+    maybeCredentialsProvider.ifPresent(subnetworkSettings::setCredentialsProvider);
+    maybeCredentialsProvider.ifPresent(zonesSettings::setCredentialsProvider);
+
+    try (var networkClient = NetworksClient.create(networkSettings.build());
+         var subnetworkClient = SubnetworksClient.create(subnetworkSettings.build());
+         var zoneClient = ZonesClient.create(zonesSettings.build())
     ) {
       networkClient.list(projectId).iterateAll().forEach(network -> {
         var data = new MagpieGcpResource.MagpieGcpResourceBuilder(mapper, network.getName())

diff --git a/magpie-json/pom.xml b/magpie-json/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>magpie-parent</artifactId>
     <groupId>io.openraven.magpie</groupId>
-    <version>0.7.4-SNAPSHOT</version>
+    <version>0.8.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/magpie-persist/pom.xml b/magpie-persist/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>magpie-parent</artifactId>
     <groupId>io.openraven.magpie</groupId>
-    <version>0.7.4-SNAPSHOT</version>
+    <version>0.8.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>