Remove dropwizard-jackson dep from core (#470)

This is a simplification / cleanup. The dependency does not appear to be required in `polaris-core`

Add custom code to `PolarisApplication` find classes directly listed in the `Discoverable`
service descriptor and register them with the ObjectMapper. This approach to finding sub-types
is consistent both with the java service descriptors (listed types actually implement the
service interface) and at the same time allows moving the Dropwizard `Discoverable` dependencies
to the polaris-service module that actually integrates with Dropwizard.

Move leaf metastore classes to the `Discoverable` service descriptor in their respective module.

Note: this fixes the cross-jar leak of EclipseLinkPolarisMetaStoreManagerFactory in service descriptors.

extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java

@@ -20,6 +20,7 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.dropwizard.jackson.Discoverable;
 import jakarta.annotation.Nonnull;
 import org.apache.polaris.core.PolarisDiagnostics;
 import org.apache.polaris.core.context.RealmContext;
@@ -34,7 +35,7 @@
  */
 @JsonTypeName("eclipse-link")
 public class EclipseLinkPolarisMetaStoreManagerFactory
-    extends LocalPolarisMetaStoreManagerFactory<PolarisEclipseLinkStore> {
+    extends LocalPolarisMetaStoreManagerFactory<PolarisEclipseLinkStore> implements Discoverable {
   @JsonProperty("conf-file")
   private String confFile;
 

polaris-service/src/main/resources/META-INF/services/org.apache.polaris.core.persistence.MetaStoreManagerFactory → extension/persistence/eclipselink/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable

@@ -17,5 +17,4 @@
 # under the License.
 #
 
-org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory
 org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory

polaris-core/build.gradle.kts

@@ -33,10 +33,6 @@ dependencies {
   constraints {
     implementation("io.airlift:aircompressor:0.27") { because("Vulnerability detected in 0.25") }
   }
-  // TODO - this is only here for the Discoverable interface
-  // We should use a different mechanism to discover the plugin implementations
-  implementation(platform(libs.dropwizard.bom))
-  implementation("io.dropwizard:dropwizard-jackson")
 
   implementation(platform(libs.jackson.bom))
   implementation("com.fasterxml.jackson.core:jackson-annotations")

polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java

@@ -19,7 +19,6 @@
 package org.apache.polaris.core.persistence;
 
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import io.dropwizard.jackson.Discoverable;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
@@ -34,7 +33,7 @@
  * configuration
  */
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
-public interface MetaStoreManagerFactory extends Discoverable {
+public interface MetaStoreManagerFactory {
 
   PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext);
 

polaris-service/build.gradle.kts

@@ -118,7 +118,7 @@ dependencies {
   testImplementation(libs.mockito.core)
   testRuntimeOnly("org.junit.platform:junit-platform-launcher")
 
-  testRuntimeOnly(project(":polaris-eclipselink"))
+  testImplementation(project(":polaris-eclipselink"))
 }
 
 if (project.properties.get("eclipseLink") == "true") {

polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java

@@ -36,6 +36,8 @@
 import io.dropwizard.core.Application;
 import io.dropwizard.core.setup.Bootstrap;
 import io.dropwizard.core.setup.Environment;
+import io.dropwizard.jackson.Discoverable;
+import io.dropwizard.jackson.DiscoverableSubtypeResolver;
 import io.micrometer.prometheusmetrics.PrometheusConfig;
 import io.micrometer.prometheusmetrics.PrometheusMeterRegistry;
 import io.opentelemetry.api.OpenTelemetry;
@@ -63,6 +65,7 @@
 import java.net.URL;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.Executors;
@@ -142,6 +145,7 @@ private static void printAsciiArt() throws IOException {
 
   @Override
   public void initialize(Bootstrap<PolarisApplicationConfig> bootstrap) {
+    registerTypes(bootstrap.getObjectMapper());
     // Enable variable substitution with environment variables
     EnvironmentVariableSubstitutor substitutor = new EnvironmentVariableSubstitutor(false);
     SubstitutingSourceProvider provider =
@@ -152,6 +156,18 @@ public void initialize(Bootstrap<PolarisApplicationConfig> bootstrap) {
     bootstrap.addCommand(new PurgeRealmsCommand());
   }
 
+  private void registerTypes(ObjectMapper mapper) {
+    // Reuse the DW service discovery method, but unlike the constructor of
+    // DiscoverableSubtypeResolver, use the first level classes from the `Discoverable`
+    // service descriptor and register them with the ObjectMapper.
+    class Discoverer extends DiscoverableSubtypeResolver {
+      List<Class<?>> discover() {
+        return discoverServices(Discoverable.class);
+      }
+    }
+    new Discoverer().discover().forEach(mapper::registerSubtypes);
+  }
+
   @Override
   public void run(PolarisApplicationConfig configuration, Environment environment) {
     MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory();

polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java

@@ -19,6 +19,7 @@
 package org.apache.polaris.service.persistence;
 
 import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.dropwizard.jackson.Discoverable;
 import jakarta.annotation.Nonnull;
 import java.util.Collections;
 import java.util.HashSet;
@@ -36,7 +37,7 @@
 
 @JsonTypeName("in-memory")
 public class InMemoryPolarisMetaStoreManagerFactory
-    extends LocalPolarisMetaStoreManagerFactory<PolarisTreeMapStore> {
+    extends LocalPolarisMetaStoreManagerFactory<PolarisTreeMapStore> implements Discoverable {
   final Set<String> bootstrappedRealms = new HashSet<>();
 
   @Override

polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable

@@ -18,7 +18,7 @@
 #
 
 org.apache.polaris.service.auth.DiscoverableAuthenticator
-org.apache.polaris.core.persistence.MetaStoreManagerFactory
+org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory
 org.apache.polaris.service.config.OAuth2ApiService
 org.apache.polaris.service.context.RealmContextResolver
 org.apache.polaris.service.context.CallContextResolver

polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationConfigurationTest.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import io.dropwizard.testing.ConfigOverride;
+import io.dropwizard.testing.ResourceHelpers;
+import io.dropwizard.testing.junit5.DropwizardAppExtension;
+import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
+import org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory;
+import org.apache.polaris.service.config.PolarisApplicationConfig;
+import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(DropwizardExtensionsSupport.class)
+public class PolarisApplicationConfigurationTest {
+
+  public static final String CONFIG_PATH =
+      ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml");
+  // Bind to random ports to support parallelism
+  public static final ConfigOverride RANDOM_APP_PORT =
+      ConfigOverride.config("server.applicationConnectors[0].port", "0");
+  public static final ConfigOverride RANDOM_ADMIN_PORT =
+      ConfigOverride.config("server.adminConnectors[0].port", "0");
+
+  @Nested
+  class DefaultMetastore {
+    private final DropwizardAppExtension<PolarisApplicationConfig> app =
+        new DropwizardAppExtension<>(
+            PolarisApplication.class, CONFIG_PATH, RANDOM_APP_PORT, RANDOM_ADMIN_PORT);
+
+    @Test
+    void testMetastoreType() {
+      assertThat(app.getConfiguration().getMetaStoreManagerFactory())
+          .isInstanceOf(InMemoryPolarisMetaStoreManagerFactory.class);
+    }
+  }
+
+  @Nested
+  class EclipseLinkMetastore {
+    private final DropwizardAppExtension<PolarisApplicationConfig> app =
+        new DropwizardAppExtension<>(
+            PolarisApplication.class,
+            CONFIG_PATH,
+            RANDOM_APP_PORT,
+            RANDOM_ADMIN_PORT,
+            ConfigOverride.config("metaStoreManager.type", "eclipse-link"),
+            ConfigOverride.config("metaStoreManager.persistence-unit", "test-unit"),
+            ConfigOverride.config("metaStoreManager.conf-file", "/test-conf-file"));
+
+    @Test
+    void testMetastoreType() {
+      assertThat(app.getConfiguration().getMetaStoreManagerFactory())
+          .isInstanceOf(EclipseLinkPolarisMetaStoreManagerFactory.class)
+          .extracting("persistenceUnitName", "confFile")
+          .containsExactly("test-unit", "/test-conf-file");
+    }
+  }
+}