Skip to content

Commit

Permalink
Abstract resolution mechanisms from service implementations
Browse files Browse the repository at this point in the history 
Currently all service implementations, both the Polaris admin service and the (Iceberg REST) catalog service need knowledge about the implementation details about _how exactly_ entities and their instances are managed. This is rather a persistence implementation concern, not a service implementation concern. Services should tell, if necessary, tell a properly abstracted builder which entities it will need.

Another concern this change tackles is to eventually allow a more efficient data model that is suitable for horizontally scalable databases without having to use pgsql with the implicitly required serializable isolation level.

This change abstracts the two related but still different entity resolution (`ResolutionManifest` and `Resolver`) via interfaces and concrete implementation classes, eliminating the hard dependency of services to concrete persistence specific implementations, and also replace the C/Go-style error handling with exceptions.

On top, this change moves types to separate packages.

This PR is part of a series of changes to abstract implementations and separate those by concern. Follow ups of this change include:
* Remove the dependency on "entity cache" from `Resolver` - consumers of `Resolver` unwrap those in all cases
* Rather unify `ResolutionManifest` and `Resolver` - those are very tightly coupled things, doing quite some work
* Implement proper exception abstraction (the changes in this PR are already better, but not as good as it could be)
* Eventually let service implementation work only with their natural keys, and not interact with many types (meta-store-manager-factory, meta-store-manager, meta-store-session, entity-manager, entity-cache).
  • Loading branch information
@snazy
snazy committed Dec 11, 2024
1 parent 9453770 commit 33b6529
Show file tree
Hide file tree
Showing 48 changed files with 2,316 additions and 1,874 deletions.
4 changes: 2 additions & 2 deletions dropwizard/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@
import org.apache.polaris.core.persistence.PolarisEntityManager;
import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
import org.apache.polaris.core.persistence.cache.EntityCache;
import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest;
import org.apache.polaris.core.persistence.resolution.ResolutionManifest;
import org.apache.polaris.core.storage.cache.StorageCredentialCache;
import org.apache.polaris.service.catalog.BasePolarisCatalog;
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
Expand Down Expand Up @@ -385,7 +385,7 @@ public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext)
public Catalog createCallContextCatalog(
CallContext context,
AuthenticatedPolarisPrincipal authenticatedPolarisPrincipal,
final PolarisResolutionManifest resolvedManifest) {
ResolutionManifest resolvedManifest) {
// This depends on the BasePolarisCatalog allowing calling initialize multiple times
// to override the previous config.
Catalog catalog =
Expand Down
4 changes: 2 additions & 2 deletions ...c/test/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapperAuthzTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@
import org.apache.polaris.core.entity.PrincipalEntity;
import org.apache.polaris.core.persistence.PolarisEntityManager;
import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest;
import org.apache.polaris.core.persistence.resolution.ResolutionManifest;
import org.apache.polaris.service.admin.PolarisAuthzTestBase;
import org.apache.polaris.service.catalog.io.DefaultFileIOFactory;
import org.apache.polaris.service.config.RealmEntityManagerFactory;
Expand Down Expand Up @@ -1688,7 +1688,7 @@ public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext)
public Catalog createCallContextCatalog(
CallContext context,
AuthenticatedPolarisPrincipal authenticatedPolarisPrincipal,
PolarisResolutionManifest resolvedManifest) {
ResolutionManifest resolvedManifest) {
Catalog catalog =
super.createCallContextCatalog(
context, authenticatedPolarisPrincipal, resolvedManifest);
Expand Down
175 changes: 108 additions & 67 deletions ...ce/src/test/java/org/apache/polaris/service/catalog/PolarisPassthroughResolutionView.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,19 @@
package org.apache.polaris.service.catalog;

import java.util.Arrays;
import java.util.Set;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal;
import org.apache.polaris.core.catalog.PolarisCatalogHelpers;
import org.apache.polaris.core.context.CallContext;
import org.apache.polaris.core.entity.PolarisBaseEntity;
import org.apache.polaris.core.entity.PolarisEntitySubType;
import org.apache.polaris.core.entity.PolarisEntityType;
import org.apache.polaris.core.persistence.EntityNotFoundException;
import org.apache.polaris.core.persistence.PolarisEntityManager;
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest;
import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView;
import org.apache.polaris.core.persistence.resolution.PolarisResolvedPathWrapper;
import org.apache.polaris.core.persistence.resolution.ResolutionManifest;
import org.apache.polaris.core.persistence.resolver.ResolverPath;

/**
Expand All @@ -39,7 +41,7 @@
* new single-use PolarisResolutionManifests for each desired resolved path without defining a fixed
* set of resolved entities that need to be checked against authorizable operations.
*/
public class PolarisPassthroughResolutionView implements PolarisResolutionManifestCatalogView {
public class PolarisPassthroughResolutionView implements ResolutionManifest {
private final PolarisEntityManager entityManager;
private final CallContext callContext;
private final AuthenticatedPolarisPrincipal authenticatedPrincipal;
Expand All @@ -58,86 +60,125 @@ public PolarisPassthroughResolutionView(

@Override
public PolarisResolvedPathWrapper getResolvedReferenceCatalogEntity() {
PolarisResolutionManifest manifest =
entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName);
manifest.resolveAll();
return manifest.getResolvedReferenceCatalogEntity();
return entityManager
.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName)
.buildResolved()
.getResolvedReferenceCatalogEntity();
}

@Override
public PolarisResolvedPathWrapper getResolvedPath(Object key) {
PolarisResolutionManifest manifest =
entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName);

if (key instanceof Namespace namespace) {
manifest.addPath(
new ResolverPath(Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE),
namespace);
manifest.resolveAll();
return manifest.getResolvedPath(namespace);
} else {
throw new IllegalStateException(
String.format(
"Trying to getResolvedPath(key) for %s with class %s", key, key.getClass()));
public PolarisResolvedPathWrapper getResolvedPath(
Namespace namespace, boolean prependRootContainer) {
try {
return entityManager
.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName)
.addPath(
new ResolverPath(Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE),
namespace)
.buildResolved()
.getResolvedPath(namespace, prependRootContainer);
} catch (EntityNotFoundException nf) {
return null;
}
}

@Override
public PolarisResolvedPathWrapper getResolvedPath(Object key, PolarisEntitySubType subType) {
PolarisResolutionManifest manifest =
entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName);

if (key instanceof TableIdentifier identifier) {
manifest.addPath(
new ResolverPath(
PolarisCatalogHelpers.tableIdentifierToList(identifier),
PolarisEntityType.TABLE_LIKE),
identifier);
manifest.resolveAll();
return manifest.getResolvedPath(identifier, subType);
} else {
throw new IllegalStateException(
String.format(
"Trying to getResolvedPath(key, subType) for %s with class %s and subType %s",
key, key.getClass(), subType));
public PolarisResolvedPathWrapper getResolvedPath(
TableIdentifier tableIdentifier, boolean prependRootContainer) {
try {
return entityManager
.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName)
.addPath(
new ResolverPath(
PolarisCatalogHelpers.tableIdentifierToList(tableIdentifier),
PolarisEntityType.TABLE_LIKE),
tableIdentifier)
.buildResolved()
.getResolvedPath(tableIdentifier, prependRootContainer);
} catch (EntityNotFoundException nf) {
return null;
}
}

@Override
public PolarisResolvedPathWrapper getPassthroughResolvedPath(Object key) {
PolarisResolutionManifest manifest =
entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName);
public PolarisResolvedPathWrapper getResolvedPath(
TableIdentifier tableIdentifier, PolarisEntitySubType subType, boolean prependRootContainer) {
try {
return entityManager
.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName)
.addPath(
new ResolverPath(
PolarisCatalogHelpers.tableIdentifierToList(tableIdentifier),
PolarisEntityType.TABLE_LIKE),
tableIdentifier)
.buildResolved()
.getResolvedPath(tableIdentifier, subType, prependRootContainer);
} catch (EntityNotFoundException nf) {
return null;
}
}

if (key instanceof Namespace namespace) {
manifest.addPassthroughPath(
new ResolverPath(Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE),
namespace);
return manifest.getPassthroughResolvedPath(namespace);
} else {
throw new IllegalStateException(
String.format(
"Trying to getResolvedPath(key) for %s with class %s", key, key.getClass()));
@Override
public PolarisResolvedPathWrapper getPassthroughResolvedPath(Namespace namespace) {
try {
return entityManager
.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName)
.addPassthroughPath(
new ResolverPath(Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE),
namespace)
.buildResolved()
.getPassthroughResolvedPath(namespace);
} catch (EntityNotFoundException nf) {
return null;
}
}

@Override
public PolarisResolvedPathWrapper getPassthroughResolvedPath(
Object key, PolarisEntitySubType subType) {
PolarisResolutionManifest manifest =
entityManager.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName);

if (key instanceof TableIdentifier identifier) {
manifest.addPassthroughPath(
new ResolverPath(
PolarisCatalogHelpers.tableIdentifierToList(identifier),
PolarisEntityType.TABLE_LIKE),
identifier);
return manifest.getPassthroughResolvedPath(identifier, subType);
} else {
throw new IllegalStateException(
String.format(
"Trying to getResolvedPath(key, subType) for %s with class %s and subType %s",
key, key.getClass(), subType));
TableIdentifier tableIdentifier, PolarisEntitySubType subType) {
try {
return entityManager
.prepareResolutionManifest(callContext, authenticatedPrincipal, catalogName)
.addPassthroughPath(
new ResolverPath(
PolarisCatalogHelpers.tableIdentifierToList(tableIdentifier),
PolarisEntityType.TABLE_LIKE),
tableIdentifier)
.buildResolved(subType)
.getPassthroughResolvedPath(tableIdentifier, subType);
} catch (EntityNotFoundException nf) {
return null;
}
}

@Override
public PolarisResolvedPathWrapper getResolvedPath(String name, boolean prependRootContainer) {
throw new UnsupportedOperationException();
}

@Override
public PolarisResolvedPathWrapper getResolvedTopLevelEntity(
String name, PolarisEntityType polarisEntityType) {
throw new UnsupportedOperationException();
}

@Override
public PolarisResolvedPathWrapper getResolvedRootContainerEntityAsPath() {
throw new UnsupportedOperationException();
}

@Override
public PolarisEntitySubType getLeafSubType(TableIdentifier tableIdentifier) {
throw new UnsupportedOperationException();
}

@Override
public Set<PolarisBaseEntity> getAllActivatedPrincipalRoleEntities() {
throw new UnsupportedOperationException();
}

@Override
public Set<PolarisBaseEntity> getAllActivatedCatalogRoleAndPrincipalRoles() {
throw new UnsupportedOperationException();
}
}
2 changes: 1 addition & 1 deletion ...ris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@
import jakarta.inject.Inject;
import org.apache.polaris.core.PolarisDiagnostics;
import org.apache.polaris.core.context.RealmContext;
import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory;
import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
import org.apache.polaris.core.persistence.local.LocalPolarisMetaStoreManagerFactory;
import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider;

/**
Expand Down
2 changes: 1 addition & 1 deletion ...olaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,10 @@
import org.apache.polaris.core.entity.PolarisGrantRecord;
import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
import org.apache.polaris.core.exceptions.AlreadyExistsException;
import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl;
import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
import org.apache.polaris.core.persistence.PrincipalSecretsGenerator;
import org.apache.polaris.core.persistence.RetryOnConcurrencyException;
import org.apache.polaris.core.persistence.impl.PolarisMetaStoreManagerImpl;
import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
import org.apache.polaris.core.storage.PolarisStorageIntegration;
import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider;
Expand Down
2 changes: 1 addition & 1 deletion ...olaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@
import org.apache.polaris.core.PolarisDiagnostics;
import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest;
import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl;
import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager;
import org.apache.polaris.core.persistence.impl.PolarisMetaStoreManagerImpl;
import org.apache.polaris.jpa.models.ModelPrincipalSecrets;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
Expand Down
2 changes: 1 addition & 1 deletion polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import java.util.List;
import java.util.Set;
import org.apache.polaris.core.entity.PolarisBaseEntity;
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
import org.apache.polaris.core.persistence.resolution.PolarisResolvedPathWrapper;

/** Interface for invoking authorization checks. */
public interface PolarisAuthorizer {
Expand Down
4 changes: 2 additions & 2 deletions polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,8 @@
import org.apache.polaris.core.entity.PolarisEntityCore;
import org.apache.polaris.core.entity.PolarisGrantRecord;
import org.apache.polaris.core.entity.PolarisPrivilege;
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
import org.apache.polaris.core.persistence.ResolvedPolarisEntity;
import org.apache.polaris.core.persistence.resolution.PolarisResolvedPathWrapper;
import org.apache.polaris.core.persistence.resolution.ResolvedPolarisEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
17 changes: 12 additions & 5 deletions polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntitySubType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ public enum PolarisEntitySubType {
// ANY_SUBTYPE is not stored but is used to indicate that any subtype entities should be
// returned, for example when doing a list operation or checking if a table like object of
// name X exists
ANY_SUBTYPE(-1, null),
ANY_SUBTYPE(-1, null, "Table or view"),
// the NULL value is used when an entity has no subtype, i.e. NOT_APPLICABLE really
NULL_SUBTYPE(0, null),
TABLE(2, PolarisEntityType.TABLE_LIKE),
VIEW(3, PolarisEntityType.TABLE_LIKE);
NULL_SUBTYPE(0, null, "(null)"),
TABLE(2, PolarisEntityType.TABLE_LIKE, "Table"),
VIEW(3, PolarisEntityType.TABLE_LIKE, "View");

// to efficiently map the code of a subtype to its corresponding subtype enum, use a reverse
// array which is initialized below
Expand Down Expand Up @@ -63,10 +63,13 @@ public enum PolarisEntitySubType {
// parent type for this entity
private final PolarisEntityType parentType;

PolarisEntitySubType(int code, PolarisEntityType parentType) {
private final String readableName;

PolarisEntitySubType(int code, PolarisEntityType parentType, String readableName) {
// remember the id of this entity
this.code = code;
this.parentType = parentType;
this.readableName = readableName;
}

/**
Expand Down Expand Up @@ -111,4 +114,8 @@ public PolarisEntityType getParentType() {

return null;
}

public String readableName() {
return readableName;
}
}
Loading

0 comments on commit 33b6529

Please sign in to comment.