Skip to content

Commit

Permalink
Abstract resolution mechanisms from service implementations
Browse files Browse the repository at this point in the history
Currently all service implementations, both the Polaris admin service and the (Iceberg REST) catalog service need knowledge about the implementation details about _how exactly_ entities and their instances are managed. This is rather a persistence implementation concern, not a service implementation concern. Services should tell, if necessary, tell a properly abstracted builder which entities it will need.

Another concern this change tackles is to eventually allow a more efficient data model that is suitable for horizontally scalable databases without having to use pgsql with the implicitly required serializable isolation level.

This change abstracts the two related but still different entity resolution (`ResolutionManifest` and `Resolver`) via interfaces and concrete implementation classes, eliminating the hard dependency of services to concrete persistence specific implementations.

Follow ups of this change shall:
* Remove the dependency on "entity cache" from `Resolver` - consumers of `Resolver` unwrap those in all cases
* Rather unify `ResolutionManifest` and `Resolver` - those are very tightly coupled things, doing quite some work
  • Loading branch information
snazy committed Dec 10, 2024
1 parent 426f5eb commit 2c3984d
Show file tree
Hide file tree
Showing 48 changed files with 2,308 additions and 1,874 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@
import jakarta.inject.Inject;
import org.apache.polaris.core.PolarisDiagnostics;
import org.apache.polaris.core.context.RealmContext;
import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory;
import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
import org.apache.polaris.core.persistence.local.LocalPolarisMetaStoreManagerFactory;
import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider;

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,10 @@
import org.apache.polaris.core.entity.PolarisGrantRecord;
import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
import org.apache.polaris.core.exceptions.AlreadyExistsException;
import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl;
import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
import org.apache.polaris.core.persistence.PrincipalSecretsGenerator;
import org.apache.polaris.core.persistence.RetryOnConcurrencyException;
import org.apache.polaris.core.persistence.impl.PolarisMetaStoreManagerImpl;
import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
import org.apache.polaris.core.storage.PolarisStorageIntegration;
import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@
import org.apache.polaris.core.PolarisDiagnostics;
import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest;
import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl;
import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager;
import org.apache.polaris.core.persistence.impl.PolarisMetaStoreManagerImpl;
import org.apache.polaris.jpa.models.ModelPrincipalSecrets;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import java.util.List;
import java.util.Set;
import org.apache.polaris.core.entity.PolarisBaseEntity;
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
import org.apache.polaris.core.persistence.resolution.PolarisResolvedPathWrapper;

/** Interface for invoking authorization checks. */
public interface PolarisAuthorizer {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,8 @@
import org.apache.polaris.core.entity.PolarisEntityCore;
import org.apache.polaris.core.entity.PolarisGrantRecord;
import org.apache.polaris.core.entity.PolarisPrivilege;
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
import org.apache.polaris.core.persistence.ResolvedPolarisEntity;
import org.apache.polaris.core.persistence.resolution.PolarisResolvedPathWrapper;
import org.apache.polaris.core.persistence.resolution.ResolvedPolarisEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ public enum PolarisEntitySubType {
// ANY_SUBTYPE is not stored but is used to indicate that any subtype entities should be
// returned, for example when doing a list operation or checking if a table like object of
// name X exists
ANY_SUBTYPE(-1, null),
ANY_SUBTYPE(-1, null, "Table or view"),
// the NULL value is used when an entity has no subtype, i.e. NOT_APPLICABLE really
NULL_SUBTYPE(0, null),
TABLE(2, PolarisEntityType.TABLE_LIKE),
VIEW(3, PolarisEntityType.TABLE_LIKE);
NULL_SUBTYPE(0, null, "(null)"),
TABLE(2, PolarisEntityType.TABLE_LIKE, "Table"),
VIEW(3, PolarisEntityType.TABLE_LIKE, "View");

// to efficiently map the code of a subtype to its corresponding subtype enum, use a reverse
// array which is initialized below
Expand Down Expand Up @@ -63,10 +63,13 @@ public enum PolarisEntitySubType {
// parent type for this entity
private final PolarisEntityType parentType;

PolarisEntitySubType(int code, PolarisEntityType parentType) {
private final String readableName;

PolarisEntitySubType(int code, PolarisEntityType parentType, String readableName) {
// remember the id of this entity
this.code = code;
this.parentType = parentType;
this.readableName = readableName;
}

/**
Expand Down Expand Up @@ -111,4 +114,8 @@ public PolarisEntityType getParentType() {

return null;
}

public String readableName() {
return readableName;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,17 @@

/** Types of entities with their id */
public enum PolarisEntityType {
NULL_TYPE(0, null, false, false),
ROOT(1, null, false, false),
PRINCIPAL(2, ROOT, true, false),
PRINCIPAL_ROLE(3, ROOT, true, false),
CATALOG(4, ROOT, false, false),
CATALOG_ROLE(5, CATALOG, true, false),
NAMESPACE(6, CATALOG, false, true),
NULL_TYPE(0, null, false, false, "(null)"),
ROOT(1, null, false, false, "Root"),
PRINCIPAL(2, ROOT, true, false, "Principal"),
PRINCIPAL_ROLE(3, ROOT, true, false, "Principal role"),
CATALOG(4, ROOT, false, false, "Catalog"),
CATALOG_ROLE(5, CATALOG, true, false, "Catalog role"),
NAMESPACE(6, CATALOG, false, true, "Namespace"),
// generic table is either a view or a real table
TABLE_LIKE(7, NAMESPACE, false, false),
TASK(8, ROOT, false, false),
FILE(9, TABLE_LIKE, false, false);
TABLE_LIKE(7, NAMESPACE, false, false, "Table/view"),
TASK(8, ROOT, false, false, "Task"),
FILE(9, TABLE_LIKE, false, false, "File");

// to efficiently map a code to its corresponding entity type, use a reverse array which
// is initialized below
Expand Down Expand Up @@ -70,13 +70,20 @@ public enum PolarisEntityType {

// parent entity type, null for an ACCOUNT
private final PolarisEntityType parentType;

PolarisEntityType(int id, PolarisEntityType parentType, boolean isGrantee, boolean sefRef) {
private final String readableName;

PolarisEntityType(
int id,
PolarisEntityType parentType,
boolean isGrantee,
boolean sefRef,
String readableName) {
// remember the id of this entity
this.code = id;
this.isGrantee = isGrantee;
this.parentType = parentType;
this.parentSelfReference = sefRef;
this.readableName = readableName;
}

/**
Expand Down Expand Up @@ -132,4 +139,8 @@ public boolean isTopLevel() {
public PolarisEntityType getParentType() {
return this.parentType;
}

public String readableName() {
return readableName;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import org.apache.polaris.core.PolarisCallContext;
import org.apache.polaris.core.context.CallContext;
import org.apache.polaris.core.persistence.PolarisObjectMapperUtil;
import org.apache.polaris.core.persistence.impl.PolarisObjectMapperUtil;

/**
* Represents an asynchronous task entity in the persistence layer. A task executor is responsible
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.polaris.core.persistence;

import java.util.Optional;
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.exceptions.NoSuchViewException;
import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.polaris.core.entity.PolarisEntitySubType;
import org.apache.polaris.core.entity.PolarisEntityType;

public class EntityNotFoundException extends RuntimeException {
private final PolarisEntityType entityType;
private final Optional<PolarisEntitySubType> subType;
private final String name;

public EntityNotFoundException(
PolarisEntityType entityType, Optional<PolarisEntitySubType> subType, String name) {
super(
subType.map(PolarisEntitySubType::readableName).orElseGet(entityType::readableName)
+ " "
+ name);
this.entityType = entityType;
this.subType = subType;
this.name = name;
}

public EntityNotFoundException(PolarisEntityType entityType, String name) {
this(entityType, Optional.empty(), name);
}

public RuntimeException asGenericIcebergNotFoundException() {
throw new NotFoundException(
"%s does not exist: %s",
subType.map(PolarisEntitySubType::readableName).orElseGet(entityType::readableName), name);
}

public RuntimeException asSpecializedIcebergNotFoundException() {
switch (entityType) {
case NAMESPACE:
return new NoSuchNamespaceException("Namespace does not exist: %s", name);
case TABLE_LIKE:
return subType
.map(
sub -> {
switch (sub) {
case TABLE:
return new NoSuchTableException("Table does not exist: %s", name);
case VIEW:
return new NoSuchViewException("View does not exist: %s", name);
default:
return asGenericIcebergNotFoundException();
}
})
.orElseGet(this::asGenericIcebergNotFoundException);
default:
return asGenericIcebergNotFoundException();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,10 @@
import org.apache.polaris.core.entity.PolarisGrantRecord;
import org.apache.polaris.core.entity.PolarisPrivilege;
import org.apache.polaris.core.persistence.cache.EntityCache;
import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest;
import org.apache.polaris.core.persistence.resolver.Resolver;
import org.apache.polaris.core.persistence.resolution.ResolutionManifestBuilder;
import org.apache.polaris.core.persistence.resolution.ResolvedPolarisEntity;
import org.apache.polaris.core.persistence.resolver.ResolverBuilder;
import org.apache.polaris.core.persistence.resolver.ResolverBuilderImpl;
import org.apache.polaris.core.storage.cache.StorageCredentialCache;

/**
Expand Down Expand Up @@ -62,11 +64,11 @@ public PolarisEntityManager(
this.credentialCache = credentialCache;
}

public Resolver prepareResolver(
public ResolverBuilder prepareResolver(
@Nonnull CallContext callContext,
@Nonnull AuthenticatedPolarisPrincipal authenticatedPrincipal,
@Nullable String referenceCatalogName) {
return new Resolver(
return new ResolverBuilderImpl(
callContext.getPolarisCallContext(),
metaStoreManager,
authenticatedPrincipal.getPrincipalEntity().getId(),
Expand All @@ -78,16 +80,17 @@ public Resolver prepareResolver(
referenceCatalogName);
}

public PolarisResolutionManifest prepareResolutionManifest(
public ResolutionManifestBuilder prepareResolutionManifest(
@Nonnull CallContext callContext,
@Nonnull AuthenticatedPolarisPrincipal authenticatedPrincipal,
@Nullable String referenceCatalogName) {
PolarisResolutionManifest manifest =
new PolarisResolutionManifest(
callContext, this, authenticatedPrincipal, referenceCatalogName);
manifest.setSimulatedResolvedRootContainerEntity(
getSimulatedResolvedRootContainerEntity(callContext));
return manifest;
return this.metaStoreManager
.newResolutionManifestBuilder(
callContext,
authenticatedPrincipal,
() -> prepareResolver(callContext, authenticatedPrincipal, referenceCatalogName),
referenceCatalogName)
.withRootContainerEntity(getSimulatedResolvedRootContainerEntity(callContext));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,12 @@
import jakarta.annotation.Nullable;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import org.apache.polaris.core.PolarisCallContext;
import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal;
import org.apache.polaris.core.auth.PolarisGrantManager;
import org.apache.polaris.core.auth.PolarisSecretsManager;
import org.apache.polaris.core.context.CallContext;
import org.apache.polaris.core.entity.PolarisBaseEntity;
import org.apache.polaris.core.entity.PolarisEntity;
import org.apache.polaris.core.entity.PolarisEntityActiveRecord;
Expand All @@ -36,6 +39,8 @@
import org.apache.polaris.core.entity.PolarisEntityType;
import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
import org.apache.polaris.core.persistence.cache.PolarisRemoteCache;
import org.apache.polaris.core.persistence.resolution.ResolutionManifestBuilder;
import org.apache.polaris.core.persistence.resolver.ResolverBuilder;
import org.apache.polaris.core.storage.PolarisCredentialVendor;

/**
Expand Down Expand Up @@ -72,6 +77,13 @@ public interface PolarisMetaStoreManager
@Nonnull
BaseResult purge(@Nonnull PolarisCallContext callCtx);

@Nonnull
ResolutionManifestBuilder newResolutionManifestBuilder(
@Nonnull CallContext callContext,
@Nonnull AuthenticatedPolarisPrincipal authenticatedPrincipal,
@Nonnull Supplier<ResolverBuilder> resolverSupplier,
@Nullable String referenceCatalogName);

/** the return for an entity lookup call */
class EntityResult extends BaseResult {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.polaris.core.persistence;
package org.apache.polaris.core.persistence.impl;

import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
Expand All @@ -32,6 +32,7 @@
import org.apache.polaris.core.entity.PolarisEntityConstants;
import org.apache.polaris.core.entity.PolarisEntityCore;
import org.apache.polaris.core.entity.PolarisEntityType;
import org.apache.polaris.core.persistence.PolarisMetaStoreSession;

/**
* Utility class used by the meta store manager to ensure that all entities which had been resolved
Expand Down
Loading

0 comments on commit 2c3984d

Please sign in to comment.