Table creation

[benjstephenson]

Is it possible to create Dynamo tables using Eletro? I can't find anything in the docs that points to it and I don't really want to have to maintain a separate schema definition for creating tables and also the one for use with the Entity classes; unless there's a way to reuse the definition with the aws-sdk?

[tywalch]

Creating the table is a little out of Electro's scope, but making a function that could translate an Electro schema type to a table definition (CloudFormation, CreateTable command, CDK, etc) would be pretty simple 👍 You could iterate over the attributes for field names and types and iterate over the indexes for GSIs and fields.

[tywalch]

Something like this:

import * as dynamodb from 'aws-cdk-lib/aws-dynamodb';
import { Stack, StackProps, RemovalPolicy } from 'aws-cdk-lib';
import { Construct } from "constructs";
import { Entity, Schema } from "electrodb";

type NonSpecificEntity = Entity<string, string, string, Schema<string, string, string>>
type NonSpecificEntityIndex = NonSpecificEntity['schema']['indexes'][string];

type TableIndexDefinition = {
    type: 'TableIndex';
    partitionKey: dynamodb.Attribute;
    sortKey?: dynamodb.Attribute;
}

type SecondaryIndexDefinition = {
    type: 'SecondaryIndex';
    indexName: string;
    partitionKey: dynamodb.Attribute;
    sortKey?: dynamodb.Attribute;
    projectionType: dynamodb.ProjectionType;
}

type LocalSecondaryIndexDefinition = {
    type: 'LocalSecondaryIndex';
    indexName: string;
    sortKey: dynamodb.Attribute;
    projectionType: dynamodb.ProjectionType;
}

type IndexDefinition = 
    | TableIndexDefinition 
    | SecondaryIndexDefinition
    | LocalSecondaryIndexDefinition;

function getTableIndexName<E extends Entity<any, any, any, any>>(entity: E): string {
    const { schema } = entity;
    
    let tableIndexName: string | null = null;
    for (let accessPattern in schema.indexes) {
        const indexDefinition = schema.indexes[accessPattern];
        if (indexDefinition.index === undefined) {
            tableIndexName = accessPattern;
            break;
        }
    }
    
    if (tableIndexName === null) {
        throw new Error('No table index found');
    }
    
    return tableIndexName;
}

function createTableIndexDefinition(indexDefinition: NonSpecificEntityIndex): TableIndexDefinition {
    return {
        type: 'TableIndex',
        partitionKey: {
            name: indexDefinition.pk.field,
            type: dynamodb.AttributeType.STRING
        },
        sortKey: indexDefinition.sk ? {
            name: indexDefinition.sk.field,
            type: dynamodb.AttributeType.STRING
        } : undefined
    }
}

function createSecondaryIndexDefinition(indexDefinition: NonSpecificEntityIndex): SecondaryIndexDefinition {
    return {
        type: 'SecondaryIndex',
        indexName: indexDefinition.index!,
        partitionKey: {
            name: indexDefinition.pk.field,
            type: dynamodb.AttributeType.STRING
        },
        sortKey: indexDefinition.sk ? {
            name: indexDefinition.sk.field,
            type: dynamodb.AttributeType.STRING
        } : undefined,
        projectionType: indexDefinition.project === 'keys_only' 
            ? dynamodb.ProjectionType.KEYS_ONLY 
            : dynamodb.ProjectionType.ALL,
    }
}

function createLocalSecondaryIndexDefinition(indexDefinition: NonSpecificEntityIndex): LocalSecondaryIndexDefinition {
    return {
        type: 'LocalSecondaryIndex',
        indexName: indexDefinition.index!,
        sortKey: {
            name: indexDefinition.sk!.field,
            type: dynamodb.AttributeType.STRING
        },
        projectionType: indexDefinition.project === 'keys_only' 
            ? dynamodb.ProjectionType.KEYS_ONLY 
            : dynamodb.ProjectionType.ALL,
    }
}

function getIndexes(entity: NonSpecificEntity): IndexDefinition[] {
    const { schema } = entity;
    
    const indexes: IndexDefinition[] = [];
    const tableIndexName = getTableIndexName(entity);
    for (const accessPattern in schema.indexes) {
        const indexDefinition = schema.indexes[accessPattern];
        if (indexDefinition.index !== undefined && indexDefinition.index === tableIndexName) {
            indexes.push(createTableIndexDefinition(indexDefinition));
        } else if (accessPattern === tableIndexName) {
            indexes.push(createLocalSecondaryIndexDefinition(indexDefinition));
        } else {
            indexes.push(createSecondaryIndexDefinition(indexDefinition));
        }
    }
    
    return indexes;
}

function getTableIndex(indexDefinitions: IndexDefinition[]) {
    for (const tableDefinition of indexDefinitions) {
        if (tableDefinition.type === 'TableIndex') {
            return tableDefinition;
        }
    }
    
    throw new Error('No table index found');
}

function getTableName(entity: NonSpecificEntity) {
    const tableName = entity.getTableName();
    if (!tableName) {
        throw new Error('No table name found');
    }
    
    return tableName;
}

interface AwsCdkElectroDBTableStackProps extends StackProps {
    entity: NonSpecificEntity; 
}

export class AwsCdkElectroDBTableStack extends Stack {
    constructor(scope: Construct, id: string, props: AwsCdkElectroDBTableStackProps) {
        super(scope, id, props);
        const { entity } = props;
        const tableName = getTableName(entity);
        const indexDefinitions = getIndexes(entity);
        const tableIndex = getTableIndex(indexDefinitions);
        
        const table = new dynamodb.Table(this, tableName, {
            partitionKey: tableIndex.partitionKey,
            billingMode: dynamodb.BillingMode.PAY_PER_REQUEST,
            removalPolicy: RemovalPolicy.DESTROY,
            sortKey: tableIndex.sortKey,
            pointInTimeRecovery: true,
            tableClass: dynamodb.TableClass.STANDARD,
            
        });
        
        for (let indexDefinition of indexDefinitions) {
            if (indexDefinition.type === 'TableIndex') {
                continue;
            }
            
            if (indexDefinition.type === 'SecondaryIndex') {
                table.addGlobalSecondaryIndex({
                    indexName: indexDefinition.indexName,
                    partitionKey: indexDefinition.partitionKey,
                    sortKey: indexDefinition.sortKey,
                    projectionType: indexDefinition.projectionType,
                });
            } else if (indexDefinition.type === 'LocalSecondaryIndex') {
                table.addLocalSecondaryIndex({
                    indexName: indexDefinition.indexName,
                    sortKey: indexDefinition.sortKey,
                    projectionType: indexDefinition.projectionType,
                });
            }
        }
    }
}

[benjstephenson]

Looks interesting, I'll have a go. Thanks :)

[dmeehan1968]

I extended this to abstract the props generation from the actual stack creation, and also to cope with multiple entities within a service using shared or individual table names (including no table names defined at entity or service level).

This can be added to the above code, with the exception of replacing the definitions of AwsCdkElectroDBTableStack and AwsCdkElectroDBTableStackProps which now takes a service rather than an individual entity:

interface DynamoTableProps {
  tableName?: string
  tableProps: TableProps
  gsiArray: GlobalSecondaryIndexProps[]
  lsiArray: LocalSecondaryIndexProps[]
}

export type NonSpecificService = Service<Record<string, NonSpecificEntity>>

export function dynamoTablePropsFromService(service: NonSpecificService, overrideProps?: Partial<TableProps>): DynamoTableProps[] {

  const tableMap = new Map<string | undefined, DynamoTableProps>()

  for (const entity of Object.values(service.entities)) {
    const props = dynamoTablePropsFromEntity(entity, overrideProps)
    tableMap.set(props.tableName, deepmerge(tableMap.get(props.tableName) ?? {}, props))
  }

  function deduplicate<T extends Array<any>>(array: T) {
    return Array.from(new Set(array.map(el => JSON.stringify(el)))).map(el => JSON.parse(el))
  }

  return Array.from(tableMap.values()).map(props => {
    return {
      ...props,
      gsiArray: deduplicate(props.gsiArray),
      lsiArray: deduplicate(props.lsiArray),
    }
  })
}

export function dynamoTablePropsFromEntity(entity: NonSpecificEntity, overrideProps?: Partial<TableProps>): DynamoTableProps {
  let tableName = entity.getTableName()

  const indexDefinitions = getIndexes(entity);
  const tableIndex = getTableIndex(indexDefinitions);

  const tableProps: TableProps = deepmerge({
    partitionKey: tableIndex.partitionKey,
    billingMode: dynamodb.BillingMode.PAY_PER_REQUEST,
    removalPolicy: RemovalPolicy.DESTROY,
    sortKey: tableIndex.sortKey,
    pointInTimeRecovery: true,
    tableClass: dynamodb.TableClass.STANDARD,
  }, overrideProps ?? {})

  const gsiArray: GlobalSecondaryIndexProps[] = []
  const lsiArray: LocalSecondaryIndexProps[] = []

  for (let indexDefinition of indexDefinitions) {
    if (indexDefinition.type === 'TableIndex') {
      continue;
    }

    if (indexDefinition.type === 'SecondaryIndex') {
      gsiArray.push({
        indexName: indexDefinition.indexName,
        partitionKey: indexDefinition.partitionKey,
        sortKey: indexDefinition.sortKey,
        projectionType: indexDefinition.projectionType,
      })
    } else if (indexDefinition.type === 'LocalSecondaryIndex') {
      lsiArray.push({
        indexName: indexDefinition.indexName,
        sortKey: indexDefinition.sortKey,
        projectionType: indexDefinition.projectionType,
      })
    }
  }

  return {
    tableName,
    tableProps,
    gsiArray,
    lsiArray,
  }
}

interface AwsCdkElectroDBTableStackProps extends StackProps {
  service: NonSpecificService
}

export class AwsCdkElectroDBTableStack extends Stack {
  constructor(scope: Construct, id: string, { service, ...props }: AwsCdkElectroDBTableStackProps) {
    super(scope, id, props);

    dynamoTablePropsFromService(service).forEach(dynamoProps => {
      const { tableName, tableProps, gsiArray, lsiArray } = dynamoProps

      if (!tableName) {
        throw new Error('No table name found');
      }

      const table = new dynamodb.Table(this, tableName, tableProps);

      for (const gsi of gsiArray) {
        table.addGlobalSecondaryIndex(gsi);
      }

      for (const lsi of lsiArray) {
        table.addLocalSecondaryIndex(lsi);
      }
    })
  }
}

The obvious issue with this approach was deduplicating any secondary indexes produced by each entity. I've not yet tried this in earnest but it seemed to help solve a problem in ensuring that the table props were derived from the Service/Entity schema.