diff --git a/backend/src/alembic/versions/13cc78f77731_add_entity_and_span_text_entity_link.py b/backend/src/alembic/versions/13cc78f77731_add_entity_and_span_text_entity_link.py new file mode 100644 index 000000000..be96c1946 --- /dev/null +++ b/backend/src/alembic/versions/13cc78f77731_add_entity_and_span_text_entity_link.py @@ -0,0 +1,112 @@ +"""add entity and span text entity link + +Revision ID: 13cc78f77731 +Revises: 2b91203d1bb6 +Create Date: 2024-06-27 16:05:14.589423 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "13cc78f77731" +down_revision: Union[str, None] = "2b91203d1bb6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "entity", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column( + "created", sa.DateTime(), server_default=sa.text("now()"), nullable=True + ), + sa.Column( + "updated", sa.DateTime(), server_default=sa.text("now()"), nullable=True + ), + sa.Column("is_human", sa.Boolean(), nullable=False), + sa.Column("knowledge_base_id", sa.String(), nullable=False), + sa.Column("project_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_entity_created"), "entity", ["created"], unique=False) + op.create_index(op.f("ix_entity_id"), "entity", ["id"], unique=False) + op.create_index(op.f("ix_entity_is_human"), "entity", ["is_human"], unique=False) + op.create_index( + op.f("ix_entity_knowledge_base_id"), + "entity", + ["knowledge_base_id"], + unique=False, + ) + op.create_index(op.f("ix_entity_name"), "entity", ["name"], unique=False) + op.create_index( + op.f("ix_entity_project_id"), "entity", ["project_id"], unique=False + ) + op.create_table( + "spantextentitylink", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("linked_entity_id", sa.Integer(), nullable=True), + sa.Column("linked_span_text_id", sa.Integer(), nullable=True), + sa.Column("is_human", sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint( + ["linked_entity_id"], ["entity.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["linked_span_text_id"], ["spantext.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_spantextentitylink_id"), "spantextentitylink", ["id"], unique=False + ) + op.create_index( + op.f("ix_spantextentitylink_is_human"), + "spantextentitylink", + ["is_human"], + unique=False, + ) + op.create_index( + op.f("ix_spantextentitylink_linked_entity_id"), + "spantextentitylink", + ["linked_entity_id"], + unique=False, + ) + op.create_index( + op.f("ix_spantextentitylink_linked_span_text_id"), + "spantextentitylink", + ["linked_span_text_id"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + op.f("ix_spantextentitylink_linked_span_text_id"), + table_name="spantextentitylink", + ) + op.drop_index( + op.f("ix_spantextentitylink_linked_entity_id"), table_name="spantextentitylink" + ) + op.drop_index( + op.f("ix_spantextentitylink_is_human"), table_name="spantextentitylink" + ) + op.drop_index(op.f("ix_spantextentitylink_id"), table_name="spantextentitylink") + op.drop_table("spantextentitylink") + op.drop_index(op.f("ix_entity_project_id"), table_name="entity") + op.drop_index(op.f("ix_entity_name"), table_name="entity") + op.drop_index(op.f("ix_entity_knowledge_base_id"), table_name="entity") + op.drop_index(op.f("ix_entity_is_human"), table_name="entity") + op.drop_index(op.f("ix_entity_id"), table_name="entity") + op.drop_index(op.f("ix_entity_created"), table_name="entity") + op.drop_table("entity") + # ### end Alembic commands ### diff --git a/backend/src/api/endpoints/entity.py b/backend/src/api/endpoints/entity.py new file mode 100644 index 000000000..c452a0e8d --- /dev/null +++ b/backend/src/api/endpoints/entity.py @@ -0,0 +1,71 @@ +from typing import List + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from api.dependencies import get_current_user, get_db_session +from app.core.authorization.authz_user import AuthzUser +from app.core.data.crud import Crud +from app.core.data.crud.entity import crud_entity +from app.core.data.dto.entity import ( + EntityMerge, + EntityRead, + EntityRelease, + EntityUpdate, +) + +router = APIRouter( + prefix="/entity", dependencies=[Depends(get_current_user)], tags=["entity"] +) + + +@router.patch( + "/{entity_id}", + response_model=EntityRead, + summary="Updates the Entity with the given ID.", +) +def update_by_id( + *, + db: Session = Depends(get_db_session), + entity_id: int, + entity: EntityUpdate, + authz_user: AuthzUser = Depends(), +) -> EntityRead: + authz_user.assert_in_same_project_as(Crud.ENTITY, entity_id) + entity.is_human = True + db_obj = crud_entity.update(db=db, id=entity_id, update_dto=entity) + return EntityRead.model_validate(db_obj) + + +# add merge endpoint +@router.put( + "/merge", + response_model=EntityRead, + summary="Merges entities and/or span texts with given IDs.", +) +def merge_entities( + *, + db: Session = Depends(get_db_session), + entity_merge: EntityMerge, + authz_user: AuthzUser = Depends(), +) -> EntityRead: + authz_user.assert_in_project(entity_merge.project_id) + db_obj = crud_entity.merge(db, entity_merge=entity_merge) + return EntityRead.model_validate(db_obj) + + +# add resolve endpoint +@router.put( + "/release", + response_model=List[EntityRead], + summary="Releases entities and/or span texts with given IDs.", +) +def release_entities( + *, + db: Session = Depends(get_db_session), + entity_release: EntityRelease, + authz_user: AuthzUser = Depends(), +) -> List[EntityRead]: + authz_user.assert_in_project(entity_release.project_id) + db_objs = crud_entity.release(db=db, entity_release=entity_release) + return [EntityRead.model_validate(db_obj) for db_obj in db_objs] diff --git a/backend/src/api/endpoints/project.py b/backend/src/api/endpoints/project.py index 7ca56577d..7619acb6a 100644 --- a/backend/src/api/endpoints/project.py +++ b/backend/src/api/endpoints/project.py @@ -15,6 +15,7 @@ from app.core.data.crud.code import crud_code from app.core.data.crud.crud_base import NoSuchElementError from app.core.data.crud.document_tag import crud_document_tag +from app.core.data.crud.entity import crud_entity from app.core.data.crud.memo import crud_memo from app.core.data.crud.project import crud_project from app.core.data.crud.project_metadata import crud_project_meta @@ -22,6 +23,7 @@ from app.core.data.dto.action import ActionQueryParameters, ActionRead from app.core.data.dto.code import CodeRead from app.core.data.dto.document_tag import DocumentTagRead +from app.core.data.dto.entity import EntityRead from app.core.data.dto.memo import AttachedObjectType, MemoCreate, MemoInDB, MemoRead from app.core.data.dto.preprocessing_job import PreprocessingJobRead from app.core.data.dto.project import ProjectCreate, ProjectRead, ProjectUpdate @@ -530,3 +532,22 @@ def find_duplicate_text_sdocs( return DuplicateFinderService().find_duplicate_text_sdocs( project_id=proj_id, max_different_words=max_different_words ) + + +@router.get( + "/{proj_id}/entity", + response_model=List[EntityRead], + summary="Returns all Entities of the Project with the given ID", +) +def get_project_entities( + *, + proj_id: int, + db: Session = Depends(get_db_session), + authz_user: AuthzUser = Depends(), +) -> List[EntityRead]: + authz_user.assert_in_project(proj_id) + + result = crud_entity.read_by_project(db=db, proj_id=proj_id) + result = [EntityRead.model_validate(entity) for entity in result] + result.sort(key=lambda c: c.id) + return result diff --git a/backend/src/app/core/data/crud/__init__.py b/backend/src/app/core/data/crud/__init__.py index 81c175185..947d8d8fb 100644 --- a/backend/src/app/core/data/crud/__init__.py +++ b/backend/src/app/core/data/crud/__init__.py @@ -8,6 +8,7 @@ from app.core.data.crud.concept_over_time_analysis import crud_cota from app.core.data.crud.current_code import crud_current_code from app.core.data.crud.document_tag import crud_document_tag +from app.core.data.crud.entity import crud_entity from app.core.data.crud.memo import crud_memo from app.core.data.crud.object_handle import crud_object_handle from app.core.data.crud.preprocessing_job import crud_prepro_job @@ -21,6 +22,7 @@ from app.core.data.crud.span_annotation import crud_span_anno from app.core.data.crud.span_group import crud_span_group from app.core.data.crud.span_text import crud_span_text +from app.core.data.crud.span_text_entity_link import crud_span_text_entity_link from app.core.data.crud.timeline_analysis import crud_timeline_analysis from app.core.data.crud.user import crud_user from app.core.data.crud.whiteboard import crud_whiteboard @@ -51,3 +53,5 @@ class Crud(Enum): COTA_ANALYSIS = crud_cota USER = crud_user WHITEBOARD = crud_whiteboard + ENTITY = crud_entity + SPAN_TEXT_ENTITY_LINK = crud_span_text_entity_link diff --git a/backend/src/app/core/data/crud/entity.py b/backend/src/app/core/data/crud/entity.py new file mode 100644 index 000000000..02b796b59 --- /dev/null +++ b/backend/src/app/core/data/crud/entity.py @@ -0,0 +1,144 @@ +from typing import List + +from fastapi.encoders import jsonable_encoder +from sqlalchemy.orm import Session + +from app.core.data.crud.crud_base import CRUDBase +from app.core.data.crud.span_text import crud_span_text +from app.core.data.crud.span_text_entity_link import crud_span_text_entity_link +from app.core.data.dto.entity import ( + EntityCreate, + EntityMerge, + EntityRelease, + EntityUpdate, +) +from app.core.data.dto.span_text_entity_link import ( + SpanTextEntityLinkCreate, +) +from app.core.data.orm.entity import EntityORM +from app.core.data.orm.span_text_entity_link import SpanTextEntityLinkORM + + +class CRUDEntity(CRUDBase[EntityORM, EntityCreate, EntityUpdate]): + def create( + self, db: Session, *, create_dto: EntityCreate, force: bool = True + ) -> EntityORM: + result = self.create_multi(db=db, create_dtos=[create_dto], force=force) + return result[0] if len(result) > 0 else None + + def create_multi( + self, db: Session, *, create_dtos: List[EntityCreate], force: bool = True + ) -> List[EntityORM]: + if len(create_dtos) == 0: + return [] + + # assumption all entities belong to the same project + project_id = create_dtos[0].project_id + + # duplicate assignments to the same span text are filtered out here + span_text_dict = {} + for i, create_dto in enumerate(create_dtos): + for span_text_id in create_dto.span_text_ids: + span_text_dict[span_text_id] = i + + ids = list(span_text_dict.keys()) + existing_links = crud_span_text_entity_link.read_multi_span_text_and_project_id( + db=db, span_text_ids=ids, project_id=project_id + ) + existing_link_ids = [link.linked_span_text_id for link in existing_links] + old_entities = [link.linked_entity_id for link in existing_links] + + if not force: + # if a span text is already assigned it should not be reassigned + for id in existing_link_ids: + del span_text_dict[id] + + # recompute create dtos + indexes_to_use = list(set(span_text_dict.values())) + reversed_span_text_dict = {} + for key, value in span_text_dict.items(): + if value not in reversed_span_text_dict: + reversed_span_text_dict[value] = [] + reversed_span_text_dict[value].append(key) + + def map_index_to_new_dto(index): + create_dto = create_dtos[index] + create_dto.span_text_ids = reversed_span_text_dict[index] + return create_dto + + create_dtos = list(map(map_index_to_new_dto, indexes_to_use)) + + # create entity db_objs + def create_db_obj(create_dto): + data = jsonable_encoder(create_dto, exclude={"span_text_ids"}) + return self.model(**data) + + db_objs = list(map(create_db_obj, create_dtos)) + db.add_all(db_objs) + db.flush() + db.commit() + + links = [] + for db_obj, create_dto in zip(db_objs, create_dtos): + for span_text_id in create_dto.span_text_ids: + links.append( + SpanTextEntityLinkCreate( + linked_entity_id=db_obj.id, linked_span_text_id=span_text_id + ) + ) + crud_span_text_entity_link.create_multi(db=db, create_dtos=links) + db.commit() + if force: + self.__remove_unused_entites(db=db, ids=list(set(old_entities))) + return db_objs + + def read_by_project(self, db: Session, proj_id: int) -> List[EntityORM]: + return db.query(self.model).filter(self.model.project_id == proj_id).all() + + def __remove_multi(self, db: Session, *, ids: List[int]) -> List[EntityORM]: + removed = db.query(EntityORM).filter(EntityORM.id.in_(ids)).all() + db.query(EntityORM).filter(EntityORM.id.in_(ids)).delete( + synchronize_session=False + ) + db.commit() + return removed + + def remove(self, db: Session, *, id: int) -> EntityORM: + pass + + def __remove_unused_entites(self, db: Session, ids: List[int]) -> List[EntityORM]: + linked_ids_result = ( + db.query(SpanTextEntityLinkORM.linked_entity_id) + .filter(SpanTextEntityLinkORM.linked_entity_id.in_(ids)) + .distinct() + .all() + ) + linked_ids = {item[0] for item in linked_ids_result} + ids = list(set(ids) - set(linked_ids)) + return self.__remove_multi(db=db, ids=ids) + + def merge(self, db: Session, entity_merge: EntityMerge) -> EntityORM: + new_entity = EntityCreate( + name=entity_merge.name, + project_id=entity_merge.project_id, + span_text_ids=entity_merge.spantext_ids, + is_human=True, + knowledge_base_id=entity_merge.knowledge_base_id, + ) + return self.create(db=db, create_dto=new_entity, force=True) + + def release(self, db: Session, entity_release: EntityRelease) -> List[EntityORM]: + new_entities = [] + for span_text_id in entity_release.spantext_ids: + span_text = crud_span_text.read(db=db, id=span_text_id) + new_entity = EntityCreate( + name=span_text.text, + project_id=entity_release.project_id, + span_text_ids=[span_text_id], + ) + new_entities.append(new_entity) + db_objs = self.create_multi(db=db, create_dtos=new_entities, force=True) + return db_objs + + +crud_entity = CRUDEntity(EntityORM) diff --git a/backend/src/app/core/data/crud/span_annotation.py b/backend/src/app/core/data/crud/span_annotation.py index 4847fca83..bb44cfb88 100644 --- a/backend/src/app/core/data/crud/span_annotation.py +++ b/backend/src/app/core/data/crud/span_annotation.py @@ -5,11 +5,14 @@ from sqlalchemy.orm import Session from app.core.data.crud.annotation_document import crud_adoc +from app.core.data.crud.code import crud_code from app.core.data.crud.crud_base import CRUDBase +from app.core.data.crud.entity import crud_entity from app.core.data.crud.span_group import crud_span_group from app.core.data.crud.span_text import crud_span_text from app.core.data.dto.action import ActionType from app.core.data.dto.code import CodeRead +from app.core.data.dto.entity import EntityCreate from app.core.data.dto.span_annotation import ( SpanAnnotationCreate, SpanAnnotationCreateWithCodeId, @@ -35,6 +38,20 @@ def create( db=db, create_dto=SpanTextCreate(text=create_dto.span_text) ) + # create the entity + code = ( + db.query(CodeORM).filter(CodeORM.id == create_dto.current_code_id).first() + ) + project_id = code.project_id + crud_entity.create( + db=db, + create_dto=EntityCreate( + name=create_dto.span_text, + project_id=project_id, + span_text_ids=[span_text_orm.id], + ), + ) + # create the SpanAnnotation (and link the SpanText via FK) dto_obj_data = jsonable_encoder(create_dto.model_dump(exclude={"span_text"})) # noinspection PyArgumentList @@ -90,6 +107,22 @@ def create_multi( ], ) + # create the entities + code = crud_code.read(db=db, id=create_dtos[0].current_code_id) + project_id = code.project_id + crud_entity.create_multi( + db=db, + create_dtos=[ + EntityCreate( + project_id=project_id, + name=dto.span_text, + span_text_ids=[id.id], + is_human=False, + ) + for id, dto in zip(span_texts_orm, create_dtos) + ], + ) + # create the SpanAnnotation (and link the SpanText via FK) dto_objs_data = [ jsonable_encoder(create_dto.model_dump(exclude={"span_text"})) diff --git a/backend/src/app/core/data/crud/span_text.py b/backend/src/app/core/data/crud/span_text.py index 424eced02..7da30af71 100644 --- a/backend/src/app/core/data/crud/span_text.py +++ b/backend/src/app/core/data/crud/span_text.py @@ -22,31 +22,31 @@ def create(self, db: Session, *, create_dto: SpanTextCreate) -> SpanTextORM: def create_multi( self, db: Session, *, create_dtos: List[SpanTextCreate] ) -> List[SpanTextORM]: - # Only create when not already present span_texts: List[SpanTextORM] = [] to_create: List[SpanTextCreate] = [] - to_create_idx: List[int] = [] - - # TODO best would be "insert all (ignore existing) followed by get all" - for i, create_dto in enumerate(create_dtos): - db_obj = self.read_by_text(db=db, text=create_dto.text) - span_texts.append(db_obj) - if db_obj is None: - to_create.append(create_dto) - to_create_idx.append(i) - if len(to_create) > 0: - created = super().create_multi(db=db, create_dtos=to_create) - for i, obj in zip(to_create_idx, created): - span_texts[i] = obj - # Ignore types: We've made sure that no `None` values remain since we've created - # span texts to replace them - return span_texts # type: ignore + + # every span text needs to be created at most once + span_text_dict = {} + for create_dto in create_dtos: + span_text_dict[create_dto.text] = create_dto + + # only create span texts when not already present + to_create = filter( + lambda x: self.read_by_text(db=db, text=x.text) is None, + span_text_dict.values(), + ) + super().create_multi(db=db, create_dtos=to_create) + + span_texts = list( + map(lambda x: self.read_by_text(db=db, text=x.text), create_dtos) + ) + return span_texts def read_by_text(self, db: Session, *, text: str) -> Optional[SpanTextORM]: - return db.query(self.model.id).filter(self.model.text == text).first() + return db.query(self.model).filter(self.model.text == text).first() def read_all_by_text(self, db: Session, *, texts: List[str]) -> List[SpanTextORM]: - return db.query(self.model.id).filter(self.model.text in texts) + return db.query(self.model).filter(self.model.text in texts) crud_span_text = CRUDSpanText(SpanTextORM) diff --git a/backend/src/app/core/data/crud/span_text_entity_link.py b/backend/src/app/core/data/crud/span_text_entity_link.py new file mode 100644 index 000000000..f280b96cb --- /dev/null +++ b/backend/src/app/core/data/crud/span_text_entity_link.py @@ -0,0 +1,123 @@ +from typing import List + +from fastapi.encoders import jsonable_encoder +from sqlalchemy.orm import Session + +from app.core.data.crud.crud_base import CRUDBase +from app.core.data.dto.span_text_entity_link import ( + SpanTextEntityLinkCreate, + SpanTextEntityLinkUpdate, +) +from app.core.data.orm.entity import EntityORM +from app.core.data.orm.span_text_entity_link import SpanTextEntityLinkORM + +# we need: +# create +# update +# delete +# read: +# by id, we can define the rest as it is needed + + +class CRUDSpanTextEntityLink( + CRUDBase[SpanTextEntityLinkORM, SpanTextEntityLinkCreate, SpanTextEntityLinkCreate] +): + def create( + self, db: Session, *, create_dto: SpanTextEntityLinkCreate + ) -> SpanTextEntityLinkORM: + dto_obj_data = jsonable_encoder(create_dto) + db_obj = self.model(**dto_obj_data) + db.add(db_obj) + db.commit() + return db_obj + + def create_multi( + self, db: Session, *, create_dtos: List[SpanTextEntityLinkCreate] + ) -> List[SpanTextEntityLinkORM]: + if len(create_dtos) == 0: + return [] + # One assumption is that all entities have the same project_id + project_id = ( + db.query(EntityORM) + .filter(EntityORM.id == create_dtos[0].linked_entity_id) + .first() + .project_id + ) + all_ids = [link.linked_span_text_id for link in create_dtos] + existing_links = self.read_multi_span_text_and_project_id( + db=db, span_text_ids=all_ids, project_id=project_id + ) + existing_ids = [d.linked_span_text_id for d in existing_links] + to_create = { + dto.linked_span_text_id: dto + for dto in create_dtos + if dto.linked_span_text_id not in existing_ids + } + to_create = to_create.values() + + if len(to_create) > 0: + db_objs = [self.model(**jsonable_encoder(dto)) for dto in to_create] + db.bulk_save_objects(db_objs) + db.commit() + if len(existing_links) > 0: + existing_links_map = { + link.linked_span_text_id: link for link in existing_links + } + for create_dto in create_dtos: + if create_dto.linked_span_text_id in existing_links_map: + existing_links_map[ + create_dto.linked_span_text_id + ].linked_entity_id = create_dto.linked_entity_id + ids = [dto.id for dto in existing_links] + update_dtos = [ + SpanTextEntityLinkUpdate( + linked_entity_id=dto.linked_entity_id, + linked_span_text_id=dto.linked_span_text_id, + ) + for dto in existing_links + ] + self.update_multi(db, ids=ids, update_dtos=update_dtos) + + def update( + self, db: Session, *, id: int, update_dto: SpanTextEntityLinkUpdate + ) -> SpanTextEntityLinkORM: + return super().update(db, id=id, update_dto=update_dto) + + def update_multi( + self, + db: Session, + *, + ids: List[int], + update_dtos: List[SpanTextEntityLinkUpdate], + ) -> SpanTextEntityLinkORM: + if len(ids) != len(update_dtos): + raise ValueError("The number of IDs must match the number of update DTOs") + + update_mappings = [] + for id, dto in zip(ids, update_dtos): + dto_data = jsonable_encoder(dto) + dto_data["id"] = id + update_mappings.append(dto_data) + + db.bulk_update_mappings(self.model, update_mappings) + db.commit() + + updated_records = db.query(self.model).filter(self.model.id.in_(ids)).all() + return updated_records + + def read_multi_span_text_and_project_id( + self, db: Session, *, span_text_ids: List[int], project_id: int + ) -> List[SpanTextEntityLinkORM]: + query = ( + db.query(SpanTextEntityLinkORM) + .join(EntityORM, SpanTextEntityLinkORM.linked_entity_id == EntityORM.id) + .filter( + SpanTextEntityLinkORM.linked_span_text_id.in_(span_text_ids), + EntityORM.project_id == project_id, + ) + .distinct() + ) + return query.all() + + +crud_span_text_entity_link = CRUDSpanTextEntityLink(SpanTextEntityLinkORM) diff --git a/backend/src/app/core/data/dto/entity.py b/backend/src/app/core/data/dto/entity.py new file mode 100644 index 000000000..68b2a55de --- /dev/null +++ b/backend/src/app/core/data/dto/entity.py @@ -0,0 +1,63 @@ +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from app.core.data.dto.span_text import SpanTextRead + +from .dto_base import UpdateDTOBase + + +# Properties shared across all DTOs +class EntityBaseDTO(BaseModel): + is_human: Optional[bool] = Field( + default=False, description="Whether the link was created by a human" + ) + knowledge_base_id: Optional[str] = Field(default="", description="Link to wikidata") + + +# Properties for creation +class EntityCreate(EntityBaseDTO): + name: str = Field(description="Name of the Entity") + project_id: int = Field(description="Project the Entity belongs to") + span_text_ids: List[int] = Field( + description="Span Text Ids which belong to this Entity" + ) + + +# Properties for updating +class EntityUpdate(EntityBaseDTO, UpdateDTOBase): + name: str = Field(description="Name of the Entity") + span_text_ids: List[int] = Field( + description="Span Text Ids which belong to this Entity" + ) + pass + + +# Properties for merging entities/span texts +# TODO entity ids löschen und im frontend nur span_text ids weitergeben +class EntityMerge(EntityBaseDTO): + name: str = Field(description="Name of the Entity") + knowledge_base_id: Optional[str] = Field("", description="Link to wikidata") + project_id: int = Field(description="Id of the current Project") + spantext_ids: List[int] = Field(description="List of Span Text IDs to merge") + + +# Properties for releasing entities/span texts +# TODO entity ids löschen und im frontend nur span_text ids weitergeben +class EntityRelease(EntityBaseDTO): + project_id: int = Field(description="Id of the current Project") + spantext_ids: List[int] = Field(description="List of Span Text IDs to release") + + +# Properties for reading (as in ORM) +class EntityRead(EntityBaseDTO): + id: int = Field(description="ID of the Entity") + name: str = Field(description="Name of the Entity") + project_id: int = Field(description="Project the Entity belongs to") + created: datetime = Field(description="Created timestamp of the Entity") + updated: datetime = Field(description="Updated timestamp of the Entity") + span_texts: List[SpanTextRead] = Field( + default=[], description="The SpanTexts belonging to this entity" + ) + model_config = ConfigDict(from_attributes=True) # TODO ask tim what this does diff --git a/backend/src/app/core/data/dto/span_text_entity_link.py b/backend/src/app/core/data/dto/span_text_entity_link.py new file mode 100644 index 000000000..8325900b7 --- /dev/null +++ b/backend/src/app/core/data/dto/span_text_entity_link.py @@ -0,0 +1,35 @@ +from typing import Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class SpanTextEntityLinkCreate(BaseModel): + linked_entity_id: Optional[int] = Field(description="ID of the linked Entity.") + linked_span_text_id: Optional[int] = Field( + description="ID of the linked span text." + ) + is_human: Optional[bool] = Field( + False, description="Whether the link was created by a human" + ) + + +class SpanTextEntityLinkUpdate(BaseModel): + linked_entity_id: Optional[int] = Field(description="ID of the linked Entity.") + linked_span_text_id: Optional[int] = Field( + description="ID of the linked span text." + ) + is_human: Optional[bool] = Field( + False, description="Whether the link was created by a human" + ) + + +class SpanTextEntityLinkRead(BaseModel): + id: int = Field(description="ID of the SpanTextEntityLink") + linked_entity_id: Optional[int] = Field(description="ID of the linked Entity.") + linked_span_text_id: Optional[int] = Field( + description="ID of the linked span text." + ) + model_config = ConfigDict(from_attributes=True) + is_human: Optional[bool] = Field( + False, description="Whether the link was created by a human" + ) diff --git a/backend/src/app/core/data/orm/entity.py b/backend/src/app/core/data/orm/entity.py new file mode 100644 index 000000000..9bea18ad0 --- /dev/null +++ b/backend/src/app/core/data/orm/entity.py @@ -0,0 +1,50 @@ +from datetime import datetime +from typing import TYPE_CHECKING, List, Optional + +from sqlalchemy import ( + Boolean, + DateTime, + ForeignKey, + Integer, + String, + func, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.core.data.orm.orm_base import ORMBase + +if TYPE_CHECKING: + from app.core.data.orm.span_text import SpanTextORM + + +class EntityORM(ORMBase): + id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) + name: Mapped[str] = mapped_column(String, nullable=False, index=True) + created: Mapped[Optional[datetime]] = mapped_column( + DateTime, server_default=func.now(), index=True + ) + updated: Mapped[Optional[datetime]] = mapped_column( + DateTime, server_default=func.now(), onupdate=func.current_timestamp() + ) + is_human: Mapped[Boolean] = mapped_column(Boolean, default=False, index=True) + knowledge_base_id: Mapped[str] = mapped_column(String, default="", index=True) + + project_id: Mapped[int] = mapped_column( + Integer, + ForeignKey("project.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + + span_texts: Mapped[List["SpanTextORM"]] = relationship( + "SpanTextORM", secondary="spantextentitylink" + ) + + +# __table_args__ = ( +# UniqueConstraint( +# "project_id", +# "name", +# name="UC_name_unique_per_project", +# ), +# ) diff --git a/backend/src/app/core/data/orm/span_text_entity_link.py b/backend/src/app/core/data/orm/span_text_entity_link.py new file mode 100644 index 000000000..086f0d9f6 --- /dev/null +++ b/backend/src/app/core/data/orm/span_text_entity_link.py @@ -0,0 +1,20 @@ +from typing import TYPE_CHECKING, Optional + +from sqlalchemy import Boolean, ForeignKey, Integer +from sqlalchemy.orm import Mapped, mapped_column + +from app.core.data.orm.orm_base import ORMBase + +if TYPE_CHECKING: + pass + + +class SpanTextEntityLinkORM(ORMBase): + id = mapped_column(Integer, primary_key=True, index=True) + linked_entity_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("entity.id", ondelete="CASCADE"), index=True + ) + linked_span_text_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("spantext.id", ondelete="CASCADE"), index=True + ) + is_human: Mapped[Boolean] = mapped_column(Boolean, default=False, index=True) diff --git a/backend/src/app/core/db/import_all_orms.py b/backend/src/app/core/db/import_all_orms.py index 6c297c54c..6445030b9 100644 --- a/backend/src/app/core/db/import_all_orms.py +++ b/backend/src/app/core/db/import_all_orms.py @@ -9,6 +9,7 @@ from app.core.data.orm.code import CodeORM from app.core.data.orm.concept_over_time_analysis import ConceptOverTimeAnalysisORM from app.core.data.orm.document_tag import DocumentTagORM +from app.core.data.orm.entity import EntityORM from app.core.data.orm.memo import MemoORM from app.core.data.orm.object_handle import ObjectHandleORM from app.core.data.orm.orm_base import ORMBase @@ -24,6 +25,7 @@ from app.core.data.orm.span_annotation import SpanAnnotationORM from app.core.data.orm.span_group import SpanGroupORM from app.core.data.orm.span_text import SpanTextORM +from app.core.data.orm.span_text_entity_link import SpanTextEntityLinkORM from app.core.data.orm.user import UserORM from app.core.data.orm.version import VersionORM from app.core.data.orm.whiteboard import WhiteboardORM diff --git a/backend/src/main.py b/backend/src/main.py index 2e1acd41b..28eeb537c 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -38,6 +38,7 @@ concept_over_time_analysis, crawler, document_tag, + entity, export, feedback, general, @@ -262,6 +263,7 @@ def invalid_error_handler(_, exc: InvalidError): app.include_router(span_group.router) app.include_router(bbox_annotation.router) app.include_router(code.router) +app.include_router(entity.router) app.include_router(memo.router) app.include_router(search.router) app.include_router(source_document_metadata.router) diff --git a/frontend/src/api/EntityHooks.ts b/frontend/src/api/EntityHooks.ts new file mode 100644 index 000000000..0ff97a23e --- /dev/null +++ b/frontend/src/api/EntityHooks.ts @@ -0,0 +1,39 @@ +import { useMutation } from "@tanstack/react-query"; +import queryClient from "../plugins/ReactQueryClient.ts"; +import { QueryKey } from "./QueryKey.ts"; +import { EntityService } from "./openapi/services/EntityService.ts"; + +const useUpdateEntity = () => + useMutation({ + mutationFn: EntityService.updateById, + onSuccess: (data) => { + queryClient.invalidateQueries({ queryKey: [QueryKey.ENTITY, data.id] }); + queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, data.project_id] }); + }, + }); + +const useMerge = () => + useMutation({ + mutationFn: EntityService.mergeEntities, + onSuccess: (data) => { + queryClient.invalidateQueries({ queryKey: [QueryKey.ENTITY, data.project_id] }); + queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, data.project_id] }); + }, + }); + +const useRelease = () => + useMutation({ + mutationFn: EntityService.releaseEntities, + onSuccess: (data) => { + queryClient.invalidateQueries({ queryKey: [QueryKey.ENTITY, data[0].project_id] }); + queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, data[0].project_id] }); + }, + }); + +const EntityHooks = { + useUpdateEntity, + useMerge, + useRelease, +}; + +export default EntityHooks; diff --git a/frontend/src/api/ProjectHooks.ts b/frontend/src/api/ProjectHooks.ts index 22723a896..8b3fd7e04 100644 --- a/frontend/src/api/ProjectHooks.ts +++ b/frontend/src/api/ProjectHooks.ts @@ -7,6 +7,7 @@ import { ActionQueryParameters } from "./openapi/models/ActionQueryParameters.ts import { ActionRead } from "./openapi/models/ActionRead.ts"; import { CodeRead } from "./openapi/models/CodeRead.ts"; import { DocumentTagRead } from "./openapi/models/DocumentTagRead.ts"; +import { EntityRead } from "./openapi/models/EntityRead.ts"; import { MemoRead } from "./openapi/models/MemoRead.ts"; import { PreprocessingJobRead } from "./openapi/models/PreprocessingJobRead.ts"; import { ProjectCreate } from "./openapi/models/ProjectCreate.ts"; @@ -146,7 +147,16 @@ const useGetAllCodes = (projectId: number, returnAll: boolean = false) => { select: returnAll ? undefined : selectEnabledCodes, }); }; - +// entities +const useGetAllEntities = (projectId: number) => { + return useQuery({ + queryKey: [QueryKey.PROJECT_ENTITIES, projectId], + queryFn: () => + ProjectService.getProjectEntities({ + projId: projectId, + }), + }); +}; // memo const useGetMemo = (projectId: number | null | undefined, userId: number | null | undefined) => useQuery({ @@ -230,6 +240,8 @@ const ProjectHooks = { useRemoveUser, // codes useGetAllCodes, + // entities, + useGetAllEntities, // memo useGetMemo, useGetAllUserMemos, diff --git a/frontend/src/api/QueryKey.ts b/frontend/src/api/QueryKey.ts index b1c8c6697..572842021 100644 --- a/frontend/src/api/QueryKey.ts +++ b/frontend/src/api/QueryKey.ts @@ -11,6 +11,8 @@ export const QueryKey = { PROJECT_SDOCS_INFINITE: "projectDocumentsInfinite", // all codes of a project (by project id) PROJECT_CODES: "projectCodes", + // all entities of a project (by project id) + PROJECT_ENTITIES: "projectEntities", // all tags of a project (by project id) PROJECT_TAGS: "projectTags", // all crawler jobs of a project (by project id) @@ -108,6 +110,9 @@ export const QueryKey = { // a single code (by code id) CODE: "code", + // a single entity (by entity id) + ENTITY: "entity", + // a single tag (by tag id) TAG: "tag", diff --git a/frontend/src/api/openapi/models/EntityMerge.ts b/frontend/src/api/openapi/models/EntityMerge.ts new file mode 100644 index 000000000..77241240a --- /dev/null +++ b/frontend/src/api/openapi/models/EntityMerge.ts @@ -0,0 +1,26 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type EntityMerge = { + /** + * Whether the link was created by a human + */ + is_human?: boolean | null; + /** + * Link to wikidata + */ + knowledge_base_id?: string | null; + /** + * Name of the Entity + */ + name: string; + /** + * Id of the current Project + */ + project_id: number; + /** + * List of Span Text IDs to merge + */ + spantext_ids: Array; +}; diff --git a/frontend/src/api/openapi/models/EntityRead.ts b/frontend/src/api/openapi/models/EntityRead.ts new file mode 100644 index 000000000..fdc41e328 --- /dev/null +++ b/frontend/src/api/openapi/models/EntityRead.ts @@ -0,0 +1,39 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { SpanTextRead } from "./SpanTextRead"; +export type EntityRead = { + /** + * Whether the link was created by a human + */ + is_human?: boolean | null; + /** + * Link to wikidata + */ + knowledge_base_id?: string | null; + /** + * ID of the Entity + */ + id: number; + /** + * Name of the Entity + */ + name: string; + /** + * Project the Entity belongs to + */ + project_id: number; + /** + * Created timestamp of the Entity + */ + created: string; + /** + * Updated timestamp of the Entity + */ + updated: string; + /** + * The SpanTexts belonging to this entity + */ + span_texts?: Array; +}; diff --git a/frontend/src/api/openapi/models/EntityRelease.ts b/frontend/src/api/openapi/models/EntityRelease.ts new file mode 100644 index 000000000..c9f7ce659 --- /dev/null +++ b/frontend/src/api/openapi/models/EntityRelease.ts @@ -0,0 +1,22 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type EntityRelease = { + /** + * Whether the link was created by a human + */ + is_human?: boolean | null; + /** + * Link to wikidata + */ + knowledge_base_id?: string | null; + /** + * Id of the current Project + */ + project_id: number; + /** + * List of Span Text IDs to release + */ + spantext_ids: Array; +}; diff --git a/frontend/src/api/openapi/models/EntityUpdate.ts b/frontend/src/api/openapi/models/EntityUpdate.ts new file mode 100644 index 000000000..3bc9d4214 --- /dev/null +++ b/frontend/src/api/openapi/models/EntityUpdate.ts @@ -0,0 +1,22 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type EntityUpdate = { + /** + * Whether the link was created by a human + */ + is_human?: boolean | null; + /** + * Link to wikidata + */ + knowledge_base_id?: string | null; + /** + * Name of the Entity + */ + name: string; + /** + * Span Text Ids which belong to this Entity + */ + span_text_ids: Array; +}; diff --git a/frontend/src/api/openapi/models/SpanTextRead.ts b/frontend/src/api/openapi/models/SpanTextRead.ts new file mode 100644 index 000000000..c30d39e4a --- /dev/null +++ b/frontend/src/api/openapi/models/SpanTextRead.ts @@ -0,0 +1,14 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type SpanTextRead = { + /** + * Code of the SpanText + */ + text?: string; + /** + * ID of the SpanText + */ + id: number; +}; diff --git a/frontend/src/api/openapi/services/EntityService.ts b/frontend/src/api/openapi/services/EntityService.ts new file mode 100644 index 000000000..ba0317fa5 --- /dev/null +++ b/frontend/src/api/openapi/services/EntityService.ts @@ -0,0 +1,70 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { EntityMerge } from "../models/EntityMerge"; +import type { EntityRead } from "../models/EntityRead"; +import type { EntityRelease } from "../models/EntityRelease"; +import type { EntityUpdate } from "../models/EntityUpdate"; +import type { CancelablePromise } from "../core/CancelablePromise"; +import { OpenAPI } from "../core/OpenAPI"; +import { request as __request } from "../core/request"; +export class EntityService { + /** + * Updates the Entity with the given ID. + * @returns EntityRead Successful Response + * @throws ApiError + */ + public static updateById({ + entityId, + requestBody, + }: { + entityId: number; + requestBody: EntityUpdate; + }): CancelablePromise { + return __request(OpenAPI, { + method: "PATCH", + url: "/entity/{entity_id}", + path: { + entity_id: entityId, + }, + body: requestBody, + mediaType: "application/json", + errors: { + 422: `Validation Error`, + }, + }); + } + /** + * Merges entities and/or span texts with given IDs. + * @returns EntityRead Successful Response + * @throws ApiError + */ + public static mergeEntities({ requestBody }: { requestBody: EntityMerge }): CancelablePromise { + return __request(OpenAPI, { + method: "PUT", + url: "/entity/merge", + body: requestBody, + mediaType: "application/json", + errors: { + 422: `Validation Error`, + }, + }); + } + /** + * Releases entities and/or span texts with given IDs. + * @returns EntityRead Successful Response + * @throws ApiError + */ + public static releaseEntities({ requestBody }: { requestBody: EntityRelease }): CancelablePromise> { + return __request(OpenAPI, { + method: "PUT", + url: "/entity/release", + body: requestBody, + mediaType: "application/json", + errors: { + 422: `Validation Error`, + }, + }); + } +} diff --git a/frontend/src/api/openapi/services/ProjectService.ts b/frontend/src/api/openapi/services/ProjectService.ts index 43a210b4a..3d07da016 100644 --- a/frontend/src/api/openapi/services/ProjectService.ts +++ b/frontend/src/api/openapi/services/ProjectService.ts @@ -7,6 +7,7 @@ import type { ActionRead } from "../models/ActionRead"; import type { Body_project_upload_project_sdoc } from "../models/Body_project_upload_project_sdoc"; import type { CodeRead } from "../models/CodeRead"; import type { DocumentTagRead } from "../models/DocumentTagRead"; +import type { EntityRead } from "../models/EntityRead"; import type { MemoCreate } from "../models/MemoCreate"; import type { MemoRead } from "../models/MemoRead"; import type { PreprocessingJobRead } from "../models/PreprocessingJobRead"; @@ -525,4 +526,21 @@ export class ProjectService { }, }); } + /** + * Returns all Entities of the Project with the given ID + * @returns EntityRead Successful Response + * @throws ApiError + */ + public static getProjectEntities({ projId }: { projId: number }): CancelablePromise> { + return __request(OpenAPI, { + method: "GET", + url: "/project/{proj_id}/entity", + path: { + proj_id: projId, + }, + errors: { + 422: `Validation Error`, + }, + }); + } } diff --git a/frontend/src/openapi.json b/frontend/src/openapi.json index 79a8de166..20dead3ff 100644 --- a/frontend/src/openapi.json +++ b/frontend/src/openapi.json @@ -951,6 +951,35 @@ } } }, + "/project/{proj_id}/entity": { + "get": { + "tags": ["project"], + "summary": "Returns all Entities of the Project with the given ID", + "operationId": "get_project_entities", + "security": [{ "OAuth2PasswordBearer": [] }], + "parameters": [ + { "name": "proj_id", "in": "path", "required": true, "schema": { "type": "integer", "title": "Proj Id" } } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { "$ref": "#/components/schemas/EntityRead" }, + "title": "Response Project-Get Project Entities" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } + } + } + } + }, "/sdoc/{sdoc_id}": { "get": { "tags": ["sourceDocument"], @@ -3001,6 +3030,83 @@ } } }, + "/entity/{entity_id}": { + "patch": { + "tags": ["entity"], + "summary": "Updates the Entity with the given ID.", + "operationId": "update_by_id", + "security": [{ "OAuth2PasswordBearer": [] }], + "parameters": [ + { "name": "entity_id", "in": "path", "required": true, "schema": { "type": "integer", "title": "Entity Id" } } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EntityUpdate" } } } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EntityRead" } } } + }, + "422": { + "description": "Validation Error", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } + } + } + } + }, + "/entity/merge": { + "put": { + "tags": ["entity"], + "summary": "Merges entities and/or span texts with given IDs.", + "operationId": "merge_entities", + "requestBody": { + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EntityMerge" } } }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EntityRead" } } } + }, + "422": { + "description": "Validation Error", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } + } + }, + "security": [{ "OAuth2PasswordBearer": [] }] + } + }, + "/entity/release": { + "put": { + "tags": ["entity"], + "summary": "Releases entities and/or span texts with given IDs.", + "operationId": "release_entities", + "requestBody": { + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EntityRelease" } } }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { "$ref": "#/components/schemas/EntityRead" }, + "type": "array", + "title": "Response Entity-Release Entities" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } + } + }, + "security": [{ "OAuth2PasswordBearer": [] }] + } + }, "/memo/{memo_id}": { "get": { "tags": ["memo"], @@ -6812,6 +6918,126 @@ "required": ["document_id"], "title": "ElasticSearchDocumentHit" }, + "EntityMerge": { + "properties": { + "is_human": { + "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "title": "Is Human", + "description": "Whether the link was created by a human", + "default": false + }, + "knowledge_base_id": { + "anyOf": [{ "type": "string" }, { "type": "null" }], + "title": "Knowledge Base Id", + "description": "Link to wikidata", + "default": "" + }, + "name": { "type": "string", "title": "Name", "description": "Name of the Entity" }, + "project_id": { "type": "integer", "title": "Project Id", "description": "Id of the current Project" }, + "spantext_ids": { + "items": { "type": "integer" }, + "type": "array", + "title": "Spantext Ids", + "description": "List of Span Text IDs to merge" + } + }, + "type": "object", + "required": ["name", "project_id", "spantext_ids"], + "title": "EntityMerge" + }, + "EntityRead": { + "properties": { + "is_human": { + "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "title": "Is Human", + "description": "Whether the link was created by a human", + "default": false + }, + "knowledge_base_id": { + "anyOf": [{ "type": "string" }, { "type": "null" }], + "title": "Knowledge Base Id", + "description": "Link to wikidata", + "default": "" + }, + "id": { "type": "integer", "title": "Id", "description": "ID of the Entity" }, + "name": { "type": "string", "title": "Name", "description": "Name of the Entity" }, + "project_id": { "type": "integer", "title": "Project Id", "description": "Project the Entity belongs to" }, + "created": { + "type": "string", + "format": "date-time", + "title": "Created", + "description": "Created timestamp of the Entity" + }, + "updated": { + "type": "string", + "format": "date-time", + "title": "Updated", + "description": "Updated timestamp of the Entity" + }, + "span_texts": { + "items": { "$ref": "#/components/schemas/SpanTextRead" }, + "type": "array", + "title": "Span Texts", + "description": "The SpanTexts belonging to this entity", + "default": [] + } + }, + "type": "object", + "required": ["id", "name", "project_id", "created", "updated"], + "title": "EntityRead" + }, + "EntityRelease": { + "properties": { + "is_human": { + "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "title": "Is Human", + "description": "Whether the link was created by a human", + "default": false + }, + "knowledge_base_id": { + "anyOf": [{ "type": "string" }, { "type": "null" }], + "title": "Knowledge Base Id", + "description": "Link to wikidata", + "default": "" + }, + "project_id": { "type": "integer", "title": "Project Id", "description": "Id of the current Project" }, + "spantext_ids": { + "items": { "type": "integer" }, + "type": "array", + "title": "Spantext Ids", + "description": "List of Span Text IDs to release" + } + }, + "type": "object", + "required": ["project_id", "spantext_ids"], + "title": "EntityRelease" + }, + "EntityUpdate": { + "properties": { + "is_human": { + "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "title": "Is Human", + "description": "Whether the link was created by a human", + "default": false + }, + "knowledge_base_id": { + "anyOf": [{ "type": "string" }, { "type": "null" }], + "title": "Knowledge Base Id", + "description": "Link to wikidata", + "default": "" + }, + "name": { "type": "string", "title": "Name", "description": "Name of the Entity" }, + "span_text_ids": { + "items": { "type": "integer" }, + "type": "array", + "title": "Span Text Ids", + "description": "Span Text Ids which belong to this Entity" + } + }, + "type": "object", + "required": ["name", "span_text_ids"], + "title": "EntityUpdate" + }, "ExportFormat": { "type": "string", "enum": ["CSV", "JSON"], "title": "ExportFormat" }, "ExportJobParameters": { "properties": { @@ -8617,6 +8843,15 @@ "type": "object", "title": "SpanGroupUpdate" }, + "SpanTextRead": { + "properties": { + "text": { "type": "string", "title": "Text", "description": "Code of the SpanText" }, + "id": { "type": "integer", "title": "Id", "description": "ID of the SpanText" } + }, + "type": "object", + "required": ["id"], + "title": "SpanTextRead" + }, "StringOperator": { "type": "string", "enum": ["STRING_CONTAINS", "STRING_EQUALS", "STRING_NOT_EQUALS", "STRING_STARTS_WITH", "STRING_ENDS_WITH"], diff --git a/frontend/src/router/routes.tsx b/frontend/src/router/routes.tsx index b687e113c..8533c781b 100644 --- a/frontend/src/router/routes.tsx +++ b/frontend/src/router/routes.tsx @@ -13,6 +13,7 @@ import CodeFrequencyAnalysis from "../views/analysis/CodeFrequency/CodeFrequency import CodeGraph from "../views/analysis/CodeGraph/CodeGraph.tsx"; import CotaDashboard from "../views/analysis/ConceptsOverTime/CotaDashboard.tsx"; import CotaView from "../views/analysis/ConceptsOverTime/CotaView.tsx"; +import EntityDashboard from "../views/analysis/EntityDashboard/EntityDashboard.tsx"; import TableDashboard from "../views/analysis/Table/TableDashboard.tsx"; import TableView from "../views/analysis/Table/TableView.tsx"; import TimelineAnalysis from "../views/analysis/TimelineAnalysis/TimelineAnalysis.tsx"; @@ -151,6 +152,10 @@ const router = createBrowserRouter([ path: "/project/:projectId/analysis/annotated-segments", element: , }, + { + path: "/project/:projectId/analysis/entity-dashboard", + element: , + }, { path: "/project/:projectId/analysis/word-frequency", element: , diff --git a/frontend/src/views/analysis/Analysis.tsx b/frontend/src/views/analysis/Analysis.tsx index fd3455952..34b3b6bff 100644 --- a/frontend/src/views/analysis/Analysis.tsx +++ b/frontend/src/views/analysis/Analysis.tsx @@ -58,6 +58,13 @@ function Analysis() { color={"#77dd77"} /> + + diff --git a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx new file mode 100644 index 000000000..ac36cbec6 --- /dev/null +++ b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx @@ -0,0 +1,99 @@ +import { Box, Button, Grid, Portal, Stack, Typography } from "@mui/material"; +import { MRT_RowSelectionState, MRT_TableOptions } from "material-react-table"; +import { useContext, useState } from "react"; +import { useParams } from "react-router-dom"; +import EntityHooks from "../../../api/EntityHooks.ts"; +import { SpanTextRead } from "../../../api/openapi/models/SpanTextRead.ts"; +import { AppBarContext } from "../../../layouts/TwoBarLayout.tsx"; +import { useAppSelector } from "../../../plugins/ReduxHooks.ts"; +import EntityTable, { EnitityTableRow, EntityTableSaveRowProps, SpanTextTableRow } from "./EntityTable.tsx"; + +function EntityDashboard() { + const appBarContainerRef = useContext(AppBarContext); + + // global client state (react router) + const projectId = parseInt(useParams<{ projectId: string }>().projectId!); + + // global client state (redux) + const isSplitView = useAppSelector((state) => state.annotatedSegments.isSplitView); + const [rowSelectionModel, setRowSelectionModel] = useState({}); + const entityMerge = EntityHooks.useMerge(); + const entityRelease = EntityHooks.useRelease(); + const entityUpdate = EntityHooks.useUpdateEntity(); + + function handleRelease(selectedSpanTexts: SpanTextRead[]): void { + const requestBody = { + requestBody: { + project_id: projectId, + spantext_ids: selectedSpanTexts.map((spantext) => spantext.id), + }, + }; + entityRelease.mutate(requestBody); + setRowSelectionModel({}); + } + + const handleUpdate: MRT_TableOptions["onEditingRowSave"] = async ({ + row, + values, + table, + }) => { + const requestBody = { + entityId: row.original.id, + requestBody: { + name: values.name, + span_text_ids: row.original.subRows.map((span_text) => span_text.id), + knowledge_base_id: values.knowledge_base_id, + }, + }; + entityUpdate.mutate(requestBody); + table.setEditingRow(null); + }; + + function handleMerge(props: EntityTableSaveRowProps): void { + props.table.setCreatingRow(null); + const name = props.values.name; + const knowledge_base_id = props.values.knowledge_base_id; + const requestBody = { + requestBody: { + name: name, + project_id: projectId, + spantext_ids: props.selectedSpanTexts.map((spantext) => spantext.id), + knowledge_base_id: knowledge_base_id, + }, + }; + entityMerge.mutate(requestBody); + setRowSelectionModel({}); + } + + return ( + + + + Entity Dashboard + + + + + { + return ( + + + + + + ); + }} + onSaveEditRow={handleUpdate} + onCreateSaveRow={handleMerge} + /> + + + + ); +} + +export default EntityDashboard; diff --git a/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx b/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx new file mode 100644 index 000000000..c869684c5 --- /dev/null +++ b/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx @@ -0,0 +1,227 @@ +import { + LiteralUnion, + MRT_ColumnDef, + MRT_RowSelectionState, + MRT_TableInstance, + MRT_TableOptions, + MaterialReactTable, + useMaterialReactTable, +} from "material-react-table"; +import { useMemo } from "react"; +import ProjectHooks from "../../../api/ProjectHooks.ts"; +import { EntityRead } from "../../../api/openapi/models/EntityRead.ts"; +import { SpanTextRead } from "../../../api/openapi/models/SpanTextRead.ts"; + +export interface EnitityTableRow extends EntityRead { + table_id: string; + subRows: SpanTextTableRow[]; + editable: boolean; +} + +export interface SpanTextTableRow extends SpanTextRead { + table_id: string; + subRows: SpanTextTableRow[]; + editable: boolean; +} + +const columns: MRT_ColumnDef[] = [ + { + accessorKey: "id", + header: "ID", + enableEditing: false, + }, + { + accessorKey: "name", + header: "Name", + enableEditing: true, + }, + { + accessorKey: "knowledge_base_id", + header: "Knowledge Base ID", + enableEditing: true, + }, + { + accessorKey: "is_human", + header: "Is Human", + enableEditing: false, + Cell: ({ cell }) => { + return cell.getValue() ? "True" : "False"; + }, + }, +]; + +export interface EntityTableActionProps { + table: MRT_TableInstance; + selectedSpanTexts: SpanTextRead[]; +} + +export interface EntityTableSaveRowProps extends EntityTableActionProps { + values: Record, string>; +} + +export interface EntityTableProps { + projectId: number; + // selection + enableMultiRowSelection?: boolean; + rowSelectionModel: MRT_RowSelectionState; + onRowSelectionChange: MRT_TableOptions["onRowSelectionChange"]; + // toolbar + renderToolbarInternalActions?: (props: EntityTableActionProps) => React.ReactNode; + renderTopToolbarCustomActions?: (props: EntityTableActionProps) => React.ReactNode; + renderBottomToolbarCustomActions?: (props: EntityTableActionProps) => React.ReactNode; + // editing + onSaveEditRow: MRT_TableOptions["onEditingRowSave"]; + onCreateSaveRow: (props: EntityTableSaveRowProps) => void; +} + +function EntityTable({ + projectId, + enableMultiRowSelection = true, + rowSelectionModel, + onRowSelectionChange, + renderToolbarInternalActions, + renderTopToolbarCustomActions, + renderBottomToolbarCustomActions, + onSaveEditRow, + onCreateSaveRow, +}: EntityTableProps) { + // global server state + const projectEntities = ProjectHooks.useGetAllEntities(projectId); + + // computed + const { projectEntitiesRows, projectSpanTextMap } = useMemo(() => { + if (!projectEntities.data) { + return { + projectEntitiesMap: {} as Record, + projectEntitiesRows: [], + projectSpanTextMap: {} as Record, + }; + } + + const projectEntitiesRows: EnitityTableRow[] = projectEntities.data.map((entity) => { + const subRows: SpanTextTableRow[] = + entity.span_texts?.map((span) => ({ + ...span, + table_id: `S-${span.id}`, + name: span.text, + subRows: [], + editable: false, + })) || []; + const table_id = `E-${entity.id}`; + const editable = true; + return { table_id, ...entity, subRows, editable }; + }); + + const projectSpanTextMap = projectEntities.data.reduce( + (acc, entity) => { + if (Array.isArray(entity.span_texts)) { + entity.span_texts.forEach((span) => { + acc[`S-${span.id}`] = span; + }); + } + return acc; + }, + {} as Record, + ); + + return { projectEntitiesRows, projectSpanTextMap }; + }, [projectEntities.data]); + + // table + const table = useMaterialReactTable({ + data: projectEntitiesRows, + columns: columns, + getRowId: (row) => `${row.table_id}`, + enableEditing: (row) => { + return row.original.editable; + }, + createDisplayMode: "modal", + editDisplayMode: "row", + onEditingRowSave: onSaveEditRow, + onCreatingRowSave: (props) => { + const selectedSpanTexts = Object.keys(props.table.getState().rowSelection) + .filter((id) => id.startsWith("S-")) + .map((spanTextId) => projectSpanTextMap[spanTextId]); + + const allSpanTexts = [...selectedSpanTexts]; + + onCreateSaveRow({ + selectedSpanTexts: allSpanTexts, + values: props.values, + table: props.table, + }); + }, + // style + muiTablePaperProps: { + elevation: 0, + style: { height: "100%", display: "flex", flexDirection: "column" }, + }, + muiTableContainerProps: { + style: { flexGrow: 1 }, + }, + // state + state: { + rowSelection: rowSelectionModel, + isLoading: projectEntities.isLoading, + showAlertBanner: projectEntities.isError, + showProgressBars: projectEntities.isFetching, + }, + // handle error + muiToolbarAlertBannerProps: projectEntities.isError + ? { + color: "error", + children: projectEntities.error.message, + } + : undefined, + // virtualization (scrolling instead of pagination) + enablePagination: false, + enableRowVirtualization: true, + // selection + enableRowSelection: true, + enableMultiRowSelection: enableMultiRowSelection, + onRowSelectionChange, + // toolbar + enableBottomToolbar: true, + renderTopToolbarCustomActions: renderTopToolbarCustomActions + ? (props) => + renderTopToolbarCustomActions({ + table: props.table, + selectedSpanTexts: Object.keys(rowSelectionModel) + .filter((id) => id.startsWith("S-")) + .map((spanTextId) => projectSpanTextMap[spanTextId]), + }) + : undefined, + renderToolbarInternalActions: renderToolbarInternalActions + ? (props) => + renderToolbarInternalActions({ + table: props.table, + selectedSpanTexts: Object.keys(rowSelectionModel) + .filter((id) => id.startsWith("S-")) + .map((spanTextId) => projectSpanTextMap[spanTextId]), + }) + : undefined, + renderBottomToolbarCustomActions: renderBottomToolbarCustomActions + ? (props) => + renderBottomToolbarCustomActions({ + table: props.table, + selectedSpanTexts: Object.keys(rowSelectionModel) + .filter((id) => id.startsWith("S-")) + .map((spanTextId) => projectSpanTextMap[spanTextId]), + }) + : undefined, + // hide columns per default + initialState: { + columnVisibility: { + id: false, + }, + }, + // tree structure + enableExpanding: true, + getSubRows: (originalRow) => originalRow.subRows, + filterFromLeafRows: true, //search for child rows and preserve parent rows + enableSubRowSelection: true, + }); + + return ; +} +export default EntityTable;