From 84a35c049bea900a79ad8f666a88fe38b44677a0 Mon Sep 17 00:00:00 2001 From: frank-ulrich-weber Date: Mon, 1 Jul 2024 17:23:35 +0200 Subject: [PATCH] [FEATURE] Query time boosting (#1120) Co-authored-by: Sebastian Meyer --- .github/phpstan.neon | 1 + Classes/Common/Indexer.php | 18 +++++--------- Classes/Common/Solr/SolrSearch.php | 24 ++++++++++++++++++- Classes/Controller/CollectionController.php | 5 +++- Classes/Controller/ListViewController.php | 5 +++- Classes/Controller/SearchController.php | 5 +++- .../Domain/Repository/DocumentRepository.php | 19 ++++++++------- 7 files changed, 53 insertions(+), 24 deletions(-) diff --git a/.github/phpstan.neon b/.github/phpstan.neon index 7307b130f..86b7418e6 100644 --- a/.github/phpstan.neon +++ b/.github/phpstan.neon @@ -2,6 +2,7 @@ parameters: ignoreErrors: - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::countByPid\(\)\.#' - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIsListed\(\)\.#' + - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIndexIndexed\(\)\.#' - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIsSortable\(\)\.#' - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByFeUserId\(\)\.#' - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByIndexName\(\)\.#' diff --git a/Classes/Common/Indexer.php b/Classes/Common/Indexer.php index dc6b194cd..98fa6885d 100644 --- a/Classes/Common/Indexer.php +++ b/Classes/Common/Indexer.php @@ -56,8 +56,7 @@ class Indexer 'sortables' => [], 'indexed' => [], 'stored' => [], - 'tokenized' => [], - 'fieldboost' => [] + 'tokenized' => [] ]; /** @@ -305,11 +304,6 @@ protected static function loadIndexConf(int $pid): void if ($indexing['index_autocomplete']) { self::$fields['autocomplete'][] = $indexing['index_name']; } - if ($indexing['index_boost'] > 0.0) { - self::$fields['fieldboost'][$indexing['index_name']] = floatval($indexing['index_boost']); - } else { - self::$fields['fieldboost'][$indexing['index_name']] = false; - } } self::$fieldsLoaded = true; } @@ -357,8 +351,8 @@ protected static function processLogical(Document $document, array $logicalUnit) } // There can be only one toplevel unit per UID, independently of backend configuration $solrDoc->setField('toplevel', $logicalUnit['id'] == $doc->toplevelId ? true : false); - $solrDoc->setField('title', $metadata['title'][0], self::$fields['fieldboost']['title']); - $solrDoc->setField('volume', $metadata['volume'][0], self::$fields['fieldboost']['volume']); + $solrDoc->setField('title', $metadata['title'][0]); + $solrDoc->setField('volume', $metadata['volume'][0]); // verify date formatting if(strtotime($metadata['date'][0])) { $solrDoc->setField('date', self::getFormattedDate($metadata['date'][0])); @@ -445,7 +439,7 @@ protected static function processPhysical(Document $document, int $page, array $ } } $solrDoc->setField('toplevel', false); - $solrDoc->setField('type', $physicalUnit['type'], self::$fields['fieldboost']['type']); + $solrDoc->setField('type', $physicalUnit['type']); $solrDoc->setField('collection', $doc->metadataArray[$doc->toplevelId]['collection']); $solrDoc->setField('location', $document->getLocation()); @@ -520,7 +514,7 @@ private static function processMetadata($document, $metadata, &$solrDoc): array !empty($data) && substr($indexName, -8) !== '_sorting' ) { - $solrDoc->setField(self::getIndexFieldName($indexName, $document->getPid()), $data, self::$fields['fieldboost'][$indexName]); + $solrDoc->setField(self::getIndexFieldName($indexName, $document->getPid()), $data); if (in_array($indexName, self::$fields['sortables'])) { // Add sortable fields to index. $solrDoc->setField($indexName . '_sorting', $metadata[$indexName . '_sorting'][0]); @@ -626,7 +620,7 @@ private static function getSolrDocument(Query $updateQuery, Document $document, $solrDoc->setField('partof', $document->getPartof()); $solrDoc->setField('root', $document->getCurrentDocument()->rootId); $solrDoc->setField('sid', $unit['id']); - $solrDoc->setField('type', $unit['type'], self::$fields['fieldboost']['type']); + $solrDoc->setField('type', $unit['type']); $solrDoc->setField('collection', $document->getCurrentDocument()->metadataArray[$document->getCurrentDocument()->toplevelId]['collection']); $solrDoc->setField('fulltext', $fullText); return $solrDoc; diff --git a/Classes/Common/Solr/SolrSearch.php b/Classes/Common/Solr/SolrSearch.php index 4c38316af..ee33c5367 100644 --- a/Classes/Common/Solr/SolrSearch.php +++ b/Classes/Common/Solr/SolrSearch.php @@ -59,6 +59,12 @@ class SolrSearch implements \Countable, \Iterator, \ArrayAccess, QueryResultInte */ private ?QueryResult $listedMetadata; + /** + * @access private + * @var QueryResult|null + */ + private ?QueryResult $indexedMetadata; + /** * @access private * @var array @@ -90,13 +96,14 @@ class SolrSearch implements \Countable, \Iterator, \ArrayAccess, QueryResultInte * * @return void */ - public function __construct(DocumentRepository $documentRepository, $collections, array $settings, array $searchParams, QueryResult $listedMetadata = null) + public function __construct(DocumentRepository $documentRepository, $collections, array $settings, array $searchParams, QueryResult $listedMetadata = null, QueryResult $indexedMetadata = null) { $this->documentRepository = $documentRepository; $this->collections = $collections; $this->settings = $settings; $this->searchParams = $searchParams; $this->listedMetadata = $listedMetadata; + $this->indexedMetadata = $indexedMetadata; } /** @@ -664,6 +671,21 @@ protected function searchSolr($parameters = [], $enableCache = true) if ($enableCache === false || ($entry = $cache->get($cacheIdentifier)) === false) { $selectQuery = $solr->service->createSelect($parameters); + $edismax = $selectQuery->getEDisMax(); + + $queryFields = ''; + + if ($this->indexedMetadata) { + foreach ($this->indexedMetadata as $metadata) { + if ($metadata->getIndexIndexed()) { + $listMetadataRecord = $metadata->getIndexName() . '_' . ($metadata->getIndexTokenized() ? 't' : 'u') . ($metadata->getIndexStored() ? 's' : 'u') . 'i'; + $queryFields .= $listMetadataRecord . '^' . $metadata->getIndexBoost() . ' '; + } + } + } + + $edismax->setQueryFields($queryFields); + $grouping = $selectQuery->getGrouping(); $grouping->addField('uid'); $grouping->setLimit(100); // Results in group (TODO: check) diff --git a/Classes/Controller/CollectionController.php b/Classes/Controller/CollectionController.php index c7ecb958b..c44822b5c 100644 --- a/Classes/Controller/CollectionController.php +++ b/Classes/Controller/CollectionController.php @@ -151,13 +151,16 @@ public function showAction(Collection $collection): void // get all metadata records to be shown in results $listedMetadata = $this->metadataRepository->findByIsListed(true); + // get all indexed metadata fields + $indexedMetadata = $this->metadataRepository->findByIndexIndexed(true); + // get all sortable metadata records $sortableMetadata = $this->metadataRepository->findByIsSortable(true); // get all documents of given collection $solrResults = null; if (is_array($searchParams) && !empty($searchParams)) { - $solrResults = $this->documentRepository->findSolrByCollection($collection, $this->settings, $searchParams, $listedMetadata); + $solrResults = $this->documentRepository->findSolrByCollection($collection, $this->settings, $searchParams, $listedMetadata, $indexedMetadata); $itemsPerPage = $this->settings['list']['paginate']['itemsPerPage']; if (empty($itemsPerPage)) { diff --git a/Classes/Controller/ListViewController.php b/Classes/Controller/ListViewController.php index 66bb47835..324e5585a 100644 --- a/Classes/Controller/ListViewController.php +++ b/Classes/Controller/ListViewController.php @@ -96,10 +96,13 @@ public function mainAction(): void // get all metadata records to be shown in results $listedMetadata = $this->metadataRepository->findByIsListed(true); + // get all indexed metadata fields + $indexedMetadata = $this->metadataRepository->findByIndexIndexed(true); + $solrResults = null; $numResults = 0; if (is_array($this->searchParams) && !empty($this->searchParams)) { - $solrResults = $this->documentRepository->findSolrByCollections($collections, $this->settings, $this->searchParams, $listedMetadata); + $solrResults = $this->documentRepository->findSolrByCollections($collections, $this->settings, $this->searchParams, $listedMetadata, $indexedMetadata); $numResults = $solrResults->getNumFound(); $itemsPerPage = $this->settings['list']['paginate']['itemsPerPage']; diff --git a/Classes/Controller/SearchController.php b/Classes/Controller/SearchController.php index ae69761d1..127943c54 100644 --- a/Classes/Controller/SearchController.php +++ b/Classes/Controller/SearchController.php @@ -172,11 +172,14 @@ public function mainAction(): void // get all metadata records to be shown in results $listedMetadata = $this->metadataRepository->findByIsListed(true); + // get all indexed metadata fields + $indexedMetadata = $this->metadataRepository->findByIndexIndexed(true); + $solrResults = null; $numResults = 0; // Do not execute the Solr search if used together with ListView plugin. if (!$listViewSearch) { - $solrResults = $this->documentRepository->findSolrWithoutCollection($this->settings, $this->searchParams, $listedMetadata); + $solrResults = $this->documentRepository->findSolrWithoutCollection($this->settings, $this->searchParams, $listedMetadata, $indexedMetadata); $numResults = $solrResults->getNumFound(); $itemsPerPage = $this->settings['list']['paginate']['itemsPerPage']; diff --git a/Classes/Domain/Repository/DocumentRepository.php b/Classes/Domain/Repository/DocumentRepository.php index 7e09a5df7..25785b878 100644 --- a/Classes/Domain/Repository/DocumentRepository.php +++ b/Classes/Domain/Repository/DocumentRepository.php @@ -579,12 +579,13 @@ public function findChildrenOfEach(array $uids) * @param array $settings * @param array $searchParams * @param QueryResult $listedMetadata + * @param QueryResult $indexedMetadata * * @return SolrSearch */ - public function findSolrByCollection(Collection $collection, $settings, $searchParams, $listedMetadata = null) + public function findSolrByCollection(Collection $collection, $settings, $searchParams, $listedMetadata = null, $indexedMetadata = null) { - return $this->findSolr([$collection], $settings, $searchParams, $listedMetadata); + return $this->findSolr([$collection], $settings, $searchParams, $listedMetadata, $indexedMetadata); } /** @@ -596,12 +597,13 @@ public function findSolrByCollection(Collection $collection, $settings, $searchP * @param array $settings * @param array $searchParams * @param QueryResult $listedMetadata + * @param QueryResult $indexedMetadata * * @return SolrSearch */ - public function findSolrByCollections($collections, $settings, $searchParams, $listedMetadata = null): SolrSearch + public function findSolrByCollections($collections, $settings, $searchParams, $listedMetadata = null, $indexedMetadata = null): SolrSearch { - return $this->findSolr($collections, $settings, $searchParams, $listedMetadata); + return $this->findSolr($collections, $settings, $searchParams, $listedMetadata, $indexedMetadata); } /** @@ -612,12 +614,13 @@ public function findSolrByCollections($collections, $settings, $searchParams, $l * @param array $settings * @param array $searchParams * @param QueryResult $listedMetadata + * @param QueryResult $indexedMetadata * * @return SolrSearch */ - public function findSolrWithoutCollection($settings, $searchParams, $listedMetadata = null): SolrSearch + public function findSolrWithoutCollection($settings, $searchParams, $listedMetadata = null, $indexedMetadata = null): SolrSearch { - return $this->findSolr([], $settings, $searchParams, $listedMetadata); + return $this->findSolr([], $settings, $searchParams, $listedMetadata, $indexedMetadata); } /** @@ -632,13 +635,13 @@ public function findSolrWithoutCollection($settings, $searchParams, $listedMetad * * @return SolrSearch */ - private function findSolr($collections, $settings, $searchParams, $listedMetadata = null): SolrSearch + private function findSolr($collections, $settings, $searchParams, $listedMetadata = null, $indexedMetadata = null): SolrSearch { // set settings global inside this repository // (may be necessary when SolrSearch calls back) $this->settings = $settings; - $search = new SolrSearch($this, $collections, $settings, $searchParams, $listedMetadata); + $search = new SolrSearch($this, $collections, $settings, $searchParams, $listedMetadata, $indexedMetadata); $search->prepare(); return $search; }