[FEATURE] Query time boosting (#1120)

Co-authored-by: Sebastian Meyer <[email protected]>

.github/phpstan.neon

@@ -2,6 +2,7 @@ parameters:
  ignoreErrors:
  - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::countByPid\(\)\.#'
  - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIsListed\(\)\.#'
+ - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIndexIndexed\(\)\.#'
  - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIsSortable\(\)\.#'
  - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByFeUserId\(\)\.#'
  - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByIndexName\(\)\.#'

Classes/Common/Indexer.php

@@ -56,8 +56,7 @@ class Indexer
  'sortables' => [],
  'indexed' => [],
  'stored' => [],
- 'tokenized' => [],
- 'fieldboost' => []
+ 'tokenized' => []
  ];
 
  /**
@@ -305,11 +304,6 @@ protected static function loadIndexConf(int $pid): void
  if ($indexing['index_autocomplete']) {
  self::$fields['autocomplete'][] = $indexing['index_name'];
  }
- if ($indexing['index_boost'] > 0.0) {
- self::$fields['fieldboost'][$indexing['index_name']] = floatval($indexing['index_boost']);
- } else {
- self::$fields['fieldboost'][$indexing['index_name']] = false;
- }
  }
  self::$fieldsLoaded = true;
  }
@@ -357,8 +351,8 @@ protected static function processLogical(Document $document, array $logicalUnit)
  }
  // There can be only one toplevel unit per UID, independently of backend configuration
  $solrDoc->setField('toplevel', $logicalUnit['id'] == $doc->toplevelId ? true : false);
- $solrDoc->setField('title', $metadata['title'][0], self::$fields['fieldboost']['title']);
- $solrDoc->setField('volume', $metadata['volume'][0], self::$fields['fieldboost']['volume']);
+ $solrDoc->setField('title', $metadata['title'][0]);
+ $solrDoc->setField('volume', $metadata['volume'][0]);
  // verify date formatting
  if(strtotime($metadata['date'][0])) {
  $solrDoc->setField('date', self::getFormattedDate($metadata['date'][0]));
@@ -445,7 +439,7 @@ protected static function processPhysical(Document $document, int $page, array $
  }
  }
  $solrDoc->setField('toplevel', false);
- $solrDoc->setField('type', $physicalUnit['type'], self::$fields['fieldboost']['type']);
+ $solrDoc->setField('type', $physicalUnit['type']);
  $solrDoc->setField('collection', $doc->metadataArray[$doc->toplevelId]['collection']);
  $solrDoc->setField('location', $document->getLocation());
 
@@ -520,7 +514,7 @@ private static function processMetadata($document, $metadata, &$solrDoc): array
  !empty($data)
  && substr($indexName, -8) !== '_sorting'
  ) {
- $solrDoc->setField(self::getIndexFieldName($indexName, $document->getPid()), $data, self::$fields['fieldboost'][$indexName]);
+ $solrDoc->setField(self::getIndexFieldName($indexName, $document->getPid()), $data);
  if (in_array($indexName, self::$fields['sortables'])) {
  // Add sortable fields to index.
  $solrDoc->setField($indexName . '_sorting', $metadata[$indexName . '_sorting'][0]);
@@ -626,7 +620,7 @@ private static function getSolrDocument(Query $updateQuery, Document $document,
  $solrDoc->setField('partof', $document->getPartof());
  $solrDoc->setField('root', $document->getCurrentDocument()->rootId);
  $solrDoc->setField('sid', $unit['id']);
- $solrDoc->setField('type', $unit['type'], self::$fields['fieldboost']['type']);
+ $solrDoc->setField('type', $unit['type']);
  $solrDoc->setField('collection', $document->getCurrentDocument()->metadataArray[$document->getCurrentDocument()->toplevelId]['collection']);
  $solrDoc->setField('fulltext', $fullText);
  return $solrDoc;

Classes/Common/Solr/SolrSearch.php

@@ -59,6 +59,12 @@ class SolrSearch implements \Countable, \Iterator, \ArrayAccess, QueryResultInte
  */
  private ?QueryResult $listedMetadata;
 
+ /**
+ * @access private
+ * @var QueryResult|null
+ */
+ private ?QueryResult $indexedMetadata;
+
  /**
  * @access private
  * @var array
@@ -90,13 +96,14 @@ class SolrSearch implements \Countable, \Iterator, \ArrayAccess, QueryResultInte
  *
  * @return void
  */
- public function __construct(DocumentRepository $documentRepository, $collections, array $settings, array $searchParams, QueryResult $listedMetadata = null)
+ public function __construct(DocumentRepository $documentRepository, $collections, array $settings, array $searchParams, QueryResult $listedMetadata = null, QueryResult $indexedMetadata = null)
  {
  $this->documentRepository = $documentRepository;
  $this->collections = $collections;
  $this->settings = $settings;
  $this->searchParams = $searchParams;
  $this->listedMetadata = $listedMetadata;
+ $this->indexedMetadata = $indexedMetadata;
  }
 
  /**
@@ -664,6 +671,21 @@ protected function searchSolr($parameters = [], $enableCache = true)
  if ($enableCache === false || ($entry = $cache->get($cacheIdentifier)) === false) {
  $selectQuery = $solr->service->createSelect($parameters);
 
+ $edismax = $selectQuery->getEDisMax();
+
+ $queryFields = '';
+
+ if ($this->indexedMetadata) {
+ foreach ($this->indexedMetadata as $metadata) {
+ if ($metadata->getIndexIndexed()) {
+ $listMetadataRecord = $metadata->getIndexName() . '_' . ($metadata->getIndexTokenized() ? 't' : 'u') . ($metadata->getIndexStored() ? 's' : 'u') . 'i';
+ $queryFields .= $listMetadataRecord . '^' . $metadata->getIndexBoost() . ' ';
+ }
+ }
+ }
+
+ $edismax->setQueryFields($queryFields);
+
  $grouping = $selectQuery->getGrouping();
  $grouping->addField('uid');
  $grouping->setLimit(100); // Results in group (TODO: check)

Classes/Controller/CollectionController.php

@@ -151,13 +151,16 @@ public function showAction(Collection $collection): void
  // get all metadata records to be shown in results
  $listedMetadata = $this->metadataRepository->findByIsListed(true);
 
+ // get all indexed metadata fields
+ $indexedMetadata = $this->metadataRepository->findByIndexIndexed(true);
+
  // get all sortable metadata records
  $sortableMetadata = $this->metadataRepository->findByIsSortable(true);
 
  // get all documents of given collection
  $solrResults = null;
  if (is_array($searchParams) && !empty($searchParams)) {
- $solrResults = $this->documentRepository->findSolrByCollection($collection, $this->settings, $searchParams, $listedMetadata);
+ $solrResults = $this->documentRepository->findSolrByCollection($collection, $this->settings, $searchParams, $listedMetadata, $indexedMetadata);
 
  $itemsPerPage = $this->settings['list']['paginate']['itemsPerPage'];
  if (empty($itemsPerPage)) {

Classes/Controller/ListViewController.php

@@ -96,10 +96,13 @@ public function mainAction(): void
  // get all metadata records to be shown in results
  $listedMetadata = $this->metadataRepository->findByIsListed(true);
 
+ // get all indexed metadata fields
+ $indexedMetadata = $this->metadataRepository->findByIndexIndexed(true);
+
  $solrResults = null;
  $numResults = 0;
  if (is_array($this->searchParams) && !empty($this->searchParams)) {
- $solrResults = $this->documentRepository->findSolrByCollections($collections, $this->settings, $this->searchParams, $listedMetadata);
+ $solrResults = $this->documentRepository->findSolrByCollections($collections, $this->settings, $this->searchParams, $listedMetadata, $indexedMetadata);
  $numResults = $solrResults->getNumFound();
 
  $itemsPerPage = $this->settings['list']['paginate']['itemsPerPage'];

Classes/Controller/SearchController.php

@@ -172,11 +172,14 @@ public function mainAction(): void
  // get all metadata records to be shown in results
  $listedMetadata = $this->metadataRepository->findByIsListed(true);
 
+ // get all indexed metadata fields
+ $indexedMetadata = $this->metadataRepository->findByIndexIndexed(true);
+
  $solrResults = null;
  $numResults = 0;
  // Do not execute the Solr search if used together with ListView plugin.
  if (!$listViewSearch) {
- $solrResults = $this->documentRepository->findSolrWithoutCollection($this->settings, $this->searchParams, $listedMetadata);
+ $solrResults = $this->documentRepository->findSolrWithoutCollection($this->settings, $this->searchParams, $listedMetadata, $indexedMetadata);
  $numResults = $solrResults->getNumFound();
 
  $itemsPerPage = $this->settings['list']['paginate']['itemsPerPage'];

Classes/Domain/Repository/DocumentRepository.php

@@ -579,12 +579,13 @@ public function findChildrenOfEach(array $uids)
  * @param array $settings
  * @param array $searchParams
  * @param QueryResult $listedMetadata
+ * @param QueryResult $indexedMetadata
  *
  * @return SolrSearch
  */
- public function findSolrByCollection(Collection $collection, $settings, $searchParams, $listedMetadata = null)
+ public function findSolrByCollection(Collection $collection, $settings, $searchParams, $listedMetadata = null, $indexedMetadata = null)
  {
- return $this->findSolr([$collection], $settings, $searchParams, $listedMetadata);
+ return $this->findSolr([$collection], $settings, $searchParams, $listedMetadata, $indexedMetadata);
  }
 
  /**
@@ -596,12 +597,13 @@ public function findSolrByCollection(Collection $collection, $settings, $searchP
  * @param array $settings
  * @param array $searchParams
  * @param QueryResult $listedMetadata
+ * @param QueryResult $indexedMetadata
  *
  * @return SolrSearch
  */
- public function findSolrByCollections($collections, $settings, $searchParams, $listedMetadata = null): SolrSearch
+ public function findSolrByCollections($collections, $settings, $searchParams, $listedMetadata = null, $indexedMetadata = null): SolrSearch
  {
- return $this->findSolr($collections, $settings, $searchParams, $listedMetadata);
+ return $this->findSolr($collections, $settings, $searchParams, $listedMetadata, $indexedMetadata);
  }
 
  /**
@@ -612,12 +614,13 @@ public function findSolrByCollections($collections, $settings, $searchParams, $l
  * @param array $settings
  * @param array $searchParams
  * @param QueryResult $listedMetadata
+ * @param QueryResult $indexedMetadata
  *
  * @return SolrSearch
  */
- public function findSolrWithoutCollection($settings, $searchParams, $listedMetadata = null): SolrSearch
+ public function findSolrWithoutCollection($settings, $searchParams, $listedMetadata = null, $indexedMetadata = null): SolrSearch
  {
- return $this->findSolr([], $settings, $searchParams, $listedMetadata);
+ return $this->findSolr([], $settings, $searchParams, $listedMetadata, $indexedMetadata);
  }
 
  /**
@@ -632,13 +635,13 @@ public function findSolrWithoutCollection($settings, $searchParams, $listedMetad
  *
  * @return SolrSearch
  */
- private function findSolr($collections, $settings, $searchParams, $listedMetadata = null): SolrSearch
+ private function findSolr($collections, $settings, $searchParams, $listedMetadata = null, $indexedMetadata = null): SolrSearch
  {
  // set settings global inside this repository
  // (may be necessary when SolrSearch calls back)
  $this->settings = $settings;
 
- $search = new SolrSearch($this, $collections, $settings, $searchParams, $listedMetadata);
+ $search = new SolrSearch($this, $collections, $settings, $searchParams, $listedMetadata, $indexedMetadata);
  $search->prepare();
  return $search;
  }