Merge branch 'main_master' into midi-player-navigation

.github/phpstan.neon

@@ -2,6 +2,7 @@ parameters:
  ignoreErrors:
  - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::countByPid\(\)\.#'
  - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIsListed\(\)\.#'
+ - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIndexIndexed\(\)\.#'
  - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIsSortable\(\)\.#'
  - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByFeUserId\(\)\.#'
  - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByIndexName\(\)\.#'

Classes/Common/Indexer.php

@@ -56,8 +56,7 @@ class Indexer
  'sortables' => [],
  'indexed' => [],
  'stored' => [],
- 'tokenized' => [],
- 'fieldboost' => []
+ 'tokenized' => []
  ];
 
  /**
@@ -305,11 +304,6 @@ protected static function loadIndexConf(int $pid): void
  if ($indexing['index_autocomplete']) {
  self::$fields['autocomplete'][] = $indexing['index_name'];
  }
- if ($indexing['index_boost'] > 0.0) {
- self::$fields['fieldboost'][$indexing['index_name']] = floatval($indexing['index_boost']);
- } else {
- self::$fields['fieldboost'][$indexing['index_name']] = false;
- }
  }
  self::$fieldsLoaded = true;
  }
@@ -357,8 +351,8 @@ protected static function processLogical(Document $document, array $logicalUnit)
  }
  // There can be only one toplevel unit per UID, independently of backend configuration
  $solrDoc->setField('toplevel', $logicalUnit['id'] == $doc->toplevelId ? true : false);
- $solrDoc->setField('title', $metadata['title'][0], self::$fields['fieldboost']['title']);
- $solrDoc->setField('volume', $metadata['volume'][0], self::$fields['fieldboost']['volume']);
+ $solrDoc->setField('title', $metadata['title'][0]);
+ $solrDoc->setField('volume', $metadata['volume'][0]);
  // verify date formatting
  if(strtotime($metadata['date'][0])) {
  $solrDoc->setField('date', self::getFormattedDate($metadata['date'][0]));
@@ -445,7 +439,7 @@ protected static function processPhysical(Document $document, int $page, array $
  }
  }
  $solrDoc->setField('toplevel', false);
- $solrDoc->setField('type', $physicalUnit['type'], self::$fields['fieldboost']['type']);
+ $solrDoc->setField('type', $physicalUnit['type']);
  $solrDoc->setField('collection', $doc->metadataArray[$doc->toplevelId]['collection']);
  $solrDoc->setField('location', $document->getLocation());
 
@@ -520,7 +514,7 @@ private static function processMetadata($document, $metadata, &$solrDoc): array
  !empty($data)
  && substr($indexName, -8) !== '_sorting'
  ) {
- $solrDoc->setField(self::getIndexFieldName($indexName, $document->getPid()), $data, self::$fields['fieldboost'][$indexName]);
+ $solrDoc->setField(self::getIndexFieldName($indexName, $document->getPid()), $data);
  if (in_array($indexName, self::$fields['sortables'])) {
  // Add sortable fields to index.
  $solrDoc->setField($indexName . '_sorting', $metadata[$indexName . '_sorting'][0]);
@@ -626,7 +620,7 @@ private static function getSolrDocument(Query $updateQuery, Document $document,
  $solrDoc->setField('partof', $document->getPartof());
  $solrDoc->setField('root', $document->getCurrentDocument()->rootId);
  $solrDoc->setField('sid', $unit['id']);
- $solrDoc->setField('type', $unit['type'], self::$fields['fieldboost']['type']);
+ $solrDoc->setField('type', $unit['type']);
  $solrDoc->setField('collection', $document->getCurrentDocument()->metadataArray[$document->getCurrentDocument()->toplevelId]['collection']);
  $solrDoc->setField('fulltext', $fullText);
  return $solrDoc;

Classes/Common/Solr/SolrSearch.php

@@ -59,6 +59,12 @@ class SolrSearch implements \Countable, \Iterator, \ArrayAccess, QueryResultInte
  */
  private ?QueryResult $listedMetadata;
 
+ /**
+ * @access private
+ * @var QueryResult|null
+ */
+ private ?QueryResult $indexedMetadata;
+
  /**
  * @access private
  * @var array
@@ -90,13 +96,14 @@ class SolrSearch implements \Countable, \Iterator, \ArrayAccess, QueryResultInte
  *
  * @return void
  */
- public function __construct(DocumentRepository $documentRepository, $collections, array $settings, array $searchParams, QueryResult $listedMetadata = null)
+ public function __construct(DocumentRepository $documentRepository, $collections, array $settings, array $searchParams, QueryResult $listedMetadata = null, QueryResult $indexedMetadata = null)
  {
  $this->documentRepository = $documentRepository;
  $this->collections = $collections;
  $this->settings = $settings;
  $this->searchParams = $searchParams;
  $this->listedMetadata = $listedMetadata;
+ $this->indexedMetadata = $indexedMetadata;
  }
 
  /**
@@ -664,6 +671,21 @@ protected function searchSolr($parameters = [], $enableCache = true)
  if ($enableCache === false || ($entry = $cache->get($cacheIdentifier)) === false) {
  $selectQuery = $solr->service->createSelect($parameters);
 
+ $edismax = $selectQuery->getEDisMax();
+
+ $queryFields = '';
+
+ if ($this->indexedMetadata) {
+ foreach ($this->indexedMetadata as $metadata) {
+ if ($metadata->getIndexIndexed()) {
+ $listMetadataRecord = $metadata->getIndexName() . '_' . ($metadata->getIndexTokenized() ? 't' : 'u') . ($metadata->getIndexStored() ? 's' : 'u') . 'i';
+ $queryFields .= $listMetadataRecord . '^' . $metadata->getIndexBoost() . ' ';
+ }
+ }
+ }
+
+ $edismax->setQueryFields($queryFields);
+
  $grouping = $selectQuery->getGrouping();
  $grouping->addField('uid');
  $grouping->setLimit(100); // Results in group (TODO: check)
@@ -837,7 +859,7 @@ private function getDocument(Document $record, array $highlighting, array $field
  */
  private function translateLanguageCode(&$doc): void
  {
- if ($doc['metadata']['language']) {
+ if (array_key_exists('language', $doc['metadata'])) {
  foreach($doc['metadata']['language'] as $indexName => $language) {
  $doc['metadata']['language'][$indexName] = Helper::getLanguageName($language);
  }

Classes/Controller/Backend/NewTenantController.php

@@ -22,11 +22,12 @@
 use Kitodo\Dlf\Domain\Repository\StructureRepository;
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Backend\View\BackendTemplateView;
+use TYPO3\CMS\Core\Core\Environment;
 use TYPO3\CMS\Core\Exception\SiteNotFoundException;
 use TYPO3\CMS\Core\Localization\LocalizationFactory;
+use TYPO3\CMS\Core\Resource\ResourceFactory;
 use TYPO3\CMS\Core\Site\Entity\NullSite;
 use TYPO3\CMS\Core\Site\SiteFinder;
-use TYPO3\CMS\Core\Utility\ExtensionManagementUtility;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Extbase\Mvc\View\ViewInterface;
 use TYPO3\CMS\Extbase\Persistence\Generic\PersistenceManager;
@@ -179,7 +180,7 @@ protected function initializeAction(): void
  public function addFormatAction(): void
  {
  // Include formats definition file.
- $formatsDefaults = include(ExtensionManagementUtility::extPath('dlf') . 'Resources/Private/Data/FormatDefaults.php');
+ $formatsDefaults = $this->getRecords('Format');
 
  $frameworkConfiguration = $this->configurationManager->getConfiguration($this->configurationManager::CONFIGURATION_TYPE_FRAMEWORK);
  // tx_dlf_formats are stored on PID = 0
@@ -221,7 +222,7 @@ public function addFormatAction(): void
  public function addMetadataAction(): void
  {
  // Include metadata definition file.
- $metadataDefaults = include(ExtensionManagementUtility::extPath('dlf') . 'Resources/Private/Data/MetadataDefaults.php');
+ $metadataDefaults = $this->getRecords('Metadata');
 
  // load language file in own array
  $metadataLabels = $this->languageFactory->getParsedData('EXT:dlf/Resources/Private/Language/locallang_metadata.xlf', $this->siteLanguages[0]->getTypo3Language());
@@ -344,7 +345,7 @@ public function addSolrCoreAction(): void
  public function addStructureAction(): void
  {
  // Include structure definition file.
- $structureDefaults = include(ExtensionManagementUtility::extPath('dlf') . 'Resources/Private/Data/StructureDefaults.php');
+ $structureDefaults = $this->getRecords('Structure');
 
  // load language file in own array
  $structureLabels = $this->languageFactory->getParsedData('EXT:dlf/Resources/Private/Language/locallang_structure.xlf', $this->siteLanguages[0]->getTypo3Language());
@@ -426,15 +427,15 @@ public function indexAction(): void
  $this->forward('error');
  }
 
- $formatsDefaults = include(ExtensionManagementUtility::extPath('dlf') . 'Resources/Private/Data/FormatDefaults.php');
+ $formatsDefaults = $this->getRecords('Format');
  $recordInfos['formats']['numCurrent'] = $this->formatRepository->countAll();
  $recordInfos['formats']['numDefault'] = count($formatsDefaults);
 
- $structuresDefaults = include(ExtensionManagementUtility::extPath('dlf') . 'Resources/Private/Data/StructureDefaults.php');
+ $structuresDefaults = $this->getRecords('Structure');
  $recordInfos['structures']['numCurrent'] = $this->structureRepository->countByPid($this->pid);
  $recordInfos['structures']['numDefault'] = count($structuresDefaults);
 
- $metadataDefaults = include(ExtensionManagementUtility::extPath('dlf') . 'Resources/Private/Data/MetadataDefaults.php');
+ $metadataDefaults = $this->getRecords('Metadata');
  $recordInfos['metadata']['numCurrent'] = $this->metadataRepository->countByPid($this->pid);
  $recordInfos['metadata']['numDefault'] = count($metadataDefaults);
 
@@ -459,15 +460,15 @@ public function errorAction(): void
  /**
  * Get language label for given key and language.
  * 
- * @access protected
+ * @access private
  *
  * @param string $index
  * @param string $lang
  * @param array $langArray
  *
  * @return string
  */
- protected function getLLL(string $index, string $lang, array $langArray): string
+ private function getLLL(string $index, string $lang, array $langArray): string
  {
  if (isset($langArray[$lang][$index][0]['target'])) {
  return $langArray[$lang][$index][0]['target'];
@@ -477,4 +478,31 @@ protected function getLLL(string $index, string $lang, array $langArray): string
  return 'Missing translation for ' . $index;
  }
  }
+
+ /**
+ * Get records from file for given record type.
+ *
+ * @access private
+ *
+ * @param string $recordType
+ *
+ * @return array
+ */
+ private function getRecords(string $recordType): array
+ {
+ $filePath = Environment::getPublicPath() . '/typo3conf/ext/dlf/Resources/Private/Data/' . $recordType . 'Defaults.json';
+
+ $resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
+ $fileObject = $resourceFactory->getFileObjectFromCombinedIdentifier($filePath);
+
+ if ($fileObject !== null) {
+ $fileContents = $fileObject->getContents();
+ $records = json_decode($fileContents, true);
+
+ if (json_last_error() === JSON_ERROR_NONE) {
+ return $records;
+ }
+ }
+ return [];
+ }
 }

Classes/Controller/CollectionController.php

@@ -151,13 +151,16 @@ public function showAction(Collection $collection): void
  // get all metadata records to be shown in results
  $listedMetadata = $this->metadataRepository->findByIsListed(true);
 
+ // get all indexed metadata fields
+ $indexedMetadata = $this->metadataRepository->findByIndexIndexed(true);
+
  // get all sortable metadata records
  $sortableMetadata = $this->metadataRepository->findByIsSortable(true);
 
  // get all documents of given collection
  $solrResults = null;
  if (is_array($searchParams) && !empty($searchParams)) {
- $solrResults = $this->documentRepository->findSolrByCollection($collection, $this->settings, $searchParams, $listedMetadata);
+ $solrResults = $this->documentRepository->findSolrByCollection($collection, $this->settings, $searchParams, $listedMetadata, $indexedMetadata);
 
  $itemsPerPage = $this->settings['list']['paginate']['itemsPerPage'];
  if (empty($itemsPerPage)) {

Classes/Controller/ListViewController.php

@@ -96,10 +96,13 @@ public function mainAction(): void
  // get all metadata records to be shown in results
  $listedMetadata = $this->metadataRepository->findByIsListed(true);
 
+ // get all indexed metadata fields
+ $indexedMetadata = $this->metadataRepository->findByIndexIndexed(true);
+
  $solrResults = null;
  $numResults = 0;
  if (is_array($this->searchParams) && !empty($this->searchParams)) {
- $solrResults = $this->documentRepository->findSolrByCollections($collections, $this->settings, $this->searchParams, $listedMetadata);
+ $solrResults = $this->documentRepository->findSolrByCollections($collections, $this->settings, $this->searchParams, $listedMetadata, $indexedMetadata);
  $numResults = $solrResults->getNumFound();
 
  $itemsPerPage = $this->settings['list']['paginate']['itemsPerPage'];

Classes/Controller/SearchController.php

@@ -172,11 +172,14 @@ public function mainAction(): void
  // get all metadata records to be shown in results
  $listedMetadata = $this->metadataRepository->findByIsListed(true);
 
+ // get all indexed metadata fields
+ $indexedMetadata = $this->metadataRepository->findByIndexIndexed(true);
+
  $solrResults = null;
  $numResults = 0;
  // Do not execute the Solr search if used together with ListView plugin.
  if (!$listViewSearch) {
- $solrResults = $this->documentRepository->findSolrWithoutCollection($this->settings, $this->searchParams, $listedMetadata);
+ $solrResults = $this->documentRepository->findSolrWithoutCollection($this->settings, $this->searchParams, $listedMetadata, $indexedMetadata);
  $numResults = $solrResults->getNumFound();
 
  $itemsPerPage = $this->settings['list']['paginate']['itemsPerPage'];

Classes/Domain/Repository/DocumentRepository.php

@@ -579,12 +579,13 @@ public function findChildrenOfEach(array $uids)
  * @param array $settings
  * @param array $searchParams
  * @param QueryResult $listedMetadata
+ * @param QueryResult $indexedMetadata
  *
  * @return SolrSearch
  */
- public function findSolrByCollection(Collection $collection, $settings, $searchParams, $listedMetadata = null)
+ public function findSolrByCollection(Collection $collection, $settings, $searchParams, $listedMetadata = null, $indexedMetadata = null)
  {
- return $this->findSolr([$collection], $settings, $searchParams, $listedMetadata);
+ return $this->findSolr([$collection], $settings, $searchParams, $listedMetadata, $indexedMetadata);
  }
 
  /**
@@ -596,12 +597,13 @@ public function findSolrByCollection(Collection $collection, $settings, $searchP
  * @param array $settings
  * @param array $searchParams
  * @param QueryResult $listedMetadata
+ * @param QueryResult $indexedMetadata
  *
  * @return SolrSearch
  */
- public function findSolrByCollections($collections, $settings, $searchParams, $listedMetadata = null): SolrSearch
+ public function findSolrByCollections($collections, $settings, $searchParams, $listedMetadata = null, $indexedMetadata = null): SolrSearch
  {
- return $this->findSolr($collections, $settings, $searchParams, $listedMetadata);
+ return $this->findSolr($collections, $settings, $searchParams, $listedMetadata, $indexedMetadata);
  }
 
  /**
@@ -612,12 +614,13 @@ public function findSolrByCollections($collections, $settings, $searchParams, $l
  * @param array $settings
  * @param array $searchParams
  * @param QueryResult $listedMetadata
+ * @param QueryResult $indexedMetadata
  *
  * @return SolrSearch
  */
- public function findSolrWithoutCollection($settings, $searchParams, $listedMetadata = null): SolrSearch
+ public function findSolrWithoutCollection($settings, $searchParams, $listedMetadata = null, $indexedMetadata = null): SolrSearch
  {
- return $this->findSolr([], $settings, $searchParams, $listedMetadata);
+ return $this->findSolr([], $settings, $searchParams, $listedMetadata, $indexedMetadata);
  }
 
  /**
@@ -632,13 +635,13 @@ public function findSolrWithoutCollection($settings, $searchParams, $listedMetad
  *
  * @return SolrSearch
  */
- private function findSolr($collections, $settings, $searchParams, $listedMetadata = null): SolrSearch
+ private function findSolr($collections, $settings, $searchParams, $listedMetadata = null, $indexedMetadata = null): SolrSearch
  {
  // set settings global inside this repository
  // (may be necessary when SolrSearch calls back)
  $this->settings = $settings;
 
- $search = new SolrSearch($this, $collections, $settings, $searchParams, $listedMetadata);
+ $search = new SolrSearch($this, $collections, $settings, $searchParams, $listedMetadata, $indexedMetadata);
  $search->prepare();
  return $search;
  }

Resources/Private/Data/FormatDefaults.json

@@ -0,0 +1,32 @@
+{
+ "MODS": {
+ "root": "mods",
+ "namespace": "http://www.loc.gov/mods/v3",
+ "class": "Kitodo\\Dlf\\Format\\Mods"
+ },
+ "TEIHDR": {
+ "root": "teiHeader",
+ "namespace": "http://www.tei-c.org/ns/1.0",
+ "class": "Kitodo\\Dlf\\Format\\TeiHeader"
+ },
+ "ALTO": {
+ "root": "alto",
+ "namespace": "http://www.loc.gov/standards/alto/ns-v2#",
+ "class": "Kitodo\\Dlf\\Format\\Alto"
+ },
+ "IIIF1": {
+ "root": "IIIF1",
+ "namespace": "http://www.shared-canvas.org/ns/context.json",
+ "class": ""
+ },
+ "IIIF2": {
+ "root": "IIIF2",
+ "namespace": "http://iiif.io/api/presentation/2/context.json",
+ "class": ""
+ },
+ "IIIF3": {
+ "root": "IIIF3",
+ "namespace": "http://iiif.io/api/presentation/3/context.json",
+ "class": ""
+ }
+}