Skip to content

Commit

Permalink
Merge pull request #813 from dvoracek-slub/dev-collection-search
Browse files Browse the repository at this point in the history 
[BUGFIX] Fix search in collection, allow multiple collections
  • Loading branch information
Alexander Bigga committed May 12, 2022
2 parents 72271f3 + 26b0a04 commit 1253ac2
Show file tree
Hide file tree
Showing 7 changed files with 1,968 additions and 22 deletions.
20 changes: 15 additions & 5 deletions Classes/Domain/Repository/DocumentRepository.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -514,7 +514,7 @@ private function findAllByUids($uids)
/**
* Find all documents with given collection from Solr
*
* @param \Kitodo\Dlf\Domain\Model\Collection $collection
* @param \TYPO3\CMS\Extbase\Persistence\Generic\QueryResult|\Kitodo\Dlf\Domain\Model\Collection $collection
* @param array $settings
* @param array $searchParams
* @param \TYPO3\CMS\Extbase\Persistence\Generic\QueryResult $listedMetadata
Expand Down Expand Up @@ -594,10 +594,20 @@ public function findSolrByCollection($collection, $settings, $searchParams, $lis

// if a collection is given, we prepare the collection query string
if ($collection) {
$collecionsQueryString = $collection->getIndexName();
$params['filterquery'][]['query'] = 'toplevel:true';
$params['filterquery'][]['query'] = 'partof:0';
$params['filterquery'][]['query'] = 'collection_faceting:("' . $collecionsQueryString . '")';
if ($collection instanceof \Kitodo\Dlf\Domain\Model\Collection) {
$collectionsQueryString = '"' . $collection->getIndexName() . '"';
} else {
$collectionsQueryString = '';
foreach ($collection as $index => $collectionEntry) {
$collectionsQueryString .= ($index > 0 ? ' OR ' : '') . '"' . $collectionEntry->getIndexName() . '"';
}
}

if (empty($query)) {
$params['filterquery'][]['query'] = 'toplevel:true';
$params['filterquery'][]['query'] = 'partof:0';
}
$params['filterquery'][]['query'] = 'collection_faceting:(' . $collectionsQueryString . ')';
}

// Set some query parameters.
Expand Down
1,726 changes: 1,726 additions & 0 deletions Tests/Fixtures/Common/documents_fulltext.solr.json
View file

Large diffs are not rendered by default.

116 changes: 116 additions & 0 deletions Tests/Fixtures/Common/documents_fulltext.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
<?xml version="1.0" encoding="utf-8"?>
<dataset>
<tx_dlf_documents>
<uid>533223312</uid>
<pid>20000</pid>
<tstamp>1652331576</tstamp>
<crdate>1652331576</crdate>
<cruser_id>0</cruser_id>
<deleted>0</deleted>
<hidden>0</hidden>
<starttime>0</starttime>
<endtime>0</endtime>
<fe_group></fe_group>
<prod_id></prod_id>
<location>https://digital.slub-dresden.de/data/kitodo/aufdesun_351357262/aufdesun_351357262_mets.xml</location>
<record_id>oai:de:slub-dresden:db:id-351357262</record_id>
<opac_id>351357262</opac_id>
<union_id>351357262</union_id>
<urn>urn:nbn:de:bsz:14-db-id3513572628</urn>
<purl>http://digital.slub-dresden.de/id351357262</purl>
<title>Auf der Suche nach Zukunft: Das Beispiel Pieschen</title>
<title_sorting>Auf der Suche nach Zukunft: Das Beispiel Pieschen</title_sorting>
<author></author>
<year>1990</year>
<place>Dresden</place>
<thumbnail>https://digital.slub-dresden.de/data/kitodo/aufdesun_351357262/aufdesun_351357262_tif/jpegs/00000001.tif.small.jpg</thumbnail>
<structure>40</structure>
<partof>0</partof>
<volume></volume>
<volume_sorting></volume_sorting>
<license></license>
<terms></terms>
<restrictions></restrictions>
<out_of_print></out_of_print>
<rights_info></rights_info>
<collections>2</collections>
<mets_label>Auf der Suche nach Zukunft: Das Beispiel Pieschen</mets_label>
<mets_orderlabel>Auf der Suche nach Zukunft: Das Beispiel Pieschen</mets_orderlabel>
<owner>3</owner>
<solrcore>1</solrcore>
<status>0</status>
<document_format>METS</document_format>
</tx_dlf_documents>

<tx_dlf_collections>
<uid>119128558</uid>
<pid>20000</pid>
<tstamp>1631522234</tstamp>
<crdate>1631522234</crdate>
<cruser_id>2</cruser_id>
<deleted>0</deleted>
<sys_language_uid>0</sys_language_uid>
<l18n_parent>0</l18n_parent>
<l18n_diffsource>...</l18n_diffsource>
<hidden>0</hidden>
<fe_group></fe_group>
<l10n_state>NULL</l10n_state>
<fe_cruser_id>0</fe_cruser_id>
<fe_admin_lock>0</fe_admin_lock>
<label>Saxonica</label>
<index_name>Saxonica</index_name>
<index_search></index_search>
<oai_name>saxonica</oai_name>
<description></description>
<thumbnail></thumbnail>
<priority>3</priority>
<documents>0</documents>
<owner>0</owner>
<status>0</status>
</tx_dlf_collections>

<tx_dlf_collections>
<uid>332405524</uid>
<pid>20000</pid>
<tstamp>1652331460</tstamp>
<crdate>1652331460</crdate>
<cruser_id>0</cruser_id>
<deleted>0</deleted>
<sys_language_uid>0</sys_language_uid>
<l18n_parent>0</l18n_parent>
<l18n_diffsource></l18n_diffsource>
<hidden>0</hidden>
<fe_group></fe_group>
<l10n_state>NULL</l10n_state>
<fe_cruser_id>0</fe_cruser_id>
<fe_admin_lock>0</fe_admin_lock>
<label>Projekt: Dresdner Hefte</label>
<index_name>Projekt: Dresdner Hefte</index_name>
<index_search></index_search>
<oai_name>projekt-dresdner-hefte</oai_name>
<description></description>
<thumbnail>0</thumbnail>
<priority>3</priority>
<documents>0</documents>
<owner>0</owner>
<status>0</status>
</tx_dlf_collections>

<tx_dlf_relations>
<uid_local>533223312</uid_local>
<uid_foreign>119128558</uid_foreign>
<tablenames />
<sorting />
<sorting_foreign />
<ident>docs_colls</ident>
</tx_dlf_relations>

<tx_dlf_relations>
<uid_local>533223312</uid_local>
<uid_foreign>332405524</uid_foreign>
<tablenames />
<sorting />
<sorting_foreign />
<ident>docs_colls</ident>
</tx_dlf_relations>
</dataset>
43 changes: 43 additions & 0 deletions Tests/Fixtures/Common/metadata.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,35 @@
<status>0</status>
</tx_dlf_metadata>

<tx_dlf_metadata>
<uid>5002</uid>
<pid>20000</pid>
<tstamp>1638557803</tstamp>
<crdate>1631532810</crdate>
<cruser_id>1</cruser_id>
<deleted>0</deleted>
<sys_language_uid>0</sys_language_uid>
<l18n_parent>0</l18n_parent>
<l18n_diffsource></l18n_diffsource>
<hidden>0</hidden>
<sorting>32</sorting>
<l10n_state></l10n_state>
<label>Sammlungen</label>
<index_name>collection</index_name>
<format>1</format>
<default_value></default_value>
<wrap></wrap>
<index_tokenized>1</index_tokenized>
<index_stored>0</index_stored>
<index_indexed>1</index_indexed>
<index_boost>1</index_boost>
<is_sortable>0</is_sortable>
<is_facet>1</is_facet>
<is_listed>0</is_listed>
<index_autocomplete>1</index_autocomplete>
<status>0</status>
</tx_dlf_metadata>

<tx_dlf_metadataformat>
<uid>5101</uid>
<pid>20000</pid>
Expand All @@ -43,6 +72,20 @@
<mandatory>0</mandatory>
</tx_dlf_metadataformat>

<tx_dlf_metadataformat>
<uid>5102</uid>
<pid>20000</pid>
<tstamp>1638557803</tstamp>
<crdate>1631532810</crdate>
<cruser_id>1</cruser_id>
<deleted>0</deleted>
<parent_id>5002</parent_id>
<encoded>5202</encoded>
<xpath>./mods:relatedItem[@type="series"]/mods:titleInfo/mods:title[@lang="ger"]</xpath>
<xpath_sorting />
<mandatory>0</mandatory>
</tx_dlf_metadataformat>

<tx_dlf_formats>
<uid>5201</uid>
<pid>0</pid>
Expand Down
17 changes: 0 additions & 17 deletions Tests/Functional/Api/OaiPmhTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,23 +71,6 @@ protected function setUpOaiSolr()
$this->persistenceManager->persistAll();
}

protected function importSolrDocuments(Solr $solr, string $path)
{
$jsonDocuments = json_decode(file_get_contents($path), true);

$updateQuery = $solr->service->createUpdate();
$documents = array_map(function ($jsonDoc) use ($updateQuery) {
$document = $updateQuery->createDocument();
foreach ($jsonDoc as $key => $value) {
$document->setField($key, $value);
}
return $document;
}, $jsonDocuments);
$updateQuery->addDocuments($documents);
$updateQuery->addCommit();
$solr->service->update($updateQuery);
}

/**
* @test
*/
Expand Down
47 changes: 47 additions & 0 deletions Tests/Functional/Common/SolrIndexingTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
use Kitodo\Dlf\Common\Doc;
use Kitodo\Dlf\Common\Indexer;
use Kitodo\Dlf\Common\Solr;
use Kitodo\Dlf\Domain\Model\Collection;
use Kitodo\Dlf\Domain\Model\SolrCore;
use Kitodo\Dlf\Domain\Repository\CollectionRepository;
use Kitodo\Dlf\Domain\Repository\DocumentRepository;
use Kitodo\Dlf\Domain\Repository\SolrCoreRepository;
use Kitodo\Dlf\Tests\Functional\FunctionalTestCase;
Expand All @@ -19,6 +21,9 @@ class SolrIndexingTest extends FunctionalTestCase
/** @var PersistenceManager */
protected $persistenceManager;

/** @var CollectionRepository */
protected $collectionRepository;

/** @var DocumentRepository */
protected $documentRepository;

Expand All @@ -34,6 +39,7 @@ public function setUp(): void

$this->persistenceManager = $this->objectManager->get(PersistenceManager::class);

$this->collectionRepository = $this->initializeRepository(CollectionRepository::class, 20000);
$this->documentRepository = $this->initializeRepository(DocumentRepository::class, 20000);
$this->solrCoreRepository = $this->initializeRepository(SolrCoreRepository::class, 20000);

Expand Down Expand Up @@ -99,6 +105,47 @@ public function canIndexAndSearchDocument()
$this->assertEquals($document->getTitle(), $result['documents'][$document->getUid()]['title']);
}

/**
* @test
*/
public function canSearchInCollections()
{
$core = $this->createSolrCore();

$this->importDataSet(__DIR__ . '/../../Fixtures/Common/documents_fulltext.xml');
$this->importSolrDocuments($core->solr, __DIR__ . '/../../Fixtures/Common/documents_1.solr.json');
$this->importSolrDocuments($core->solr, __DIR__ . '/../../Fixtures/Common/documents_fulltext.solr.json');

$collections = $this->collectionRepository->findCollectionsBySettings([
'index_name' => ['Musik', 'Projekt: Dresdner Hefte'],
]);
$musik = $collections[0];
$dresdnerHefte = $collections[1];

$settings = [
'solrcore' => $core->solr->core,
'storagePid' => 20000,
];

// No query: Only list toplevel result(s) in collection(s)
$musikResults = $this->documentRepository->findSolrByCollection($musik, $settings, []);
$dresdnerHefteResults = $this->documentRepository->findSolrByCollection($dresdnerHefte, $settings, []);
$multiCollectionResults = $this->documentRepository->findSolrByCollection($collections, $settings, []);
$this->assertGreaterThanOrEqual(1, $musikResults['solrResults']['numFound']);
$this->assertGreaterThanOrEqual(1, $dresdnerHefteResults['solrResults']['numFound']);
$this->assertEquals('533223312LOG_0000', $dresdnerHefteResults['solrResults']['documents'][0]['id']);
$this->assertEquals(
// Assuming there's no overlap
$dresdnerHefteResults['solrResults']['numFound'] + $musikResults['solrResults']['numFound'],
$multiCollectionResults['solrResults']['numFound']
);

// With query: List all results
$metadataResults = $this->documentRepository->findSolrByCollection($dresdnerHefte, $settings, ['query' => 'Dresden']);
$fulltextResults = $this->documentRepository->findSolrByCollection($dresdnerHefte, $settings, ['query' => 'Dresden', 'fulltext' => '1']);
$this->assertGreaterThan($metadataResults['solrResults']['numFound'], $fulltextResults['solrResults']['numFound']);
}

protected function createSolrCore(): object
{
$coreName = Solr::createCore();
Expand Down
21 changes: 21 additions & 0 deletions Tests/Functional/FunctionalTestCase.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
namespace Kitodo\Dlf\Tests\Functional;

use GuzzleHttp\Client as HttpClient;
use Kitodo\Dlf\Common\Solr;
use Symfony\Component\Yaml\Yaml;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager;
Expand Down Expand Up @@ -145,4 +146,24 @@ protected function initializeRepository(string $className, int $storagePid)

return $repository;
}

protected function importSolrDocuments(Solr $solr, string $path)
{
$jsonDocuments = json_decode(file_get_contents($path), true);

$updateQuery = $solr->service->createUpdate();
$documents = array_map(function ($jsonDoc) use ($updateQuery) {
$document = $updateQuery->createDocument();
foreach ($jsonDoc as $key => $value) {
$document->setField($key, $value);
}
if (isset($jsonDoc['collection'])) {
$document->setField('collection_faceting', $jsonDoc['collection']);
}
return $document;
}, $jsonDocuments);
$updateQuery->addDocuments($documents);
$updateQuery->addCommit();
$solr->service->update($updateQuery);
}
}

0 comments on commit 1253ac2

Please sign in to comment.