Merge branch 'ReDoOCR' into 4.x-ocr

csidirop · Nov 28, 2023 · cd389bf · cd389bf
2 parents acf1f7e + 09ee1c8
commit cd389bf
Show file tree

Hide file tree

Showing 4 changed files with 71 additions and 40 deletions.
diff --git a/Classes/Controller/PageViewController.php b/Classes/Controller/PageViewController.php
@@ -144,27 +144,36 @@ protected function getFulltext($page)
         // Get fulltext link:
         $fileGrpsFulltext = GeneralUtility::trimExplode(',', $this->extConf['fileGrpFulltext']);
 
-        while ($fileGrpFulltext = array_shift($fileGrpsFulltext)) {
-            //check if and where fulltext is present:
-            if (!empty($this->document->getDoc()->physicalStructureInfo[$this->document->getDoc()->physicalStructure[$page]]['files'][$fileGrpFulltext])) { //fulltext is remote present
-                $fulltext['url'] = $this->document->getDoc()->getFileLocation($this->document->getDoc()->physicalStructureInfo[$this->document->getDoc()->physicalStructure[$page]]['files'][$fileGrpFulltext]);
-                if ($this->settings['useInternalProxy']) {
-                    // Configure @action URL for form.
-                    $uri = $this->uriBuilder->reset()
-                        ->setTargetPageUid($GLOBALS['TSFE']->id)
-                        ->setCreateAbsoluteUri(!empty($this->settings['forceAbsoluteUrl']) ? true : false)
-                        ->setArguments([
-                            'eID' => 'tx_dlf_pageview_proxy',
-                            'url' => $fulltext['url'],
-                            'uHash' => GeneralUtility::hmac($fulltext['url'], 'PageViewProxy')
-                            ])
-                        ->build();
-
-                    $fulltext['url'] = $uri;
+        if (PageViewController::getOCRengine(Doc::$extKey) == "originalremote") {
+            //check if remote fulltext exists:
+            while ($fileGrpFulltext = array_shift($fileGrpsFulltext)) { 
+                if (!empty($this->document->getDoc()->physicalStructureInfo[$this->document->getDoc()->physicalStructure[$page]]['files'][$fileGrpFulltext])) { //fulltext is remote present
+                    $fulltext['url'] = $this->document->getDoc()->getFileLocation($this->document->getDoc()->physicalStructureInfo[$this->document->getDoc()->physicalStructure[$page]]['files'][$fileGrpFulltext]);
+                    if ($this->settings['useInternalProxy']) {
+                        // Configure @action URL for form.
+                        $uri = $this->uriBuilder->reset()
+                            ->setTargetPageUid($GLOBALS['TSFE']->id)
+                            ->setCreateAbsoluteUri(!empty($this->settings['forceAbsoluteUrl']) ? true : false)
+                            ->setArguments([
+                                'eID' => 'tx_dlf_pageview_proxy',
+                                'url' => $fulltext['url'],
+                                'uHash' => GeneralUtility::hmac($fulltext['url'], 'PageViewProxy')
+                                ])
+                            ->build();
+
+                        $fulltext['url'] = $uri;
+                    }
+                    $fulltext['mimetype'] = $this->document->getDoc()->getFileMimeType($this->document->getDoc()->physicalStructureInfo[$this->document->getDoc()->physicalStructure[$page]]['files'][$fileGrpFulltext]);
+                    setcookie('tx-dlf-ocr-remotepresent', "Y", ['SameSite' => 'lax']);
+                    break;
+                } else { //no fulltext present
+                    $this->logger->notice('No full-text file found for page "' . $page . '" in fileGrp "' . $fileGrpFulltext . '"');
+                    setcookie('tx-dlf-ocr-remotepresent', "N", ['SameSite' => 'lax']);
                 }
-                $fulltext['mimetype'] = $this->document->getDoc()->getFileMimeType($this->document->getDoc()->physicalStructureInfo[$this->document->getDoc()->physicalStructure[$page]]['files'][$fileGrpFulltext]);
-                break;
-            } else if (FullTextGenerator::checkLocal(Doc::$extKey, $this->document, $page)) { //fulltext is locally present
+            }
+        } else {
+            //check if local fulltext exists:
+            if (FullTextGenerator::checkLocal(Doc::$extKey, $this->document, $page)) { //fulltext is locally present
                 //check server protocol (https://stackoverflow.com/a/14270161):
                 if ( isset($_SERVER['HTTPS'])  &&  ($_SERVER['HTTPS'] == 'on' || $_SERVER['HTTPS'] == 1)
                     ||  isset($_SERVER['HTTP_X_FORWARDED_PROTO'])  &&  $_SERVER['HTTP_X_FORWARDED_PROTO'] == 'https') {
@@ -174,10 +183,9 @@ protected function getFulltext($page)
                 }
                 $fulltext['url'] = $protocol . $_SERVER['HTTP_HOST'] . "/" . FullTextGenerator::getPageLocalPath(Doc::$extKey, $this->document, $page);
                 $fulltext['mimetype'] = "text/xml";
-            } else { //no fulltext present
-                $this->logger->notice('No full-text file found for page "' . $page . '" in fileGrp "' . $fileGrpFulltext . '"');
             }
         }
+
         if (empty($fulltext)) {
             $this->logger->notice('No full-text file found for page "' . $page . '" in fileGrps "' . $this->extConf['fileGrpFulltext'] . '"');
         }
@@ -390,14 +398,12 @@ protected function checkFulltextAvailability(int $page):array {
      */
     public static function getOCRengine(string $extKey):string {
         $conf = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get($extKey);
-        $ocrEngine = '';
 
-        if(!is_null($_COOKIE['tx-dlf-ocrEngine']) && str_contains(self::$ocrEngines, $_COOKIE['tx-dlf-ocrEngine'])){
-            $ocrEngine = $_COOKIE['tx-dlf-ocrEngine'];
+        if(!is_null($_COOKIE['tx-dlf-ocrEngine']) && (str_contains(self::$ocrEngines, $ocrEngine=$_COOKIE['tx-dlf-ocrEngine']) || $ocrEngine == "originalremote")){
+            return $ocrEngine;
         } else {
-            $ocrEngine = $conf['ocrEngine'] ; //get default default value
+            return $conf['ocrEngine'] ; //get default default value
         }
-        return $ocrEngine;
     }
 
     /**
@@ -408,19 +414,23 @@ public static function getOCRengine(string $extKey):string {
      * @return void
      */
     protected function generateFullText():void {
-        //OCR all pages: type=book
+        if(($engine = PageViewController::getOCRengine(Doc::$extKey)) == "originalremote") {
+            return;
+        }
+
+        // OCR all pages: (type=book)
         if($_POST["request"]["type"] == "book") {
             //collect all image urls:
             $images = array();
             for ($i=1; $i <= $this->document->getDoc()->numPages; $i++) {
                 $images[$i] = $this->getImage($i)["url"];
             }
-            FullTextGenerator::createBookFullText(Doc::$extKey, $this->document, $images, self::getOCRengine(Doc::$extKey));
+            FullTextGenerator::createBookFullText(Doc::$extKey, $this->document, $images, $engine);
             return;
         }
 
-        //OCR only this page
-        FullTextGenerator::createPageFullText(Doc::$extKey, $this->document, $this->getImage($this->requestData['page'])["url"], $this->requestData['page'], self::getOCRengine(Doc::$extKey));
+        // OCR only this page:
+        FullTextGenerator::createPageFullText(Doc::$extKey, $this->document, $this->getImage($this->requestData['page'])["url"], $this->requestData['page'], $engine);
     }
 
     /**

diff --git a/Classes/Plugin/FullTextGenerator.php b/Classes/Plugin/FullTextGenerator.php
@@ -65,7 +65,7 @@ protected static function getPageLocalId(Doc $doc, int $pageNum):string {
   public static function getDocLocalPath(string $extKey, Document $document):string {
     $conf = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ExtensionConfiguration::class)->get($extKey);
 
-    return $conf['fulltextFolder'] . "/" . self::generateUniqueDocLocalPath($document);
+    return $conf['fulltextFolder'] . self::generateUniqueDocLocalPath($document);
   }
 
   /**
@@ -212,7 +212,7 @@ protected static function generatePageOCR(string $extKey, array $conf, Document
     $outputFolderPath = "$documentPath/$ocrEngine";                   //Fulltextfolder (eg. fileadmin/fulltextFolder/URN/nbn/de/bsz/180/digosi/30/tesseract-basic/)
     $origMetsPath     = $documentPath."/".self::getDocLocalId($doc).".xml"; //Path to original METS (eg. fileadmin/fulltextFolder/URN/nbn/de/bsz/180/digosi/30/log59088.xml)
     $newMetsPath      = $outputFolderPath."/".self::getDocLocalId($doc).".xml"; //Path to updated METS
-    $outputPath       = "$outputFolderPath/$pageId.xml";              //Fulltextfile path
+    $outputPath       = "$outputFolderPath/$pageId.xml";              //Fulltextfile path (eg. fileadmin/fulltextFolder/URN/nbn/de/bsz/180/digosi/30/tesseract-basic/log59088_295.xml)
     $tmpOutputFolderPath = $conf['fulltextTempFolder'] . self::generateUniqueDocLocalPath($document) . "/$ocrEngine"; //(eg. fileadmin/_temp_/ocrTempFolder/fulltext/URN/nbn/de/bsz/180/digosi/30/tesseract-basic)
     $tmpImagePath     = $conf['fulltextImagesFolder'] . self::generateUniqueDocLocalPath($document) . "/$pageId"; //Imagefile path (eg. fileadmin/_temp_/ocrTempFolder/images/URN/nbn/de/bsz/180/digosi/30/log59088_1)
     $tmpOutputPath    = $tmpOutputFolderPath . "/$pageId";            //Fulltextfile temporary path (eg. fileadmin/_temp_/ocrTempFolder/fulltext/URN/nbn/de/bsz/180/digosi/30/tesseract-basic/log59088_295)

diff --git a/Classes/Plugin/Tools/FullTextGenerationScripts/OCRmain.sh b/Classes/Plugin/Tools/FullTextGenerationScripts/OCRmain.sh
@@ -73,7 +73,7 @@ done
 SECONDS=0 #messure time
 
 # Run given OCR-Engine:
-$ocrEngine --pageId $pageId --imagePath $imagePath --outputPath $tmpOutputPath --tmpImagePath $tmpImagePath
+$ocrEngine --pageId $pageId --pageNum $pageNum --imagePath $imagePath --outputPath $tmpOutputPath --tmpImagePath $tmpImagePath
 
 # Move temporary output file to final location, if it is not already there:
 if [ "$outputPath" != "$tmpOutputPath" ]; then 
@@ -83,7 +83,7 @@ fi
 
 # Update METS file:
 if [ "$ocrUpdateMets" == "1" ]; then
-	./typo3conf/ext/dlf/Classes/Plugin/Tools/FullTextGenerationScripts/UpdateMets.sh --pageId $pageId --outputPath $outputPath --url $url --ocrEngine $ocrEngine --ocrIndexMets $ocrIndexMets
+	./typo3conf/ext/dlf/Classes/Plugin/Tools/FullTextGenerationScripts/UpdateMets.sh --pageId $pageId --pageNum $pageNum --outputPath $outputPath --url $url --ocrEngine $ocrEngine --ocrIndexMets $ocrIndexMets
 fi
 
 echo -e "OCR completed in $SECONDS seconds"

diff --git a/Classes/Plugin/Tools/FullTextGenerationScripts/UpdateMets.sh b/Classes/Plugin/Tools/FullTextGenerationScripts/UpdateMets.sh
@@ -9,6 +9,7 @@ set -euo pipefail # exit on: error, undefined variable, pipefail
 while [ $# -gt 0 ] ; do
   case $1 in
   --pageId)       pageId="$2" ;;     # Page ID (eg. log59088_1)
+  --pageNum)      pageNum="$2" ;;    # Page number (eg. 1)
   --url)          url="$2" ;;        # Alto URL (eg. http://localhost/fileadmin/fulltextFolder//URN/nbn/de/bsz/180/digosi/30/tesseract-basic/log59088_1.xml)
   --outputPath)   outputPath="$2" ;; # Fulltextfile path (eg. /var/www/typo3/public/fileadmin/fulltextfolder/URN/nbn/de/bsz/180/digosi/30/tesseract-basic/log59088_1.xml)
   --ocrEngine)    ocrEngine="$2" ;;  # OCR-Engine (eg. /var/www/typo3/public/typo3conf/ext/dlf/Classes/Plugin/Tools/FullTextGenerationScripts/tesseract-basic.sh)
@@ -21,7 +22,7 @@ done
 
 # Extract some values from parameters:
 docLocalId=$(rev <<< "$pageId" | cut -d _ -f 2- | rev) # (eg. log_59088_1 -> log_59088)
-pageNum=$(rev <<< "$pageId" | cut -d _ -f 1 | rev) # (eg. log_59088_1 -> 1)
+#pageNum=$(rev <<< "$pageId" | cut -d _ -f 1 | rev) # (eg. log_59088_1 -> 1)
 outputFolder=$(rev <<< "$outputPath" | cut -d / -f 2- | rev) # (eg. /var/www/typo3/public/fileadmin/fulltextfolder/URN/nbn/de/bsz/180/digosi/30/tesseract-basic)
 ocrEngine=$(rev <<< "$ocrEngine" | cut -d '/' -f 1 | cut -d '.' -f 2- | rev) # (eg. tesseract-basic)
 metsUrl=$(rev <<< "$url" | cut -d / -f 2- | rev)"/$docLocalId.xml"
@@ -49,10 +50,30 @@ if [ $oai ] ; then
   mv mets_tmp.xml mets.xml
 fi
 
-# Update METS with given ALTO file:
-ocrd --log-level INFO workspace add --force --file-grp FULLTEXT --file-id "fulltext-$pageId" --page-id="fulltext-$pageNum" --mimetype text/xml "$url"
-xmlstarlet ed -L -a "//mets:file[@ID='fulltext-$pageId']" -t attr -n "CREATED" -v "$(date +%Y-%m-%dT%H:%M:%S%z)" mets.xml # Add Date attribute to file node
-xmlstarlet ed -L -a "//mets:file[@ID='fulltext-$pageId']" -t attr -n "SOFTWARE" -v "DFG-Viewer-OCR-On-Demand-$ocrEngine" mets.xml # Add OCR-ENGINE attribute to file node
+# Check if there is already a FULLTEXT section for the given pageId:
+# 1. Get all FILEIDs from structMap for given page number:
+physID=$(xmlstarlet sel -N mets="http://www.loc.gov/METS/" -t -v "//mets:div[@ORDER='$pageNum']/@ID" mets.xml)
+fileIdList=($(xmlstarlet sel -N mets="http://www.loc.gov/METS/" -t -v '//mets:structMap[@TYPE="PHYSICAL"]/mets:div/mets:div[@ID="'$physID'"]/mets:fptr/@FILEID' -n mets.xml ));
+# 2. Check if there is already a FULLTEXT section for the given fileId:
+updated=0 # Flag to check if METS was updated
+for fileId in "${fileIdList[@]}"; do
+  if [[ -n $(xmlstarlet sel -N mets="http://www.loc.gov/METS/" -t -v '//mets:fileSec/mets:fileGrp[@USE="FULLTEXT"]/mets:file[@ID="'$fileId'"]/mets:FLocat/@xlink:href' -n mets.xml) ]] ; then
+    updated=1 # Set flag to 1
+
+    # Update METS by updating existing elements with given ALTO file:
+    ocrd --log-level INFO workspace add --force --file-grp FULLTEXT --file-id "$fileId" --page-id="$physID" --mimetype text/xml "$url"
+    xmlstarlet ed -L -N mets="http://www.loc.gov/METS/" -u "//mets:file[@ID='$fileId']/@CREATED" -v "$(date +%Y-%m-%dT%H:%M:%S%z)" -i "//mets:file[@ID='$fileId'][not(@CREATED)]" -t attr -n "CREATED" -v "$(date +%Y-%m-%dT%H:%M:%S%z)" mets.xml  # Add/Update date attribute to file node
+    xmlstarlet ed -L -N mets="http://www.loc.gov/METS/" -u "//mets:file[@ID='$fileId']/@SOFTWARE" -v "DFG-Viewer-OCR-On-Demand-$ocrEngine" -i "//mets:file[@ID='$fileId'][not(@SOFTWARE)]" -t attr -n "SOFTWARE" -v "DFG-Viewer-OCR-On-Demand-$ocrEngine" mets.xml  # Add OCR-ENGINE attribute to file node
+  fi
+done
+
+if [[ $updated == 0 ]]; then # No FULLTEXT section for fileId
+  # Update METS by adding given ALTO file:
+  ocrd --log-level INFO workspace add --force --file-grp FULLTEXT --file-id "fulltext-$pageId" --page-id="$physID" --mimetype text/xml "$url"
+  xmlstarlet ed -L -N mets="http://www.loc.gov/METS/" -a "//mets:file[@ID='fulltext-$pageId']" -t attr -n "CREATED" -v "$(date +%Y-%m-%dT%H:%M:%S%z)" mets.xml # Add Date attribute to file node
+  xmlstarlet ed -L -N mets="http://www.loc.gov/METS/" -a "//mets:file[@ID='fulltext-$pageId']" -t attr -n "SOFTWARE" -v "DFG-Viewer-OCR-On-Demand-$ocrEngine" mets.xml # Add OCR-ENGINE attribute to file node
+  # ocrd workspace update-page --order "$pageNum" "$physID" # Update physical structMap if needed
+fi
 
 # Validate METS:
 #apt -y install libxml2-utils