More fixes to text analyzer

src/core/module/content-rect.js

@@ -109,7 +109,8 @@ export async function getContentRect(pdfDocument, structuredCharsProvider) {
   let { view } = await pdfDocument.getPage(numPages === 2 ? 1 : 0);
   let width = view[2] - view[0];
   rect[0] = 0;
-  rect[2] = width;
-
+  // Note: Even if page width/height is the same, some pages, in the same PDF,
+  // can have bigger mediaBox with a smaller cropBox applied on it
+  rect[2] = Infinity;
   return rect;
 }

src/core/module/page-label.js

@@ -190,6 +190,12 @@ function getLabelSequence(words) {
   return sequences.sort((a, b) => b.length - a.length)[0];
 }
 
+function getClusterMaxDistance(cluster, property) {
+  let min = Math.min(...cluster.map(x => x[property]));
+  let max = Math.max(...cluster.map(x => x[property]));
+  return max - min;
+}
+
 export async function getPageLabel(pdfDocument, structuredCharsProvider, pageIndex, metadataPagesField) {
   const NEXT_PREV_PAGES = 2;
   let numPages = pdfDocument.catalog.numPages;
@@ -221,14 +227,27 @@ export async function getPageLabel(pdfDocument, structuredCharsProvider, pageInd
     }
   }
 
-  let yClusters = getClusters(candidateWords, 'relativeY', 5);
+  let eps = 5;
+
+  let yClusters = getClusters(candidateWords, 'relativeY', eps);
 
   let bestSequence = [];
   for (let yCluster of yClusters) {
-    let clusters = getClusters(yCluster, 'relativeX', 5);
+    let clusters = getClusters(yCluster, 'relativeX', eps);
     for (let cluster of clusters) {
+      // Ignore clusters with too many values
+      if (cluster.length > eps * 5) {
+        continue;
+      }
       let sequence = getLabelSequence(cluster);
-      if (sequence && sequence.length > bestSequence) {
+
+      if (
+        sequence &&
+        sequence.length > bestSequence &&
+        // Make sure the final sequence page label min and max distance doesn't surpass eps
+        getClusterMaxDistance(sequence, 'relativeY') <= eps &&
+        getClusterMaxDistance(sequence, 'relativeX') <= eps
+      ) {
         bestSequence = sequence;
       }
     }

src/core/module/reference-extractor.js

@@ -19,7 +19,7 @@ function removeASCIISymbolsAndNumbers(inputString) {
 }
 
 function getReferencesTitleOffset(chars) {
-  let titles = ['references', 'bibliography', 'literature', 'bibliographie'];
+  let titles = ['references', 'bibliography', 'literature', 'bibliographie', 'literatur'];
 
   let paragraphs = []; // This will hold the start and end indices of each paragraph
   let start = 0; // Start index of the current paragraph
@@ -244,6 +244,10 @@ function extractByLayout(chars) {
 
   let clusters = getClusters(deltas, 'delta', 1);
 
+  if (!clusters.length) {
+    return [];
+  }
+
   let paragraphBreaks = clusters[0];
 
   // Extracting by layout depends on first line of each reference being shifted more on the left
@@ -320,11 +324,11 @@ function extractByLayout(chars) {
     references.push({
       text,
       chars,
-      position
+      position,
     });
   }
 
-  return references
+  return references;
 }
 
 function extractByParagraphSpacing(chars) {

src/core/module/structure.js

@@ -906,7 +906,9 @@ function split(chars, reflowRTL) {
   }
 
   for (let char of chars) {
-    if (char.lineBreakAfter && !char.paragraphBreakAfter && dashChars.has(char.c)) {
+    // OCRed PDFs sometimes result in each line being a separate paragraph
+    // while, normal PDFs only need this when paragraph is wrapped to another column
+    if (char.lineBreakAfter /*&& !char.paragraphBreakAfter*/ && dashChars.has(char.c)) {
       char.ignorable = true;
     }
   }