[api-minor] Don't normalize the text used in the text layer.

Some arabic chars like \ufe94 could be searched in a pdf, hence it must be normalized
when creating the search query. So to avoid to duplicate the normalization code,
everything is moved in the find controller.
The previous code to normalize text was using NFKC but with a hardcoded map, hence it
has been replaced by the use of normalize("NFKC") (it helps to reduce the bundle size
by 30kb).
In playing with this \ufe94 char, I noticed that the bidi algorithm wasn't taking into
account some RTL unicode ranges, the generated font wasn't embedding the mapping this
char and the unicode ranges in the OS/2 table weren't up-to-date.

When normalized some chars can be replaced by several ones and it induced to have
some extra chars in the text layer. To avoid any regression, when copying some text
from the text layer, a copied string is normalized (NFKC) before being put in the
clipboard (it works like this in either Acrobat or Chrome).

web/pdf_find_controller.js

@@ -18,8 +18,8 @@
 /** @typedef {import("./interfaces").IPDFLinkService} IPDFLinkService */
 
 import { binarySearchFirstItem, scrollIntoView } from "./ui_utils.js";
+import { getCharacterType, getNormalizeWithNFKC } from "./pdf_find_utils.js";
 import { createPromiseCapability } from "pdfjs-lib";
-import { getCharacterType } from "./pdf_find_utils.js";
 
 const FindState = {
   FOUND: 0,
@@ -126,12 +126,7 @@ function normalize(text) {
   } else {
     // Compile the regular expression for text normalization once.
     const replace = Object.keys(CHARACTERS_TO_NORMALIZE).join("");
-    const toNormalizeWithNFKC =
-      "\u2460-\u2473" + // Circled numbers.
-      "\u24b6-\u24ff" + // Circled letters/numbers.
-      "\u3244-\u32bf" + // Circled ideograms/numbers.
-      "\u32d0-\u32fe" + // Circled ideograms.
-      "\uff00-\uffef"; // Halfwidth, fullwidth forms.
+    const toNormalizeWithNFKC = getNormalizeWithNFKC();
 
     // 3040-309F: Hiragana
     // 30A0-30FF: Katakana
@@ -840,6 +835,7 @@ class PDFFindController {
     }
 
     let promise = Promise.resolve();
+    const textOptions = { disableNormalization: true };
     for (let i = 0, ii = this._linkService.pagesCount; i < ii; i++) {
       const extractTextCapability = createPromiseCapability();
       this._extractTextPromises[i] = extractTextCapability.promise;
@@ -848,7 +844,7 @@ class PDFFindController {
         return this._pdfDocument
           .getPage(i + 1)
           .then(pdfPage => {
-            return pdfPage.getTextContent();
+            return pdfPage.getTextContent(textOptions);
           })
           .then(
             textContent => {

web/pdf_find_utils.js

@@ -112,4 +112,46 @@ function getCharacterType(charCode) {
   return CharacterType.ALPHA_LETTER;
 }
 
-export { CharacterType, getCharacterType };
+let NormalizeWithNFKC;
+function getNormalizeWithNFKC() {
+  /* eslint-disable no-irregular-whitespace */
+  NormalizeWithNFKC ||= ` ¨ª¯²-µ¸-º¼-¾IJ-ijĿ-ŀʼnſDŽ-njDZ-dzʰ-ʸ˘-˝ˠ-ˤʹͺ;΄-΅·ϐ-ϖϰ-ϲϴ-ϵϹևٵ-ٸक़-य़ড়-ঢ়য়ਲ਼ਸ਼ਖ਼-ਜ਼ਫ਼ଡ଼-ଢ଼ำຳໜ-ໝ༌གྷཌྷདྷབྷཛྷཀྵჼᴬ-ᴮᴰ-ᴺᴼ-ᵍᵏ-ᵪᵸᶛ-ᶿẚ-ẛάέήίόύώΆ᾽-῁ΈΉ῍-῏ΐΊ῝-῟ΰΎ῭-`ΌΏ´-῾ - ‑‗․-… ″-‴‶-‷‼‾⁇-⁉⁗ ⁰-ⁱ⁴-₎ₐ-ₜ₨℀-℃℅-ℇ℉-ℓℕ-№ℙ-ℝ℠-™ℤΩℨK-ℭℯ-ℱℳ-ℹ℻-⅀ⅅ-ⅉ⅐-ⅿ↉∬-∭∯-∰〈-〉①-⓪⨌⩴-⩶⫝̸ⱼ-ⱽⵯ⺟⻳⼀-⿕　〶〸-〺゛-゜ゟヿㄱ-ㆎ㆒-㆟㈀-㈞㈠-㉇㉐-㉾㊀-㏿ꚜ-ꚝꝰꟲ-ꟴꟸ-ꟹꭜ-ꭟꭩ豈-嗀塚晴凞-羽蘒諸逸-都飯-舘並-龎ff-stﬓ-ﬗיִײַ-זּטּ-לּמּנּ-סּףּ-פּצּ-ﮱﯓ-ﴽﵐ-ﶏﶒ-ﷇﷰ-﷼︐-︙︰-﹄﹇-﹒﹔-﹦﹨-﹫ﹰ-ﹲﹴﹶ-ﻼ！-ﾾￂ-ￇￊ-ￏￒ-ￗￚ-ￜ￠-￦`;
+
+  if (typeof PDFJSDev === "undefined" || PDFJSDev.test("TESTING")) {
+    const ranges = [];
+    const range = [];
+    const diacriticsRegex = /^\p{M}$/u;
+    // Some chars must be replaced by their NFKC counterpart during a search.
+    for (let i = 0; i < 65536; i++) {
+      const c = String.fromCharCode(i);
+      if (c.normalize("NFKC") !== c && !diacriticsRegex.test(c)) {
+        if (range.length !== 2) {
+          range[0] = range[1] = i;
+          continue;
+        }
+        if (range[1] + 1 !== i) {
+          if (range[0] === range[1]) {
+            ranges.push(String.fromCharCode(range[0]));
+          } else {
+            ranges.push(
+              `${String.fromCharCode(range[0])}-${String.fromCharCode(
+                range[1]
+              )}`
+            );
+          }
+          range[0] = range[1] = i;
+        } else {
+          range[1] = i;
+        }
+      }
+    }
+    if (ranges.join("") !== NormalizeWithNFKC) {
+      throw new Error(
+        "getNormalizeWithNFKC - update the `NormalizeWithNFKC` string."
+      );
+    }
+  }
+  return NormalizeWithNFKC;
+}
+
+export { CharacterType, getCharacterType, getNormalizeWithNFKC };

web/pdf_page_view.js

@@ -368,6 +368,7 @@ class PDFPageView {
       if (!textLayer.renderingDone) {
         const readableStream = pdfPage.streamTextContent({
           includeMarkedContent: true,
+          disableNormalization: true,
         });
         textLayer.setTextContentSource(readableStream);
       }

web/pdf_viewer.js

@@ -665,6 +665,8 @@ class PDFViewer {
       }
       buffer.length = 0;
       const page = await this.pdfDocument.getPage(pageNum);
+      // By default getTextContent pass disableNormalization equals to false
+      // which is fine because we want a normalized string.
       const { items } = await page.getTextContent();
       for (const item of items) {
         if (item.str) {

web/text_highlighter.js

@@ -208,9 +208,20 @@ class TextHighlighter {
       return;
     }
 
+    let lastDivIdx = -1;
+    let lastOffset = -1;
     for (let i = i0; i < i1; i++) {
       const match = matches[i];
       const begin = match.begin;
+      if (begin.divIdx === lastDivIdx && begin.offset === lastOffset) {
+        // It's possible to be in this situation if we searched for a 'f' and we
+        // have a ligature 'ff' in the text. The 'ff' has to be highlighted two
+        // times.
+        continue;
+      }
+      lastDivIdx = begin.divIdx;
+      lastOffset = begin.offset;
+
       const end = match.end;
       const isSelected = isSelectedPage && i === selectedMatchIdx;
       const highlightSuffix = isSelected ? " selected" : "";

web/text_layer_builder.js

@@ -20,7 +20,8 @@
 // eslint-disable-next-line max-len
 /** @typedef {import("./text_accessibility.js").TextAccessibilityManager} TextAccessibilityManager */
 
-import { renderTextLayer, updateTextLayer } from "pdfjs-lib";
+import { normalizeUnicode, renderTextLayer, updateTextLayer } from "pdfjs-lib";
+import { removeNullCharacters } from "./ui_utils.js";
 
 /**
  * @typedef {Object} TextLayerBuilderOptions
@@ -212,6 +213,16 @@ class TextLayerBuilder {
       }
       end.classList.remove("active");
     });
+
+    div.addEventListener("copy", event => {
+      const selection = document.getSelection();
+      event.clipboardData.setData(
+        "text/plain",
+        removeNullCharacters(normalizeUnicode(selection.toString()))
+      );
+      event.preventDefault();
+      event.stopPropagation();
+    });
   }
 }