Merge pull request #17701 from calixteman/alt_text_ai

[Editor] Add the possibility to query some ML stuff to guess an alt text for an image

src/display/editor/alt_text.js

@@ -76,6 +76,10 @@ class AltText {
     this.#altTextWasFromKeyBoard = false;
   }
 
+  isEmpty() {
+    return !this.#altText && !this.#altTextDecorative;
+  }
+
   get data() {
     return {
       altText: this.#altText,

src/display/editor/editor.js

@@ -970,6 +970,10 @@ class AnnotationEditor {
     this.#altText.data = data;
   }
 
+  hasAltText() {
+    return !this.#altText?.isEmpty();
+  }
+
   /**
    * Render this editor in a div.
    * @returns {HTMLDivElement | null}

src/display/editor/stamp.js

@@ -431,6 +431,42 @@ class StampEditor extends AnnotationEditor {
     const bitmap = this.#isSvg
       ? this.#bitmap
       : this.#scaleBitmap(width, height);
+
+    if (this._uiManager.hasMLManager && !this.hasAltText()) {
+      const offscreen = new OffscreenCanvas(width, height);
+      const ctx = offscreen.getContext("2d");
+      ctx.drawImage(
+        bitmap,
+        0,
+        0,
+        bitmap.width,
+        bitmap.height,
+        0,
+        0,
+        width,
+        height
+      );
+      offscreen.convertToBlob().then(blob => {
+        const fileReader = new FileReader();
+        fileReader.onload = () => {
+          const url = fileReader.result;
+          this._uiManager
+            .mlGuess({
+              service: "image-to-text",
+              request: {
+                imageData: url,
+              },
+            })
+            .then(response => {
+              const altText = response?.output || "";
+              if (this.parent && altText && !this.hasAltText()) {
+                this.altTextData = { altText, decorative: false };
+              }
+            });
+        };
+        fileReader.readAsDataURL(blob);
+      });
+    }
     const ctx = canvas.getContext("2d");
     ctx.filter = this._uiManager.hcmFilter;
     ctx.drawImage(

src/display/editor/tools.js

@@ -563,6 +563,8 @@ class AnnotationEditorUIManager {
 
   #mainHighlightColorPicker = null;
 
+  #mlManager = null;
+
   #mode = AnnotationEditorType.NONE;
 
   #selectedEditors = new Set();
@@ -749,7 +751,8 @@ class AnnotationEditorUIManager {
     eventBus,
     pdfDocument,
     pageColors,
-    highlightColors
+    highlightColors,
+    mlManager
   ) {
     this.#container = container;
     this.#viewer = viewer;
@@ -763,6 +766,7 @@ class AnnotationEditorUIManager {
     this.#filterFactory = pdfDocument.filterFactory;
     this.#pageColors = pageColors;
     this.#highlightColors = highlightColors || null;
+    this.#mlManager = mlManager || null;
     this.viewParameters = {
       realScale: PixelsPerInch.PDF_TO_CSS_UNITS,
       rotation: 0,
@@ -797,6 +801,14 @@ class AnnotationEditorUIManager {
     }
   }
 
+  async mlGuess(data) {
+    return this.#mlManager?.guess(data) || null;
+  }
+
+  get hasMLManager() {
+    return !!this.#mlManager;
+  }
+
   get hcmFilter() {
     return shadow(
       this,