mirrors/pdf.js
1
0
Fork 0
mirror of https://github.com/mozilla/pdf.js.git synced 2025-04-22 16:18:08 +02:00
Code Activity

Implement the new alt text flow (bug 1909604)

Browse source 
For the Firefox pdf viewer, we want to use AI to guess an alt-text when adding an image to a pdf.
For now the telemtry stuff is not implemented and will come soon.
In order to test it locally:
 - set enableAltText, enableFakeMLManager and enableUpdatedAddImage to true.
or in Firefox:
 - set browser.ml.enable, pdfjs.enableAltText and pdfjs.enableUpdatedAddImage to true.
This commit is contained in:
Calixte Denizet 2024-07-24 09:34:32 +02:00
parent 8378c40f4c
commit ed22d934e5
22 changed files with 1366 additions and 91 deletions
Download patch file Download diff file Expand all files Collapse all files

150
src/display/editor/alt_text.js
View file

@ -16,7 +16,7 @@
import { noContextMenu } from "../display_utils.js";
class AltText {
#altText = "";
#altText = null;
#altTextDecorative = false;
@ -28,12 +28,21 @@ class AltText {
#altTextWasFromKeyBoard = false;
#badge = null;
#editor = null;
#guessedText = null;
#textWithDisclaimer = null;
#useNewAltTextFlow = false;
static _l10nPromise = null;
constructor(editor) {
this.#editor = editor;
this.#useNewAltTextFlow = editor._uiManager.useNewAltTextFlow;
}
static initialize(l10nPromise) {
@ -43,9 +52,17 @@ class AltText {
async render() {
const altText = (this.#altTextButton = document.createElement("button"));
altText.className = "altText";
const msg = await AltText._l10nPromise.get(
"pdfjs-editor-alt-text-button-label"
);
let msg;
if (this.#useNewAltTextFlow) {
altText.classList.add("new");
msg = await AltText._l10nPromise.get(
"pdfjs-editor-new-alt-text-missing-button-label"
);
} else {
msg = await AltText._l10nPromise.get(
"pdfjs-editor-alt-text-button-label"
);
}
altText.textContent = msg;
altText.setAttribute("aria-label", msg);
altText.tabIndex = "0";
@ -84,9 +101,62 @@ class AltText {
}
isEmpty() {
if (this.#useNewAltTextFlow) {
return this.#altText === null;
}
return !this.#altText && !this.#altTextDecorative;
}
hasData() {
if (this.#useNewAltTextFlow) {
return this.#altText !== null || !!this.#guessedText;
}
return this.isEmpty();
}
get guessedText() {
return this.#guessedText;
}
async setGuessedText(guessedText) {
if (this.#altText !== null) {
// The user provided their own alt text, so we don't want to overwrite it.
return;
}
this.#guessedText = guessedText;
this.#textWithDisclaimer = await AltText._l10nPromise.get(
"pdfjs-editor-new-alt-text-generated-alt-text-with-disclaimer"
)({ generatedAltText: guessedText });
this.#setState();
}
toggleAltTextBadge(visibility = false) {
if (!this.#useNewAltTextFlow || this.#altText) {
this.#badge?.remove();
this.#badge = null;
return;
}
if (!this.#badge) {
const badge = (this.#badge = document.createElement("div"));
badge.className = "noAltTextBadge";
this.#editor.div.append(badge);
}
this.#badge.classList.toggle("hidden", !visibility);
}
serialize(isForCopying) {
let altText = this.#altText;
if (!isForCopying && this.#guessedText === altText) {
altText = this.#textWithDisclaimer;
}
return {
altText,
decorative: this.#altTextDecorative,
guessedText: this.#guessedText,
textWithDisclaimer: this.#textWithDisclaimer,
};
}
get data() {
return {
altText: this.#altText,
@ -97,12 +167,24 @@ class AltText {
/**
* Set the alt text data.
*/
set data({ altText, decorative }) {
set data({
altText,
decorative,
guessedText,
textWithDisclaimer,
cancel = false,
}) {
if (guessedText) {
this.#guessedText = guessedText;
this.#textWithDisclaimer = textWithDisclaimer;
}
if (this.#altText === altText && this.#altTextDecorative === decorative) {
return;
}
this.#altText = altText;
this.#altTextDecorative = decorative;
if (!cancel) {
this.#altText = altText;
this.#altTextDecorative = decorative;
}
this.#setState();
}
@ -121,6 +203,8 @@ class AltText {
this.#altTextButton?.remove();
this.#altTextButton = null;
this.#altTextTooltip = null;
this.#badge?.remove();
this.#badge = null;
}
async #setState() {
@ -128,18 +212,48 @@ class AltText {
if (!button) {
return;
}
if (!this.#altText && !this.#altTextDecorative) {
button.classList.remove("done");
this.#altTextTooltip?.remove();
return;
}
button.classList.add("done");
AltText._l10nPromise
.get("pdfjs-editor-alt-text-edit-button-label")
.then(msg => {
button.setAttribute("aria-label", msg);
});
if (this.#useNewAltTextFlow) {
// If we've an alt text, we get an "added".
// If we've a guessed text and the alt text has never been set, we get a
// "to-review" been set.
// Otherwise, we get a "missing".
const type =
(this.#altText && "added") ||
(this.#altText === null && this.guessedText && "to-review") ||
"missing";
button.classList.toggle("done", !!this.#altText);
AltText._l10nPromise
.get(`pdfjs-editor-new-alt-text-${type}-button-label`)
.then(msg => {
button.setAttribute("aria-label", msg);
// We can't just use button.textContent here, because it would remove
// the existing tooltip element.
for (const child of button.childNodes) {
if (child.nodeType === Node.TEXT_NODE) {
child.textContent = msg;
break;
}
}
});
if (!this.#altText) {
this.#altTextTooltip?.remove();
return;
}
} else {
if (!this.#altText && !this.#altTextDecorative) {
button.classList.remove("done");
this.#altTextTooltip?.remove();
return;
}
button.classList.add("done");
AltText._l10nPromise
.get("pdfjs-editor-alt-text-edit-button-label")
.then(msg => {
button.setAttribute("aria-label", msg);
});
}
let tooltip = this.#altTextTooltip;
if (!tooltip) {
this.#altTextTooltip = tooltip = document.createElement("span");

68
src/display/editor/editor.js
View file

@ -58,8 +58,6 @@ class AnnotationEditor {
#boundFocusout = this.focusout.bind(this);
#editToolbar = null;
#focusedResizerName = "";
#hasBeenClicked = false;
@ -80,6 +78,8 @@ class AnnotationEditor {
#telemetryTimeouts = null;
_editToolbar = null;
_initialOptions = Object.create(null);
_isVisible = true;
@ -210,6 +210,9 @@ class AnnotationEditor {
"pdfjs-editor-alt-text-button-label",
"pdfjs-editor-alt-text-edit-button-label",
"pdfjs-editor-alt-text-decorative-tooltip",
"pdfjs-editor-new-alt-text-added-button-label",
"pdfjs-editor-new-alt-text-missing-button-label",
"pdfjs-editor-new-alt-text-to-review-button-label",
"pdfjs-editor-resizer-label-topLeft",
"pdfjs-editor-resizer-label-topMiddle",
"pdfjs-editor-resizer-label-topRight",
@ -223,6 +226,18 @@ class AnnotationEditor {
l10n.get(str.replaceAll(/([A-Z])/g, c => `-${c.toLowerCase()}`)),
])
);
// The string isn't in the above list because the string has a parameter
// (i.e. the guessed text) and we must pass it to the l10n function to get
// the correct translation.
AnnotationEditor._l10nPromise.set(
"pdfjs-editor-new-alt-text-generated-alt-text-with-disclaimer",
l10n.get.bind(
l10n,
"pdfjs-editor-new-alt-text-generated-alt-text-with-disclaimer"
)
);
if (options?.strings) {
for (const str of options.strings) {
AnnotationEditor._l10nPromise.set(str, l10n.get(str));
@ -952,6 +967,9 @@ class AnnotationEditor {
this.fixAndSetPosition();
}
/**
* Called when the alt text dialog is closed.
*/
altTextFinish() {
this.#altText?.finish();
}
@ -961,24 +979,24 @@ class AnnotationEditor {
* @returns {Promise<EditorToolbar|null>}
*/
async addEditToolbar() {
if (this.#editToolbar || this.#isInEditMode) {
return this.#editToolbar;
if (this._editToolbar || this.#isInEditMode) {
return this._editToolbar;
}
this.#editToolbar = new EditorToolbar(this);
this.div.append(this.#editToolbar.render());
this._editToolbar = new EditorToolbar(this);
this.div.append(this._editToolbar.render());
if (this.#altText) {
this.#editToolbar.addAltTextButton(await this.#altText.render());
this._editToolbar.addAltTextButton(await this.#altText.render());
}
return this.#editToolbar;
return this._editToolbar;
}
removeEditToolbar() {
if (!this.#editToolbar) {
if (!this._editToolbar) {
return;
}
this.#editToolbar.remove();
this.#editToolbar = null;
this._editToolbar.remove();
this._editToolbar = null;
// We destroy the alt text but we don't null it because we want to be able
// to restore it in case the user undoes the deletion.
@ -1016,8 +1034,24 @@ class AnnotationEditor {
this.#altText.data = data;
}
get guessedAltText() {
return this.#altText?.guessedText;
}
async setGuessedAltText(text) {
await this.#altText?.setGuessedText(text);
}
serializeAltText(isForCopying) {
return this.#altText?.serialize(isForCopying);
}
hasAltText() {
return !this.#altText?.isEmpty();
return !!this.#altText && !this.#altText.isEmpty();
}
hasAltTextData() {
return this.#altText?.hasData() ?? false;
}
/**
@ -1558,18 +1592,19 @@ class AnnotationEditor {
select() {
this.makeResizable();
this.div?.classList.add("selectedEditor");
if (!this.#editToolbar) {
if (!this._editToolbar) {
this.addEditToolbar().then(() => {
if (this.div?.classList.contains("selectedEditor")) {
// The editor can have been unselected while we were waiting for the
// edit toolbar to be created, hence we want to be sure that this
// editor is still selected.
this.#editToolbar?.show();
this._editToolbar?.show();
}
});
return;
}
this.#editToolbar?.show();
this._editToolbar?.show();
this.#altText?.toggleAltTextBadge(false);
}
/**
@ -1585,7 +1620,8 @@ class AnnotationEditor {
preventScroll: true,
});
}
this.#editToolbar?.hide();
this._editToolbar?.hide();
this.#altText?.toggleAltTextBadge(true);
}
/**

136
src/display/editor/stamp.js
View file

@ -36,8 +36,6 @@ class StampEditor extends AnnotationEditor {
#canvas = null;
#hasMLBeenQueried = false;
#observer = null;
#resizeTimeoutId = null;
@ -98,6 +96,14 @@ class StampEditor extends AnnotationEditor {
});
}
/** @inheritdoc */
altTextFinish() {
if (this._uiManager.useNewAltTextFlow) {
this.div.hidden = false;
}
super.altTextFinish();
}
#getBitmapFetched(data, fromId = false) {
if (!data) {
this.remove();
@ -117,7 +123,13 @@ class StampEditor extends AnnotationEditor {
#getBitmapDone() {
this.#bitmapPromise = null;
this._uiManager.enableWaiting(false);
if (this.#canvas) {
if (!this.#canvas) {
return;
}
if (this._uiManager.useNewAltTextFlow && this.#bitmap) {
this._editToolbar.hide();
this._uiManager.editAltText(this, /* firstTime = */ true);
} else {
this.div.focus();
}
}
@ -329,7 +341,9 @@ class StampEditor extends AnnotationEditor {
this._uiManager.enableWaiting(false);
const canvas = (this.#canvas = document.createElement("canvas"));
div.append(canvas);
div.hidden = false;
if (!this._uiManager.useNewAltTextFlow) {
div.hidden = false;
}
this.#drawBitmap(width, height);
this.#createObserver();
if (!this.#hasBeenAddedInUndoStack) {
@ -348,6 +362,77 @@ class StampEditor extends AnnotationEditor {
}
}
copyCanvas(maxDimension, createImageData = false) {
const { width: bitmapWidth, height: bitmapHeight } = this.#bitmap;
const canvas = document.createElement("canvas");
let bitmap = this.#bitmap;
let width = bitmapWidth,
height = bitmapHeight;
if (bitmapWidth > maxDimension || bitmapHeight > maxDimension) {
const ratio = Math.min(
maxDimension / bitmapWidth,
maxDimension / bitmapHeight
);
width = Math.floor(bitmapWidth * ratio);
height = Math.floor(bitmapHeight * ratio);
if (!this.#isSvg) {
bitmap = this.#scaleBitmap(width, height);
}
}
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext("2d", {
willReadFrequently: true,
});
ctx.filter = this._uiManager.hcmFilter;
if (createImageData && this._uiManager.hcmFilter !== "none") {
const offscreen = new OffscreenCanvas(width, height);
const offscreenCtx = offscreen.getContext("2d", {
willReadFrequently: true,
});
offscreenCtx.drawImage(
bitmap,
0,
0,
bitmap.width,
bitmap.height,
0,
0,
width,
height
);
const data = offscreenCtx.getImageData(0, 0, width, height).data;
ctx.drawImage(offscreen, 0, 0);
return { canvas, imageData: { width, height, data } };
}
ctx.drawImage(
bitmap,
0,
0,
bitmap.width,
bitmap.height,
0,
0,
width,
height
);
let imageData = null;
if (createImageData) {
imageData = {
width,
height,
data: ctx.getImageData(0, 0, width, height).data,
};
}
return { canvas, imageData };
}
/**
* When the dimensions of the div change the inner canvas must
* renew its dimensions, hence it must redraw its own contents.
@ -425,43 +510,6 @@ class StampEditor extends AnnotationEditor {
return bitmap;
}
async #mlGuessAltText(bitmap, width, height) {
if (this.#hasMLBeenQueried) {
return;
}
this.#hasMLBeenQueried = true;
const isMLEnabled = await this._uiManager.isMLEnabledFor("altText");
if (!isMLEnabled || this.hasAltText()) {
return;
}
const offscreen = new OffscreenCanvas(width, height);
const ctx = offscreen.getContext("2d", { willReadFrequently: true });
ctx.drawImage(
bitmap,
0,
0,
bitmap.width,
bitmap.height,
0,
0,
width,
height
);
const response = await this._uiManager.mlGuess({
service: "moz-image-to-text",
request: {
data: ctx.getImageData(0, 0, width, height).data,
width,
height,
channels: 4,
},
});
const altText = response?.output || "";
if (this.parent && altText && !this.hasAltText()) {
this.altTextData = { altText, decorative: false };
}
}
#drawBitmap(width, height) {
width = Math.ceil(width);
height = Math.ceil(height);
@ -475,8 +523,6 @@ class StampEditor extends AnnotationEditor {
? this.#bitmap
: this.#scaleBitmap(width, height);
this.#mlGuessAltText(bitmap, width, height);
const ctx = canvas.getContext("2d");
ctx.filter = this._uiManager.hcmFilter;
ctx.drawImage(
@ -616,11 +662,11 @@ class StampEditor extends AnnotationEditor {
// of this annotation and the clipboard doesn't support ImageBitmaps,
// hence we serialize the bitmap to a data url.
serialized.bitmapUrl = this.#serializeBitmap(/* toUrl = */ true);
serialized.accessibilityData = this.altTextData;
serialized.accessibilityData = this.serializeAltText(true);
return serialized;
}
const { decorative, altText } = this.altTextData;
const { decorative, altText } = this.serializeAltText(false);
if (!decorative && altText) {
serialized.accessibilityData = { type: "Figure", alt: altText };
}

12
src/display/editor/tools.js
View file

@ -851,6 +851,10 @@ class AnnotationEditorUIManager {
}
}
hasMLManager() {
return !!this.#mlManager;
}
async mlGuess(data) {
return this.#mlManager?.guess(data) || null;
}
@ -859,6 +863,10 @@ class AnnotationEditorUIManager {
return !!(await this.#mlManager?.isEnabledFor(name));
}
get mlManager() {
return this.#mlManager;
}
get useNewAltTextFlow() {
return this.#enableUpdatedAddImage;
}
@ -912,8 +920,8 @@ class AnnotationEditorUIManager {
this.#mainHighlightColorPicker = colorPicker;
}
editAltText(editor) {
this.#altTextManager?.editAltText(this, editor);
editAltText(editor, firstTime = false) {
this.#altTextManager?.editAltText(this, editor, firstTime);
}
switchToMode(mode, callback) {