Skip to content

Commit 686be4a

Browse files
calixtemancalixteman
committed
Make tagged images visible for screen readers (bug 1708040)
The idea is to insert a span in the text layer with an aria-role set to img and use the bounding box provided by the attribute field in the tag dict in order to have non-null dimensions for the image to make it "visible".
1 parent 4b906ad commit 686be4a

12 files changed

+239
-9
lines changed

‎src/core/struct_tree.js

+30-2
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515

1616
import { AnnotationPrefix, stringToPDFString, warn } from "../shared/util.js";
1717
import { Dict, isName, Name, Ref, RefSetCache } from "./primitives.js";
18+
import { lookupNormalRect, stringToAsciiOrUTF16BE } from "./core_utils.js";
1819
import { NumberTree } from "./name_number_tree.js";
19-
import { stringToAsciiOrUTF16BE } from "./core_utils.js";
2020
import { writeObject } from "./writer.js";
2121

2222
const MAX_DEPTH = 40;
@@ -751,10 +751,38 @@ class StructTreePage {
751751
obj.role = node.role;
752752
obj.children = [];
753753
parent.children.push(obj);
754-
const alt = node.dict.get("Alt");
754+
let alt = node.dict.get("Alt");
755+
if (typeof alt !== "string") {
756+
alt = node.dict.get("ActualText");
757+
}
755758
if (typeof alt === "string") {
756759
obj.alt = stringToPDFString(alt);
757760
}
761+
762+
const a = node.dict.get("A");
763+
if (a instanceof Dict) {
764+
const bbox = lookupNormalRect(a.getArray("BBox"), null);
765+
if (bbox) {
766+
obj.bbox = bbox;
767+
} else {
768+
const width = a.get("Width");
769+
const height = a.get("Height");
770+
if (
771+
typeof width === "number" &&
772+
width > 0 &&
773+
typeof height === "number" &&
774+
height > 0
775+
) {
776+
obj.bbox = [0, 0, width, height];
777+
}
778+
}
779+
// TODO: If the bbox is not available, we should try to get it from
780+
// the content stream.
781+
// For example when rendering on the canvas the commands between the
782+
// beginning and the end of the marked-content sequence, we can
783+
// compute the overall bbox.
784+
}
785+
758786
const lang = node.dict.get("Lang");
759787
if (typeof lang === "string") {
760788
obj.lang = stringToPDFString(lang);

‎src/display/editor/annotation_editor_layer.js

+3-2
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,8 @@ class AnnotationEditorLayer {
395395
const { target } = event;
396396
if (
397397
target === this.#textLayer.div ||
398-
(target.classList.contains("endOfContent") &&
398+
((target.getAttribute("role") === "img" ||
399+
target.classList.contains("endOfContent")) &&
399400
this.#textLayer.div.contains(target))
400401
) {
401402
const { isMac } = FeatureTest.platform;
@@ -413,7 +414,7 @@ class AnnotationEditorLayer {
413414
HighlightEditor.startHighlighting(
414415
this,
415416
this.#uiManager.direction === "ltr",
416-
event
417+
{ target: this.#textLayer.div, x: event.x, y: event.y }
417418
);
418419
this.#textLayer.div.addEventListener(
419420
"pointerup",

‎test/integration/accessibility_spec.mjs

+40
Original file line numberDiff line numberDiff line change
@@ -241,4 +241,44 @@ describe("accessibility", () => {
241241
);
242242
});
243243
});
244+
245+
describe("Figure in the content stream", () => {
246+
let pages;
247+
248+
beforeAll(async () => {
249+
pages = await loadAndWait("bug1708040.pdf", ".textLayer");
250+
});
251+
252+
afterAll(async () => {
253+
await closePages(pages);
254+
});
255+
256+
it("must check that an image is correctly inserted in the text layer", async () => {
257+
await Promise.all(
258+
pages.map(async ([browserName, page]) => {
259+
expect(await isStructTreeVisible(page))
260+
.withContext(`In ${browserName}`)
261+
.toBeTrue();
262+
263+
const spanId = await page.evaluate(() => {
264+
const el = document.querySelector(
265+
`.structTree span[role="figure"]`
266+
);
267+
return el.getAttribute("aria-owns") || null;
268+
});
269+
270+
expect(spanId).withContext(`In ${browserName}`).not.toBeNull();
271+
272+
const ariaLabel = await page.evaluate(id => {
273+
const img = document.querySelector(`#${id} > span[role="img"]`);
274+
return img.getAttribute("aria-label");
275+
}, spanId);
276+
277+
expect(ariaLabel)
278+
.withContext(`In ${browserName}`)
279+
.toEqual("A logo of a fox and a globe");
280+
})
281+
);
282+
});
283+
});
244284
});

‎test/integration/highlight_editor_spec.mjs

+47
Original file line numberDiff line numberDiff line change
@@ -2053,4 +2053,51 @@ describe("Highlight Editor", () => {
20532053
);
20542054
});
20552055
});
2056+
2057+
describe("Free Highlight with an image in the struct tree", () => {
2058+
let pages;
2059+
2060+
beforeAll(async () => {
2061+
pages = await loadAndWait(
2062+
"bug1708040.pdf",
2063+
".annotationEditorLayer",
2064+
null,
2065+
null,
2066+
{ highlightEditorColors: "red=#AB0000" }
2067+
);
2068+
});
2069+
2070+
afterAll(async () => {
2071+
await closePages(pages);
2072+
});
2073+
2074+
it("must check that it's possible to draw on an image in a struct tree", async () => {
2075+
await Promise.all(
2076+
pages.map(async ([browserName, page]) => {
2077+
await switchToHighlight(page);
2078+
2079+
const rect = await getRect(page, `.textLayer span[role="img"]`);
2080+
2081+
const x = rect.x + rect.width / 2;
2082+
const y = rect.y + rect.height / 2;
2083+
const clickHandle = await waitForPointerUp(page);
2084+
await page.mouse.move(x, y);
2085+
await page.mouse.down();
2086+
await page.mouse.move(rect.x - 1, rect.y - 1);
2087+
await page.mouse.up();
2088+
await awaitPromise(clickHandle);
2089+
2090+
await page.waitForSelector(getEditorSelector(0));
2091+
const usedColor = await page.evaluate(() => {
2092+
const highlight = document.querySelector(
2093+
`.page[data-page-number = "1"] .canvasWrapper > svg.highlight`
2094+
);
2095+
return highlight.getAttribute("fill");
2096+
});
2097+
2098+
expect(usedColor).withContext(`In ${browserName}`).toEqual("#AB0000");
2099+
})
2100+
);
2101+
});
2102+
});
20562103
});

‎test/pdfs/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -664,3 +664,4 @@
664664
!issue18561.pdf
665665
!highlights.pdf
666666
!highlight.pdf
667+
!bug1708040.pdf

‎test/pdfs/bug1708040.pdf

68.7 KB
Binary file not shown.

‎test/unit/api_spec.js

+2
Original file line numberDiff line numberDiff line change
@@ -3807,11 +3807,13 @@ Caron Broadcasting, Inc., an Ohio corporation (“Lessee”).`)
38073807
role: "Figure",
38083808
children: [{ type: "content", id: "p406R_mc11" }],
38093809
alt: "d h c s logo",
3810+
bbox: [57.75, 676, 133.35, 752],
38103811
},
38113812
{
38123813
role: "Figure",
38133814
children: [{ type: "content", id: "p406R_mc1" }],
38143815
alt: "Great Seal of the State of California",
3816+
bbox: [481.5, 678, 544.5, 741],
38153817
},
38163818
{
38173819
role: "P",

‎test/unit/struct_tree_spec.js

+44
Original file line numberDiff line numberDiff line change
@@ -107,4 +107,48 @@ describe("struct tree", function () {
107107
await loadingTask.destroy();
108108
});
109109
});
110+
111+
it("parses structure with a figure and its bounding box", async function () {
112+
const filename = "bug1708040.pdf";
113+
const params = buildGetDocumentParams(filename);
114+
const loadingTask = getDocument(params);
115+
const doc = await loadingTask.promise;
116+
const page = await doc.getPage(1);
117+
const struct = await page.getStructTree();
118+
equalTrees(
119+
{
120+
children: [
121+
{
122+
role: "Document",
123+
children: [
124+
{
125+
role: "Sect",
126+
children: [
127+
{
128+
role: "P",
129+
children: [{ type: "content", id: "p21R_mc0" }],
130+
lang: "EN-US",
131+
},
132+
{
133+
role: "P",
134+
children: [{ type: "content", id: "p21R_mc1" }],
135+
lang: "EN-US",
136+
},
137+
{
138+
role: "Figure",
139+
children: [{ type: "content", id: "p21R_mc2" }],
140+
alt: "A logo of a fox and a globe\u0000",
141+
bbox: [72, 287.782, 456, 695.032],
142+
},
143+
],
144+
},
145+
],
146+
},
147+
],
148+
role: "Root",
149+
},
150+
struct
151+
);
152+
await loadingTask.destroy();
153+
});
110154
});

‎web/annotation_editor_layer_builder.css

+4
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@
7171

7272
&:not(.free) span {
7373
cursor: var(--editorHighlight-editing-cursor);
74+
75+
&[role="img"] {
76+
cursor: var(--editorFreeHighlight-editing-cursor);
77+
}
7478
}
7579

7680
&.free span {

‎web/pdf_page_view.js

+10-4
Original file line numberDiff line numberDiff line change
@@ -474,10 +474,13 @@ class PDFPageView {
474474
}
475475

476476
const treeDom = await this.structTreeLayer?.render();
477-
if (treeDom && this.canvas && treeDom.parentNode !== this.canvas) {
478-
// Pause translation when inserting the structTree in the DOM.
477+
if (treeDom) {
479478
this.l10n.pause();
480-
this.canvas.append(treeDom);
479+
this.structTreeLayer?.addElementsToTextLayer();
480+
if (this.canvas && treeDom.parentNode !== this.canvas) {
481+
// Pause translation when inserting the structTree in the DOM.
482+
this.canvas.append(treeDom);
483+
}
481484
this.l10n.resume();
482485
}
483486
this.structTreeLayer?.show();
@@ -1068,7 +1071,10 @@ class PDFPageView {
10681071
await this.#finishRenderTask(renderTask);
10691072

10701073
if (this.textLayer || this.annotationLayer) {
1071-
this.structTreeLayer ||= new StructTreeLayerBuilder(pdfPage);
1074+
this.structTreeLayer ||= new StructTreeLayerBuilder(
1075+
pdfPage,
1076+
viewport.rawDims
1077+
);
10721078
}
10731079

10741080
this.#renderTextLayer();

‎web/struct_tree_layer_builder.js

+53-1
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,13 @@ class StructTreeLayerBuilder {
8282

8383
#elementAttributes = new Map();
8484

85-
constructor(pdfPage) {
85+
#rawDims;
86+
87+
#elementsToAddToTextLayer = null;
88+
89+
constructor(pdfPage, rawDims) {
8690
this.#promise = pdfPage.getStructTree();
91+
this.#rawDims = rawDims;
8792
}
8893

8994
async render() {
@@ -156,6 +161,50 @@ class StructTreeLayerBuilder {
156161
}
157162
}
158163

164+
#addImageInTextLayer(node, element) {
165+
const { alt, bbox, children } = node;
166+
const child = children?.[0];
167+
if (!this.#rawDims || !alt || !bbox || child?.type !== "content") {
168+
return false;
169+
}
170+
171+
const { id } = child;
172+
if (!id) {
173+
return false;
174+
}
175+
176+
// We cannot add the created element to the text layer immediately, as the
177+
// text layer might not be ready yet. Instead, we store the element and add
178+
// it later in `addElementsToTextLayer`.
179+
180+
element.setAttribute("aria-owns", id);
181+
const img = document.createElement("span");
182+
(this.#elementsToAddToTextLayer ||= new Map()).set(id, img);
183+
img.setAttribute("role", "img");
184+
img.setAttribute("aria-label", removeNullCharacters(alt));
185+
186+
const { pageHeight, pageX, pageY } = this.#rawDims;
187+
const calc = "calc(var(--scale-factor)*";
188+
const { style } = img;
189+
style.width = `${calc}${bbox[2] - bbox[0]}px)`;
190+
style.height = `${calc}${bbox[3] - bbox[1]}px)`;
191+
style.left = `${calc}${bbox[0] - pageX}px)`;
192+
style.top = `${calc}${pageHeight - bbox[3] + pageY}px)`;
193+
194+
return true;
195+
}
196+
197+
addElementsToTextLayer() {
198+
if (!this.#elementsToAddToTextLayer) {
199+
return;
200+
}
201+
for (const [id, img] of this.#elementsToAddToTextLayer) {
202+
document.getElementById(id)?.append(img);
203+
}
204+
this.#elementsToAddToTextLayer.clear();
205+
this.#elementsToAddToTextLayer = null;
206+
}
207+
159208
#walk(node) {
160209
if (!node) {
161210
return null;
@@ -171,6 +220,9 @@ class StructTreeLayerBuilder {
171220
} else if (PDF_ROLE_TO_HTML_ROLE[role]) {
172221
element.setAttribute("role", PDF_ROLE_TO_HTML_ROLE[role]);
173222
}
223+
if (role === "Figure" && this.#addImageInTextLayer(node, element)) {
224+
return element;
225+
}
174226
}
175227

176228
this.#setAttributes(node, element);

‎web/text_layer_builder.css

+5
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@
5252
}
5353
/*#endif*/
5454

55+
span[role="img"] {
56+
user-select: none;
57+
cursor: default;
58+
}
59+
5560
.highlight {
5661
--highlight-bg-color: rgb(180 0 170 / 0.25);
5762
--highlight-selected-bg-color: rgb(0 100 0 / 0.25);

0 commit comments

Comments
 (0)