Skip to content

Commit 40eaa66

Browse files
calixtemancalixteman
committed
Make tagged images visible for screen readers (bug 1708040)
The idea is to insert a span in the text layer with an aria-role set to img and use the bounding box provided by the attribute field in the tag dict in order to have non-null dimensions for the image to make it "visible".
1 parent e3fd62d commit 40eaa66

12 files changed

+221
-6
lines changed

‎src/core/struct_tree.js

+30-2
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515

1616
import { AnnotationPrefix, stringToPDFString, warn } from "../shared/util.js";
1717
import { Dict, isName, Name, Ref, RefSetCache } from "./primitives.js";
18+
import { lookupNormalRect, stringToAsciiOrUTF16BE } from "./core_utils.js";
1819
import { NumberTree } from "./name_number_tree.js";
19-
import { stringToAsciiOrUTF16BE } from "./core_utils.js";
2020
import { writeObject } from "./writer.js";
2121

2222
const MAX_DEPTH = 40;
@@ -751,10 +751,38 @@ class StructTreePage {
751751
obj.role = node.role;
752752
obj.children = [];
753753
parent.children.push(obj);
754-
const alt = node.dict.get("Alt");
754+
let alt = node.dict.get("Alt");
755+
if (typeof alt !== "string") {
756+
alt = node.dict.get("ActualText");
757+
}
755758
if (typeof alt === "string") {
756759
obj.alt = stringToPDFString(alt);
757760
}
761+
762+
const a = node.dict.get("A");
763+
if (a instanceof Dict) {
764+
const bbox = lookupNormalRect(a.getArray("BBox"), null);
765+
if (bbox) {
766+
obj.bbox = bbox;
767+
} else {
768+
const width = a.get("Width");
769+
const height = a.get("Height");
770+
if (
771+
typeof width === "number" &&
772+
width > 0 &&
773+
typeof height === "number" &&
774+
height > 0
775+
) {
776+
obj.bbox = [0, 0, width, height];
777+
}
778+
}
779+
// TODO: If the bbox is not available, we should try to get it from
780+
// the content stream.
781+
// For example when rendering on the canvas the commands between the
782+
// beginning and the end of the marked-content sequence, we can
783+
// compute the overall bbox.
784+
}
785+
758786
const lang = node.dict.get("Lang");
759787
if (typeof lang === "string") {
760788
obj.lang = stringToPDFString(lang);

‎src/display/editor/annotation_editor_layer.js

+3-2
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,8 @@ class AnnotationEditorLayer {
395395
const { target } = event;
396396
if (
397397
target === this.#textLayer.div ||
398-
(target.classList.contains("endOfContent") &&
398+
((target.getAttribute("role") === "img" ||
399+
target.classList.contains("endOfContent")) &&
399400
this.#textLayer.div.contains(target))
400401
) {
401402
const { isMac } = FeatureTest.platform;
@@ -413,7 +414,7 @@ class AnnotationEditorLayer {
413414
HighlightEditor.startHighlighting(
414415
this,
415416
this.#uiManager.direction === "ltr",
416-
event
417+
{ target: this.#textLayer.div, x: event.x, y: event.y }
417418
);
418419
this.#textLayer.div.addEventListener(
419420
"pointerup",

‎test/integration/accessibility_spec.mjs

+40
Original file line numberDiff line numberDiff line change
@@ -241,4 +241,44 @@ describe("accessibility", () => {
241241
);
242242
});
243243
});
244+
245+
describe("Figure in the content stream", () => {
246+
let pages;
247+
248+
beforeAll(async () => {
249+
pages = await loadAndWait("bug1708040.pdf", ".textLayer");
250+
});
251+
252+
afterAll(async () => {
253+
await closePages(pages);
254+
});
255+
256+
it("must check that an image is correctly inserted in the text layer", async () => {
257+
await Promise.all(
258+
pages.map(async ([browserName, page]) => {
259+
expect(await isStructTreeVisible(page))
260+
.withContext(`In ${browserName}`)
261+
.toBeTrue();
262+
263+
const spanId = await page.evaluate(() => {
264+
const el = document.querySelector(
265+
`.structTree span[role="figure"]`
266+
);
267+
return el.getAttribute("aria-owns") || null;
268+
});
269+
270+
expect(spanId).withContext(`In ${browserName}`).not.toBeNull();
271+
272+
const ariaLabel = await page.evaluate(id => {
273+
const img = document.querySelector(`#${id} > span[role="img"]`);
274+
return img.getAttribute("aria-label");
275+
}, spanId);
276+
277+
expect(ariaLabel)
278+
.withContext(`In ${browserName}`)
279+
.toEqual("A logo of a fox and a globe");
280+
})
281+
);
282+
});
283+
});
244284
});

‎test/integration/highlight_editor_spec.mjs

+47
Original file line numberDiff line numberDiff line change
@@ -2020,4 +2020,51 @@ describe("Highlight Editor", () => {
20202020
);
20212021
});
20222022
});
2023+
2024+
describe("Free Highlight with an image in the struct tree", () => {
2025+
let pages;
2026+
2027+
beforeAll(async () => {
2028+
pages = await loadAndWait(
2029+
"bug1708040.pdf",
2030+
".annotationEditorLayer",
2031+
null,
2032+
null,
2033+
{ highlightEditorColors: "red=#AB0000" }
2034+
);
2035+
});
2036+
2037+
afterAll(async () => {
2038+
await closePages(pages);
2039+
});
2040+
2041+
it("must check that it's possible to draw on an image in a struct tree", async () => {
2042+
await Promise.all(
2043+
pages.map(async ([browserName, page]) => {
2044+
await switchToHighlight(page);
2045+
2046+
const rect = await getRect(page, `.textLayer span[role="img"]`);
2047+
2048+
const x = rect.x + rect.width / 2;
2049+
const y = rect.y + rect.height / 2;
2050+
const clickHandle = await waitForPointerUp(page);
2051+
await page.mouse.move(x, y);
2052+
await page.mouse.down();
2053+
await page.mouse.move(rect.x - 1, rect.y - 1);
2054+
await page.mouse.up();
2055+
await awaitPromise(clickHandle);
2056+
2057+
await page.waitForSelector(getEditorSelector(0));
2058+
const usedColor = await page.evaluate(() => {
2059+
const highlight = document.querySelector(
2060+
`.page[data-page-number = "1"] .canvasWrapper > svg.highlight`
2061+
);
2062+
return highlight.getAttribute("fill");
2063+
});
2064+
2065+
expect(usedColor).withContext(`In ${browserName}`).toEqual("#AB0000");
2066+
})
2067+
);
2068+
});
2069+
});
20232070
});

‎test/pdfs/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -664,3 +664,4 @@
664664
!issue18561.pdf
665665
!highlights.pdf
666666
!highlight.pdf
667+
!bug1708040.pdf

‎test/pdfs/bug1708040.pdf

68.7 KB
Binary file not shown.

‎test/unit/api_spec.js

+2
Original file line numberDiff line numberDiff line change
@@ -3807,11 +3807,13 @@ Caron Broadcasting, Inc., an Ohio corporation (“Lessee”).`)
38073807
role: "Figure",
38083808
children: [{ type: "content", id: "p406R_mc11" }],
38093809
alt: "d h c s logo",
3810+
bbox: [57.75, 676, 133.35, 752],
38103811
},
38113812
{
38123813
role: "Figure",
38133814
children: [{ type: "content", id: "p406R_mc1" }],
38143815
alt: "Great Seal of the State of California",
3816+
bbox: [481.5, 678, 544.5, 741],
38153817
},
38163818
{
38173819
role: "P",

‎test/unit/struct_tree_spec.js

+44
Original file line numberDiff line numberDiff line change
@@ -107,4 +107,48 @@ describe("struct tree", function () {
107107
await loadingTask.destroy();
108108
});
109109
});
110+
111+
it("parses structure with a figure and its bounding box", async function () {
112+
const filename = "bug1708040.pdf";
113+
const params = buildGetDocumentParams(filename);
114+
const loadingTask = getDocument(params);
115+
const doc = await loadingTask.promise;
116+
const page = await doc.getPage(1);
117+
const struct = await page.getStructTree();
118+
equalTrees(
119+
{
120+
children: [
121+
{
122+
role: "Document",
123+
children: [
124+
{
125+
role: "Sect",
126+
children: [
127+
{
128+
role: "P",
129+
children: [{ type: "content", id: "p21R_mc0" }],
130+
lang: "EN-US",
131+
},
132+
{
133+
role: "P",
134+
children: [{ type: "content", id: "p21R_mc1" }],
135+
lang: "EN-US",
136+
},
137+
{
138+
role: "Figure",
139+
children: [{ type: "content", id: "p21R_mc2" }],
140+
alt: "A logo of a fox and a globe\u0000",
141+
bbox: [72, 287.782, 456, 695.032],
142+
},
143+
],
144+
},
145+
],
146+
},
147+
],
148+
role: "Root",
149+
},
150+
struct
151+
);
152+
await loadingTask.destroy();
153+
});
110154
});

‎web/annotation_editor_layer_builder.css

+4
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@
7171

7272
&:not(.free) span {
7373
cursor: var(--editorHighlight-editing-cursor);
74+
75+
&[role="img"] {
76+
cursor: var(--editorFreeHighlight-editing-cursor);
77+
}
7478
}
7579

7680
&.free span {

‎web/pdf_page_view.js

+4-1
Original file line numberDiff line numberDiff line change
@@ -1068,7 +1068,10 @@ class PDFPageView {
10681068
await this.#finishRenderTask(renderTask);
10691069

10701070
if (this.textLayer || this.annotationLayer) {
1071-
this.structTreeLayer ||= new StructTreeLayerBuilder(pdfPage);
1071+
this.structTreeLayer ||= new StructTreeLayerBuilder(
1072+
pdfPage,
1073+
viewport.rawDims
1074+
);
10721075
}
10731076

10741077
this.#renderTextLayer();

‎web/struct_tree_layer_builder.js

+41-1
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,11 @@ class StructTreeLayerBuilder {
8282

8383
#elementAttributes = new Map();
8484

85-
constructor(pdfPage) {
85+
#rawDims;
86+
87+
constructor(pdfPage, rawDims) {
8688
this.#promise = pdfPage.getStructTree();
89+
this.#rawDims = rawDims;
8790
}
8891

8992
async render() {
@@ -156,6 +159,40 @@ class StructTreeLayerBuilder {
156159
}
157160
}
158161

162+
#addImageInTextLayer(node, element) {
163+
const { alt, bbox, children } = node;
164+
const child = children?.[0];
165+
if (!this.#rawDims || !alt || !bbox || child?.type !== "content") {
166+
return false;
167+
}
168+
169+
const { id } = child;
170+
if (!id) {
171+
return false;
172+
}
173+
const span = document.getElementById(id);
174+
if (!span) {
175+
return false;
176+
}
177+
178+
element.setAttribute("aria-owns", id);
179+
180+
const img = document.createElement("span");
181+
span.append(img);
182+
img.setAttribute("role", "img");
183+
img.setAttribute("aria-label", removeNullCharacters(alt));
184+
185+
const { pageHeight, pageX, pageY } = this.#rawDims;
186+
const calc = "calc(var(--scale-factor)*";
187+
const { style } = img;
188+
style.width = `${calc}${bbox[2] - bbox[0]}px)`;
189+
style.height = `${calc}${bbox[3] - bbox[1]}px)`;
190+
style.left = `${calc}${bbox[0] - pageX}px)`;
191+
style.top = `${calc}${pageHeight - bbox[3] + pageY}px)`;
192+
193+
return true;
194+
}
195+
159196
#walk(node) {
160197
if (!node) {
161198
return null;
@@ -171,6 +208,9 @@ class StructTreeLayerBuilder {
171208
} else if (PDF_ROLE_TO_HTML_ROLE[role]) {
172209
element.setAttribute("role", PDF_ROLE_TO_HTML_ROLE[role]);
173210
}
211+
if (role === "Figure" && this.#addImageInTextLayer(node, element)) {
212+
return element;
213+
}
174214
}
175215

176216
this.#setAttributes(node, element);

‎web/text_layer_builder.css

+5
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@
5252
}
5353
/*#endif*/
5454

55+
span[role="img"] {
56+
user-select: none;
57+
cursor: default;
58+
}
59+
5560
.highlight {
5661
--highlight-bg-color: rgb(180 0 170 / 0.25);
5762
--highlight-selected-bg-color: rgb(0 100 0 / 0.25);

0 commit comments

Comments
 (0)