Include text from shape selector in thumbnail alt text

Include extracted document text from shape selectors in the alt text of
thumbnails.

Author: robertknight

src/sidebar/components/Annotation/Annotation.tsx

@@ -124,7 +124,12 @@ function Annotation({
         replyCount={replyCount}
         threadIsCollapsed={threadIsCollapsed}
       />
-      {targetShape && <AnnotationThumbnail tag={annotation.$tag} />}
+      {targetShape && (
+        <AnnotationThumbnail
+          tag={annotation.$tag}
+          textInImage={targetShape.text}
+        />
+      )}
       {annotationQuote && (
         <AnnotationQuote
           quote={annotationQuote}

src/sidebar/components/Annotation/AnnotationThumbnail.tsx

@@ -31,7 +31,14 @@ function BitmapImage({ alt, bitmap, classes, scale = 1.0 }: BitmapImageProps) {
       width={bitmap.width}
       height={bitmap.height}
       role="img"
+      // The `alt` attribute on an `<img>` maps to aria-label. We might want to
+      // split this into a separate concise label and longer description in
+      // future.
       aria-label={alt}
+      // Set the title attribute to make it easy to inspect the alt text on
+      // desktop. Screen readers will only read `aria-label` since it has the
+      // same value.
+      title={alt}
       className={classes}
       style={{
         width: `${bitmap.width / scale}px`,
@@ -44,9 +51,17 @@ function BitmapImage({ alt, bitmap, classes, scale = 1.0 }: BitmapImageProps) {
 export type AnnotationThumbnailProps = {
   tag: string;
   thumbnailService: ThumbnailService;
+
+  /**
+   * Text contained in the thumbnail.
+   *
+   * This is used when generating alt text for the thumbnail.
+   */
+  textInImage?: string;
 };
 
 function AnnotationThumbnail({
+  textInImage,
   tag,
   thumbnailService,
 }: AnnotationThumbnailProps) {
@@ -69,14 +84,21 @@ function AnnotationThumbnail({
     }
   }, [error, devicePixelRatio, tag, thumbnail, thumbnailService]);
 
+  let altText;
+  if (textInImage) {
+    altText = `Thumbnail. Contains text: ${textInImage}`;
+  } else {
+    altText = 'Thumbnail';
+  }
+
   return (
     <div
       className="flex flex-row justify-center"
       data-testid="thumbnail-container"
     >
       {thumbnail && (
         <BitmapImage
-          alt="annotated content thumbnail"
+          alt={altText}
           bitmap={thumbnail}
           classes="border rounded-md"
           scale={devicePixelRatio}

src/sidebar/components/Annotation/test/Annotation-test.js

@@ -157,12 +157,14 @@ describe('Annotation', () => {
             right: 10,
             bottom: 0,
           },
+          text: 'Some text',
         },
       ];
       const wrapper = createComponent({ annotation });
       const thumbnail = wrapper.find('AnnotationThumbnail');
       assert.isTrue(thumbnail.exists());
       assert.equal(thumbnail.prop('tag'), annotation.$tag);
+      assert.equal(thumbnail.prop('textInImage'), 'Some text');
     });
   });
 

src/sidebar/components/Annotation/test/AnnotationThumbnail-test.js

@@ -38,6 +38,24 @@ describe('AnnotationThumbnail', () => {
     assert.isTrue(wrapper.exists('BitmapImage'));
   });
 
+  [
+    {
+      textInImage: undefined,
+      expectedAlt: 'Thumbnail',
+    },
+    {
+      textInImage: 'Foo bar',
+      expectedAlt: 'Thumbnail. Contains text: Foo bar',
+    },
+  ].forEach(({ textInImage, expectedAlt }) => {
+    it('sets alt text for thumbnail', () => {
+      fakeThumbnailService.get.returns(fakeThumbnail);
+      const wrapper = createComponent({ textInImage });
+      const image = wrapper.find('canvas');
+      assert.equal(image.prop('aria-label'), expectedAlt);
+    });
+  });
+
   it('requests thumbnail and then renders it if not cached', async () => {
     const wrapper = createComponent();
     assert.calledOnce(fakeThumbnailService.fetch);

src/sidebar/helpers/annotation-metadata.ts

@@ -4,7 +4,6 @@ import type {
   EPUBContentSelector,
   PageSelector,
   SavedAnnotation,
-  Shape,
   ShapeSelector,
   TextQuoteSelector,
 } from '../../types/api';
@@ -401,11 +400,11 @@ export function quote(annotation: APIAnnotationData): string | null {
  * This will return `null` if the annotation is associated with a text
  * selection instead of a shape.
  */
-export function shape(annotation: APIAnnotationData): Shape | null {
+export function shape(annotation: APIAnnotationData): ShapeSelector | null {
   const shapeSelector = annotation.target[0]?.selector?.find(
     s => s.type === 'ShapeSelector',
   ) as ShapeSelector | undefined;
-  return shapeSelector?.shape ?? null;
+  return shapeSelector ?? null;
 }
 
 /**

src/sidebar/helpers/test/annotation-metadata-test.js

@@ -726,15 +726,18 @@ describe('sidebar/helpers/annotation-metadata', () => {
           },
         ],
         expected: {
-          type: 'rect',
-          left: 0,
-          top: 10,
-          right: 10,
-          bottom: 0,
+          type: 'ShapeSelector',
+          shape: {
+            type: 'rect',
+            left: 0,
+            top: 10,
+            right: 10,
+            bottom: 0,
+          },
         },
       },
     ].forEach(({ selectors, expected }) => {
-      it('returns shape from shape selector', () => {
+      it('returns shape selector', () => {
         const annotation = {
           target: [
             {