Video gallery (#9052)

* video support

* tests and backend changes

* undo main merge

* upload fix

* Revert "undo main merge"

This reverts commit e2a26e6.

* type fixes

* format

* pr fixes

* Update gradio/components/gallery.py

Co-authored-by: Abubakar Abid <[email protected]>

* Update gradio/components/gallery.py

Co-authored-by: Abubakar Abid <[email protected]>

* type fix

* thumbnails

* thumbnail type

* remove thumbnail generation

* add changeset

* test fixes

* test fixes

* python test fix

* python test fixc

* fix

* fix

* story fix

---------

Co-authored-by: Abubakar Abid <[email protected]>
Co-authored-by: gradio-pr-bot <[email protected]>

Author: 3 people

.changeset/light-bats-arrive.md

@@ -0,0 +1,7 @@
+---
+"@gradio/core": minor
+"@gradio/gallery": minor
+"gradio": minor
+---
+
+feat:Video gallery

demo/gallery_component/files/cheetah.jpg

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: gallery_component"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", "    cheetahs = [\n", "        \"https://upload.wikimedia.org/wikipedia/commons/0/09/TheCheethcat.jpg\",\n", "        \"https://nationalzoo.si.edu/sites/default/files/animals/cheetah-003.jpg\",\n", "        \"https://img.etimg.com/thumb/msid-50159822,width-650,imgsize-129520,,resizemode-4,quality-100/.jpg\",\n", "        \"https://nationalzoo.si.edu/sites/default/files/animals/cheetah-002.jpg\",\n", "        \"https://images.theconversation.com/files/375893/original/file-20201218-13-a8h8uq.jpg?ixlib=rb-1.1.0&rect=16%2C407%2C5515%2C2924&q=45&auto=format&w=496&fit=clip\",\n", "    ]\n", "    gr.Gallery(value=cheetahs, columns=4)\n", "\n", "demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: gallery_component"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/cheetah.jpg https://github.com/gradio-app/gradio/raw/main/demo/gallery_component/files/cheetah.jpg\n", "!wget -q -O files/world.mp4 https://github.com/gradio-app/gradio/raw/main/demo/gallery_component/files/world.mp4"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", "    gallery_items = [\n", "        (\"https://upload.wikimedia.org/wikipedia/commons/0/09/TheCheethcat.jpg\", \"cheetah1\"),\n", "        (\"https://nationalzoo.si.edu/sites/default/files/animals/cheetah-003.jpg\", \"cheetah2\"),\n", "        (\"https://videos.pexels.com/video-files/3209828/3209828-uhd_2560_1440_25fps.mp4\", \"world\"),\n", "        (\"files/cheetah.jpg\", \"cheetah3\"),\n", "        (\"files/world.mp4\", \"world2\")\n", "    ]\n", "    gr.Gallery(value=gallery_items, columns=4)\n", "\n", "demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/gallery_component/files/world.mp4

@@ -1,13 +1,13 @@
 import gradio as gr
 
 with gr.Blocks() as demo:
-    cheetahs = [
-        "https://upload.wikimedia.org/wikipedia/commons/0/09/TheCheethcat.jpg",
-        "https://nationalzoo.si.edu/sites/default/files/animals/cheetah-003.jpg",
-        "https://img.etimg.com/thumb/msid-50159822,width-650,imgsize-129520,,resizemode-4,quality-100/.jpg",
-        "https://nationalzoo.si.edu/sites/default/files/animals/cheetah-002.jpg",
-        "https://images.theconversation.com/files/375893/original/file-20201218-13-a8h8uq.jpg?ixlib=rb-1.1.0&rect=16%2C407%2C5515%2C2924&q=45&auto=format&w=496&fit=clip",
+    gallery_items = [
+        ("https://upload.wikimedia.org/wikipedia/commons/0/09/TheCheethcat.jpg", "cheetah1"),
+        ("https://nationalzoo.si.edu/sites/default/files/animals/cheetah-003.jpg", "cheetah2"),
+        ("https://videos.pexels.com/video-files/3209828/3209828-uhd_2560_1440_25fps.mp4", "world"),
+        ("files/cheetah.jpg", "cheetah3"),
+        ("files/world.mp4", "world2")
     ]
-    gr.Gallery(value=cheetahs, columns=4)
+    gr.Gallery(value=gallery_items, columns=4)
 
 demo.launch()

demo/gallery_component/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: gallery_component_events"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", "    cheetahs = [\n", "        \"https://gradio-builds.s3.amazonaws.com/assets/cheetah-003.jpg\",\n", "        \"https://gradio-builds.s3.amazonaws.com/assets/lite-logo.png\",\n", "        \"https://gradio-builds.s3.amazonaws.com/assets/TheCheethcat.jpg\",\n", "    ]\n", "    with gr.Row():\n", "        with gr.Column():\n", "            gal = gr.Gallery(columns=4, interactive=True, label=\"Input Gallery\")\n", "            btn = gr.Button()\n", "        with gr.Column():\n", "            output_gal = gr.Gallery(columns=4, interactive=True, label=\"Output Gallery\")\n", "    with gr.Row():\n", "        textbox = gr.Json(label=\"uploaded files\")\n", "        num_upload = gr.Number(value=0, label=\"Num Upload\")\n", "        num_change = gr.Number(value=0, label=\"Num Change\")\n", "        select_output = gr.Textbox(label=\"Select Data\")\n", "        gal.upload(lambda v,n: (v, v, n+1), [gal, num_upload], [textbox, output_gal, num_upload])\n", "        gal.change(lambda v,n: (v, v, n+1), [gal, num_change], [textbox, output_gal, num_change])\n", "\n", "    btn.click(lambda: cheetahs, None, [output_gal])\n", "\n", "    def select(select_data: gr.SelectData):\n", "        return select_data.value['image']['url']\n", "\n", "    output_gal.select(select, None, select_output)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: gallery_component_events"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", "    files = [\n", "        \"https://gradio-builds.s3.amazonaws.com/assets/cheetah-003.jpg\",\n", "        \"https://gradio-static-files.s3.amazonaws.com/world.mp4\",\n", "        \"https://gradio-builds.s3.amazonaws.com/assets/TheCheethcat.jpg\",\n", "    ]\n", "    with gr.Row():\n", "        with gr.Column():\n", "            gal = gr.Gallery(columns=4, interactive=True, label=\"Input Gallery\")\n", "            btn = gr.Button()\n", "        with gr.Column():\n", "            output_gal = gr.Gallery(columns=4, interactive=True, label=\"Output Gallery\")\n", "    with gr.Row():\n", "        textbox = gr.Json(label=\"uploaded files\")\n", "        num_upload = gr.Number(value=0, label=\"Num Upload\")\n", "        num_change = gr.Number(value=0, label=\"Num Change\")\n", "        select_output = gr.Textbox(label=\"Select Data\")\n", "        gal.upload(lambda v,n: (v, v, n+1), [gal, num_upload], [textbox, output_gal, num_upload])\n", "        gal.change(lambda v,n: (v, v, n+1), [gal, num_change], [textbox, output_gal, num_change])\n", "\n", "    btn.click(lambda: files, None, [output_gal])\n", "\n", "    def select(select_data: gr.SelectData):\n", "        return select_data.value['image']['url'] if 'image' in select_data.value else select_data.value['video']['url']\n", "\n", "    output_gal.select(select, None, select_output)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/gallery_component/run.py

@@ -1,9 +1,9 @@
 import gradio as gr
 
 with gr.Blocks() as demo:
-    cheetahs = [
+    files = [
         "https://gradio-builds.s3.amazonaws.com/assets/cheetah-003.jpg",
-        "https://gradio-builds.s3.amazonaws.com/assets/lite-logo.png",
+        "https://gradio-static-files.s3.amazonaws.com/world.mp4",
         "https://gradio-builds.s3.amazonaws.com/assets/TheCheethcat.jpg",
     ]
     with gr.Row():
@@ -20,10 +20,10 @@
         gal.upload(lambda v,n: (v, v, n+1), [gal, num_upload], [textbox, output_gal, num_upload])
         gal.change(lambda v,n: (v, v, n+1), [gal, num_change], [textbox, output_gal, num_change])
 
-    btn.click(lambda: cheetahs, None, [output_gal])
+    btn.click(lambda: files, None, [output_gal])
 
     def select(select_data: gr.SelectData):
-        return select_data.value['image']['url']
+        return select_data.value['image']['url'] if 'image' in select_data.value else select_data.value['video']['url']
 
     output_gal.select(select, None, select_output)
 

demo/gallery_component_events/run.ipynb

@@ -17,6 +17,7 @@
 import numpy as np
 import PIL.Image
 from gradio_client import handle_file
+from gradio_client import utils as client_utils
 from gradio_client.documentation import document
 from gradio_client.utils import is_http_url_like
 
@@ -28,24 +29,29 @@
 if TYPE_CHECKING:
     from gradio.components import Timer
 
-GalleryImageType = Union[np.ndarray, PIL.Image.Image, Path, str]
-CaptionedGalleryImageType = tuple[GalleryImageType, str]
+GalleryMediaType = Union[np.ndarray, PIL.Image.Image, Path, str]
+CaptionedGalleryMediaType = tuple[GalleryMediaType, str]
 
 
 class GalleryImage(GradioModel):
     image: FileData
     caption: Optional[str] = None
 
 
+class GalleryVideo(GradioModel):
+    video: FileData
+    caption: Optional[str] = None
+
+
 class GalleryData(GradioRootModel):
-    root: list[GalleryImage]
+    root: list[Union[GalleryImage, GalleryVideo]]
 
 
 @document()
 class Gallery(Component):
     """
-    Creates a gallery component that allows displaying a grid of images, and optionally captions. If used as an input, the user can upload images to the gallery.
-    If used as an output, the user can click on individual images to view them at a higher resolution.
+    Creates a gallery component that allows displaying a grid of images or videos, and optionally captions. If used as an input, the user can upload images or videos to the gallery.
+    If used as an output, the user can click on individual images or videos to view them at a higher resolution.
 
     Demos: fake_gan
     """
@@ -63,6 +69,7 @@ def __init__(
         ) = None,
         *,
         format: str = "webp",
+        file_types: list[str] | None = None,
         label: str | None = None,
         every: Timer | float | None = None,
         inputs: Component | Sequence[Component] | set[Component] | None = None,
@@ -92,8 +99,9 @@ def __init__(
     ):
         """
         Parameters:
-            value: List of images to display in the gallery by default. If callable, the function will be called whenever the app loads to set the initial value of the component.
+            value: List of images or videos to display in the gallery by default. If callable, the function will be called whenever the app loads to set the initial value of the component.
             format: Format to save images before they are returned to the frontend, such as 'jpeg' or 'png'. This parameter only applies to images that are returned from the prediction function as numpy arrays or PIL Images. The format should be supported by the PIL library.
+            file_types: List of file extensions or types of files to be uploaded (e.g. ['image', 'video']), when this is used as an input component. "image" allows only image files to be uploaded, "video" allows only video files to be uploaded, ".mp4" allows only mp4 files to be uploaded, etc. If None, any image and video files types are allowed.
             label: The label for this component. Appears above the component and is also used as the header if there are a table of examples for this component. If None and used in a `gr.Interface`, the label will be the name of the parameter this component is assigned to.
             every: Continously calls `value` to recalculate it if `value` is a function (has no effect otherwise). Can provide a Timer whose tick resets `value`, or a float that provides the regular interval for the reset Timer.
             inputs: Components that are used as inputs to calculate `value` if `value` is a function (has no effect otherwise). `value` is recalculated any time the inputs change.
@@ -134,6 +142,7 @@ def __init__(
         self.selected_index = selected_index
         self.type = type
         self.show_fullscreen_button = show_fullscreen_button
+        self.file_types = file_types
 
         self.show_share_button = (
             (utils.get_space() is not None)
@@ -167,27 +176,31 @@ def preprocess(
     ):
         """
         Parameters:
-            payload: a list of images, or list of (image, caption) tuples
+            payload: a list of images or videos, or list of (media, caption) tuples
         Returns:
-            Passes the list of images as a list of (image, caption) tuples, or a list of (image, None) tuples if no captions are provided (which is usually the case). The image can be a `str` file path, a `numpy` array, or a `PIL.Image` object depending on `type`.
+            Passes the list of images or videos as a list of (media, caption) tuples, or a list of (media, None) tuples if no captions are provided (which is usually the case). Images can be a `str` file path, a `numpy` array, or a `PIL.Image` object depending on `type`.  Videos are always `str` file path.
         """
         if payload is None or not payload.root:
             return None
         data = []
         for gallery_element in payload.root:
-            image = self.convert_to_type(gallery_element.image.path, self.type)  # type: ignore
-            data.append((image, gallery_element.caption))
+            media = (
+                gallery_element.video.path
+                if (type(gallery_element) is GalleryVideo)
+                else self.convert_to_type(gallery_element.image.path, self.type)  # type: ignore
+            )
+            data.append((media, gallery_element.caption))
         return data
 
     def postprocess(
         self,
-        value: list[GalleryImageType | CaptionedGalleryImageType] | None,
+        value: list[GalleryMediaType | CaptionedGalleryMediaType] | None,
     ) -> GalleryData:
         """
         Parameters:
-            value: Expects the function to return a `list` of images, or `list` of (image, `str` caption) tuples. Each image can be a `str` file path, a `numpy` array, or a `PIL.Image` object.
+            value: Expects the function to return a `list` of images or videos, or `list` of (media, `str` caption) tuples. Each image can be a `str` file path, a `numpy` array, or a `PIL.Image` object. Each video can be a `str` file path.
         Returns:
-            a list of images, or list of (image, caption) tuples
+            a list of images or videos, or list of (media, caption) tuples
         """
         if value is None:
             return GalleryData(root=[])
@@ -197,6 +210,7 @@ def _save(img):
             url = None
             caption = None
             orig_name = None
+            mime_type = None
             if isinstance(img, (tuple, list)):
                 img, caption = img
             if isinstance(img, np.ndarray):
@@ -211,6 +225,7 @@ def _save(img):
                 file_path = str(utils.abspath(file))
             elif isinstance(img, str):
                 file_path = img
+                mime_type = client_utils.get_mimetype(file_path)
                 if is_http_url_like(img):
                     url = img
                     orig_name = Path(urlparse(img).path).name
@@ -220,12 +235,29 @@ def _save(img):
             elif isinstance(img, Path):
                 file_path = str(img)
                 orig_name = img.name
+                mime_type = client_utils.get_mimetype(file_path)
             else:
                 raise ValueError(f"Cannot process type as image: {type(img)}")
-            return GalleryImage(
-                image=FileData(path=file_path, url=url, orig_name=orig_name),
-                caption=caption,
-            )
+            if mime_type is not None and "video" in mime_type:
+                return GalleryVideo(
+                    video=FileData(
+                        path=file_path,
+                        url=url,
+                        orig_name=orig_name,
+                        mime_type=mime_type,
+                    ),
+                    caption=caption,
+                )
+            else:
+                return GalleryImage(
+                    image=FileData(
+                        path=file_path,
+                        url=url,
+                        orig_name=orig_name,
+                        mime_type=mime_type,
+                    ),
+                    caption=caption,
+                )
 
         if wasm_utils.IS_WASM:
             for img in value:

demo/gallery_component_events/run.py

@@ -112,7 +112,7 @@
 		"drop_file": "Drop File Here",
 		"drop_image": "Drop Image Here",
 		"drop_video": "Drop Video Here",
-		"drop_gallery": "Drop Image(s) Here",
+		"drop_gallery": "Drop Media Here",
 		"paste_clipboard": "Paste from Clipboard"
 	}
 }

gradio/components/gallery.py

@@ -86,6 +86,14 @@
 				},
 				caption: "A fast cheetah"
 			},
+			{
+				video: {
+					path: "https://gradio-builds.s3.amazonaws.com/demo-files/world.mp4",
+					url: "https://gradio-builds.s3.amazonaws.com/demo-files/world.mp4",
+					orig_name: "world.mp4"
+				},
+				caption: "The world"
+			},
 			{
 				image: {
 					path: "https://gradio-builds.s3.amazonaws.com/demo-files/cheetah-002.jpg",
@@ -139,7 +147,7 @@
 	play={async ({ canvasElement }) => {
 		const canvas = within(canvasElement);
 
-		const image = canvas.getByLabelText("Thumbnail 1 of 7");
+		const image = canvas.getByLabelText("Thumbnail 1 of 8");
 		await userEvent.click(image);
 		const expand_btn = canvas.getByRole("button", {
 			name: "View in full screen"