experimental: Support video upload for animations

apps/builder/app/builder/shared/assets/asset-utils.ts

@@ -2,6 +2,14 @@ import type { Asset, FontAsset, ImageAsset } from "@webstudio-is/sdk";
 import { nanoid } from "nanoid";
 import type { UploadingFileData } from "~/shared/nano-states";
 
+const videoExtensionToMime = [
+  [".mp4", "video/mp4"],
+  [".webm", "video/webm"],
+  [".mpg", "video/mpeg"],
+  [".mpeg", "video/mpeg"],
+  [".mov", "video/quicktime"],
+] as const;
+
 const extensionToMime = new Map([
   [".gif", "image/gif"],
   [".ico", "image/x-icon"],
@@ -10,8 +18,14 @@ const extensionToMime = new Map([
   [".png", "image/png"],
   [".svg", "image/svg+xml"],
   [".webp", "image/webp"],
+  // Support video formats as images
+  ...videoExtensionToMime,
 ] as const);
 
+export const isVideoFormat = (format: string) => {
+  return videoExtensionToMime.some(([extension]) => extension.includes(format));
+};
+
 const extensions = [...extensionToMime.keys()];
 
 export const imageMimeTypes = [...extensionToMime.values()];
@@ -102,6 +116,27 @@ export const uploadingFileDataToAsset = (
   );
   const format = mimeType.split("/")[1];
 
+  if (mimeType.startsWith("video/")) {
+    // Use image type for now
+    const asset: ImageAsset = {
+      id: fileData.assetId,
+      name: fileData.objectURL,
+      format,
+      type: "image",
+      description: "",
+      createdAt: "",
+      projectId: "",
+      size: 0,
+
+      meta: {
+        width: Number.NaN,
+        height: Number.NaN,
+      },
+    };
+
+    return asset;
+  }
+
   if (mimeType.startsWith("image/")) {
     const asset: ImageAsset = {
       id: fileData.assetId,

apps/builder/app/builder/shared/assets/use-assets.tsx

@@ -145,6 +145,24 @@ const $assetContainers = computed(
 
 export type UploadData = ActionData;
 
+const getVideoDimensions = async (file: File) => {
+  return new Promise<{ width: number; height: number }>((resolve, reject) => {
+    const url = URL.createObjectURL(file);
+    const vid = document.createElement("video");
+    vid.preload = "metadata";
+    vid.src = url;
+
+    vid.onloadedmetadata = () => {
+      URL.revokeObjectURL(url);
+      resolve({ width: vid.videoWidth, height: vid.videoHeight });
+    };
+    vid.onerror = () => {
+      URL.revokeObjectURL(url);
+      reject(new Error("Invalid video file"));
+    };
+  });
+};
+
 const uploadAsset = async ({
   authToken,
   projectId,
@@ -198,8 +216,17 @@ const uploadAsset = async ({
       headers.set("Content-Type", "application/json");
     }
 
+    let width = undefined;
+    let height = undefined;
+
+    if (mimeType.startsWith("video/") && fileOrUrl instanceof File) {
+      const videoSize = await getVideoDimensions(fileOrUrl);
+      width = videoSize.width;
+      height = videoSize.height;
+    }
+
     const uploadResponse = await fetch(
-      restAssetsUploadPath({ name: metaData.name }),
+      restAssetsUploadPath({ name: metaData.name, width, height }),
       {
         method: "POST",
         body,

apps/builder/app/builder/shared/image-manager/image-thumbnail.tsx

@@ -7,6 +7,7 @@ import { Filename } from "./filename";
 import { Image } from "./image";
 import brokenImage from "~/shared/images/broken-image-placeholder.svg";
 import { theme } from "@webstudio-is/design-system";
+import { isVideoFormat } from "../assets/asset-utils";
 
 const StyledWebstudioImage = styled(Image, {
   position: "absolute",
@@ -34,6 +35,32 @@ const StyledWebstudioImage = styled(Image, {
   },
 });
 
+const StyledWebstudioVideo = styled("video", {
+  position: "absolute",
+  width: "100%",
+  height: "100%",
+  objectFit: "contain",
+
+  // This is shown only if an image was not loaded and broken
+  // From the spec:
+  // - The pseudo-elements generated by ::before and ::after are contained by the element's formatting box,
+  //   and thus don't apply to "replaced" elements such as <img>, or to <br> elements
+  // Not in spec but supported by all browsers:
+  // - broken image is not a "replaced" element so this style is applied
+  "&::after": {
+    content: "' '",
+    position: "absolute",
+    width: "100%",
+    height: "100%",
+    left: 0,
+    top: 0,
+    backgroundSize: "contain",
+    backgroundRepeat: "no-repeat",
+    backgroundPosition: "center",
+    backgroundImage: `url(${brokenImage})`,
+  },
+});
+
 const ThumbnailContainer = styled(Box, {
   position: "relative",
   display: "flex",
@@ -120,18 +147,29 @@ export const ImageThumbnail = ({
           onChange?.(assetContainer);
         }}
       >
-        <StyledWebstudioImage
-          assetId={assetContainer.asset.id}
-          name={assetContainer.asset.name}
-          objectURL={
-            assetContainer.status === "uploading"
-              ? assetContainer.objectURL
-              : undefined
-          }
-          alt={description ?? name}
-          // width={64} used for Image optimizations it should be approximately equal to the width of the picture on the screen in px
-          width={64}
-        />
+        {isVideoFormat(assetContainer.asset.format) ? (
+          <StyledWebstudioVideo
+            width={64}
+            src={
+              assetContainer.status === "uploading"
+                ? assetContainer.objectURL
+                : `/cgi/image/${assetContainer.asset.name}?format=raw`
+            }
+          />
+        ) : (
+          <StyledWebstudioImage
+            assetId={assetContainer.asset.id}
+            name={assetContainer.asset.name}
+            objectURL={
+              assetContainer.status === "uploading"
+                ? assetContainer.objectURL
+                : undefined
+            }
+            alt={description ?? name}
+            // width={64} used for Image optimizations it should be approximately equal to the width of the picture on the screen in px
+            width={64}
+          />
+        )}
       </Thumbnail>
       <Box
         css={{

apps/builder/app/routes/rest.assets_.$name.tsx

@@ -21,6 +21,8 @@ export const action = async (
 
   const { request, params } = props;
 
+  // await new Promise((resolve) => setTimeout(resolve, 20000));
+
   if (params.name === undefined) {
     throw new Error("Name is undefined");
   }
@@ -57,12 +59,31 @@ export const action = async (
         body = imageRequest.body;
       }
 
+      const url = new URL(request.url);
+      const contentTypeArr = contentType?.split(";")[0]?.split("/") ?? [];
+
+      const format =
+        contentTypeArr[0] === "video" ? contentTypeArr[1] : undefined;
+
+      const width = url.searchParams.has("width")
+        ? parseInt(url.searchParams.get("width")!, 10)
+        : undefined;
+      const height = url.searchParams.has("height")
+        ? parseInt(url.searchParams.get("height")!, 10)
+        : undefined;
+
+      const assetInfoFallback =
+        height !== undefined && width !== undefined && format !== undefined
+          ? { width, height, format }
+          : undefined;
+
       const context = await createContext(request);
       const asset = await uploadFile(
         params.name,
         body,
         createAssetClient(),
-        context
+        context,
+        assetInfoFallback
       );
       return {
         uploadedAssets: [asset],

apps/builder/app/shared/router-utils/path-utils.ts

@@ -123,7 +123,27 @@ export const restAssetsPath = () => {
   return `/rest/assets`;
 };
 
-export const restAssetsUploadPath = ({ name }: { name: string }) => {
+export const restAssetsUploadPath = ({
+  name,
+  width,
+  height,
+}: {
+  name: string;
+  width?: number | undefined;
+  height?: number | undefined;
+}) => {
+  const urlSearchParams = new URLSearchParams();
+  if (width !== undefined) {
+    urlSearchParams.set("width", String(width));
+  }
+  if (height !== undefined) {
+    urlSearchParams.set("height", String(height));
+  }
+
+  if (urlSearchParams.size > 0) {
+    return `/rest/assets/${name}?${urlSearchParams.toString()}`;
+  }
+
   return `/rest/assets/${name}`;
 };
 

packages/asset-uploader/src/client.ts

@@ -4,6 +4,9 @@ export type AssetClient = {
   uploadFile: (
     name: string,
     type: string,
-    data: AsyncIterable<Uint8Array>
+    data: AsyncIterable<Uint8Array>,
+    assetInfoFallback:
+      | { width: number; height: number; format: string }
+      | undefined
   ) => Promise<AssetData>;
 };

packages/asset-uploader/src/clients/s3/s3.ts

@@ -26,7 +26,12 @@ export const createS3Client = (options: S3ClientOptions): AssetClient => {
     uriEscapePath: false,
   });
 
-  const uploadFile: AssetClient["uploadFile"] = async (name, type, data) => {
+  const uploadFile: AssetClient["uploadFile"] = async (
+    name,
+    type,
+    data,
+    assetInfoFallback
+  ) => {
     return uploadToS3({
       signer,
       name,
@@ -36,6 +41,7 @@ export const createS3Client = (options: S3ClientOptions): AssetClient => {
       endpoint: options.endpoint,
       bucket: options.bucket,
       acl: options.acl,
+      assetInfoFallback,
     });
   };
 

packages/asset-uploader/src/clients/s3/upload.ts

@@ -13,6 +13,7 @@ export const uploadToS3 = async ({
   endpoint,
   bucket,
   acl,
+  assetInfoFallback,
 }: {
   signer: SignatureV4;
   name: string;
@@ -22,6 +23,9 @@ export const uploadToS3 = async ({
   endpoint: string;
   bucket: string;
   acl?: string;
+  assetInfoFallback:
+    | { width: number; height: number; format: string }
+    | undefined;
 }): Promise<AssetData> => {
   const limitSize = createSizeLimiter(maxSize, name);
 
@@ -65,8 +69,20 @@ export const uploadToS3 = async ({
     throw Error(`Cannot upload file ${name}`);
   }
 
+  if (type.startsWith("video") && assetInfoFallback !== undefined) {
+    return {
+      size: data.byteLength,
+      format: assetInfoFallback?.format,
+      meta: {
+        width: assetInfoFallback?.width ?? 0,
+        height: assetInfoFallback?.height ?? 0,
+      },
+    };
+  }
+
   const assetData = await getAssetData({
-    type: type.startsWith("image") ? "image" : "font",
+    type:
+      type.startsWith("image") || type.startsWith("video") ? "image" : "font",
     size: data.byteLength,
     data: new Uint8Array(data),
     name,

packages/asset-uploader/src/upload.ts

@@ -96,7 +96,10 @@ export const uploadFile = async (
   name: string,
   data: ReadableStream<Uint8Array>,
   client: AssetClient,
-  context: AppContext
+  context: AppContext,
+  assetInfoFallback:
+    | { width: number; height: number; format: string }
+    | undefined
 ): Promise<Asset> => {
   let file = await context.postgrest.client
     .from("File")
@@ -117,7 +120,8 @@ export const uploadFile = async (
       name,
       file.data.format,
       // global web streams types do not define ReadableStream as async iterable
-      data as unknown as AsyncIterable<Uint8Array>
+      data as unknown as AsyncIterable<Uint8Array>,
+      assetInfoFallback
     );
     const { meta, format, size } = assetData;
     file = await context.postgrest.client