feat(server/fileuploads): use a presigned url to upload large files

packages/frontend-2/components/viewer/comments/Editor.vue

@@ -45,6 +45,7 @@ import { useAttachments } from '~~/lib/core/composables/fileUpload'
 import { useInjectedViewerState } from '~~/lib/viewer/composables/setup'
 import { isSuccessfullyUploaded } from '~~/lib/core/api/blobStorage'
 import { canInviteToProject } from '~~/lib/projects/helpers/permissions'
+import { acceptedFileExtensions } from '@speckle/shared'
 
 const emit = defineEmits<{
   (e: 'update:modelValue', val: Optional<CommentEditorValue>): void
@@ -73,36 +74,7 @@ const acceptValue = ref(
   [
     UniqueFileTypeSpecifier.AnyImage,
     UniqueFileTypeSpecifier.AnyVideo,
-    '.pdf',
-    '.zip',
-    '.7z',
-    '.pptx',
-    '.ifc',
-    '.dwg',
-    '.dxf',
-    '.3dm',
-    '.ghx',
-    '.gh',
-    '.rvt',
-    '.pla',
-    '.pln',
-    '.obj',
-    '.blend',
-    '.3ds',
-    '.max',
-    '.mtl',
-    '.stl',
-    '.md',
-    '.txt',
-    '.csv',
-    '.xlsx',
-    '.xls',
-    '.doc',
-    '.docx',
-    '.svg',
-    '.eps',
-    '.gwb',
-    '.skp'
+    ...acceptedFileExtensions.map((fileExtension) => `.${fileExtension}`)
   ].join(',')
 )
 

packages/frontend-2/lib/common/generated/gql/graphql.ts

@@ -1029,6 +1029,32 @@ export type FileUpload = {
   userId: Scalars['String']['output'];
 };
 
+export type FileUploadMutations = {
+  __typename?: 'FileUploadMutations';
+  /**
+   * Generate a pre-signed url to which a file can be uploaded.
+   * After uploading the file, call mutation startFileImport to register the completed upload.
+   */
+  generateUploadUrl: GenerateFileUploadUrlOutput;
+  /**
+   * Before calling this mutation, call generateUploadUrl to get the
+   * pre-signed url and blobId. Then upload the file to that url.
+   * Once the upload to the pre-signed url is completed, this mutation should be
+   * called to register the completed upload and create the blob metadata.
+   */
+  startFileImport: FileUpload;
+};
+
+
+export type FileUploadMutationsGenerateUploadUrlArgs = {
+  input: GenerateFileUploadUrlInput;
+};
+
+
+export type FileUploadMutationsStartFileImportArgs = {
+  input: StartFileImportInput;
+};
+
 export type GendoAiRender = {
   __typename?: 'GendoAIRender';
   camera?: Maybe<Scalars['JSONObject']['output']>;
@@ -1064,6 +1090,17 @@ export type GendoAiRenderInput = {
   versionId: Scalars['ID']['input'];
 };
 
+export type GenerateFileUploadUrlInput = {
+  fileName: Scalars['String']['input'];
+  projectId: Scalars['String']['input'];
+};
+
+export type GenerateFileUploadUrlOutput = {
+  __typename?: 'GenerateFileUploadUrlOutput';
+  fileId: Scalars['String']['output'];
+  url: Scalars['String']['output'];
+};
+
 export type InvitableCollaboratorsFilter = {
   search?: InputMaybe<Scalars['String']['input']>;
 };
@@ -1459,6 +1496,7 @@ export type Mutation = {
    * @deprecated Part of the old API surface and will be removed in the future. Use VersionMutations.moveToModel instead.
    */
   commitsMove: Scalars['Boolean']['output'];
+  fileUploadMutations: FileUploadMutations;
   /**
    * Delete a pending invite
    * Note: The required scope to invoke this is not given out to app or personal access tokens
@@ -3283,6 +3321,17 @@ export const SortDirection = {
 } as const;
 
 export type SortDirection = typeof SortDirection[keyof typeof SortDirection];
+export type StartFileImportInput = {
+  /**
+   * The etag is returned by the blob storage provider in the response body after a successful upload.
+   * It is used to verify the integrity of the uploaded file.
+   */
+  etag: Scalars['String']['input'];
+  fileId: Scalars['String']['input'];
+  modelId: Scalars['String']['input'];
+  projectId: Scalars['String']['input'];
+};
+
 export type Stream = {
   __typename?: 'Stream';
   /**
@@ -7734,8 +7783,10 @@ export type AllObjectTypes = {
   CountOnlyCollection: CountOnlyCollection,
   CurrencyBasedPrices: CurrencyBasedPrices,
   FileUpload: FileUpload,
+  FileUploadMutations: FileUploadMutations,
   GendoAIRender: GendoAiRender,
   GendoAIRenderCollection: GendoAiRenderCollection,
+  GenerateFileUploadUrlOutput: GenerateFileUploadUrlOutput,
   LegacyCommentViewerData: LegacyCommentViewerData,
   LimitedUser: LimitedUser,
   LimitedWorkspace: LimitedWorkspace,
@@ -8211,6 +8262,10 @@ export type FileUploadFieldArgs = {
   uploadDate: {},
   userId: {},
 }
+export type FileUploadMutationsFieldArgs = {
+  generateUploadUrl: FileUploadMutationsGenerateUploadUrlArgs,
+  startFileImport: FileUploadMutationsStartFileImportArgs,
+}
 export type GendoAiRenderFieldArgs = {
   camera: {},
   createdAt: {},
@@ -8230,6 +8285,10 @@ export type GendoAiRenderCollectionFieldArgs = {
   items: {},
   totalCount: {},
 }
+export type GenerateFileUploadUrlOutputFieldArgs = {
+  fileId: {},
+  url: {},
+}
 export type LegacyCommentViewerDataFieldArgs = {
   camPos: {},
   filters: {},
@@ -8359,6 +8418,7 @@ export type MutationFieldArgs = {
   commitUpdate: MutationCommitUpdateArgs,
   commitsDelete: MutationCommitsDeleteArgs,
   commitsMove: MutationCommitsMoveArgs,
+  fileUploadMutations: {},
   inviteDelete: MutationInviteDeleteArgs,
   inviteResend: MutationInviteResendArgs,
   modelMutations: {},
@@ -9341,8 +9401,10 @@ export type AllObjectFieldArgTypes = {
   CountOnlyCollection: CountOnlyCollectionFieldArgs,
   CurrencyBasedPrices: CurrencyBasedPricesFieldArgs,
   FileUpload: FileUploadFieldArgs,
+  FileUploadMutations: FileUploadMutationsFieldArgs,
   GendoAIRender: GendoAiRenderFieldArgs,
   GendoAIRenderCollection: GendoAiRenderCollectionFieldArgs,
+  GenerateFileUploadUrlOutput: GenerateFileUploadUrlOutputFieldArgs,
   LegacyCommentViewerData: LegacyCommentViewerDataFieldArgs,
   LimitedUser: LimitedUserFieldArgs,
   LimitedWorkspace: LimitedWorkspaceFieldArgs,

packages/server/assets/fileuploads/typedefs/fileuploads.graphql

@@ -75,6 +75,48 @@ type FileUpload {
   model: Model
 }
 
+input GenerateFileUploadUrlInput {
+  projectId: String!
+  fileName: String!
+}
+
+type GenerateFileUploadUrlOutput {
+  url: String!
+  fileId: String!
+}
+
+input StartFileImportInput {
+  projectId: String!
+  modelId: String!
+
+  fileId: String!
+  """
+  The etag is returned by the blob storage provider in the response body after a successful upload.
+  It is used to verify the integrity of the uploaded file.
+  """
+  etag: String!
+}
+
+type FileUploadMutations {
+  """
+  Generate a pre-signed url to which a file can be uploaded.
+  After uploading the file, call mutation startFileImport to register the completed upload.
+  """
+  generateUploadUrl(input: GenerateFileUploadUrlInput!): GenerateFileUploadUrlOutput!
+
+  """
+  Before calling this mutation, call generateUploadUrl to get the
+  pre-signed url and blobId. Then upload the file to that url.
+  Once the upload to the pre-signed url is completed, this mutation should be
+  called to register the completed upload and create the blob metadata.
+  """
+  startFileImport(input: StartFileImportInput!): FileUpload!
+}
+
+extend type Mutation {
+  fileUploadMutations: FileUploadMutations!
+}
+
 enum ProjectPendingModelsUpdatedMessageType {
   CREATED
   UPDATED

packages/server/codegen.yml

@@ -31,6 +31,7 @@ generates:
         ProjectInviteMutations: '@/modules/core/helpers/graphTypes#MutationsObjectGraphQLReturn'
         ModelMutations: '@/modules/core/helpers/graphTypes#MutationsObjectGraphQLReturn'
         VersionMutations: '@/modules/core/helpers/graphTypes#MutationsObjectGraphQLReturn'
+        FileUploadMutations: '@/modules/core/helpers/graphTypes#MutationsObjectGraphQLReturn'
         CommentMutations: '@/modules/core/helpers/graphTypes#MutationsObjectGraphQLReturn'
         AutomateMutations: '@/modules/core/helpers/graphTypes#MutationsObjectGraphQLReturn'
         AdminMutations: '@/modules/core/helpers/graphTypes#MutationsObjectGraphQLReturn'

packages/server/modules/blobstorage/clients/objectStorage.ts

@@ -5,14 +5,28 @@ import {
   getS3Region,
   getS3SecretKey
 } from '@/modules/shared/helpers/envHelper'
-import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3'
-import { Optional } from '@speckle/shared'
+import {
+  HeadObjectCommand,
+  PutObjectCommand,
+  S3Client,
+  S3ClientConfig
+} from '@aws-sdk/client-s3'
+import { getSignedUrl as s3GetSignedUrl } from '@aws-sdk/s3-request-presigner'
+import type { Optional } from '@speckle/shared'
+import {
+  GetBlobMetadataFromStorage,
+  GetSignedUrl
+} from '@/modules/blobstorage/domain/operations'
 
 export type ObjectStorage = {
   client: S3Client
   bucket: string
 }
 
+export type GetProjectObjectStorage = (args: {
+  projectId: string
+}) => Promise<ObjectStorage>
+
 export type GetObjectStorageParams = {
   credentials: S3ClientConfig['credentials']
   endpoint: S3ClientConfig['endpoint']
@@ -57,3 +71,34 @@ export const getMainObjectStorage = (): ObjectStorage => {
   mainObjectStorage = getObjectStorage(mainParams)
   return mainObjectStorage
 }
+
+export const getSignedUrlFactory = (deps: {
+  objectStorage: ObjectStorage
+}): GetSignedUrl => {
+  const { objectStorage } = deps
+  const { client, bucket } = objectStorage
+  return async (params) => {
+    const { objectKey, urlExpiryDurationSeconds } = params
+    const command = new PutObjectCommand({ Bucket: bucket, Key: objectKey })
+    return s3GetSignedUrl(client, command, { expiresIn: urlExpiryDurationSeconds })
+  }
+}
+
+export const getBlobMetadataFromStorage = (deps: {
+  objectStorage: ObjectStorage
+}): GetBlobMetadataFromStorage => {
+  const { objectStorage } = deps
+  const { client, bucket } = objectStorage
+
+  return async (params) => {
+    const { objectKey } = params
+
+    // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/s3/command/HeadObjectCommand/
+    const headObjectCommand = new HeadObjectCommand({ Bucket: bucket, Key: objectKey })
+    const metadata = await client.send(headObjectCommand)
+    return {
+      contentLength: metadata.ContentLength,
+      eTag: metadata.ETag
+    }
+  }
+}

packages/server/modules/blobstorage/domain/operations.ts

@@ -2,7 +2,7 @@ import {
   BlobStorageItem,
   BlobStorageItemInput
 } from '@/modules/blobstorage/domain/types'
-import { MaybeNullOrUndefined, Nullable } from '@speckle/shared'
+import { MaybeNullOrUndefined, Nullable, Optional } from '@speckle/shared'
 import type { Readable } from 'stream'
 import { StoreFileStream } from '@/modules/blobstorage/domain/storageOperations'
 
@@ -52,3 +52,27 @@ export type UploadFileStream = (
 ) => Promise<{ blobId: string; fileName: string; fileHash: string }>
 
 export { StoreFileStream }
+
+export type GeneratePresignedUrl = (params: {
+  projectId: string
+  userId: string
+  blobId: string
+  fileName: string
+  urlExpiryDurationSeconds: number
+}) => Promise<string>
+
+export type GetSignedUrl = (params: {
+  objectKey: string
+  urlExpiryDurationSeconds: number
+}) => Promise<string>
+
+export type GetBlobMetadataFromStorage = (params: {
+  objectKey: string
+}) => Promise<{ eTag: Optional<string>; contentLength: Optional<number> }>
+
+export type RegisterCompletedUpload = (params: {
+  projectId: string
+  blobId: string
+  expectedETag: string
+  maximumFileSize: number
+}) => Promise<BlobStorageItem>

packages/server/modules/blobstorage/domain/types.ts

@@ -1,6 +1,12 @@
 import { Nullable } from '@speckle/shared'
 import { SetOptional } from 'type-fest'
 
+export enum BlobUploadStatus {
+  Pending = 0,
+  Completed = 1,
+  Error = 2
+}
+
 export type BlobStorageItem = {
   id: string
   streamId: string
@@ -9,7 +15,7 @@ export type BlobStorageItem = {
   fileName: string
   fileType: string
   fileSize: Nullable<number>
-  uploadStatus: number
+  uploadStatus: number | BlobUploadStatus
   uploadError: Nullable<string>
   createdAt: Date
   fileHash: Nullable<string>

packages/server/modules/blobstorage/errors/index.ts

@@ -0,0 +1,13 @@
+import { BaseError } from '@/modules/shared/errors'
+
+export class StoredBlobAccessError extends BaseError {
+  static code = 'STORED_BLOB_ACCESS_ERROR'
+  static defaultMessage = 'An issue occurred while attempting to access a stored blob.'
+  static statusCode = 400
+}
+
+export class AlreadyRegisteredBlobError extends BaseError {
+  static code = 'ALREADY_REGISTERED_BLOB_ERROR'
+  static defaultMessage = 'The blob is already registered as having been uploaded.'
+  static statusCode = 400
+}