feat(storage): add delimiter support to list API (aws-amplify#13517)

* feat: add types

* feat: enable delimiter

* chore: add unit tests

* chore: bump bundle size

* chore: add tsdocs

* chore: address feedback

* chore: address feedback

* chore: address feedback

Author: israx

packages/aws-amplify/package.json

@@ -485,7 +485,7 @@
 			"name": "[Storage] list (S3)",
 			"path": "./dist/esm/storage/index.mjs",
 			"import": "{ list }",
-			"limit": "14.94 kB"
+			"limit": "15.04 kB"
 		},
 		{
 			"name": "[Storage] remove (S3)",

packages/storage/__tests__/providers/s3/apis/list.test.ts

@@ -512,4 +512,167 @@ describe('list API', () => {
 			}
 		});
 	});
+
+	describe('with delimiter', () => {
+		const mockedContents = [
+			{
+				Key: 'photos/',
+				...listObjectClientBaseResultItem,
+			},
+			{
+				Key: 'photos/2023.png',
+				...listObjectClientBaseResultItem,
+			},
+			{
+				Key: 'photos/2024.png',
+				...listObjectClientBaseResultItem,
+			},
+		];
+		const mockedCommonPrefixes = [
+			{ Prefix: 'photos/2023/' },
+			{ Prefix: 'photos/2024/' },
+			{ Prefix: 'photos/2025/' },
+		];
+
+		const expectedExcludedSubpaths = mockedCommonPrefixes.map(
+			({ Prefix }) => Prefix,
+		);
+
+		const mockedPath = 'photos/';
+
+		beforeEach(() => {
+			mockListObject.mockResolvedValueOnce({
+				Contents: mockedContents,
+				CommonPrefixes: mockedCommonPrefixes,
+			});
+		});
+		afterEach(() => {
+			jest.clearAllMocks();
+			mockListObject.mockClear();
+		});
+
+		it('should return excludedSubpaths when "exclude" strategy is passed in the request', async () => {
+			const { items, excludedSubpaths } = await list({
+				path: mockedPath,
+				options: {
+					subpathStrategy: { strategy: 'exclude' },
+				},
+			});
+			expect(items).toHaveLength(3);
+			expect(excludedSubpaths).toEqual(expectedExcludedSubpaths);
+			expect(listObjectsV2).toHaveBeenCalledTimes(1);
+			await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
+				listObjectClientConfig,
+				{
+					Bucket: bucket,
+					MaxKeys: 1000,
+					Prefix: mockedPath,
+					Delimiter: '/',
+				},
+			);
+		});
+
+		it('should return excludedSubpaths when "exclude" strategy and listAll are passed in the request', async () => {
+			const { items, excludedSubpaths } = await list({
+				path: mockedPath,
+				options: {
+					subpathStrategy: { strategy: 'exclude' },
+					listAll: true,
+				},
+			});
+			expect(items).toHaveLength(3);
+			expect(excludedSubpaths).toEqual(expectedExcludedSubpaths);
+			expect(listObjectsV2).toHaveBeenCalledTimes(1);
+			await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
+				listObjectClientConfig,
+				{
+					Bucket: bucket,
+					MaxKeys: 1000,
+					Prefix: mockedPath,
+					Delimiter: '/',
+				},
+			);
+		});
+
+		it('should return excludedSubpaths when "exclude" strategy and pageSize are passed in the request', async () => {
+			const { items, excludedSubpaths } = await list({
+				path: mockedPath,
+				options: {
+					subpathStrategy: { strategy: 'exclude' },
+					pageSize: 3,
+				},
+			});
+			expect(items).toHaveLength(3);
+			expect(excludedSubpaths).toEqual(expectedExcludedSubpaths);
+			expect(listObjectsV2).toHaveBeenCalledTimes(1);
+			await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
+				listObjectClientConfig,
+				{
+					Bucket: bucket,
+					MaxKeys: 3,
+					Prefix: mockedPath,
+					Delimiter: '/',
+				},
+			);
+		});
+
+		it('should listObjectsV2 contain a custom Delimiter when "exclude" with delimiter is passed', async () => {
+			await list({
+				path: mockedPath,
+				options: {
+					subpathStrategy: {
+						strategy: 'exclude',
+						delimiter: '-',
+					},
+				},
+			});
+			expect(listObjectsV2).toHaveBeenCalledTimes(1);
+			await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
+				listObjectClientConfig,
+				{
+					Bucket: bucket,
+					MaxKeys: 1000,
+					Prefix: mockedPath,
+					Delimiter: '-',
+				},
+			);
+		});
+
+		it('should listObjectsV2 contain an undefined Delimiter when "include" strategy is passed', async () => {
+			await list({
+				path: mockedPath,
+				options: {
+					subpathStrategy: {
+						strategy: 'include',
+					},
+				},
+			});
+			expect(listObjectsV2).toHaveBeenCalledTimes(1);
+			await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
+				listObjectClientConfig,
+				{
+					Bucket: bucket,
+					MaxKeys: 1000,
+					Prefix: mockedPath,
+					Delimiter: undefined,
+				},
+			);
+		});
+
+		it('should listObjectsV2 contain an undefined Delimiter when no options are passed', async () => {
+			await list({
+				path: mockedPath,
+			});
+			expect(listObjectsV2).toHaveBeenCalledTimes(1);
+			await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
+				listObjectClientConfig,
+				{
+					Bucket: bucket,
+					MaxKeys: 1000,
+					Prefix: mockedPath,
+					Delimiter: undefined,
+				},
+			);
+		});
+	});
 });

packages/storage/src/providers/s3/apis/internal/list.ts

@@ -28,7 +28,9 @@ import {
 } from '../../utils/client';
 import { getStorageUserAgentValue } from '../../utils/userAgent';
 import { logger } from '../../../../utils';
-import { STORAGE_INPUT_PREFIX } from '../../utils/constants';
+import { DEFAULT_DELIMITER, STORAGE_INPUT_PREFIX } from '../../utils/constants';
+import { CommonPrefix } from '../../utils/client/types';
+import { StorageSubpathStrategy } from '../../../../types';
 
 const MAX_PAGE_SIZE = 1000;
 
@@ -79,13 +81,15 @@ export const list = async (
 		Prefix: isInputWithPrefix ? `${generatedPrefix}${objectKey}` : objectKey,
 		MaxKeys: options?.listAll ? undefined : options?.pageSize,
 		ContinuationToken: options?.listAll ? undefined : options?.nextToken,
+		Delimiter: getDelimiter(options.subpathStrategy),
 	};
 	logger.debug(`listing items from "${listParams.Prefix}"`);
 
 	const listInputArgs: ListInputArgs = {
 		s3Config,
 		listParams,
 	};
+
 	if (options.listAll) {
 		if (isInputWithPrefix) {
 			return _listAllWithPrefix({
@@ -176,23 +180,29 @@ const _listAllWithPath = async ({
 	listParams,
 }: ListInputArgs): Promise<ListAllWithPathOutput> => {
 	const listResult: ListOutputItemWithPath[] = [];
+	const excludedSubpaths: string[] = [];
 	let continuationToken = listParams.ContinuationToken;
 	do {
-		const { items: pageResults, nextToken: pageNextToken } =
-			await _listWithPath({
-				s3Config,
-				listParams: {
-					...listParams,
-					ContinuationToken: continuationToken,
-					MaxKeys: MAX_PAGE_SIZE,
-				},
-			});
+		const {
+			items: pageResults,
+			excludedSubpaths: pageExcludedSubpaths,
+			nextToken: pageNextToken,
+		} = await _listWithPath({
+			s3Config,
+			listParams: {
+				...listParams,
+				ContinuationToken: continuationToken,
+				MaxKeys: MAX_PAGE_SIZE,
+			},
+		});
 		listResult.push(...pageResults);
+		excludedSubpaths.push(...(pageExcludedSubpaths ?? []));
 		continuationToken = pageNextToken;
 	} while (continuationToken);
 
 	return {
 		items: listResult,
+		excludedSubpaths,
 	};
 };
 
@@ -206,27 +216,56 @@ const _listWithPath = async ({
 		listParamsClone.MaxKeys = MAX_PAGE_SIZE;
 	}
 
-	const response: ListObjectsV2Output = await listObjectsV2(
+	const {
+		Contents: contents,
+		NextContinuationToken: nextContinuationToken,
+		CommonPrefixes: commonPrefixes,
+	}: ListObjectsV2Output = await listObjectsV2(
 		{
 			...s3Config,
 			userAgentValue: getStorageUserAgentValue(StorageAction.List),
 		},
 		listParamsClone,
 	);
 
-	if (!response?.Contents) {
+	const excludedSubpaths =
+		commonPrefixes && mapCommonPrefixesToExcludedSubpaths(commonPrefixes);
+
+	if (!contents) {
 		return {
 			items: [],
+			excludedSubpaths,
 		};
 	}
 
 	return {
-		items: response.Contents.map(item => ({
+		items: contents.map(item => ({
 			path: item.Key!,
 			eTag: item.ETag,
 			lastModified: item.LastModified,
 			size: item.Size,
 		})),
-		nextToken: response.NextContinuationToken,
+		nextToken: nextContinuationToken,
+		excludedSubpaths,
 	};
 };
+
+const mapCommonPrefixesToExcludedSubpaths = (
+	commonPrefixes: CommonPrefix[],
+): string[] => {
+	return commonPrefixes.reduce((mappedSubpaths, { Prefix }) => {
+		if (Prefix) {
+			mappedSubpaths.push(Prefix);
+		}
+
+		return mappedSubpaths;
+	}, [] as string[]);
+};
+
+const getDelimiter = (
+	subpathStrategy?: StorageSubpathStrategy,
+): string | undefined => {
+	if (subpathStrategy?.strategy === 'exclude') {
+		return subpathStrategy?.delimiter ?? DEFAULT_DELIMITER;
+	}
+};

packages/storage/src/providers/s3/utils/constants.ts

@@ -23,3 +23,5 @@ export const UPLOADS_STORAGE_KEY = '__uploadInProgress';
 export const STORAGE_INPUT_PREFIX = 'prefix';
 export const STORAGE_INPUT_KEY = 'key';
 export const STORAGE_INPUT_PATH = 'path';
+
+export const DEFAULT_DELIMITER = '/';

packages/storage/src/types/index.ts

@@ -29,6 +29,7 @@ export {
 	StorageRemoveOptions,
 	StorageListAllOptions,
 	StorageListPaginateOptions,
+	StorageSubpathStrategy,
 } from './options';
 export {
 	StorageItem,

packages/storage/src/types/options.ts

@@ -10,12 +10,65 @@ export interface StorageOptions {
 
 export type StorageListAllOptions = StorageOptions & {
 	listAll: true;
+	subpathStrategy?: StorageSubpathStrategy;
 };
 
 export type StorageListPaginateOptions = StorageOptions & {
 	listAll?: false;
 	pageSize?: number;
 	nextToken?: string;
+	subpathStrategy?: StorageSubpathStrategy;
 };
 
 export type StorageRemoveOptions = StorageOptions;
+
+export type StorageSubpathStrategy =
+	| {
+			/**
+			 * Default behavior. Includes all subpaths for a given page in the result.
+			 */
+			strategy: 'include';
+	  }
+	| {
+			/**
+			 * When passed, the output of the list API will provide a list of `excludedSubpaths`
+			 * that are delimited by the `/` (by default) character.
+			 *
+			 *
+			 * @example
+			 * ```ts
+			 *	const { excludedSubpaths } = await list({
+			 *		path: 'photos/',
+			 *		options: {
+			 *			subpathStrategy: {
+			 *				strategy: 'exclude',
+			 *			}
+			 *		}
+			 *	});
+			 *
+			 *	console.log(excludedSubpaths);
+			 *
+			 * ```
+			 */
+			strategy: 'exclude';
+			/**
+			 * Deliminate with with a custom delimiter character.
+			 *
+			 * @example
+			 * ```ts
+			 *	const { excludedSubpaths } = await list({
+			 *		path: 'photos/',
+			 *		options: {
+			 *			subpathStrategy: {
+			 *				strategy: 'exclude',
+			 *				delimiter: '-'
+			 *			}
+			 *		}
+			 *	});
+			 *
+			 *	console.log(excludedSubpaths);
+			 *
+			 * ```
+			 */
+			delimiter?: string;
+	  };

packages/storage/src/types/outputs.ts

@@ -70,4 +70,8 @@ export interface StorageListOutput<Item extends StorageItem> {
 	 * List of items returned by the list API.
 	 */
 	items: Item[];
+	/**
+	 * List of excluded subpaths when `exclude` is passed as part of the `subpathStrategy` of the input options.
+	 */
+	excludedSubpaths?: string[];
 }