feat!(website): allow selecting fields to download in metadata and default to a shorter selection

kubernetes/loculus/templates/_common-metadata.tpl

@@ -12,6 +12,7 @@ fields:
     type: string
     notSearchable: true
     hideOnSequenceDetailsPage: true
+    includeInDownloadsByDefault: true
   - name: accession
     type: string
     notSearchable: true
@@ -82,6 +83,7 @@ fields:
     autocomplete: true
     displayName: Data use terms
     initiallyVisible: true
+    includeInDownloadsByDefault: true
     customDisplay:
       type: dataUseTerms
     header: Data use terms
@@ -96,6 +98,7 @@ fields:
     type: string
     notSearchable: true
     header: Data use terms
+    includeInDownloadsByDefault: true
     customDisplay:
       type: link
       url: "__value__"
@@ -243,6 +246,9 @@ organisms:
   {{- if .order }}
   order: {{ .order }}
   {{- end }}
+  {{- if .includeInDownloadsByDefault }}
+  includeInDownloadsByDefault: {{ .includeInDownloadsByDefault }}
+  {{- end }}
   {{- if .customDisplay }}
   customDisplay:
     type: {{ quote .customDisplay.type }}

kubernetes/loculus/values.yaml

@@ -65,6 +65,8 @@ defaultOrganismConfig: &defaultOrganismConfig
     # initiallyVisible: Whether it appear in searchUI by default (default false)
     # generateIndex: Whether the field should be indexed for search (default false, only allowed for string fields)
     # autocomplete: Whether the field should be used for search autocomplete (default false, only allowed for string fields and probably generateIndex should be true)
+    ## Download related
+    # includeInDownloadsByDefault: Whether this field should be selected by default in download options (default false)
     ## Ingest related
     # ingest: Which NCBI field to map to this field (optional)
     ## Preprocessing related
@@ -75,6 +77,7 @@ defaultOrganismConfig: &defaultOrganismConfig
         header: Sample details
         ingest: ncbiCollectionDate
         order: 10
+        includeInDownloadsByDefault: true
         preprocessing:
           function: parse_date_into_range
           inputs:
@@ -147,6 +150,7 @@ defaultOrganismConfig: &defaultOrganismConfig
         type: date
         rangeSearch: true
         noInput: true
+        includeInDownloadsByDefault: true
       - name: ncbiUpdateDate
         type: date
         displayName: NCBI update date
@@ -174,6 +178,7 @@ defaultOrganismConfig: &defaultOrganismConfig
         generateIndex: true
         autocomplete: true
         initiallyVisible: true
+        includeInDownloadsByDefault: true
         header: Sample details
         order: 20
         ingest: country
@@ -528,6 +533,7 @@ defaultOrganismConfig: &defaultOrganismConfig
         desired: true
         enableSubstringSearch: true
         order: 40
+        includeInDownloadsByDefault: true
         truncateColumnDisplayTo: 25
         ingest: ncbiSubmitterNames
         preprocessing:
@@ -542,6 +548,7 @@ defaultOrganismConfig: &defaultOrganismConfig
         truncateColumnDisplayTo: 15
         header: Authors
         ingest: ncbiSubmitterAffiliation
+        includeInDownloadsByDefault: true
       - name: ncbiSubmitterCountry
         displayName: NCBI submitter country
         generateIndex: true
@@ -1238,6 +1245,7 @@ defaultOrganisms:
           generateIndex: true
           autocomplete: true
           initiallyVisible: true
+          includeInDownloadsByDefault: true
           preprocessing:
             inputs: {input: nextclade.clade}
         - name: totalStopCodons

website/src/components/SearchPage/DownloadDialog/DownloadDialog.spec.tsx

@@ -1,12 +1,20 @@
 import { render, screen } from '@testing-library/react';
 import userEvent from '@testing-library/user-event';
-import { describe, expect, test } from 'vitest';
+import { describe, expect, test, vi } from 'vitest';
 
 import { DownloadDialog } from './DownloadDialog.tsx';
 import { DownloadUrlGenerator } from './DownloadUrlGenerator.ts';
 import { FieldFilter, SelectFilter, type SequenceFilter } from './SequenceFilters.tsx';
+import type { Metadata } from '../../../types/config.ts';
 import type { ReferenceGenomesSequenceNames, ReferenceAccession } from '../../../types/referencesGenomes.ts';
 
+// Mock the FieldSelectorModal to avoid errors in tests
+vi.mock('./FieldSelector/FieldSelectorModal.tsx', () => ({
+    getDefaultSelectedFields: () => ['field1', 'field2'],
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    FieldSelectorModal: vi.fn(() => null),
+}));
+
 const defaultAccession: ReferenceAccession = {
     name: 'main',
     insdcAccessionFull: undefined,
@@ -21,6 +29,23 @@ const defaultReferenceGenome: ReferenceGenomesSequenceNames = {
 const defaultLapisUrl = 'https://lapis';
 const defaultOrganism = 'ebola';
 
+// Mock metadata for testing
+const mockMetadata: Metadata[] = [
+    {
+        name: 'field1',
+        displayName: 'Field 1',
+        type: 'string',
+        header: 'Group 1',
+        includeInDownloadsByDefault: true,
+    },
+    {
+        name: 'field2',
+        displayName: 'Field 2',
+        type: 'string',
+        header: 'Group 1',
+    },
+];
+
 async function renderDialog({
     downloadParams = new SelectFilter(new Set()),
     allowSubmissionOfConsensusSequences = true,
@@ -37,6 +62,7 @@ async function renderDialog({
             referenceGenomesSequenceNames={defaultReferenceGenome}
             allowSubmissionOfConsensusSequences={allowSubmissionOfConsensusSequences}
             dataUseTermsEnabled={dataUseTermsEnabled}
+            metadata={mockMetadata}
         />,
     );
 
@@ -78,7 +104,7 @@ describe('DownloadDialog', () => {
         let [path, query] = getDownloadHref()?.split('?') ?? [];
         expect(path).toBe(`${defaultLapisUrl}/sample/details`);
         expect(query).toMatch(
-            /downloadAsFile=true&downloadFileBasename=ebola_metadata_\d{4}-\d{2}-\d{2}T\d{4}&versionStatus=LATEST_VERSION&isRevocation=false&dataUseTerms=OPEN&dataFormat=tsv&accession=accession1&accession=accession2&field1=value1/,
+            /downloadAsFile=true&downloadFileBasename=ebola_metadata_\d{4}-\d{2}-\d{2}T\d{4}&versionStatus=LATEST_VERSION&isRevocation=false&dataUseTerms=OPEN&dataFormat=tsv&fields=accessionVersion%2Cfield1%2Cfield2&accession=accession1&accession=accession2&field1=value1/,
         );
 
         await userEvent.click(screen.getByLabelText(olderVersionsLabel));
@@ -108,7 +134,7 @@ describe('DownloadDialog', () => {
         let [path, query] = getDownloadHref()?.split('?') ?? [];
         expect(path).toBe(`${defaultLapisUrl}/sample/details`);
         expect(query).toMatch(
-            /downloadAsFile=true&downloadFileBasename=ebola_metadata_\d{4}-\d{2}-\d{2}T\d{4}&versionStatus=LATEST_VERSION&isRevocation=false&dataUseTerms=OPEN&dataFormat=tsv&accessionVersion=SEQID1&accessionVersion=SEQID2/,
+            /downloadAsFile=true&downloadFileBasename=ebola_metadata_\d{4}-\d{2}-\d{2}T\d{4}&versionStatus=LATEST_VERSION&isRevocation=false&dataUseTerms=OPEN&dataFormat=tsv&fields=accessionVersion%2Cfield1%2Cfield2&accessionVersion=SEQID1&accessionVersion=SEQID2/,
         );
 
         await userEvent.click(screen.getByLabelText(olderVersionsLabel));

website/src/components/SearchPage/DownloadDialog/DownloadDialog.tsx

@@ -4,8 +4,10 @@ import { DownloadDialogButton } from './DowloadDialogButton.tsx';
 import { DownloadButton } from './DownloadButton.tsx';
 import { DownloadForm } from './DownloadForm.tsx';
 import { type DownloadUrlGenerator, type DownloadOption } from './DownloadUrlGenerator.ts';
+import { getDefaultSelectedFields } from './FieldSelector/FieldSelectorModal.tsx';
 import type { SequenceFilter } from './SequenceFilters.tsx';
 import { routes } from '../../../routes/routes.ts';
+import type { Metadata } from '../../../types/config.ts';
 import type { ReferenceGenomesSequenceNames } from '../../../types/referencesGenomes.ts';
 import { ActiveFilters } from '../../common/ActiveFilters.tsx';
 import { BaseDialog } from '../../common/BaseDialog.tsx';
@@ -16,6 +18,7 @@ type DownloadDialogProps = {
     referenceGenomesSequenceNames: ReferenceGenomesSequenceNames;
     allowSubmissionOfConsensusSequences: boolean;
     dataUseTermsEnabled: boolean;
+    metadata: Metadata[];
 };
 
 export const DownloadDialog: FC<DownloadDialogProps> = ({
@@ -24,6 +27,7 @@ export const DownloadDialog: FC<DownloadDialogProps> = ({
     referenceGenomesSequenceNames,
     allowSubmissionOfConsensusSequences,
     dataUseTermsEnabled,
+    metadata,
 }) => {
     const [isOpen, setIsOpen] = useState(false);
 
@@ -32,6 +36,7 @@ export const DownloadDialog: FC<DownloadDialogProps> = ({
 
     const [downloadOption, setDownloadOption] = useState<DownloadOption | undefined>();
     const [agreedToDataUseTerms, setAgreedToDataUseTerms] = useState(dataUseTermsEnabled ? false : true);
+    const [selectedFields, setSelectedFields] = useState<string[]>(getDefaultSelectedFields(metadata));
 
     return (
         <>
@@ -44,6 +49,9 @@ export const DownloadDialog: FC<DownloadDialogProps> = ({
                         onChange={setDownloadOption}
                         allowSubmissionOfConsensusSequences={allowSubmissionOfConsensusSequences}
                         dataUseTermsEnabled={dataUseTermsEnabled}
+                        metadata={metadata}
+                        selectedFields={selectedFields}
+                        onSelectedFieldsChange={setSelectedFields}
                     />
                     {dataUseTermsEnabled && (
                         <div className='mb-4 py-4'>

website/src/components/SearchPage/DownloadDialog/DownloadForm.tsx

@@ -2,22 +2,38 @@ import { type FC, useEffect, useState } from 'react';
 
 import type { DownloadDataType } from './DownloadDataType.ts';
 import type { DownloadOption } from './DownloadUrlGenerator.ts';
+import { FieldSelectorButton } from './FieldSelector/FieldSelectorButton.tsx';
+import { FieldSelectorModal } from './FieldSelector/FieldSelectorModal.tsx';
 import { DropdownOptionBlock, RadioOptionBlock } from './OptionBlock.tsx';
 import { routes } from '../../../routes/routes.ts';
+import { ACCESSION_VERSION_FIELD } from '../../../settings.ts';
+import type { Metadata } from '../../../types/config.ts';
 import type { ReferenceGenomesSequenceNames } from '../../../types/referencesGenomes.ts';
 
 type DownloadFormProps = {
     referenceGenomesSequenceNames: ReferenceGenomesSequenceNames;
     onChange: (value: DownloadOption) => void;
     allowSubmissionOfConsensusSequences: boolean;
     dataUseTermsEnabled: boolean;
+    metadata: Metadata[];
+    selectedFields: string[];
+    onSelectedFieldsChange: (fields: string[]) => void;
 };
 
+// Helper function to ensure accessionVersion is always the first field
+function ensureAccessionVersionField(fields: string[]): string[] {
+    const fieldsWithoutAccessionVersion = fields.filter((field) => field !== ACCESSION_VERSION_FIELD);
+    return [ACCESSION_VERSION_FIELD, ...fieldsWithoutAccessionVersion];
+}
+
 export const DownloadForm: FC<DownloadFormProps> = ({
     referenceGenomesSequenceNames,
     onChange,
     allowSubmissionOfConsensusSequences,
     dataUseTermsEnabled,
+    metadata,
+    selectedFields,
+    onSelectedFieldsChange,
 }) => {
     const [includeRestricted, setIncludeRestricted] = useState(0);
     const [includeOldData, setIncludeOldData] = useState(0);
@@ -27,6 +43,8 @@ export const DownloadForm: FC<DownloadFormProps> = ({
     const [alignedNucleotideSequence, setAlignedNucleotideSequence] = useState(0);
     const [alignedAminoAcidSequence, setAlignedAminoAcidSequence] = useState(0);
 
+    const [isFieldSelectorOpen, setIsFieldSelectorOpen] = useState(false);
+
     const isMultiSegmented = referenceGenomesSequenceNames.nucleotideSequences.length > 1;
 
     useEffect(() => {
@@ -66,6 +84,7 @@ export const DownloadForm: FC<DownloadFormProps> = ({
             includeOldData: includeOldData === 1,
             includeRestricted: includeRestricted === 1,
             compression: compressionOptions[compression],
+            fields: dataType === 0 ? ensureAccessionVersionField(selectedFields) : undefined, // Always include accessionVersion as first field
         });
     }, [
         includeRestricted,
@@ -79,9 +98,21 @@ export const DownloadForm: FC<DownloadFormProps> = ({
         referenceGenomesSequenceNames.nucleotideSequences,
         referenceGenomesSequenceNames.genes,
         onChange,
+        selectedFields,
     ]);
 
-    const metadataOption = { label: <>Metadata</> };
+    const metadataOption = {
+        label: (
+            <div className='flex items-center gap-3'>
+                <span>Metadata</span>
+                <FieldSelectorButton
+                    onClick={() => setIsFieldSelectorOpen(true)}
+                    selectedFieldsCount={selectedFields.length}
+                    disabled={dataType !== 0}
+                />
+            </div>
+        ),
+    };
     const dataTypeOptions = allowSubmissionOfConsensusSequences
         ? [
               metadataOption,
@@ -185,6 +216,14 @@ export const DownloadForm: FC<DownloadFormProps> = ({
                 selected={compression}
                 onSelect={setCompression}
             />
+
+            <FieldSelectorModal
+                isOpen={isFieldSelectorOpen}
+                onClose={() => setIsFieldSelectorOpen(false)}
+                metadata={metadata}
+                initialSelectedFields={selectedFields}
+                onSave={onSelectedFieldsChange}
+            />
         </div>
     );
 };

website/src/components/SearchPage/DownloadDialog/DownloadUrlGenerator.spec.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from 'vitest';
+
+import { DownloadUrlGenerator } from './DownloadUrlGenerator';
+import { FieldFilter } from './SequenceFilters';
+import type { ConsolidatedMetadataFilters } from '../../../utils/search';
+
+describe('DownloadUrlGenerator', () => {
+    const organism = 'test-organism';
+    const lapisUrl = 'https://example.com/api';
+    const dataUseTermsEnabled = true;
+
+    it('includes selected fields in the URL for metadata downloads', () => {
+        const generator = new DownloadUrlGenerator(organism, lapisUrl, dataUseTermsEnabled);
+
+        const sequenceFilter = new FieldFilter({}, {}, [] as ConsolidatedMetadataFilters);
+
+        const result = generator.generateDownloadUrl(sequenceFilter, {
+            dataType: { type: 'metadata' },
+            includeOldData: false,
+            includeRestricted: false,
+            compression: undefined,
+            fields: ['field1', 'field2', 'field3'],
+        });
+
+        // Check that the fields parameter is included in the URL
+        expect(result.params.get('fields')).toBe('field1,field2,field3');
+    });
+
+    it('does not include fields parameter for non-metadata downloads', () => {
+        const generator = new DownloadUrlGenerator(organism, lapisUrl, dataUseTermsEnabled);
+
+        const sequenceFilter = new FieldFilter({}, {}, [] as ConsolidatedMetadataFilters);
+
+        const result = generator.generateDownloadUrl(sequenceFilter, {
+            dataType: { type: 'unalignedNucleotideSequences' },
+            includeOldData: false,
+            includeRestricted: false,
+            compression: undefined,
+            fields: ['field1', 'field2', 'field3'],
+        });
+
+        // Check that the fields parameter is not included in the URL
+        expect(result.params.has('fields')).toBe(false);
+    });
+
+    it('does not include fields parameter when fields array is empty', () => {
+        const generator = new DownloadUrlGenerator(organism, lapisUrl, dataUseTermsEnabled);
+
+        const sequenceFilter = new FieldFilter({}, {}, [] as ConsolidatedMetadataFilters);
+
+        const result = generator.generateDownloadUrl(sequenceFilter, {
+            dataType: { type: 'metadata' },
+            includeOldData: false,
+            includeRestricted: false,
+            compression: undefined,
+            fields: [],
+        });
+
+        // Check that the fields parameter is not included in the URL
+        expect(result.params.has('fields')).toBe(false);
+    });
+
+    it('does not include fields parameter when fields are undefined', () => {
+        const generator = new DownloadUrlGenerator(organism, lapisUrl, dataUseTermsEnabled);
+
+        const sequenceFilter = new FieldFilter({}, {}, [] as ConsolidatedMetadataFilters);
+
+        const result = generator.generateDownloadUrl(sequenceFilter, {
+            dataType: { type: 'metadata' },
+            includeOldData: false,
+            includeRestricted: false,
+            compression: undefined,
+            fields: undefined,
+        });
+
+        // Check that the fields parameter is not included in the URL
+        expect(result.params.has('fields')).toBe(false);
+    });
+});