fix from-icloud csv error and import memories

process icloud metadata (csv files) in separate pass.
fix logic to detect csv files and ignore unprocessed csv files causing
upload errors.

Also added option to import memories as albums, defaulting to false.

fixes #853

Author: Cedric Reginster

adapters/folder/DATA/icloud-takeout/Memories/Spécial memory.csv renamed to adapters/folder/DATA/icloud-takeout/Memories/1. April 2025.csv

@@ -17,30 +17,45 @@ type iCloudMeta struct {
 	originalCreationDate time.Time
 }
 
-func UseICloudAlbum(m *gen.SyncMap[string, iCloudMeta], fsys fs.FS, filename string) (string, error) {
+func UseICloudMemory(m *gen.SyncMap[string, iCloudMeta], fsys fs.FS, filename string) (string, error) {
 	file, err := fsys.Open(filename)
 	if err != nil {
 		return "", err
 	}
 	defer file.Close()
+	albumName := "Memory " + strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename))
+
+	return albumName, useAlbum(m, file, albumName)
+}
 
+func UseICloudAlbum(m *gen.SyncMap[string, iCloudMeta], fsys fs.FS, filename string) (string, error) {
+	file, err := fsys.Open(filename)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
 	albumName := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename))
+
+	return albumName, useAlbum(m, file, albumName)
+}
+
+func useAlbum(m *gen.SyncMap[string, iCloudMeta], file fs.File, albumName string) error {
 	reader := csv.NewReader(file)
 	records, err := reader.ReadAll()
 	if err != nil {
-		return "", errors.Join(err, errors.New("failed to read all csv records"))
+		return errors.Join(err, errors.New("failed to read all csv records"))
 	}
 	for _, record := range records[1:] {
 		if len(record) != 1 {
-			return "", errors.Join(err, errors.New("invalid record"))
+			return errors.Join(err, errors.New("invalid record"))
 		}
 		fileName := record[0]
 		meta, _ := m.Load(fileName)
 		meta.albums = append(meta.albums, assets.Album{Title: albumName})
 		m.Store(fileName, meta)
 	}
 
-	return albumName, nil
+	return nil
 }
 
 // Example:

adapters/folder/icloud.go

@@ -82,7 +82,8 @@ type ImportFolderOptions struct {
 	PicasaAlbum bool
 
 	// Use icloud takeout metadata (albums & creation date)
-	ICloudTakeout bool
+	ICloudTakeout          bool
+	ICloudMemoriesAsAlbums bool
 
 	// local time zone
 	TZ *time.Location
@@ -131,7 +132,6 @@ func (o *ImportFolderOptions) AddFromFolderFlags(cmd *cobra.Command, parent *cob
 		cmd.Flags().Var(&o.ManageBurst, "manage-burst", "Manage burst photos. Possible values: NoStack, Stack, StackKeepRaw, StackKeepJPEG")
 		cmd.Flags().BoolVar(&o.ManageEpsonFastFoto, "manage-epson-fastfoto", false, "Manage Epson FastFoto file (default: false)")
 		cmd.Flags().BoolVar(&o.PicasaAlbum, "album-picasa", false, "Use Picasa album name found in .picasa.ini file (default: false)")
-		cmd.Flags().BoolVar(&o.ICloudTakeout, "icloud-takeout", false, "Use metadata from icloud takeout (Albums & original creation dates) (default: false)")
 	}
 }
 
@@ -151,9 +151,11 @@ func (o *ImportFolderOptions) AddFromICloudFlags(cmd *cobra.Command, parent *cob
 		`.DS_Store/`,         // Mac OS custom attributes
 		`/._*`,               // MacOS resource files
 		`.photostructure/`,   // PhotoStructure
+		`Recently Deleted/`,  // ICloud recently deleted
 	)
 
 	o.ICloudTakeout = true
+	cmd.Flags().BoolVar(&o.ICloudMemoriesAsAlbums, "memories", false, "Import icloud memories as albums (default: false)")
 	o.PicasaAlbum = false
 	cmd.Flags().StringVar(&o.ImportIntoAlbum, "into-album", "", "Specify an album to import all files into")
 

adapters/folder/options.go

@@ -42,6 +42,7 @@ type LocalAssetBrowser struct {
 	requiresDateInformation bool                              // true if we need to read the date from the file for the options
 	picasaAlbums            *gen.SyncMap[string, PicasaAlbum] // ap[string]PicasaAlbum
 	icloudMetas             *gen.SyncMap[string, iCloudMeta]
+	icloudMetaPass          bool
 }
 
 func NewLocalFiles(ctx context.Context, l *fileevent.Recorder, flags *ImportFolderOptions, fsyss ...fs.FS) (*LocalAssetBrowser, error) {
@@ -64,6 +65,7 @@ func NewLocalFiles(ctx context.Context, l *fileevent.Recorder, flags *ImportFold
 	}
 	if flags.ICloudTakeout {
 		la.icloudMetas = gen.NewSyncMap[string, iCloudMeta]()
+		la.icloudMetaPass = true
 	}
 
 	if flags.InfoCollector == nil {
@@ -101,6 +103,14 @@ func (la *LocalAssetBrowser) Browse(ctx context.Context) chan *assets.Group {
 	gOut := make(chan *assets.Group)
 	go func() {
 		defer close(gOut)
+		// two passes for icloud takouts
+		if la.icloudMetaPass {
+			for _, fsys := range la.fsyss {
+				la.concurrentParseDir(ctx, fsys, ".", gOut)
+			}
+			la.wg.Wait()
+			la.icloudMetaPass = false
+		}
 		for _, fsys := range la.fsyss {
 			la.concurrentParseDir(ctx, fsys, ".", gOut)
 		}
@@ -152,42 +162,55 @@ func (la *LocalAssetBrowser) parseDir(ctx context.Context, fsys fs.FS, dir strin
 			continue
 		}
 
-		if la.flags.BannedFiles.Match(name) {
-			la.log.Record(ctx, fileevent.DiscoveredDiscarded, fshelper.FSName(fsys, entry.Name()), "reason", "banned file")
-			continue
+		// process csv files on icloud meta pass
+		if la.icloudMetaPass && ext == icloudMetadataExt {
+			if strings.HasSuffix(strings.ToLower(dir), "albums") {
+				a, err := UseICloudAlbum(la.icloudMetas, fsys, name)
+				if err != nil {
+					la.log.Record(ctx, fileevent.Error, fshelper.FSName(fsys, name), "error", err.Error())
+				} else {
+					la.log.Log().Info("iCloud album detected", "file", fshelper.FSName(fsys, name), "album", a)
+				}
+				continue
+			}
+			if la.flags.ICloudMemoriesAsAlbums && strings.HasSuffix(strings.ToLower(dir), "memories") {
+				a, err := UseICloudMemory(la.icloudMetas, fsys, name)
+				if err != nil {
+					la.log.Record(ctx, fileevent.Error, fshelper.FSName(fsys, name), "error", err.Error())
+				} else {
+					la.log.Log().Info("iCloud memory detected", "file", fshelper.FSName(fsys, name), "album", a)
+				}
+				continue
+			}
+			// iCloud photo details (csv). File name pattern: "Photo Details.csv"
+			if strings.HasPrefix(strings.ToLower(base), "photo details") {
+				err := UseICloudPhotoDetails(la.icloudMetas, fsys, name)
+				if err != nil {
+					la.log.Record(ctx, fileevent.Error, fshelper.FSName(fsys, name), "error", err.Error())
+				} else {
+					la.log.Log().Info("iCloud photo details detected", "file", fshelper.FSName(fsys, name))
+				}
+				continue
+			}
 		}
 
-		if la.flags.SupportedMedia.IsUseLess(name) {
-			la.log.Record(ctx, fileevent.DiscoveredUseless, fshelper.FSName(fsys, entry.Name()))
+		// skip all other files in icloud meta pass
+		if la.icloudMetaPass {
 			continue
 		}
 
-		// iCloud albums
-		if la.flags.ICloudTakeout && strings.ToLower(dir) == "albums" && ext == icloudMetadataExt {
-			a, err := UseICloudAlbum(la.icloudMetas, fsys, name)
-			if err != nil {
-				la.log.Record(ctx, fileevent.Error, fshelper.FSName(fsys, name), "error", err.Error())
-			} else {
-				la.log.Log().Info("iCloud album detected", "file", fshelper.FSName(fsys, name), "album", a)
-			}
+		// silently ignore .csv files after icloud meta pass
+		if la.flags.ICloudTakeout && !la.icloudMetaPass && ext == icloudMetadataExt {
 			continue
 		}
 
-		// iCloud memories
-		if la.flags.ICloudTakeout && strings.ToLower(dir) == "memories" && ext == ".csv" {
-			// ignore
-			la.log.Record(ctx, fileevent.DiscoveredDiscarded, fshelper.FSName(fsys, name), "reason", "iCloud memories ignored")
+		if la.flags.BannedFiles.Match(name) {
+			la.log.Record(ctx, fileevent.DiscoveredDiscarded, fshelper.FSName(fsys, entry.Name()), "reason", "banned file")
 			continue
 		}
 
-		// iCloud photo details (csv). File name pattern: "Photo Details.csv"
-		if la.flags.ICloudTakeout && strings.HasPrefix(strings.ToLower(base), "photo details") && ext == ".csv" {
-			err := UseICloudPhotoDetails(la.icloudMetas, fsys, name)
-			if err != nil {
-				la.log.Record(ctx, fileevent.Error, fshelper.FSName(fsys, name), "error", err.Error())
-			} else {
-				la.log.Log().Info("iCloud photo details detected", "file", fshelper.FSName(fsys, name))
-			}
+		if la.flags.SupportedMedia.IsUseLess(name) {
+			la.log.Record(ctx, fileevent.DiscoveredUseless, fshelper.FSName(fsys, entry.Name()))
 			continue
 		}
 

adapters/folder/readFolder.go

@@ -122,10 +122,11 @@ func TestLocalAssets(t *testing.T) {
 		{
 			name: "icloud-takeout",
 			flags: ImportFolderOptions{
-				SupportedMedia: filetypes.DefaultSupportedMedia,
-				Recursive:      true,
-				ICloudTakeout:  true,
-				TZ:             time.Local,
+				SupportedMedia:         filetypes.DefaultSupportedMedia,
+				Recursive:              true,
+				ICloudTakeout:          true,
+				ICloudMemoriesAsAlbums: true,
+				TZ:                     time.Local,
 				InclusionFlags: cliflags.InclusionFlags{
 					DateRange: cliflags.InitDateRange(time.Local, "2023-10-06"),
 				},
@@ -139,11 +140,11 @@ func TestLocalAssets(t *testing.T) {
 				"Photos/photo_wo_exif.jpg",
 			},
 			expectedAlbums: map[string][]string{
-				"Spécial album":   {"Photos/photo1.jpg", "Photos/photo2.jpg"},
-				"Spécial album 2": {"Photos/photo2.jpg"},
+				"Spécial album":        {"Photos/photo1.jpg", "Photos/photo2.jpg"},
+				"Spécial album 2":      {"Photos/photo2.jpg"},
+				"Memory 1. April 2025": {"Photos/photo1.jpg", "Photos/photo2.jpg"},
 			},
 			expectedCounts: fileevent.NewCounts().
-				Set(fileevent.DiscoveredDiscarded, 1).
 				Set(fileevent.DiscoveredImage, 3).
 				Set(fileevent.Uploaded, 3).
 				Value(),

adapters/folder/readFolderWithFIles_test.go

@@ -127,6 +127,5 @@ func (ic *ImmichClient) GetSupportedMediaTypes(ctx context.Context) (filetypes.S
 	}
 	sm[".mp"] = filetypes.TypeUseless
 	sm[".json"] = filetypes.TypeSidecar
-	sm[".csv"] = filetypes.TypeMeta
 	return sm, err
 }

immich/ping.go

@@ -15,7 +15,6 @@ const (
 	TypeImage   = "image"
 	TypeSidecar = "sidecar"
 	TypeUseless = "useless"
-	TypeMeta    = "meta"
 	TypeUnknown = ""
 )
 
@@ -28,7 +27,6 @@ var DefaultSupportedMedia = SupportedMedia{
 	".xmp":  TypeSidecar,
 	".json": TypeSidecar,
 	".mp":   TypeUseless,
-	".csv":  TypeMeta,
 }
 
 func (sm SupportedMedia) TypeFromName(name string) string {