add support for icloud takeouts

supports albums and original creation dates.
request copy of your data from: https://privacy.apple.com

Author: Cedric Reginster

adapters/folder/DATA/icloud-takeout/Albums/Spécial album 2.csv

@@ -0,0 +1,2 @@
+imgName
+photo2.jpg

adapters/folder/DATA/icloud-takeout/Albums/Spécial album.csv

@@ -0,0 +1,3 @@
+imgName
+photo1.jpg
+photo2.jpg

adapters/folder/DATA/icloud-takeout/Memories/Spécial memory.csv

@@ -0,0 +1,3 @@
+imgName
+photo1.jpg
+photo2.jpg

adapters/folder/DATA/icloud-takeout/Photos/Photo Details.csv

@@ -0,0 +1,2 @@
+imgName,fileChecksum,favorite,hidden,deleted,originalCreationDate,viewCount,importDate
+photo_wo_exif.jpg,ignored-checksum,no,no,no,"Friday October 06,2023 12:11 PM GMT",10,"Friday October 06,2023 12:11 PM GMT"

adapters/folder/DATA/icloud-takeout/Photos/photo1.jpg

@@ -0,0 +1,78 @@
+package folder
+
+import (
+	"encoding/csv"
+	"errors"
+	"io/fs"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/simulot/immich-go/internal/assets"
+	"github.com/simulot/immich-go/internal/gen"
+)
+
+type iCloudMeta struct {
+	albums               []assets.Album
+	originalCreationDate time.Time
+}
+
+func UseICloudAlbum(m *gen.SyncMap[string, iCloudMeta], fsys fs.FS, filename string) (string, error) {
+	file, err := fsys.Open(filename)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	albumName := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename))
+	reader := csv.NewReader(file)
+	records, err := reader.ReadAll()
+	if err != nil {
+		return "", errors.Join(err, errors.New("failed to read all csv records"))
+	}
+	for _, record := range records[1:] {
+		if len(record) != 1 {
+			return "", errors.Join(err, errors.New("invalid record"))
+		}
+		fileName := record[0]
+		meta, _ := m.Load(fileName)
+		meta.albums = append(meta.albums, assets.Album{Title: albumName})
+		m.Store(fileName, meta)
+	}
+
+	return albumName, nil
+}
+
+// Example:
+// imgName,fileChecksum,favorite,hidden,deleted,originalCreationDate,viewCount,importDate
+// IMG_7938.HEIC,AfQj57ORF2JIumUCjO+PawZ9nqPg,no,no,no,"Saturday June 4,2022 12:11 PM GMT",10,"Saturday June 4,2022 12:11 PM GMT"
+func UseICloudPhotoDetails(m *gen.SyncMap[string, iCloudMeta], fsys fs.FS, filename string) error {
+	file, err := fsys.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	reader := csv.NewReader(file)
+	records, err := reader.ReadAll()
+	if err != nil {
+		return errors.Join(err, errors.New("failed to read all csv records"))
+	}
+	// skip header
+	for _, record := range records[1:] {
+		if len(record) != 8 {
+			return errors.Join(err, errors.New("invalid record"))
+		}
+		fileName := record[0]
+		originalCreationDate := record[5]
+		t, err := time.Parse("Monday January 2,2006 15:04 PM GMT", originalCreationDate)
+		if err != nil {
+			return errors.Join(err, errors.New("invalid original creation date"))
+		}
+		meta, _ := m.Load(fileName)
+		meta.originalCreationDate = t
+		m.Store(fileName, meta)
+	}
+
+	return nil
+}

adapters/folder/DATA/icloud-takeout/Photos/photo2.jpg

@@ -79,6 +79,9 @@ type ImportFolderOptions struct {
 	// Use picasa albums
 	PicasaAlbum bool
 
+	// Use icloud takeout metadata (albums & creation date)
+	ICloudTakeout bool
+
 	// local time zone
 	TZ *time.Location
 }
@@ -123,6 +126,7 @@ func (o *ImportFolderOptions) AddFromFolderFlags(cmd *cobra.Command, parent *cob
 		cmd.Flags().Var(&o.ManageBurst, "manage-burst", "Manage burst photos. Possible values: NoStack, Stack, StackKeepRaw, StackKeepJPEG")
 		cmd.Flags().BoolVar(&o.ManageEpsonFastFoto, "manage-epson-fastfoto", false, "Manage Epson FastFoto file (default: false)")
 		cmd.Flags().BoolVar(&o.PicasaAlbum, "album-picasa", false, "Use Picasa album name found in .picasa.ini file (default: false)")
+		cmd.Flags().BoolVar(&o.ICloudTakeout, "icloud-takeout", false, "Use metadata from icloud takeout (Albums & original creation dates) (default: false)")
 	}
 }
 

adapters/folder/DATA/icloud-takeout/Photos/photo_wo_exif.jpg

@@ -39,6 +39,7 @@ type LocalAssetBrowser struct {
 	groupers                []groups.Grouper
 	requiresDateInformation bool                              // true if we need to read the date from the file for the options
 	picasaAlbums            *gen.SyncMap[string, PicasaAlbum] // ap[string]PicasaAlbum
+	icloudMetas             *gen.SyncMap[string, iCloudMeta]
 }
 
 func NewLocalFiles(ctx context.Context, l *fileevent.Recorder, flags *ImportFolderOptions, fsyss ...fs.FS) (*LocalAssetBrowser, error) {
@@ -59,6 +60,9 @@ func NewLocalFiles(ctx context.Context, l *fileevent.Recorder, flags *ImportFold
 	if flags.PicasaAlbum {
 		la.picasaAlbums = gen.NewSyncMap[string, PicasaAlbum]() // make(map[string]PicasaAlbum)
 	}
+	if flags.ICloudTakeout {
+		la.icloudMetas = gen.NewSyncMap[string, iCloudMeta]()
+	}
 
 	if flags.InfoCollector == nil {
 		flags.InfoCollector = filenames.NewInfoCollector(flags.TZ, flags.SupportedMedia)
@@ -140,6 +144,8 @@ func (la *LocalAssetBrowser) parseDir(ctx context.Context, fsys fs.FS, dir strin
 	for _, entry := range entries {
 		base := entry.Name()
 		name := path.Join(dir, base)
+		ext := filepath.Ext(base)
+
 		if entry.IsDir() {
 			continue
 		}
@@ -154,6 +160,35 @@ func (la *LocalAssetBrowser) parseDir(ctx context.Context, fsys fs.FS, dir strin
 			continue
 		}
 
+		// iCloud albums
+		if la.flags.ICloudTakeout && strings.ToLower(dir) == "albums" && ext == ".csv" {
+			a, err := UseICloudAlbum(la.icloudMetas, fsys, name)
+			if err != nil {
+				la.log.Record(ctx, fileevent.Error, fshelper.FSName(fsys, name), "error", err.Error())
+			} else {
+				la.log.Log().Info("iCloud album detected", "file", fshelper.FSName(fsys, name), "album", a)
+			}
+			continue
+		}
+
+		// iCloud memories
+		if la.flags.ICloudTakeout && strings.ToLower(dir) == "memories" && ext == ".csv" {
+			// ignore
+			la.log.Record(ctx, fileevent.DiscoveredDiscarded, fshelper.FSName(fsys, name), "reason", "iCloud memories ignored")
+			continue
+		}
+
+		// iCloud photo details (csv). File name pattern: "Photo Details.csv"
+		if la.flags.ICloudTakeout && strings.HasPrefix(strings.ToLower(base), "photo details") && ext == ".csv" {
+			err := UseICloudPhotoDetails(la.icloudMetas, fsys, name)
+			if err != nil {
+				la.log.Record(ctx, fileevent.Error, fshelper.FSName(fsys, name), "error", err.Error())
+			} else {
+				la.log.Log().Info("iCloud photo details detected", "file", fshelper.FSName(fsys, name))
+			}
+			continue
+		}
+
 		if la.flags.PicasaAlbum && (strings.ToLower(base) == ".picasa.ini" || strings.ToLower(base) == "picasa.ini") {
 			a, err := ReadPicasaIni(fsys, name)
 			if err != nil {
@@ -165,7 +200,6 @@ func (la *LocalAssetBrowser) parseDir(ctx context.Context, fsys fs.FS, dir strin
 			continue
 		}
 
-		ext := filepath.Ext(base)
 		mediaType := la.flags.SupportedMedia.TypeFromExt(ext)
 
 		if mediaType == filetypes.TypeUnknown {
@@ -290,6 +324,16 @@ func (la *LocalAssetBrowser) parseDir(ctx context.Context, fsys fs.FS, dir strin
 
 			// Read metadata from the file only id needed (date range or take date from filename)
 			if la.requiresDateInformation {
+				// try to get date from icloud takeout meta
+				if a.CaptureDate.IsZero() && la.flags.ICloudTakeout {
+					meta, ok := la.icloudMetas.Load(a.OriginalFileName)
+					if ok {
+						a.FromApplication = &assets.Metadata{
+							DateTaken: meta.originalCreationDate,
+						}
+						a.CaptureDate = a.FromApplication.DateTaken
+					}
+				}
 				if a.CaptureDate.IsZero() {
 					// no date in XMP, JSON, try reading the metadata
 					f, err := a.OpenFile()
@@ -309,6 +353,7 @@ func (la *LocalAssetBrowser) parseDir(ctx context.Context, fsys fs.FS, dir strin
 						}
 						f.Close()
 					}
+
 				}
 			}
 
@@ -347,6 +392,12 @@ func (la *LocalAssetBrowser) parseDir(ctx context.Context, fsys fs.FS, dir strin
 						done = true
 					}
 				}
+				if la.flags.ICloudTakeout {
+					if meta, ok := la.icloudMetas.Load(a.OriginalFileName); ok {
+						a.Albums = meta.albums
+						done = true
+					}
+				}
 				if !done && la.flags.UsePathAsAlbumName != FolderModeNone && la.flags.UsePathAsAlbumName != "" {
 					Album := ""
 					switch la.flags.UsePathAsAlbumName {

adapters/folder/icloud.go

@@ -119,6 +119,35 @@ func TestLocalAssets(t *testing.T) {
 				Set(fileevent.DiscoveredImage, 2).Set(fileevent.DiscoveredDiscarded, 1).Set(fileevent.Uploaded, 1).
 				Set(fileevent.INFO, 1).Value(),
 		},
+		{
+			name: "icloud-takeout",
+			flags: ImportFolderOptions{
+				SupportedMedia: filetypes.DefaultSupportedMedia,
+				Recursive:      true,
+				ICloudTakeout:  true,
+				TZ:             time.Local,
+				InclusionFlags: cliflags.InclusionFlags{
+					DateRange: cliflags.InitDateRange(time.Local, "2023-10-06"),
+				},
+			},
+			fsys: []fs.FS{
+				os.DirFS("DATA/icloud-takeout"),
+			},
+			expectedFiles: []string{
+				"Photos/photo1.jpg",
+				"Photos/photo2.jpg",
+				"Photos/photo_wo_exif.jpg",
+			},
+			expectedAlbums: map[string][]string{
+				"Spécial album":   {"Photos/photo1.jpg", "Photos/photo2.jpg"},
+				"Spécial album 2": {"Photos/photo2.jpg"},
+			},
+			expectedCounts: fileevent.NewCounts().
+				Set(fileevent.DiscoveredDiscarded, 1).
+				Set(fileevent.DiscoveredImage, 3).
+				Set(fileevent.Uploaded, 3).
+				Value(),
+		},
 	}
 
 	logFile := configuration.DefaultLogFile()

adapters/folder/options.go

@@ -127,5 +127,6 @@ func (ic *ImmichClient) GetSupportedMediaTypes(ctx context.Context) (filetypes.S
 	}
 	sm[".mp"] = filetypes.TypeUseless
 	sm[".json"] = filetypes.TypeSidecar
+	sm[".csv"] = filetypes.TypeMeta
 	return sm, err
 }

adapters/folder/readFolder.go

@@ -15,6 +15,7 @@ const (
 	TypeImage   = "image"
 	TypeSidecar = "sidecar"
 	TypeUseless = "useless"
+	TypeMeta    = "meta"
 	TypeUnknown = ""
 )
 
@@ -27,6 +28,7 @@ var DefaultSupportedMedia = SupportedMedia{
 	".xmp":  TypeSidecar,
 	".json": TypeSidecar,
 	".mp":   TypeUseless,
+	".csv":  TypeMeta,
 }
 
 func (sm SupportedMedia) TypeFromName(name string) string {