diff --git a/import.go b/import.go index 332e2ed..147764a 100644 --- a/import.go +++ b/import.go @@ -4,7 +4,6 @@ import ( "flag" "fmt" "io" - "log" "os" "path/filepath" "strconv" @@ -17,6 +16,22 @@ import ( "github.com/evanoberholster/imagemeta" ) +type importConfig struct { + From string + To string + Filter string + Start uint + End uint + MaxWorkers int +} + +type importSummary struct { + processed int + copied int + skipped int + failed int +} + // Copy a file from src to dst func copyFile(src, dst string) (err error) { sfi, err := os.Stat(src) @@ -70,7 +85,9 @@ func copyFileContents(src, dst string, mtime time.Time) (err error) { if err := os.Chtimes(dst, time.Now(), mtime); err != nil { return err } - err = out.Sync() + if err = out.Sync(); err != nil { + return + } return } @@ -84,166 +101,223 @@ func findTagInAllIfds(index *exif.IfdIndex, tagName string) (string, error) { if err != nil { return "", err } - return valueRaw.(string), nil + value, ok := valueRaw.(string) + if !ok { + return "", fmt.Errorf("tag %s is not a string", tagName) + } + return value, nil } } return "", fmt.Errorf("tag not found") } -func main() { - var from, to, filter string - flag.StringVar(&from, "from", "", "Source path") - flag.StringVar(&to, "to", "", "Destination path") - flag.StringVar(&filter, "filter", "", "Optional file type filter") - - var start, end uint - flag.UintVar(&start, "start", uint(0), "Start date") - flag.UintVar(&end, "end", ^uint(0), "End date") - flag.Parse() - - if from == "" || to == "" { - fmt.Fprintf(os.Stderr, "Error: Need source and target directory (use '--from' and '--to')\n\n") - flag.Usage() - os.Exit(-1) +func parseFlags(args []string) (importConfig, error) { + var cfg importConfig + fs := flag.NewFlagSet("file-importer", flag.ContinueOnError) + fs.StringVar(&cfg.From, "from", "", "Source path") + fs.StringVar(&cfg.To, "to", "", "Destination path") + fs.StringVar(&cfg.Filter, "filter", "", "Optional file type filter") + fs.UintVar(&cfg.Start, "start", uint(0), "Start date") + fs.UintVar(&cfg.End, "end", ^uint(0), "End date") + fs.IntVar(&cfg.MaxWorkers, "workers", 10, "Maximum number of concurrent workers") + if err := fs.Parse(args); err != nil { + return importConfig{}, err } + if cfg.From == "" || cfg.To == "" { + return importConfig{}, fmt.Errorf("need source and target directory (use '--from' and '--to')") + } + if cfg.MaxWorkers < 1 { + return importConfig{}, fmt.Errorf("--workers must be >= 1") + } + cfg.Filter = strings.ToLower(cfg.Filter) + return cfg, nil +} - // Read the source directory - fmt.Printf("Importing files from %s -> %s\n", from, to) - files, err := os.ReadDir(from) +func resolveTimestamp(path string, fi os.FileInfo, logf func(string, ...any)) time.Time { + file, err := os.Open(path) if err != nil { - log.Fatal(err) + logf("%s: error opening file: %v", fi.Name(), err) + return fi.ModTime() } + defer file.Close() - // Create a channel to distribute the work - fileChan := make(chan os.FileInfo, len(files)) - var wg sync.WaitGroup + var timestampValue time.Time + var dateTimeString, offsetString string + var dtErr, offErr error - // Limit the number of concurrent goroutines - const maxGoroutines = 10 - guard := make(chan struct{}, maxGoroutines) + // 1. Try standard EXIF extraction (works for JPEG, TIFF, CR2, etc.) + rawExif, err := exif.SearchAndExtractExifWithReader(file) + if err == nil { + im, err := exifcommon.NewIfdMappingWithStandard() + if err == nil { + ti := exif.NewTagIndex() + _, index, err := exif.Collect(im, ti, rawExif) + if err == nil { + dateTimeString, dtErr = findTagInAllIfds(&index, "DateTimeOriginal") + offsetString, offErr = findTagInAllIfds(&index, "OffsetTimeOriginal") + if offErr != nil { + offsetString, _ = findTagInAllIfds(&index, "OffsetTime") + } + } + } + } - for _, f := range files { - // Skip directories - if f.IsDir() { - continue + // 2. Fallback for CR3 and other formats using imagemeta + if dtErr != nil || dateTimeString == "" { + if _, err := file.Seek(0, 0); err == nil { + md, err := imagemeta.DecodeCR3(file) + if err == nil { + timestampValue = md.DateTimeOriginal() + } } + } - // Filter for file extension - ext := strings.Trim(filepath.Ext(f.Name()), ".") - if filter != "" && ext != filter { - continue + // 3. Process the extracted strings with timezone logic + if timestampValue.IsZero() && dateTimeString != "" { + layout := "2006:01:02 15:04:05" + if offsetString != "" { + // Attempt to parse with timezone offset + timestampValue, err = time.Parse(layout+"-07:00", dateTimeString+offsetString) + if err != nil { + logf("%s: error parsing DateTimeOriginal with offset: %v", fi.Name(), err) + } } - info, err := f.Info() - if err != nil { - fmt.Printf("Error getting info for %s: %v\n", f.Name(), err) - continue + // Fallback: parse as local time if no offset or if offset parsing failed + if timestampValue.IsZero() { + timestampValue, err = time.ParseInLocation(layout, dateTimeString, time.Local) + if err != nil { + logf("%s: error parsing DateTimeOriginal: %v", fi.Name(), err) + } } - fileChan <- info } - close(fileChan) - for f := range fileChan { - ext := strings.Trim(filepath.Ext(f.Name()), ".") - wg.Add(1) - guard <- struct{}{} // Block if guard channel is full + // 4. Final fallback to ModTime + if timestampValue.IsZero() { + if dtErr != nil || dateTimeString == "" { + logf("%s: no EXIF data found, using ModTime", fi.Name()) + } else { + logf("%s: failed to parse EXIF, using ModTime", fi.Name()) + } + timestampValue = fi.ModTime() + } + return timestampValue +} - go func(f os.FileInfo, ext string) { - defer wg.Done() - defer func() { <-guard }() // Release guard slot +func processFile(cfg importConfig, fi os.FileInfo, timestamp time.Time, logf func(string, ...any)) error { + ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(fi.Name())), ".") + timestampFolder := timestamp.Format("2006-01-02") + folder := filepath.Join(cfg.To, timestampFolder+"-"+ext) + if err := os.MkdirAll(folder, 0o755); err != nil { + return fmt.Errorf("%s: create folder %s failed: %w", fi.Name(), folder, err) + } - // Filter for EXIF DateTime if it exists, otherwise ModTime - file, err := os.Open(filepath.Join(from, f.Name())) - if err != nil { - fmt.Printf("%s: Error opening file: %v\n", f.Name(), err) - return - } - defer file.Close() + fromFile := filepath.Join(cfg.From, fi.Name()) + toFile := filepath.Join(folder, fi.Name()) + logf("Copying %s -> %s (%s)", fromFile, toFile, timestamp) + if err := copyFile(fromFile, toFile); err != nil { + return fmt.Errorf("%s: copy failed: %w", fi.Name(), err) + } + return nil +} - var timestampValue time.Time - var dateTimeString, offsetString string - var dtErr, offErr error +func runImport(cfg importConfig, out io.Writer) (importSummary, error) { + files, err := os.ReadDir(cfg.From) + if err != nil { + return importSummary{}, err + } - // 1. Try standard EXIF extraction (works for JPEG, TIFF, CR2, etc.) - rawExif, err := exif.SearchAndExtractExifWithReader(file) - if err == nil { - im, err := exifcommon.NewIfdMappingWithStandard() - if err == nil { - ti := exif.NewTagIndex() - _, index, err := exif.Collect(im, ti, rawExif) - if err == nil { - dateTimeString, dtErr = findTagInAllIfds(&index, "DateTimeOriginal") - offsetString, offErr = findTagInAllIfds(&index, "OffsetTimeOriginal") - if offErr != nil { - offsetString, _ = findTagInAllIfds(&index, "OffsetTime") - } - } - } - } + fmt.Fprintf(out, "Importing files from %s -> %s\n", cfg.From, cfg.To) - // 2. Fallback for CR3 and other formats using imagemeta - if dtErr != nil || dateTimeString == "" { - file.Seek(0, 0) - md, err := imagemeta.DecodeCR3(file) - if err == nil { - // Imagemeta handles the extraction differently - timestampValue = md.DateTimeOriginal() - } - } + var ( + mu sync.Mutex + summary importSummary + ) + logf := func(format string, args ...any) { + mu.Lock() + defer mu.Unlock() + fmt.Fprintf(out, format+"\n", args...) + } - // 3. Process the extracted strings with our timezone logic - if timestampValue.IsZero() && dateTimeString != "" { - layout := "2006:01:02 15:04:05" - if offsetString != "" { - // Attempt to parse with timezone offset - timestampValue, err = time.Parse(layout+"-07:00", dateTimeString+offsetString) - if err != nil { - fmt.Printf("%s: Error parsing DateTimeOriginal with offset: %v\n", f.Name(), err) - } - } + jobs := make(chan os.FileInfo) + var wg sync.WaitGroup + for range cfg.MaxWorkers { + wg.Add(1) + go func() { + defer wg.Done() + for fi := range jobs { + mu.Lock() + summary.processed++ + mu.Unlock() - // Fallback: parse as local time if no offset or if offset parsing failed - if timestampValue.IsZero() { - timestampValue, err = time.ParseInLocation(layout, dateTimeString, time.Local) - if err != nil { - fmt.Printf("%s: Error parsing DateTimeOriginal: %v\n", f.Name(), err) - } + timestamp := resolveTimestamp(filepath.Join(cfg.From, fi.Name()), fi, logf) + i, _ := strconv.Atoi(timestamp.Format("20060102")) + if uint(i) < cfg.Start || uint(i) > cfg.End { + mu.Lock() + summary.skipped++ + mu.Unlock() + continue } - } - // 4. Final fallback to ModTime - if timestampValue.IsZero() { - if dtErr != nil || dateTimeString == "" { - fmt.Printf("%s: No EXIF data found, using ModTime\n", f.Name()) - } else { - fmt.Printf("%s: Failed to parse EXIF, using ModTime\n", f.Name()) + err := processFile(cfg, fi, timestamp, logf) + if err != nil { + logf("%v", err) + mu.Lock() + summary.failed++ + mu.Unlock() + continue } - timestampValue = f.ModTime() + mu.Lock() + summary.copied++ + mu.Unlock() } + }() + } - i, _ := strconv.Atoi(timestampValue.Format("20060102")) - if uint(i) < start || uint(i) > end { - return - } + for _, f := range files { + if f.IsDir() { + continue + } + ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(f.Name())), ".") + if cfg.Filter != "" && ext != cfg.Filter { + continue + } + info, err := f.Info() + if err != nil { + logf("Error getting info for %s: %v", f.Name(), err) + mu.Lock() + summary.failed++ + mu.Unlock() + continue + } + jobs <- info + } + close(jobs) + wg.Wait() - // Create folder if needed - timestamp := timestampValue.Format("2006-01-02") - folder := filepath.Join(to, timestamp+"-"+strings.ToLower(ext)) - if err := os.MkdirAll(folder, 0755); err != nil { - fmt.Printf("%s: Error creating folder %s: %v\n", f.Name(), folder, err) - return - } + fmt.Fprintf( + out, + "Done. processed=%d copied=%d skipped=%d failed=%d\n", + summary.processed, + summary.copied, + summary.skipped, + summary.failed, + ) - // Copy the file - fromFile := filepath.Join(from, f.Name()) - toFile := filepath.Join(folder, f.Name()) - fmt.Printf("Copying %s -> %s (%s)\n", fromFile, toFile, timestampValue) - err = copyFile(fromFile, toFile) - if err != nil { - fmt.Printf("Copy file failed: %q\n", err) - } - }(f, ext) + if summary.failed > 0 { + return summary, fmt.Errorf("import completed with %d failures", summary.failed) } + return summary, nil +} - wg.Wait() +func main() { + cfg, err := parseFlags(os.Args[1:]) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(2) + } + if _, err := runImport(cfg, os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } } diff --git a/import_integration_test.go b/import_integration_test.go new file mode 100644 index 0000000..c64f9e6 --- /dev/null +++ b/import_integration_test.go @@ -0,0 +1,154 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestParseFlagsNormalizesFilterAndWorkers(t *testing.T) { + cfg, err := parseFlags([]string{"--from", "/src", "--to", "/dst", "--filter", "JPG", "--workers", "3"}) + if err != nil { + t.Fatalf("parseFlags returned error: %v", err) + } + if cfg.Filter != "jpg" { + t.Fatalf("expected lower-cased filter, got: %q", cfg.Filter) + } + if cfg.MaxWorkers != 3 { + t.Fatalf("expected workers=3, got: %d", cfg.MaxWorkers) + } +} + +func TestParseFlagsRejectsInvalidWorkers(t *testing.T) { + _, err := parseFlags([]string{"--from", "/src", "--to", "/dst", "--workers", "0"}) + if err == nil { + t.Fatal("expected error for workers=0") + } + if !strings.Contains(err.Error(), "--workers") { + t.Fatalf("expected workers validation error, got: %v", err) + } +} + +func TestRunImportAppliesFilterAndDateRange(t *testing.T) { + root := t.TempDir() + from := filepath.Join(root, "from") + to := filepath.Join(root, "to") + if err := os.MkdirAll(from, 0o755); err != nil { + t.Fatalf("mkdir from failed: %v", err) + } + if err := os.MkdirAll(to, 0o755); err != nil { + t.Fatalf("mkdir to failed: %v", err) + } + + inRange := filepath.Join(from, "in_range.jpg") + outOfRange := filepath.Join(from, "out_range.jpg") + filtered := filepath.Join(from, "ignored.txt") + + mustWriteFile(t, inRange, "jpg content") + mustWriteFile(t, outOfRange, "older jpg") + mustWriteFile(t, filtered, "text content") + + mustSetMtime(t, inRange, time.Date(2024, 3, 5, 10, 0, 0, 0, time.UTC)) + mustSetMtime(t, outOfRange, time.Date(2024, 3, 1, 10, 0, 0, 0, time.UTC)) + mustSetMtime(t, filtered, time.Date(2024, 3, 5, 10, 0, 0, 0, time.UTC)) + + cfg := importConfig{ + From: from, + To: to, + Filter: "jpg", + Start: 20240303, + End: 20240331, + MaxWorkers: 2, + } + + var out bytes.Buffer + summary, err := runImport(cfg, &out) + if err != nil { + t.Fatalf("runImport returned error: %v\noutput:\n%s", err, out.String()) + } + + if summary.processed != 2 { + t.Fatalf("expected processed=2 (only jpg files), got: %d", summary.processed) + } + if summary.copied != 1 { + t.Fatalf("expected copied=1, got: %d", summary.copied) + } + if summary.skipped != 1 { + t.Fatalf("expected skipped=1, got: %d", summary.skipped) + } + if summary.failed != 0 { + t.Fatalf("expected failed=0, got: %d", summary.failed) + } + + copiedPath := filepath.Join(to, "2024-03-05-jpg", "in_range.jpg") + if got := readFileString(t, copiedPath); got != "jpg content" { + t.Fatalf("copied file content mismatch: %q", got) + } + + notCopiedPath := filepath.Join(to, "2024-03-01-jpg", "out_range.jpg") + if _, err := os.Stat(notCopiedPath); !os.IsNotExist(err) { + t.Fatalf("expected out-of-range file to not be copied, stat err=%v", err) + } + + ignoredPath := filepath.Join(to, "2024-03-05-txt", "ignored.txt") + if _, err := os.Stat(ignoredPath); !os.IsNotExist(err) { + t.Fatalf("expected filtered file to not be copied, stat err=%v", err) + } +} + +func TestRunImportNormalizesExtensionFolderName(t *testing.T) { + root := t.TempDir() + from := filepath.Join(root, "from") + to := filepath.Join(root, "to") + if err := os.MkdirAll(from, 0o755); err != nil { + t.Fatalf("mkdir from failed: %v", err) + } + if err := os.MkdirAll(to, 0o755); err != nil { + t.Fatalf("mkdir to failed: %v", err) + } + + src := filepath.Join(from, "IMG_0001.JPG") + mustWriteFile(t, src, "upper extension") + mustSetMtime(t, src, time.Date(2024, 7, 8, 9, 10, 11, 0, time.UTC)) + + cfg := importConfig{ + From: from, + To: to, + Filter: "jpg", + Start: 20240708, + End: 20240708, + MaxWorkers: 1, + } + + var out bytes.Buffer + summary, err := runImport(cfg, &out) + if err != nil { + t.Fatalf("runImport returned error: %v\noutput:\n%s", err, out.String()) + } + if summary.copied != 1 { + t.Fatalf("expected copied=1, got: %d", summary.copied) + } + + dst := filepath.Join(to, "2024-07-08-jpg", "IMG_0001.JPG") + if got := readFileString(t, dst); got != "upper extension" { + t.Fatalf("copied file content mismatch: %q", got) + } +} + +func TestRunImportReturnsErrorForMissingSourceDir(t *testing.T) { + root := t.TempDir() + cfg := importConfig{ + From: filepath.Join(root, "missing"), + To: filepath.Join(root, "to"), + MaxWorkers: 1, + } + + var out bytes.Buffer + _, err := runImport(cfg, &out) + if err == nil { + t.Fatal("expected error for missing source directory") + } +}