feat: only scan files which have new paths or updated modification dates

music-player/migrations/000003_song_file_modified_date.down.sql

@@ -0,0 +1 @@
+ALTER TABLE songs DROP COLUMN modified_date;

music-player/migrations/000003_song_file_modified_date.up.sql

@@ -0,0 +1,5 @@
+ALTER TABLE songs ADD modified_date bigint;
+UPDATE songs SET modified_date = 0;
+ALTER TABLE songs ALTER COLUMN modified_date SET NOT NULL;
+
+CREATE INDEX filename_timestamp ON songs (base_path, relative_path, modified_date);

music-player/pkg/read/audio.go

@@ -9,8 +9,8 @@ import (
   duration "github.com/felamaslen/go-music-player/pkg/read/duration"
 )
 
-func ReadFile(basePath string, fileName string) (song *Song, err error) {
-  fullPath := filepath.Join(basePath, fileName)
+func ReadFile(basePath string, scannedFile *File) (song *Song, err error) {
+  fullPath := filepath.Join(basePath, scannedFile.RelativePath)
   file, errFile := os.Open(fullPath)
   if errFile != nil {
     return &Song{}, errFile
@@ -31,7 +31,8 @@ func ReadFile(basePath string, fileName string) (song *Song, err error) {
     Duration: durationTime,
     DurationOk: durationOk,
     BasePath: basePath,
-    RelativePath: fileName,
+    RelativePath: scannedFile.RelativePath,
+    ModifiedDate: scannedFile.ModifiedDate,
   }
 
   return &result, nil

music-player/pkg/read/audio_test.go

@@ -17,9 +17,11 @@ var _ = Describe("reading audio files", func() {
   basePath := path.Join(rootDir, read.TestDirectory)
 
   Context("when the file is ogg vorbis", func() {
-
     It("should get the expected info from the file", func() {
-      result, err := read.ReadFile(basePath, read.TestSong.RelativePath)
+      result, err := read.ReadFile(basePath, &read.File{
+	RelativePath: read.TestSong.RelativePath,
+	ModifiedDate: 102118,
+      })
 
       Expect(err).To(BeNil())
 
@@ -31,6 +33,7 @@ var _ = Describe("reading audio files", func() {
 	DurationOk: true,
 	BasePath: basePath,
 	RelativePath: "file_example_OOG_1MG.ogg",
+	ModifiedDate: 102118,
       }))
     })
   })

music-player/pkg/read/files.go

@@ -5,10 +5,12 @@ import (
 	"path/filepath"
 
 	config "github.com/felamaslen/go-music-player/pkg/config"
+	"github.com/felamaslen/go-music-player/pkg/database"
 	"github.com/felamaslen/go-music-player/pkg/logger"
+	"github.com/jmoiron/sqlx"
 )
 
-func ReadMultipleFiles(basePath string, files chan string) chan *Song {
+func ReadMultipleFiles(basePath string, files chan *File) chan *Song {
   var l = logger.CreateLogger(config.GetConfig().LogLevel)
 
   songs := make(chan *Song)
@@ -23,13 +25,13 @@ func ReadMultipleFiles(basePath string, files chan string) chan *Song {
       select {
       case file, more := <- files:
         if more {
-          l.Verbose("Reading file: %s\n", file)
+          l.Verbose("Reading file: %s\n", file.RelativePath)
           song, err := ReadFile(basePath, file)
 
           if err == nil {
             songs <- song
           } else {
-            l.Error("Error reading file (%s): %s\n", file, err)
+            l.Error("Error reading file (%s): %s\n", file.RelativePath, err)
           }
         } else {
           return
@@ -46,47 +48,89 @@ func isValidFile(file string) bool {
   return filepath.Ext(file) == ".ogg"
 }
 
-func recursiveDirScan(l *logger.Logger, directory string, output *chan string, root bool, basePath string) {
-  if (root) {
-    l.Verbose("Scanning root directory: %s\n", directory)
+func recursiveDirScan(
+  db *sqlx.DB,
+  l *logger.Logger,
+  output *chan *File,
+  rootDirectory string,
+  relativePath string,
+  isRoot bool,
+) {
+  directoryToScan := filepath.Join(rootDirectory, relativePath)
+
+  if (isRoot) {
+    l.Verbose("Scanning root directory: %s\n", directoryToScan)
 
     defer func() {
       l.Verbose("Finished recursive directory scan")
       close(*output)
     }()
   } else {
-    l.Debug("Scanning subdirectory: %s\n", directory)
+    l.Debug("Scanning subdirectory: %s\n", directoryToScan)
   }
 
-
-  files, err := ioutil.ReadDir(directory)
+  files, err := ioutil.ReadDir(directoryToScan)
 
   if err != nil {
-    l.Fatal("Error scanning directory: (%s): %s", directory, err)
+    l.Fatal("Error scanning directory: (%s): %s", directoryToScan, err)
     return
   }
 
   for _, file := range(files) {
-    absolutePath := filepath.Join(directory, file.Name())
-    relativePath := filepath.Join(basePath, file.Name())
+    fileRelativePath := filepath.Join(relativePath, file.Name())
 
     if file.IsDir() {
-      recursiveDirScan(l, absolutePath, output, false, relativePath)
+      recursiveDirScan(
+        db,
+        l,
+        output,
+        rootDirectory,
+        fileRelativePath,
+        false,
+      )
     } else if isValidFile(file.Name()) {
-      l.Verbose("Found file: %s\n", relativePath)
-
-      *output <- relativePath
+      modifiedDate := file.ModTime().Unix()
+
+      var existingCount = 0
+
+      err := db.Get(
+        &existingCount,
+        `
+        select count(*) from songs
+        where base_path = $1 and relative_path = $2 and modified_date = $3
+        `,
+        rootDirectory,
+        fileRelativePath,
+        modifiedDate,
+      )
+
+      if err == nil && existingCount == 0 {
+        l.Verbose("Found file: %s\n", fileRelativePath)
+
+        *output <- &File{
+          RelativePath: fileRelativePath,
+          ModifiedDate: modifiedDate,
+        }
+      }
     }
   }
 }
 
-func ScanDirectory(directory string) chan string {
+func ScanDirectory(directory string) chan *File {
+  db := database.GetConnection()
   l := logger.CreateLogger(config.GetConfig().LogLevel)
 
-  files := make(chan string)
+  files := make(chan *File)
   
   go func() {
-    recursiveDirScan(l, directory, &files, true, "")
+    recursiveDirScan(
+      db,
+      l,
+      &files,
+      directory,
+      "",
+      true,
+    )
   }()
 
   return files

music-player/pkg/read/files_test.go

@@ -1,25 +1,38 @@
 package read_test
 
 import (
+	"os"
+	"path"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/felamaslen/go-music-player/pkg/database"
 	"github.com/felamaslen/go-music-player/pkg/read"
-	_ "github.com/felamaslen/go-music-player/pkg/testing"
+	setup "github.com/felamaslen/go-music-player/pkg/testing"
 )
 
 var _ = Describe("reading files", func() {
 
+  db := database.GetConnection()
+
+  BeforeEach(func() {
+    setup.PrepareDatabaseForTesting()
+  })
+
   Describe("reading file info", func() {
     var results []*read.Song
 
     BeforeEach(func() {
       results = nil
-      files := make(chan string, 1)
+      files := make(chan *read.File, 1)
 
       go func() {
 	defer close(files)
-	files <- read.TestSong.RelativePath
+	files <- &read.File{
+	  RelativePath: read.TestSong.RelativePath,
+	  ModifiedDate: 100123,
+	}
       }()
 
       outputChan := read.ReadMultipleFiles(read.TestDirectory, files)
@@ -42,14 +55,17 @@ var _ = Describe("reading files", func() {
     })
 
     It("should get the correct info from the file", func() {
-      Expect(*results[0]).To(Equal(read.TestSong))
+      var expectedResult = read.TestSong
+      expectedResult.ModifiedDate = 100123
+
+      Expect(*results[0]).To(Equal(expectedResult))
     })
   })
 
   Describe("scanning a directory recursively", func() {
-    var results []string
+    var results []*read.File
 
-    BeforeEach(func() {
+    var testScanDirectory = func() {
       results = nil
       files := read.ScanDirectory(read.TestDirectory)
 
@@ -64,13 +80,42 @@ var _ = Describe("reading files", func() {
 	  done = !more
 	}
       }
+    }
+
+    Context("when the database is empty", func() {
+      BeforeEach(testScanDirectory)
+
+      It("should return a channel with all the files in the directory", func() {
+	Expect(results).To(HaveLen(2))
+	Expect(results[0].RelativePath).To(Equal(read.TestSong.RelativePath))
+	Expect(results[1].RelativePath).To(Equal(read.TestSongNested.RelativePath))
+      })
     })
 
-    It("should return a channel with all the files in the directory", func() {
-      Expect(results).To(Equal([]string{
-	read.TestSong.RelativePath,
-	read.TestSongNested.RelativePath,
-      }))
+    Context("when the database already contains one of the files", func() {
+      BeforeEach(func() {
+	info, _ := os.Stat(path.Join(read.TestSong.BasePath, read.TestSong.RelativePath))
+	
+	db.MustExec(
+	  `
+	  insert into songs (title, artist, album, base_path, relative_path, modified_date)
+	  values ($1, $2, $3, $4, $5, $6)
+	  `,
+	  "old title",
+	  "old artist",
+	  "old album",
+	  read.TestSong.BasePath,
+	  read.TestSong.RelativePath,
+	  info.ModTime().Unix(),
+	)
+
+	testScanDirectory()
+      })
+
+      It("should only return those files which do not exist in the database", func() {
+	Expect(results).To(HaveLen(1))
+	Expect(results[0].RelativePath).To(Equal(read.TestSongNested.RelativePath))
+      })
     })
   })
 })

music-player/pkg/read/types.go

@@ -8,4 +8,10 @@ type Song struct {
   DurationOk bool
   BasePath string 	`db:"base_path"`
   RelativePath string 	`db:"relative_path"`
+  ModifiedDate int64 	`db:"modified_date"` 
+}
+
+type File struct {
+  RelativePath string
+  ModifiedDate int64
 }

music-player/pkg/repository/scan.go

@@ -31,14 +31,23 @@ func InsertMusicIntoDatabase(songs chan *read.Song) {
 
       query, err := db.Query(
         `
-        insert into songs (title, artist, album, duration, base_path, relative_path)
-        values ($1, $2, $3, $4, $5, $6)
+        insert into songs (
+          title
+          ,artist
+          ,album
+          ,duration
+          ,base_path
+          ,relative_path
+          ,modified_date
+        )
+        values ($1, $2, $3, $4, $5, $6, $7)
         on conflict (base_path, relative_path) do update
         set
           title = excluded.title
           ,artist = excluded.artist
           ,album = excluded.album
           ,duration = excluded.duration
+          ,modified_date = excluded.modified_date
         `,
         song.Title,
         song.Artist,
@@ -46,6 +55,7 @@ func InsertMusicIntoDatabase(songs chan *read.Song) {
         duration,
         song.BasePath,
         song.RelativePath,
+        song.ModifiedDate,
       )
 
       query.Close()

music-player/pkg/repository/scan_test.go

@@ -33,6 +33,7 @@ var _ = Describe("scanning repository", func() {
 	  DurationOk: true,
 	  BasePath: "/path/to",
 	  RelativePath: "file.ogg",
+	  ModifiedDate: 8876,
 	}
 
 	songs <- &read.Song{
@@ -43,6 +44,7 @@ var _ = Describe("scanning repository", func() {
 	  DurationOk: true,
 	  BasePath: "/different/path",
 	  RelativePath: "otherFile.ogg",
+	  ModifiedDate: 11883,
 	}
       }()
 
@@ -62,7 +64,7 @@ var _ = Describe("scanning repository", func() {
 	var song read.Song
 
 	rows, _ := db.Queryx(`
-	select title, artist, album, duration, base_path, relative_path
+	select title, artist, album, duration, base_path, relative_path, modified_date
 	from songs
 	order by title
 	`)
@@ -77,6 +79,7 @@ var _ = Describe("scanning repository", func() {
 	  Duration: 431,
 	  BasePath: "/path/to",
 	  RelativePath: "file.ogg",
+	  ModifiedDate: 8876,
 	}))
 
 	rows.Next()
@@ -89,6 +92,7 @@ var _ = Describe("scanning repository", func() {
 	  Duration: 256,
 	  BasePath: "/different/path",
 	  RelativePath: "otherFile.ogg",
+	  ModifiedDate: 11883,
 	}))
 
 	rows.Close()
@@ -99,14 +103,15 @@ var _ = Describe("scanning repository", func() {
       BeforeEach(func() {
 	db.MustExec(
 	  `
-	  insert into songs (title, artist, album, base_path, relative_path)
-	  values ($1, $2, $3, $4, $5)
+	  insert into songs (title, artist, album, base_path, relative_path, modified_date)
+	  values ($1, $2, $3, $4, $5, $6)
 	  `,
 	  "my title",
 	  "my artist",
 	  "my album",
 	  "/path/to",
 	  "file.ogg",
+	  7782,
 	)
 
 	testInsertSongs()
@@ -124,7 +129,7 @@ var _ = Describe("scanning repository", func() {
 
       It("should upsert the existing item", func() {
 	rows, _ := db.Queryx(`
-	select title, artist, album, duration, base_path, relative_path from songs
+	select title, artist, album, duration, base_path, relative_path, modified_date from songs
 	where base_path = '/path/to' and relative_path = 'file.ogg'
 	`)
 
@@ -136,6 +141,7 @@ var _ = Describe("scanning repository", func() {
 	Expect(song.Artist).To(Equal("The Beatles"))
 	Expect(song.Album).To(Equal(""))
 	Expect(song.Duration).To(Equal(431))
+	Expect(song.ModifiedDate).To(Equal(int64(8876)))
 
 	rows.Close()
       })

music-player/pkg/testing/testing.go

@@ -1,7 +1,6 @@
 package testing
 
 import (
-	"fmt"
 	"os"
 	"path"
 	"runtime"
@@ -12,7 +11,6 @@ import (
 func ChangeToRootDir() {
   _, filename, _, _ := runtime.Caller(0)
   dir := path.Join(path.Dir(filename), "../..")
-  fmt.Printf("Changing dir to %v\n", dir)
   err := os.Chdir(dir)
   if err != nil {
     panic(err)