feat: remember errors when scanning files

    * feat: skip files with errors on next scan
    * feat: insert scan errors into database
    * refactor: common package for types
    * feat: repository to insert a scan error
    * feat: migration for scan_errors table
    * fix: conflicting environment variable in makefile

gmus-backend/Makefile

@@ -1,6 +1,6 @@
-NAME 	:= docker.fela.space/gmus-backend
+IMAGE := docker.fela.space/gmus-backend
 TAG 	:= $$(git log -1 --pretty=%H)
-IMG 	:= ${NAME}:${TAG}
+IMG 	:= ${IMAGE}:${TAG}
 
 build.scan:
 	go build -o bin/gmus.scan ./cmd/gmus.scan

gmus-backend/migrations/000006_create_scan_errors_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE scan_errors;

gmus-backend/migrations/000006_create_scan_errors_table.up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE scan_errors (
+  id SERIAL PRIMARY KEY,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  base_path VARCHAR NOT NULL,
+  relative_path VARCHAR NOT NULL,
+  error VARCHAR NOT NULL
+);
+
+CREATE INDEX scan_errors_base_path ON scan_errors (base_path);
+CREATE INDEX scan_errors_relative_path ON scan_errors (relative_path);

gmus-backend/pkg/read/audio.go

@@ -6,27 +6,28 @@ import (
 
 	tag "github.com/dhowden/tag"
 	duration "github.com/felamaslen/gmus-backend/pkg/read/duration"
+	"github.com/felamaslen/gmus-backend/pkg/types"
 )
 
-func ReadFile(basePath string, scannedFile *File) (song *Song, err error) {
+func ReadFile(basePath string, scannedFile *types.File) (song *types.Song, err error) {
 	fullPath := filepath.Join(basePath, scannedFile.RelativePath)
 	file, errFile := os.Open(fullPath)
 	if errFile != nil {
-		return &Song{}, errFile
+		return &types.Song{}, errFile
 	}
 
 	defer file.Close()
 
 	tags, errTags := tag.ReadFrom(file)
 	if errTags != nil {
-		return &Song{}, errTags
+		return &types.Song{}, errTags
 	}
 
 	durationSeconds := duration.GetSongDurationSeconds(file, tags)
 
 	trackNumber, _ := tags.Track()
 
-	result := Song{
+	result := types.Song{
 		TrackNumber:  trackNumber,
 		Title:        tags.Title(),
 		Artist:       tags.Artist(),

gmus-backend/pkg/read/audio_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/felamaslen/gmus-backend/pkg/read"
 	_ "github.com/felamaslen/gmus-backend/pkg/testing"
+	"github.com/felamaslen/gmus-backend/pkg/types"
 )
 
 var _ = Describe("reading audio files", func() {
@@ -18,14 +19,14 @@ var _ = Describe("reading audio files", func() {
 
 	Context("when the file is ogg vorbis", func() {
 		It("should get the expected info from the file", func() {
-			result, err := read.ReadFile(basePath, &read.File{
+			result, err := read.ReadFile(basePath, &types.File{
 				RelativePath: read.TestSong.RelativePath,
 				ModifiedDate: 102118,
 			})
 
 			Expect(err).To(BeNil())
 
-			Expect(*result).To(Equal(read.Song{
+			Expect(*result).To(Equal(types.Song{
 				TrackNumber:  23,
 				Title:        "Impact Moderato",
 				Artist:       "Kevin MacLeod",

gmus-backend/pkg/read/files.go

@@ -3,10 +3,13 @@ package read
 import (
 	"io/ioutil"
 	"path/filepath"
+	"time"
 
 	config "github.com/felamaslen/gmus-backend/pkg/config"
 	"github.com/felamaslen/gmus-backend/pkg/database"
 	"github.com/felamaslen/gmus-backend/pkg/logger"
+	"github.com/felamaslen/gmus-backend/pkg/repository"
+	"github.com/felamaslen/gmus-backend/pkg/types"
 	"github.com/jmoiron/sqlx"
 	"github.com/lib/pq"
 )
@@ -14,10 +17,11 @@ import (
 const BATCH_SIZE = 100
 const LOG_EVERY = 100
 
-func ReadMultipleFiles(basePath string, files chan *File) chan *Song {
+func ReadMultipleFiles(basePath string, files chan *types.File) chan *types.Song {
+	var db = database.GetConnection()
 	var l = logger.CreateLogger(config.GetConfig().LogLevel)
 
-	songs := make(chan *Song)
+	songs := make(chan *types.Song)
 
 	go func() {
 		defer func() {
@@ -36,6 +40,12 @@ func ReadMultipleFiles(basePath string, files chan *File) chan *Song {
 						songs <- song
 					} else {
 						l.Error("[READ] Error (%s): %v\n", file.RelativePath, err)
+						repository.InsertScanError(db, &types.ScanError{
+							CreatedAt:    time.Now(),
+							BasePath:     basePath,
+							RelativePath: file.RelativePath,
+							Error:        err.Error(),
+						})
 					}
 				} else {
 					return
@@ -55,7 +65,7 @@ func isValidFile(file string) bool {
 func recursiveDirScan(
 	db *sqlx.DB,
 	l *logger.Logger,
-	allFiles *chan *File,
+	allFiles *chan *types.File,
 	rootDirectory string,
 	relativePath string,
 	isRoot bool,
@@ -93,7 +103,7 @@ func recursiveDirScan(
 				false,
 			)
 		} else if isValidFile(file.Name()) {
-			*allFiles <- &File{
+			*allFiles <- &types.File{
 				RelativePath: fileRelativePath,
 				ModifiedDate: file.ModTime().Unix(),
 			}
@@ -104,13 +114,13 @@ func recursiveDirScan(
 func batchFilterFiles(
 	db *sqlx.DB,
 	l *logger.Logger,
-	filteredOutput *chan *File,
-	allFiles *chan *File,
+	filteredOutput *chan *types.File,
+	allFiles *chan *types.File,
 	basePath string,
 ) {
 	defer close(*filteredOutput)
 
-	var batch [BATCH_SIZE]*File
+	var batch [BATCH_SIZE]*types.File
 	var batchSize = 0
 	var numFiltered = 0
 
@@ -129,32 +139,13 @@ func batchFilterFiles(
 			modifiedDates = append(modifiedDates, batch[i].ModifiedDate)
 		}
 
-		newOrUpdatedFiles, err := db.Queryx(
-			`
-      select r.relative_path, r.modified_date
-      from (
-        select * from unnest($1::varchar[], $2::bigint[])
-        as t(relative_path, modified_date)
-      ) r
-
-      left join songs on
-        songs.base_path = $3
-        and songs.relative_path = r.relative_path
-        and songs.modified_date = r.modified_date
-
-      where songs.id is null
-      `,
-			relativePaths,
-			modifiedDates,
-			basePath,
-		)
-
+		newOrUpdatedFiles, err := repository.SelectNewOrUpdatedFiles(db, relativePaths, modifiedDates, basePath)
 		if err != nil {
 			l.Fatal("[FILTER] Fatal error! %v\n", err)
 		}
 
 		for newOrUpdatedFiles.Next() {
-			var file File
+			var file types.File
 			newOrUpdatedFiles.StructScan(&file)
 
 			l.Verbose("[NEW] %s\n", file.RelativePath)
@@ -190,12 +181,12 @@ func batchFilterFiles(
 	}
 }
 
-func ScanDirectory(directory string) chan *File {
+func ScanDirectory(directory string) chan *types.File {
 	db := database.GetConnection()
 	l := logger.CreateLogger(config.GetConfig().LogLevel)
 
-	filteredOutput := make(chan *File)
-	allFiles := make(chan *File)
+	filteredOutput := make(chan *types.File)
+	allFiles := make(chan *types.File)
 
 	go func() {
 		batchFilterFiles(db, l, &filteredOutput, &allFiles, directory)

gmus-backend/pkg/read/files_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/felamaslen/gmus-backend/pkg/database"
 	"github.com/felamaslen/gmus-backend/pkg/read"
 	setup "github.com/felamaslen/gmus-backend/pkg/testing"
+	"github.com/felamaslen/gmus-backend/pkg/types"
 )
 
 var _ = Describe("reading files", func() {
@@ -21,49 +22,90 @@ var _ = Describe("reading files", func() {
 	})
 
 	Describe("reading file info", func() {
-		var results []*read.Song
+		var results []*types.Song
+		var files chan *types.File
 
 		BeforeEach(func() {
 			results = nil
-			files := make(chan *read.File, 1)
+			files = make(chan *types.File, 1)
+		})
 
-			go func() {
-				defer close(files)
-				files <- &read.File{
-					RelativePath: read.TestSong.RelativePath,
-					ModifiedDate: 100123,
-				}
-			}()
+		Context("when all the files are readable", func() {
+			BeforeEach(func() {
+				go func() {
+					defer close(files)
+					files <- &types.File{
+						RelativePath: read.TestSong.RelativePath,
+						ModifiedDate: 100123,
+					}
+				}()
 
-			outputChan := read.ReadMultipleFiles(read.TestDirectory, files)
+				outputChan := read.ReadMultipleFiles(read.TestDirectory, files)
 
-			outputDone := false
+				outputDone := false
 
-			for !outputDone {
-				select {
-				case result, more := <-outputChan:
-					if more {
-						results = append(results, result)
+				for !outputDone {
+					select {
+					case result, more := <-outputChan:
+						if more {
+							results = append(results, result)
+						}
+						outputDone = !more
 					}
-					outputDone = !more
 				}
-			}
-		})
+			})
+
+			It("should return the correct number of results", func() {
+				Expect(results).To(HaveLen(1))
+			})
+
+			It("should get the correct info from the file", func() {
+				var expectedResult = read.TestSong
+				expectedResult.ModifiedDate = 100123
 
-		It("should return the correct number of results", func() {
-			Expect(results).To(HaveLen(1))
+				Expect(*results[0]).To(Equal(expectedResult))
+			})
 		})
 
-		It("should get the correct info from the file", func() {
-			var expectedResult = read.TestSong
-			expectedResult.ModifiedDate = 100123
+		Context("when an error occurs reading a file", func() {
+			BeforeEach(func() {
+				go func() {
+					defer close(files)
+					files <- &types.File{
+						RelativePath: "test/file/does/not/exist.ogg",
+						ModifiedDate: 123,
+					}
+				}()
+
+				outputChan := read.ReadMultipleFiles(read.TestDirectory, files)
 
-			Expect(*results[0]).To(Equal(expectedResult))
+				outputDone := false
+
+				for !outputDone {
+					select {
+					case _, more := <-outputChan:
+						outputDone = !more
+					}
+				}
+			})
+
+			It("should add the file to the scan_errors table", func() {
+				var scanError []*types.ScanError
+				db.Select(&scanError, "select relative_path, base_path, error from scan_errors")
+
+				Expect(scanError).To(HaveLen(1))
+
+				Expect(scanError[0]).To(Equal(&types.ScanError{
+					RelativePath: "test/file/does/not/exist.ogg",
+					BasePath:     read.TestDirectory,
+					Error:        "open pkg/read/testdata/test/file/does/not/exist.ogg: no such file or directory",
+				}))
+			})
 		})
 	})
 
 	Describe("scanning a directory recursively", func() {
-		var results []*read.File
+		var results []*types.File
 
 		var testScanDirectory = func() {
 			results = nil
@@ -104,9 +146,9 @@ var _ = Describe("reading files", func() {
 
 				db.MustExec(
 					`
-	  insert into songs (title, artist, album, base_path, relative_path, modified_date)
-	  values ($1, $2, $3, $4, $5, $6)
-	  `,
+					insert into songs (title, artist, album, base_path, relative_path, modified_date)
+					values ($1, $2, $3, $4, $5, $6)
+					`,
 					"old title",
 					"old artist",
 					"old album",
@@ -123,5 +165,21 @@ var _ = Describe("reading files", func() {
 				Expect(results[0].RelativePath).To(Equal(read.TestSongNested.RelativePath))
 			})
 		})
+
+		Context("when an error previously occurred scanning one of the files", func() {
+			BeforeEach(func() {
+				db.MustExec(`
+				insert into scan_errors (base_path, relative_path, error)
+				values ($1, $2, $3)
+				`, read.TestSong.BasePath, read.TestSong.RelativePath, "A bad thing happened")
+
+				testScanDirectory()
+			})
+
+			It("should only return those files which did not have errors marked against them", func() {
+				Expect(results).To(HaveLen(1))
+				Expect(results[0].RelativePath).To(Equal(read.TestSongNested.RelativePath))
+			})
+		})
 	})
 })

gmus-backend/pkg/read/test_file_info.go

@@ -1,8 +1,10 @@
 package read
 
+import "github.com/felamaslen/gmus-backend/pkg/types"
+
 const TestDirectory = "pkg/read/testdata"
 
-var TestSong = Song{
+var TestSong = types.Song{
 	TrackNumber:  23,
 	Title:        "Impact Moderato",
 	Artist:       "Kevin MacLeod",
@@ -12,7 +14,7 @@ var TestSong = Song{
 	RelativePath: "file_example_OOG_1MG.ogg",
 }
 
-var TestSongNested = Song{
+var TestSongNested = types.Song{
 	TrackNumber:  14,
 	Title:        "Clementi: Piano Sonata in D major, Op 25 No 6 - Movement 2: Un poco andante",
 	Artist:       "Howard Shelley",

gmus-backend/pkg/repository/player.go

@@ -3,32 +3,32 @@ package repository
 import (
 	"database/sql"
 
-	"github.com/felamaslen/gmus-backend/pkg/read"
+	"github.com/felamaslen/gmus-backend/pkg/types"
 	"github.com/jmoiron/sqlx"
 )
 
-func GetNextSong(db *sqlx.DB, prevSongId int) (nextSong *read.Song, err error) {
-	nextSong = &read.Song{}
+func GetNextSong(db *sqlx.DB, prevSongId int) (nextSong *types.Song, err error) {
+	nextSong = &types.Song{}
 	err = db.QueryRowx(
 		querySelectNextSong,
 		prevSongId,
 	).StructScan(nextSong)
 	if err != nil && err == sql.ErrNoRows {
 		err = nil
-		nextSong = &read.Song{Id: 0}
+		nextSong = &types.Song{Id: 0}
 	}
 	return
 }
 
-func GetPrevSong(db *sqlx.DB, nextSongId int) (prevSong *read.Song, err error) {
-	prevSong = &read.Song{}
+func GetPrevSong(db *sqlx.DB, nextSongId int) (prevSong *types.Song, err error) {
+	prevSong = &types.Song{}
 	err = db.QueryRowx(
 		querySelectPrevSong,
 		nextSongId,
 	).StructScan(prevSong)
 	if err != nil && err == sql.ErrNoRows {
 		err = nil
-		prevSong = &read.Song{Id: 0}
+		prevSong = &types.Song{Id: 0}
 	}
 	return
 }

gmus-backend/pkg/repository/queries.go

@@ -169,3 +169,26 @@ order by
 
 limit 1
 `
+
+const queryInsertScanError = `
+insert into scan_errors (created_at, base_path, relative_path, error)
+values ($1, $2, $3, $4)
+`
+
+const querySelectNewOrUpdatedFiles = `
+with all_files as (
+	select * from unnest($1::varchar[], $2::bigint[])
+	as t(relative_path, modified_date)
+)
+select r.relative_path, r.modified_date
+from all_files r
+left join songs on
+	songs.base_path = $3
+	and songs.relative_path = r.relative_path
+	and songs.modified_date = r.modified_date
+left join scan_errors e on
+	e.base_path = $3
+	and e.relative_path = r.relative_path
+where songs.id is null
+	and e.id is null
+`

gmus-backend/pkg/repository/scan.go

@@ -0,0 +1,23 @@
+package repository
+
+import (
+	"github.com/felamaslen/gmus-backend/pkg/types"
+	"github.com/lib/pq"
+
+	"github.com/jmoiron/sqlx"
+)
+
+func InsertScanError(db *sqlx.DB, scanError *types.ScanError) (err error) {
+	_, err = db.Exec(queryInsertScanError, scanError.CreatedAt, scanError.BasePath, scanError.RelativePath, scanError.Error)
+	return
+}
+
+func SelectNewOrUpdatedFiles(db *sqlx.DB, relativePaths pq.StringArray, modifiedDates pq.Int64Array, basePath string) (result *sqlx.Rows, err error) {
+	result, err = db.Queryx(
+		querySelectNewOrUpdatedFiles,
+		relativePaths,
+		modifiedDates,
+		basePath,
+	)
+	return
+}

gmus-backend/pkg/repository/scan_test.go

@@ -0,0 +1,50 @@
+package repository_test
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/felamaslen/gmus-backend/pkg/database"
+	"github.com/felamaslen/gmus-backend/pkg/repository"
+	setup "github.com/felamaslen/gmus-backend/pkg/testing"
+	"github.com/felamaslen/gmus-backend/pkg/types"
+)
+
+var _ = Describe("scan repository", func() {
+	db := database.GetConnection()
+
+	BeforeEach(func() {
+		setup.PrepareDatabaseForTesting()
+	})
+
+	Describe("InsertScanError", func() {
+		BeforeEach(func() {
+			repository.InsertScanError(db, &types.ScanError{
+				CreatedAt:    time.Date(2021, 4, 20, 11, 17, 25, 0, time.UTC),
+				BasePath:     "/my/base/path",
+				RelativePath: "path/to/file.ogg",
+				Error:        "File does not exist or something",
+			})
+		})
+
+		It("should insert the error into the database", func() {
+			var result []*types.ScanError
+			db.Select(&result, `
+			select created_at, base_path, relative_path, error
+			from scan_errors
+			`)
+
+			Expect(result).To(HaveLen(1))
+			Expect(result[0]).To(Equal(&types.ScanError{
+				CreatedAt:    time.Date(2021, 4, 20, 11, 17, 25, 0, time.UTC).In(time.Local),
+				BasePath:     "/my/base/path",
+				RelativePath: "path/to/file.ogg",
+				Error:        "File does not exist or something",
+			}))
+		})
+	})
+
+	// Note; SelectNewOrUpdatedFiles logic is tested via pkg/read/files_test.go
+})

gmus-backend/pkg/repository/songs.go

@@ -1,15 +1,15 @@
 package repository
 
 import (
-	"github.com/felamaslen/gmus-backend/pkg/read"
+	"github.com/felamaslen/gmus-backend/pkg/types"
 	"github.com/jmoiron/sqlx"
 	"github.com/lib/pq"
 )
 
 const BATCH_SIZE = 100
 
-func SelectSong(db *sqlx.DB, ids []int) (songs *[]*read.Song, err error) {
-	songs = &[]*read.Song{}
+func SelectSong(db *sqlx.DB, ids []int) (songs *[]*types.Song, err error) {
+	songs = &[]*types.Song{}
 	var idsArray pq.Int64Array
 	for _, id := range ids {
 		idsArray = append(idsArray, int64(id))
@@ -53,14 +53,14 @@ func SelectAlbumsByArtist(db *sqlx.DB, artist string) (albums *[]string, err err
 	return
 }
 
-func SelectSongsByArtist(db *sqlx.DB, artist string) (songs *[]*read.SongExternal, err error) {
-	songs = &[]*read.SongExternal{}
+func SelectSongsByArtist(db *sqlx.DB, artist string) (songs *[]*types.SongExternal, err error) {
+	songs = &[]*types.SongExternal{}
 	err = db.Select(songs, querySelectSongsByArtist, artist)
 
 	return
 }
 
-func BatchUpsertSongs(db *sqlx.DB, batch *[BATCH_SIZE]*read.Song, batchSize int) error {
+func BatchUpsertSongs(db *sqlx.DB, batch *[BATCH_SIZE]*types.Song, batchSize int) error {
 	var trackNumbers pq.Int64Array
 	var titles pq.StringArray
 	var artists pq.StringArray

gmus-backend/pkg/repository/songs_test.go

@@ -5,9 +5,9 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/felamaslen/gmus-backend/pkg/database"
-	"github.com/felamaslen/gmus-backend/pkg/read"
 	"github.com/felamaslen/gmus-backend/pkg/repository"
 	setup "github.com/felamaslen/gmus-backend/pkg/testing"
+	"github.com/felamaslen/gmus-backend/pkg/types"
 )
 
 var _ = Describe("songs repository", func() {
@@ -55,7 +55,7 @@ var _ = Describe("songs repository", func() {
 			Expect(err).To(BeNil())
 
 			Expect(*result).To(HaveLen(1))
-			Expect((*result)[0]).To(Equal(&read.Song{
+			Expect((*result)[0]).To(Equal(&types.Song{
 				Id:           int(id),
 				TrackNumber:  7,
 				Title:        "Hey Jude",
@@ -77,7 +77,7 @@ var _ = Describe("songs repository", func() {
 			Expect(err).To(BeNil())
 
 			Expect(*result).To(HaveLen(2))
-			Expect((*result)[0]).To(Equal(&read.Song{
+			Expect((*result)[0]).To(Equal(&types.Song{
 				Id:           int(id),
 				TrackNumber:  7,
 				Title:        "Hey Jude",
@@ -89,7 +89,7 @@ var _ = Describe("songs repository", func() {
 				ModifiedDate: 8876,
 			}))
 
-			Expect((*result)[1]).To(Equal(&read.Song{
+			Expect((*result)[1]).To(Equal(&types.Song{
 				Id:           int(id2),
 				TrackNumber:  13,
 				Title:        "Track 1",
@@ -113,7 +113,7 @@ var _ = Describe("songs repository", func() {
 	})
 
 	Describe("BatchUpsertSongs", func() {
-		songs := [100]*read.Song{
+		songs := [100]*types.Song{
 			{
 				TrackNumber:  1,
 				Title:        "Title A",
@@ -142,7 +142,7 @@ var _ = Describe("songs repository", func() {
 			})
 
 			It("should insert the batch into the database", func() {
-				var result []*read.Song
+				var result []*types.Song
 				db.Select(&result, testQuerySelectAllSongs)
 
 				Expect(result).To(HaveLen(2))
@@ -152,8 +152,8 @@ var _ = Describe("songs repository", func() {
 		})
 
 		Context("when the songs already exist", func() {
-			var result []*read.Song
-			var modifiedBatch [100]*read.Song
+			var result []*types.Song
+			var modifiedBatch [100]*types.Song
 
 			modifiedBatch[0] = songs[0]
 			modifiedBatch[1] = songs[1]
@@ -164,7 +164,7 @@ var _ = Describe("songs repository", func() {
 				repository.BatchUpsertSongs(db, &songs, 2)
 				repository.BatchUpsertSongs(db, &modifiedBatch, 2)
 
-				result = []*read.Song{}
+				result = []*types.Song{}
 				db.Select(&result, testQuerySelectSong12)
 			})
 

gmus-backend/pkg/server/fetch.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/felamaslen/gmus-backend/pkg/database"
 	"github.com/felamaslen/gmus-backend/pkg/logger"
-	"github.com/felamaslen/gmus-backend/pkg/read"
 	"github.com/felamaslen/gmus-backend/pkg/repository"
+	"github.com/felamaslen/gmus-backend/pkg/types"
 	"github.com/go-redis/redis"
 	"github.com/jmoiron/sqlx"
 )
@@ -63,8 +63,8 @@ func routeFetchAlbums(l *logger.Logger, rdb redis.Cmdable, w http.ResponseWriter
 }
 
 type SongsResponse struct {
-	Artist string                `json:"artist"`
-	Songs  *[]*read.SongExternal `json:"songs"`
+	Artist string                 `json:"artist"`
+	Songs  *[]*types.SongExternal `json:"songs"`
 }
 
 func routeFetchSongs(l *logger.Logger, rdb redis.Cmdable, w http.ResponseWriter, r *http.Request) error {
@@ -119,7 +119,7 @@ func routeFetchSongInfo(l *logger.Logger, rdb redis.Cmdable, w http.ResponseWrit
 
 	song := (*songs)[0]
 
-	response, err := json.Marshal(read.SongExternal{
+	response, err := json.Marshal(types.SongExternal{
 		Id:          id,
 		TrackNumber: song.TrackNumber,
 		Title:       song.Title,
@@ -161,9 +161,9 @@ func routeFetchMultiSongInfo(l *logger.Logger, rdb redis.Cmdable, w http.Respons
 		return err
 	}
 
-	songsArray := []read.SongExternal{}
+	songsArray := []types.SongExternal{}
 	for _, song := range *songs {
-		songsArray = append(songsArray, read.SongExternal{
+		songsArray = append(songsArray, types.SongExternal{
 			Id:          song.Id,
 			TrackNumber: song.TrackNumber,
 			Title:       song.Title,
@@ -186,14 +186,14 @@ type NullResponse struct {
 	Id int `json:"id"`
 }
 
-func respondWithSongOrNull(db *sqlx.DB, w http.ResponseWriter, song *read.Song) error {
+func respondWithSongOrNull(db *sqlx.DB, w http.ResponseWriter, song *types.Song) error {
 	if song.Id == 0 {
 		response, _ := json.Marshal(NullResponse{})
 		w.Write(response)
 		return nil
 	}
 
-	response, err := json.Marshal(read.SongExternal{
+	response, err := json.Marshal(types.SongExternal{
 		Id:          song.Id,
 		TrackNumber: song.TrackNumber,
 		Title:       song.Title,

gmus-backend/pkg/services/scanner.go

@@ -6,18 +6,19 @@ import (
 	"github.com/felamaslen/gmus-backend/pkg/logger"
 	"github.com/felamaslen/gmus-backend/pkg/read"
 	"github.com/felamaslen/gmus-backend/pkg/repository"
+	"github.com/felamaslen/gmus-backend/pkg/types"
 )
 
 const LOG_EVERY = 100
 
 const BATCH_SIZE = 100
 
-func UpsertSongsFromChannel(songs chan *read.Song) {
+func UpsertSongsFromChannel(songs chan *types.Song) {
 	var l = logger.CreateLogger(config.GetConfig().LogLevel)
 
 	db := database.GetConnection()
 
-	var batch [BATCH_SIZE]*read.Song
+	var batch [BATCH_SIZE]*types.Song
 	var batchSize = 0
 	var numAdded = 0
 

gmus-backend/pkg/services/scanner_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/felamaslen/gmus-backend/pkg/read"
 	"github.com/felamaslen/gmus-backend/pkg/services"
 	setup "github.com/felamaslen/gmus-backend/pkg/testing"
+	"github.com/felamaslen/gmus-backend/pkg/types"
 )
 
 var _ = Describe("Music scanner service", func() {
@@ -18,14 +19,14 @@ var _ = Describe("Music scanner service", func() {
 	})
 
 	Describe("UpsertSongsFromChannel", func() {
-		var songs chan *read.Song
+		var songs chan *types.Song
 
 		var testScanSongs = func() {
-			songs = make(chan *read.Song)
+			songs = make(chan *types.Song)
 
 			go func() {
 				defer close(songs)
-				songs <- &read.Song{
+				songs <- &types.Song{
 					TrackNumber:  7,
 					Title:        "Hey Jude",
 					Artist:       "The Beatles",
@@ -36,7 +37,7 @@ var _ = Describe("Music scanner service", func() {
 					ModifiedDate: 8876,
 				}
 
-				songs <- &read.Song{
+				songs <- &types.Song{
 					TrackNumber:  11,
 					Title:        "Starman",
 					Artist:       "David Bowie",
@@ -61,7 +62,7 @@ var _ = Describe("Music scanner service", func() {
 			})
 
 			It("should insert both songs", func() {
-				var songs []read.Song
+				var songs []types.Song
 
 				db.Select(&songs, `
 	select track_number, title, artist, album, duration, base_path, relative_path, modified_date
@@ -69,7 +70,7 @@ var _ = Describe("Music scanner service", func() {
 	order by title
 	`)
 
-				Expect(songs[0]).To(Equal(read.Song{
+				Expect(songs[0]).To(Equal(types.Song{
 					TrackNumber:  7,
 					Title:        "Hey Jude",
 					Artist:       "The Beatles",
@@ -80,7 +81,7 @@ var _ = Describe("Music scanner service", func() {
 					ModifiedDate: 8876,
 				}))
 
-				Expect(songs[1]).To(Equal(read.Song{
+				Expect(songs[1]).To(Equal(types.Song{
 					TrackNumber:  11,
 					Title:        "Starman",
 					Artist:       "David Bowie",
@@ -122,7 +123,7 @@ var _ = Describe("Music scanner service", func() {
 			})
 
 			It("should upsert the existing item", func() {
-				var songs []read.Song
+				var songs []types.Song
 				db.Select(&songs, `
 	select
 	  track_number
@@ -154,7 +155,7 @@ var _ = Describe("Music scanner service", func() {
 		It("should recursively scan files from a directory and add them to the database", func() {
 			services.ScanAndInsert(read.TestDirectory)
 
-			var songs []read.Song
+			var songs []types.Song
 			err := db.Select(&songs, `
 	select title, artist, album, duration, base_path, relative_path
 	from songs
@@ -164,7 +165,7 @@ var _ = Describe("Music scanner service", func() {
 
 			Expect(songs).To(HaveLen(2))
 
-			Expect(read.Song{
+			Expect(types.Song{
 				Title:        read.TestSong.Title,
 				Artist:       read.TestSong.Artist,
 				Album:        read.TestSong.Album,
@@ -173,7 +174,7 @@ var _ = Describe("Music scanner service", func() {
 				RelativePath: read.TestSong.RelativePath,
 			}).To(BeElementOf(songs))
 
-			Expect(read.Song{
+			Expect(types.Song{
 				Title:        read.TestSongNested.Title,
 				Artist:       read.TestSongNested.Artist,
 				Album:        read.TestSongNested.Album,

gmus-backend/pkg/testing/testing.go

@@ -27,4 +27,5 @@ func PrepareDatabaseForTesting() {
 	db := database.GetConnection()
 
 	db.MustExec("truncate table songs")
+	db.MustExec("truncate table scan_errors")
 }

gmus-backend/pkg/types/errors.go

@@ -0,0 +1,11 @@
+package types
+
+import "time"
+
+type ScanError struct {
+	Id           int       `db:"id"`
+	CreatedAt    time.Time `db:"created_at"`
+	BasePath     string    `db:"base_path"`
+	RelativePath string    `db:"relative_path"`
+	Error        string    `db:"error"`
+}

gmus-backend/pkg/read/types.go → gmus-backend/pkg/types/song.go

@@ -1,4 +1,4 @@
-package read
+package types
 
 type Song struct {
 	Id           int    `db:"id"`