diff --git a/internal/calibre/tasks/import.go b/internal/calibre/tasks/import.go index 899ba9e2..751fb957 100644 --- a/internal/calibre/tasks/import.go +++ b/internal/calibre/tasks/import.go @@ -7,6 +7,7 @@ import ( "io/fs" "lybbrio/internal/calibre" "lybbrio/internal/ent" + "lybbrio/internal/ent/book" "lybbrio/internal/ent/bookfile" "lybbrio/internal/ent/schema/filetype" "lybbrio/internal/ent/schema/ksuid" @@ -19,106 +20,6 @@ import ( "github.com/rs/zerolog/log" ) -type importOutputCtxKey string - -const importOutputKey importOutputCtxKey = "importOutput" - -var ignorableFilenames []string = []string{ - "cover", - "metadata", -} - -type importContext struct { - visitedAuthors map[int64]ksuid.ID - visitedTags map[int64]ksuid.ID - visitedPublishers map[int64]ksuid.ID - visitedLanguages map[int64]ksuid.ID - visitedSeries map[int64]ksuid.ID - failedBooks []string -} - -func newImportContext() *importContext { - return &importContext{ - visitedAuthors: make(map[int64]ksuid.ID), - visitedTags: make(map[int64]ksuid.ID), - visitedPublishers: make(map[int64]ksuid.ID), - visitedLanguages: make(map[int64]ksuid.ID), - visitedSeries: make(map[int64]ksuid.ID), - } -} - -func (c *importContext) String() string { - var ret strings.Builder - if len(c.failedBooks) > 0 { - ret.WriteString("Failed Books:\n") - for _, book := range c.failedBooks { - ret.WriteString(fmt.Sprintf("\t%s\n", book)) - } - } - return ret.String() -} - -func (c *importContext) AddFailedBook(book string) { - c.failedBooks = append(c.failedBooks, book) -} - -func (c *importContext) AuthorVisited(id int64) (ksuid.ID, bool) { - ret, ok := c.visitedAuthors[id] - return ret, ok -} - -func (c *importContext) AddAuthorVisited(id int64, ksuid ksuid.ID) { - c.visitedAuthors[id] = ksuid -} - -func (c *importContext) TagVisited(id int64) (ksuid.ID, bool) { - ret, ok := c.visitedTags[id] - return ret, ok -} - -func (c *importContext) AddTagVisited(id int64, ksuid ksuid.ID) { - c.visitedTags[id] = ksuid -} - -func (c *importContext) PublisherVisited(id int64) (ksuid.ID, bool) { - ret, ok := c.visitedPublishers[id] - return ret, ok -} - -func (c *importContext) AddPublisherVisited(id int64, ksuid ksuid.ID) { - c.visitedPublishers[id] = ksuid -} - -func (c *importContext) LanguageVisited(id int64) (ksuid.ID, bool) { - ret, ok := c.visitedLanguages[id] - return ret, ok -} - -func (c *importContext) AddLanguageVisited(id int64, ksuid ksuid.ID) { - c.visitedLanguages[id] = ksuid -} - -func (c *importContext) SeriesVisited(id int64) (ksuid.ID, bool) { - ret, ok := c.visitedSeries[id] - return ret, ok -} - -func (c *importContext) AddSeriesVisited(id int64, ksuid ksuid.ID) { - c.visitedSeries[id] = ksuid -} - -func importContextFrom(ctx context.Context) *importContext { - output := ctx.Value(importOutputKey) - if output == nil { - return nil - } - return output.(*importContext) -} - -func importContextTo(ctx context.Context, output *importContext) context.Context { - return context.WithValue(ctx, importOutputKey, output) -} - func ImportTask(cal calibre.Calibre, client *ent.Client) scheduler.TaskFunc { return func(ctx context.Context, task *ent.Task, cb scheduler.ProgressCallback) (string, error) { log := log.Ctx(ctx) @@ -127,7 +28,7 @@ func ImportTask(cal calibre.Calibre, client *ent.Client) scheduler.TaskFunc { ic := newImportContext() ctx = importContextTo(ctx, ic) - err := ImportBooks(cal, client, ctx, cb) + err := importBooks(ctx, cal, client, cb) if err != nil { return "", err } @@ -136,122 +37,129 @@ func ImportTask(cal calibre.Calibre, client *ent.Client) scheduler.TaskFunc { } } -func ImportBooks(cal calibre.Calibre, client *ent.Client, ctx context.Context, cb scheduler.ProgressCallback) error { - books, err := cal.GetBooks(ctx) +func importBooks(ctx context.Context, cal calibre.Calibre, client *ent.Client, cb scheduler.ProgressCallback) error { + calibreBooks, err := cal.GetBooks(ctx) if err != nil { return err } - total := len(books) - for idx, book := range books { - ic := importContextFrom(ctx) - - bookCreate := client.Book.Create(). - SetTitle(book.Title). - SetSort(book.Sort). - SetCalibreID(book.ID). - SetIsbn(book.ISBN). - SetPath(book.Path). - SetDescription(book.Comments.Text) - if book.PubDate != nil { - bookCreate.SetPublishedDate(*book.PubDate) - } - if book.SeriesIndex != nil { - bookCreate.SetSeriesIndex(*book.SeriesIndex) - } - - entBook, err := bookCreate. - Save(ctx) - if err != nil { - if ent.IsConstraintError(err) { - log.Debug().Err(err). - Str("book", book.Title). - Int64("bookID", book.ID). - Msg("Book already exists") - } else { - log.Warn().Err(err). - Str("book", book.Title). - Int64("bookID", book.ID). - Msg("Failed to create book") - ic.AddFailedBook(book.Title) - } - if err := cb(float64(idx+1) / (float64(total))); err != nil { - log.Warn().Err(err). - Str("book", book.Title). - Int64("bookID", book.ID). - Msg("Failed to update progress") - } - continue - } - - err = createOrAttachAuthors(client, ctx, entBook, book.Authors) + ic := importContextFrom(ctx) + total := len(calibreBooks) + for idx, calBook := range calibreBooks { + err := importBook(ctx, cal, client, *calBook) if err != nil { log.Warn().Err(err). - Str("book", book.Title). - Int64("bookID", book.ID). - Msg("Failed to create authors") + Str("book", calBook.Title). + Int64("bookID", calBook.ID). + Msg("Failed to import book") } - - err = createIdentifiers(ctx, client, entBook, book.Identifiers) - if err != nil { + if err := cb(float64(idx+1) / (float64(total))); err != nil { + ic.AddFailedBook(calBook.Title) log.Warn().Err(err). - Str("book", book.Title). - Int64("bookID", book.ID). - Msg("Failed to create identifiers") + Str("book", calBook.Title). + Int64("bookID", calBook.ID). + Msg("Failed to update progress") } + } - err = createOrAttachTags(ctx, client, entBook, book.Tags) - if err != nil { - log.Warn().Err(err). - Str("book", book.Title). - Int64("bookID", book.ID). - Msg("Failed to create tags") - } + return nil +} - err = createOrAttachPublishers(ctx, client, entBook, book.Publisher) - if err != nil { - log.Warn().Err(err). - Str("book", book.Title). - Int64("bookID", book.ID). - Msg("Failed to create publishers") - } +func importBook(ctx context.Context, cal calibre.Calibre, client *ent.Client, calBook calibre.Book) error { - err = createOrAttachLanguages(ctx, client, entBook, book.Languages) - if err != nil { - log.Warn().Err(err). - Str("book", book.Title). - Int64("bookID", book.ID). - Msg("Failed to create languages") - } + bookCreate := client.Book.Create(). + SetTitle(calBook.Title). + SetSort(calBook.Sort). + SetCalibreID(calBook.ID). + SetIsbn(calBook.ISBN). + SetPath(calBook.Path). + SetDescription(calBook.Comments.Text) + if calBook.PubDate != nil { + bookCreate.SetPublishedDate(*calBook.PubDate) + } + if calBook.SeriesIndex != nil { + bookCreate.SetSeriesIndex(*calBook.SeriesIndex) + } - err = createOrAttachSeriesList(ctx, client, entBook, book.Series) - if err != nil { - log.Warn().Err(err). - Str("book", book.Title). - Int64("bookID", book.ID). - Msg("Failed to create series") + var entBook *ent.Book + var err error + entBook, err = bookCreate. + Save(ctx) + if err != nil { + if ent.IsConstraintError(err) { + log.Debug().Err(err). + Str("book", calBook.Title). + Int64("bookID", calBook.ID). + Msg("Book already exists") + entBook, err = client.Book.Query(). + Where(book.CalibreIDEQ(int64(calBook.ID))). + Only(ctx) + if err != nil { + return fmt.Errorf("failed to query existing book: %w", err) + } + } else { + return fmt.Errorf("failed to create book: %w", err) } + } - err = registerBookFiles(ctx, cal, client, entBook, *book) - if err != nil { - log.Warn().Err(err). - Str("book", book.Title). - Int64("bookID", book.ID). - Msg("Failed register book files") - ic.AddFailedBook(book.Title) - } + err = createOrAttachAuthors(ctx, client, entBook, calBook.Authors) + if err != nil { + log.Warn().Err(err). + Str("book", calBook.Title). + Int64("bookID", calBook.ID). + Msg("Failed to create authors") + } - if err := cb(float64(idx+1) / (float64(total))); err != nil { - log.Warn().Err(err). - Str("book", book.Title). - Int64("bookID", book.ID). - Msg("Failed to update progress") - } + err = createIdentifiers(ctx, client, entBook, calBook.Identifiers) + if err != nil { + log.Warn().Err(err). + Str("book", calBook.Title). + Int64("bookID", calBook.ID). + Msg("Failed to create identifiers") + } + + err = createOrAttachTags(ctx, client, entBook, calBook.Tags) + if err != nil { + log.Warn().Err(err). + Str("book", calBook.Title). + Int64("bookID", calBook.ID). + Msg("Failed to create tags") + } + + err = createOrAttachPublishers(ctx, client, entBook, calBook.Publisher) + if err != nil { + log.Warn().Err(err). + Str("book", calBook.Title). + Int64("bookID", calBook.ID). + Msg("Failed to create publishers") } + err = createOrAttachLanguages(ctx, client, entBook, calBook.Languages) + if err != nil { + log.Warn().Err(err). + Str("book", calBook.Title). + Int64("bookID", calBook.ID). + Msg("Failed to create languages") + } + + err = createOrAttachSeriesList(ctx, client, entBook, calBook.Series) + if err != nil { + log.Warn().Err(err). + Str("book", calBook.Title). + Int64("bookID", calBook.ID). + Msg("Failed to create series") + } + + err = registerBookFiles(ctx, cal, client, entBook, calBook) + if err != nil { + log.Warn().Err(err). + Str("book", calBook.Title). + Int64("bookID", calBook.ID). + Msg("Failed register book files") + } return nil } -func createOrAttachAuthors(client *ent.Client, ctx context.Context, book *ent.Book, authors []calibre.Author) error { +func createOrAttachAuthors(ctx context.Context, client *ent.Client, book *ent.Book, authors []calibre.Author) error { log := log.Ctx(ctx).With().Str("book", book.Title).Str("bookID", book.ID.String()).Logger() for _, a := range authors { err := createOrAttachAuthor(ctx, client, book, a) @@ -454,7 +362,7 @@ func createOrAttachSeries(ctx context.Context, client *ent.Client, book *ent.Boo func registerBookFiles(ctx context.Context, cal calibre.Calibre, client *ent.Client, book *ent.Book, calBook calibre.Book) error { log := log.Ctx(ctx).With().Str("book", calBook.Title).Str("bookID", book.ID.String()).Logger() path := calBook.FullPath(cal) - log.Info().Str("path", path).Msg("Registering book files") + log.Trace().Str("path", path).Msg("Registering book files") files, err := os.ReadDir(calBook.FullPath(cal)) if err != nil { return fmt.Errorf("failed to read book directory: %w", err) @@ -466,6 +374,9 @@ func registerBookFiles(ctx context.Context, cal calibre.Calibre, client *ent.Cli path := filepath.Join(calBook.FullPath(cal), file.Name()) err := registerBookFile(ctx, client, book.ID, path, file) if err != nil { + if inner := errors.Unwrap(err); ent.IsConstraintError(inner) { + continue + } log.Warn().Err(err). Str("file", file.Name()). Msg("Failed to register file") @@ -487,9 +398,6 @@ func registerBookFile(ctx context.Context, client *ent.Client, bookID ksuid.ID, } info, err := file.Info() if err != nil { - log.Warn().Err(err). - Str("file", file.Name()). - Msg("Failed to get file info") return fmt.Errorf("failed to get file info: %w", err) } size := info.Size() diff --git a/internal/calibre/tasks/import_context.go b/internal/calibre/tasks/import_context.go new file mode 100644 index 00000000..26972a48 --- /dev/null +++ b/internal/calibre/tasks/import_context.go @@ -0,0 +1,108 @@ +package tasks + +import ( + "context" + "fmt" + "lybbrio/internal/ent/schema/ksuid" + "strings" +) + +type importOutputCtxKey string + +const importOutputKey importOutputCtxKey = "importOutput" + +var ignorableFilenames []string = []string{ + "cover", + "metadata", +} + +type importContext struct { + visitedAuthors map[int64]ksuid.ID + visitedTags map[int64]ksuid.ID + visitedPublishers map[int64]ksuid.ID + visitedLanguages map[int64]ksuid.ID + visitedSeries map[int64]ksuid.ID + failedBooks []string +} + +func newImportContext() *importContext { + return &importContext{ + visitedAuthors: make(map[int64]ksuid.ID), + visitedTags: make(map[int64]ksuid.ID), + visitedPublishers: make(map[int64]ksuid.ID), + visitedLanguages: make(map[int64]ksuid.ID), + visitedSeries: make(map[int64]ksuid.ID), + } +} + +func (c *importContext) String() string { + var ret strings.Builder + if len(c.failedBooks) > 0 { + ret.WriteString("Failed Books:\n") + for _, book := range c.failedBooks { + ret.WriteString(fmt.Sprintf("\t%s\n", book)) + } + } + return ret.String() +} + +func (c *importContext) AddFailedBook(book string) { + c.failedBooks = append(c.failedBooks, book) +} + +func (c *importContext) AuthorVisited(id int64) (ksuid.ID, bool) { + ret, ok := c.visitedAuthors[id] + return ret, ok +} + +func (c *importContext) AddAuthorVisited(id int64, ksuid ksuid.ID) { + c.visitedAuthors[id] = ksuid +} + +func (c *importContext) TagVisited(id int64) (ksuid.ID, bool) { + ret, ok := c.visitedTags[id] + return ret, ok +} + +func (c *importContext) AddTagVisited(id int64, ksuid ksuid.ID) { + c.visitedTags[id] = ksuid +} + +func (c *importContext) PublisherVisited(id int64) (ksuid.ID, bool) { + ret, ok := c.visitedPublishers[id] + return ret, ok +} + +func (c *importContext) AddPublisherVisited(id int64, ksuid ksuid.ID) { + c.visitedPublishers[id] = ksuid +} + +func (c *importContext) LanguageVisited(id int64) (ksuid.ID, bool) { + ret, ok := c.visitedLanguages[id] + return ret, ok +} + +func (c *importContext) AddLanguageVisited(id int64, ksuid ksuid.ID) { + c.visitedLanguages[id] = ksuid +} + +func (c *importContext) SeriesVisited(id int64) (ksuid.ID, bool) { + ret, ok := c.visitedSeries[id] + return ret, ok +} + +func (c *importContext) AddSeriesVisited(id int64, ksuid ksuid.ID) { + c.visitedSeries[id] = ksuid +} + +func importContextFrom(ctx context.Context) *importContext { + output := ctx.Value(importOutputKey) + if output == nil { + return nil + } + return output.(*importContext) +} + +func importContextTo(ctx context.Context, output *importContext) context.Context { + return context.WithValue(ctx, importOutputKey, output) +} diff --git a/internal/calibre/tasks/import_context_test.go b/internal/calibre/tasks/import_context_test.go new file mode 100644 index 00000000..d56df5cc --- /dev/null +++ b/internal/calibre/tasks/import_context_test.go @@ -0,0 +1,112 @@ +package tasks + +import ( + "context" + "testing" + + "lybbrio/internal/ent/schema/ksuid" + + "github.com/stretchr/testify/require" +) + +func Test_AddFailedBook(t *testing.T) { + require := require.New(t) + c := newImportContext() + c.AddFailedBook("test") + require.Equal([]string{"test"}, c.failedBooks) +} + +func Test_AuthorVisited(t *testing.T) { + require := require.New(t) + c := newImportContext() + expected := ksuid.MustNew("aut") + c.AddAuthorVisited(1, expected) + kid, ok := c.AuthorVisited(1) + require.True(ok) + require.Equal(expected, kid) + + _, ok = c.AuthorVisited(2) + require.False(ok) +} + +func Test_TagVisited(t *testing.T) { + require := require.New(t) + c := newImportContext() + expected := ksuid.MustNew("tag") + c.AddTagVisited(1, expected) + kid, ok := c.TagVisited(1) + require.True(ok) + require.Equal(expected, kid) + + _, ok = c.TagVisited(2) + require.False(ok) +} + +func Test_PublisherVisited(t *testing.T) { + require := require.New(t) + c := newImportContext() + expected := ksuid.MustNew("pub") + c.AddPublisherVisited(1, expected) + kid, ok := c.PublisherVisited(1) + require.True(ok) + require.Equal(expected, kid) + + _, ok = c.PublisherVisited(2) + require.False(ok) +} + +func Test_LanguageVisited(t *testing.T) { + require := require.New(t) + c := newImportContext() + expected := ksuid.MustNew("lan") + c.AddLanguageVisited(1, expected) + kid, ok := c.LanguageVisited(1) + require.True(ok) + require.Equal(expected, kid) + + _, ok = c.LanguageVisited(2) + require.False(ok) +} + +func Test_SeriesVisited(t *testing.T) { + require := require.New(t) + c := newImportContext() + expected := ksuid.MustNew("ser") + c.AddSeriesVisited(1, expected) + kid, ok := c.SeriesVisited(1) + require.True(ok) + require.Equal(expected, kid) + + _, ok = c.SeriesVisited(2) + require.False(ok) +} + +func Test_ImportContext(t *testing.T) { + require := require.New(t) + ic := newImportContext() + ctx := context.Background() + ctx = importContextTo(ctx, ic) + require.NotNil(ctx) + + ic2 := importContextFrom(ctx) + require.NotNil(ic2) + require.Equal(ic, ic2) +} + +func Test_ImportContext_ReturnsNilWhenNotAttached(t *testing.T) { + require := require.New(t) + ctx := context.Background() + ic := importContextFrom(ctx) + require.Nil(ic) +} + +func Test_ImportContext_String(t *testing.T) { + require := require.New(t) + ic := newImportContext() + require.Empty(ic.String()) + ic.AddFailedBook("test") + ic.AddFailedBook("test2") + + require.Contains(ic.String(), "test") + require.Contains(ic.String(), "test2") +} diff --git a/internal/ent/schema/bookfile.go b/internal/ent/schema/bookfile.go index 0a7a65dc..ef6cb093 100644 --- a/internal/ent/schema/bookfile.go +++ b/internal/ent/schema/bookfile.go @@ -37,7 +37,8 @@ func (BookFile) Fields() []ent.Field { field.Text("name"). NotEmpty(), field.Text("path"). - NotEmpty(), + NotEmpty(). + Unique(), field.Int64("size"). Positive(). Comment("Size in bytes"), diff --git a/internal/tasks/library_scan.go b/internal/tasks/library_scan.go deleted file mode 100644 index 3503fc28..00000000 --- a/internal/tasks/library_scan.go +++ /dev/null @@ -1,88 +0,0 @@ -package tasks - -import ( - "context" - "fmt" - "lybbrio/internal/ent" - "lybbrio/internal/ent/schema/filetype" - "lybbrio/internal/scheduler" - "os" - "path/filepath" - - "github.com/pirmd/epub" - "github.com/rs/zerolog/log" -) - -func getBookFileCount(ctx context.Context, libraryPath string) (int64, error) { - var ret int64 - err := filepath.WalkDir(libraryPath, func(path string, d os.DirEntry, err error) error { - if err != nil { - return fmt.Errorf("Could not walk library path: %w", err) - } - if d.IsDir() { - return nil - } - fileType := filetype.FromExtension(d.Name()) - if fileType == filetype.Unknown { - return nil - } - ret++ - return nil - }) - return ret, err -} - -func LibraryScanTask(client *ent.Client, libraryPath string) scheduler.TaskFunc { - return func(ctx context.Context, task *ent.Task, cb scheduler.ProgressCallback) (string, error) { - log := log.Ctx(ctx).With().Str("task_id", task.ID.String()).Logger() - bookCount, err := getBookFileCount(ctx, libraryPath) - if err != nil { - return "", fmt.Errorf("Could not get book count: %w", err) - } - log.Info().Int64("book_count", bookCount).Msg("Found books in library") - - err = filepath.WalkDir(libraryPath, func(path string, d os.DirEntry, err error) error { - if err != nil { - return fmt.Errorf("Could not walk library path: %w", err) - } - if d.IsDir() { - return nil - } - fileType := filetype.FromExtension(d.Name()) - if fileType == filetype.Unknown { - return nil - } - switch fileType { - case filetype.EPUB: - fallthrough - case filetype.KEPUB: - err = epubMetadata(ctx, path) - if err != nil { - return fmt.Errorf("Could not get epub metadata: %w", err) - } - } - - // TODO: Add book to database - return nil - }) - if err != nil { - return "", fmt.Errorf("Could not walk library path: %w", err) - } - return "", nil - } -} - -func epubMetadata(ctx context.Context, path string) error { - ep, err := epub.Open(path) - if err != nil { - return fmt.Errorf("Could not open epub: %w", err) - } - defer ep.Close() - info, err := ep.Information() - if err != nil { - return fmt.Errorf("Could not get epub information: %w", err) - } - log := log.Ctx(ctx).With().Str("title", info.Title[0]).Logger() - log.Info().Msg("Found epub") - return nil -}