Skip to content

Commit

Permalink
Calibre Import
Browse files Browse the repository at this point in the history 
Signed-off-by: Russell Troxel <[email protected]>
  • Loading branch information
@rtrox
rtrox committed Jan 13, 2024
1 parent ed083a1 commit eae9d96
Show file tree
Hide file tree
Showing 5 changed files with 331 additions and 290 deletions.
310 changes: 109 additions & 201 deletions internal/calibre/tasks/import.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io/fs"
"lybbrio/internal/calibre"
"lybbrio/internal/ent"
"lybbrio/internal/ent/book"
"lybbrio/internal/ent/bookfile"
"lybbrio/internal/ent/schema/filetype"
"lybbrio/internal/ent/schema/ksuid"
Expand All @@ -19,106 +20,6 @@ import (
"github.com/rs/zerolog/log"
)

type importOutputCtxKey string

const importOutputKey importOutputCtxKey = "importOutput"

var ignorableFilenames []string = []string{
"cover",
"metadata",
}

type importContext struct {
visitedAuthors map[int64]ksuid.ID
visitedTags map[int64]ksuid.ID
visitedPublishers map[int64]ksuid.ID
visitedLanguages map[int64]ksuid.ID
visitedSeries map[int64]ksuid.ID
failedBooks []string
}

func newImportContext() *importContext {
return &importContext{
visitedAuthors: make(map[int64]ksuid.ID),
visitedTags: make(map[int64]ksuid.ID),
visitedPublishers: make(map[int64]ksuid.ID),
visitedLanguages: make(map[int64]ksuid.ID),
visitedSeries: make(map[int64]ksuid.ID),
}
}

func (c *importContext) String() string {
var ret strings.Builder
if len(c.failedBooks) > 0 {
ret.WriteString("Failed Books:\n")
for _, book := range c.failedBooks {
ret.WriteString(fmt.Sprintf("\t%s\n", book))
}
}
return ret.String()
}

func (c *importContext) AddFailedBook(book string) {
c.failedBooks = append(c.failedBooks, book)
}

func (c *importContext) AuthorVisited(id int64) (ksuid.ID, bool) {
ret, ok := c.visitedAuthors[id]
return ret, ok
}

func (c *importContext) AddAuthorVisited(id int64, ksuid ksuid.ID) {
c.visitedAuthors[id] = ksuid
}

func (c *importContext) TagVisited(id int64) (ksuid.ID, bool) {
ret, ok := c.visitedTags[id]
return ret, ok
}

func (c *importContext) AddTagVisited(id int64, ksuid ksuid.ID) {
c.visitedTags[id] = ksuid
}

func (c *importContext) PublisherVisited(id int64) (ksuid.ID, bool) {
ret, ok := c.visitedPublishers[id]
return ret, ok
}

func (c *importContext) AddPublisherVisited(id int64, ksuid ksuid.ID) {
c.visitedPublishers[id] = ksuid
}

func (c *importContext) LanguageVisited(id int64) (ksuid.ID, bool) {
ret, ok := c.visitedLanguages[id]
return ret, ok
}

func (c *importContext) AddLanguageVisited(id int64, ksuid ksuid.ID) {
c.visitedLanguages[id] = ksuid
}

func (c *importContext) SeriesVisited(id int64) (ksuid.ID, bool) {
ret, ok := c.visitedSeries[id]
return ret, ok
}

func (c *importContext) AddSeriesVisited(id int64, ksuid ksuid.ID) {
c.visitedSeries[id] = ksuid
}

func importContextFrom(ctx context.Context) *importContext {
output := ctx.Value(importOutputKey)
if output == nil {
return nil
}
return output.(*importContext)
}

func importContextTo(ctx context.Context, output *importContext) context.Context {
return context.WithValue(ctx, importOutputKey, output)
}

func ImportTask(cal calibre.Calibre, client *ent.Client) scheduler.TaskFunc {
return func(ctx context.Context, task *ent.Task, cb scheduler.ProgressCallback) (string, error) {
log := log.Ctx(ctx)
Expand All @@ -127,7 +28,7 @@ func ImportTask(cal calibre.Calibre, client *ent.Client) scheduler.TaskFunc {
ic := newImportContext()
ctx = importContextTo(ctx, ic)

err := ImportBooks(cal, client, ctx, cb)
err := importBooks(ctx, cal, client, cb)
if err != nil {
return "", err
}
Expand All @@ -136,122 +37,129 @@ func ImportTask(cal calibre.Calibre, client *ent.Client) scheduler.TaskFunc {
}
}

func ImportBooks(cal calibre.Calibre, client *ent.Client, ctx context.Context, cb scheduler.ProgressCallback) error {
books, err := cal.GetBooks(ctx)
func importBooks(ctx context.Context, cal calibre.Calibre, client *ent.Client, cb scheduler.ProgressCallback) error {
calibreBooks, err := cal.GetBooks(ctx)
if err != nil {
return err
}
total := len(books)
for idx, book := range books {
ic := importContextFrom(ctx)

bookCreate := client.Book.Create().
SetTitle(book.Title).
SetSort(book.Sort).
SetCalibreID(book.ID).
SetIsbn(book.ISBN).
SetPath(book.Path).
SetDescription(book.Comments.Text)
if book.PubDate != nil {
bookCreate.SetPublishedDate(*book.PubDate)
}
if book.SeriesIndex != nil {
bookCreate.SetSeriesIndex(*book.SeriesIndex)
}

entBook, err := bookCreate.
Save(ctx)
if err != nil {
if ent.IsConstraintError(err) {
log.Debug().Err(err).
Str("book", book.Title).
Int64("bookID", book.ID).
Msg("Book already exists")
} else {
log.Warn().Err(err).
Str("book", book.Title).
Int64("bookID", book.ID).
Msg("Failed to create book")
ic.AddFailedBook(book.Title)
}
if err := cb(float64(idx+1) / (float64(total))); err != nil {
log.Warn().Err(err).
Str("book", book.Title).
Int64("bookID", book.ID).
Msg("Failed to update progress")
}
continue
}

err = createOrAttachAuthors(client, ctx, entBook, book.Authors)
ic := importContextFrom(ctx)
total := len(calibreBooks)
for idx, calBook := range calibreBooks {
err := importBook(ctx, cal, client, *calBook)
if err != nil {
log.Warn().Err(err).
Str("book", book.Title).
Int64("bookID", book.ID).
Msg("Failed to create authors")
Str("book", calBook.Title).
Int64("bookID", calBook.ID).
Msg("Failed to import book")
}

err = createIdentifiers(ctx, client, entBook, book.Identifiers)
if err != nil {
if err := cb(float64(idx+1) / (float64(total))); err != nil {
ic.AddFailedBook(calBook.Title)
log.Warn().Err(err).
Str("book", book.Title).
Int64("bookID", book.ID).
Msg("Failed to create identifiers")
Str("book", calBook.Title).
Int64("bookID", calBook.ID).
Msg("Failed to update progress")
}
}

err = createOrAttachTags(ctx, client, entBook, book.Tags)
if err != nil {
log.Warn().Err(err).
Str("book", book.Title).
Int64("bookID", book.ID).
Msg("Failed to create tags")
}
return nil
}

err = createOrAttachPublishers(ctx, client, entBook, book.Publisher)
if err != nil {
log.Warn().Err(err).
Str("book", book.Title).
Int64("bookID", book.ID).
Msg("Failed to create publishers")
}
func importBook(ctx context.Context, cal calibre.Calibre, client *ent.Client, calBook calibre.Book) error {

err = createOrAttachLanguages(ctx, client, entBook, book.Languages)
if err != nil {
log.Warn().Err(err).
Str("book", book.Title).
Int64("bookID", book.ID).
Msg("Failed to create languages")
}
bookCreate := client.Book.Create().
SetTitle(calBook.Title).
SetSort(calBook.Sort).
SetCalibreID(calBook.ID).
SetIsbn(calBook.ISBN).
SetPath(calBook.Path).
SetDescription(calBook.Comments.Text)
if calBook.PubDate != nil {
bookCreate.SetPublishedDate(*calBook.PubDate)
}
if calBook.SeriesIndex != nil {
bookCreate.SetSeriesIndex(*calBook.SeriesIndex)
}

err = createOrAttachSeriesList(ctx, client, entBook, book.Series)
if err != nil {
log.Warn().Err(err).
Str("book", book.Title).
Int64("bookID", book.ID).
Msg("Failed to create series")
var entBook *ent.Book
var err error
entBook, err = bookCreate.
Save(ctx)
if err != nil {
if ent.IsConstraintError(err) {
log.Debug().Err(err).
Str("book", calBook.Title).
Int64("bookID", calBook.ID).
Msg("Book already exists")
entBook, err = client.Book.Query().
Where(book.CalibreIDEQ(int64(calBook.ID))).
Only(ctx)
if err != nil {
return fmt.Errorf("failed to query existing book: %w", err)
}
} else {
return fmt.Errorf("failed to create book: %w", err)
}
}

err = registerBookFiles(ctx, cal, client, entBook, *book)
if err != nil {
log.Warn().Err(err).
Str("book", book.Title).
Int64("bookID", book.ID).
Msg("Failed register book files")
ic.AddFailedBook(book.Title)
}
err = createOrAttachAuthors(ctx, client, entBook, calBook.Authors)
if err != nil {
log.Warn().Err(err).
Str("book", calBook.Title).
Int64("bookID", calBook.ID).
Msg("Failed to create authors")
}

if err := cb(float64(idx+1) / (float64(total))); err != nil {
log.Warn().Err(err).
Str("book", book.Title).
Int64("bookID", book.ID).
Msg("Failed to update progress")
}
err = createIdentifiers(ctx, client, entBook, calBook.Identifiers)
if err != nil {
log.Warn().Err(err).
Str("book", calBook.Title).
Int64("bookID", calBook.ID).
Msg("Failed to create identifiers")
}

err = createOrAttachTags(ctx, client, entBook, calBook.Tags)
if err != nil {
log.Warn().Err(err).
Str("book", calBook.Title).
Int64("bookID", calBook.ID).
Msg("Failed to create tags")
}

err = createOrAttachPublishers(ctx, client, entBook, calBook.Publisher)
if err != nil {
log.Warn().Err(err).
Str("book", calBook.Title).
Int64("bookID", calBook.ID).
Msg("Failed to create publishers")
}

err = createOrAttachLanguages(ctx, client, entBook, calBook.Languages)
if err != nil {
log.Warn().Err(err).
Str("book", calBook.Title).
Int64("bookID", calBook.ID).
Msg("Failed to create languages")
}

err = createOrAttachSeriesList(ctx, client, entBook, calBook.Series)
if err != nil {
log.Warn().Err(err).
Str("book", calBook.Title).
Int64("bookID", calBook.ID).
Msg("Failed to create series")
}

err = registerBookFiles(ctx, cal, client, entBook, calBook)
if err != nil {
log.Warn().Err(err).
Str("book", calBook.Title).
Int64("bookID", calBook.ID).
Msg("Failed register book files")
}
return nil
}

func createOrAttachAuthors(client *ent.Client, ctx context.Context, book *ent.Book, authors []calibre.Author) error {
func createOrAttachAuthors(ctx context.Context, client *ent.Client, book *ent.Book, authors []calibre.Author) error {
log := log.Ctx(ctx).With().Str("book", book.Title).Str("bookID", book.ID.String()).Logger()
for _, a := range authors {
err := createOrAttachAuthor(ctx, client, book, a)
Expand Down Expand Up @@ -454,7 +362,7 @@ func createOrAttachSeries(ctx context.Context, client *ent.Client, book *ent.Boo
func registerBookFiles(ctx context.Context, cal calibre.Calibre, client *ent.Client, book *ent.Book, calBook calibre.Book) error {
log := log.Ctx(ctx).With().Str("book", calBook.Title).Str("bookID", book.ID.String()).Logger()
path := calBook.FullPath(cal)
log.Info().Str("path", path).Msg("Registering book files")
log.Trace().Str("path", path).Msg("Registering book files")
files, err := os.ReadDir(calBook.FullPath(cal))
if err != nil {
return fmt.Errorf("failed to read book directory: %w", err)
Expand All @@ -466,6 +374,9 @@ func registerBookFiles(ctx context.Context, cal calibre.Calibre, client *ent.Cli
path := filepath.Join(calBook.FullPath(cal), file.Name())
err := registerBookFile(ctx, client, book.ID, path, file)
if err != nil {
if inner := errors.Unwrap(err); ent.IsConstraintError(inner) {
continue
}
log.Warn().Err(err).
Str("file", file.Name()).
Msg("Failed to register file")
Expand All @@ -487,9 +398,6 @@ func registerBookFile(ctx context.Context, client *ent.Client, bookID ksuid.ID,
}
info, err := file.Info()
if err != nil {
log.Warn().Err(err).
Str("file", file.Name()).
Msg("Failed to get file info")
return fmt.Errorf("failed to get file info: %w", err)
}
size := info.Size()
Expand Down
Loading

0 comments on commit eae9d96

Please sign in to comment.