Skip to content

Commit

Permalink
Updates to new library scanner and adding jsdoc types
Browse files Browse the repository at this point in the history
  • Loading branch information
advplyr committed Aug 27, 2023
1 parent ea1d051 commit 2c8448d
Show file tree
Hide file tree
Showing 10 changed files with 416 additions and 16 deletions.
4 changes: 2 additions & 2 deletions server/Server.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,10 @@ const Logger = require('./Logger')
const Auth = require('./Auth')
const Watcher = require('./Watcher')
const Scanner = require('./scanner/Scanner')
const LibraryScanner = require('./scanner/LibraryScanner')
const Database = require('./Database')
const SocketAuthority = require('./SocketAuthority')

const routes = require('./routes/index')

const ApiRouter = require('./routers/ApiRouter')
const HlsRouter = require('./routers/HlsRouter')

Expand Down Expand Up @@ -78,6 +77,7 @@ class Server {
this.rssFeedManager = new RssFeedManager()

this.scanner = new Scanner(this.coverManager, this.taskManager)
this.libraryScanner = new LibraryScanner(this.coverManager, this.taskManager)
this.cronManager = new CronManager(this.scanner, this.podcastManager)

// Routers
Expand Down
2 changes: 2 additions & 0 deletions server/controllers/LibraryController.js
Original file line number Diff line number Diff line change
Expand Up @@ -977,6 +977,8 @@ class LibraryController {
}
res.sendStatus(200)
await this.scanner.scan(req.library, options)
// TODO: New library scanner
// await this.libraryScanner.scan(req.library, options)
await Database.resetLibraryIssuesFilterData(req.library.id)
Logger.info('[LibraryController] Scan complete')
}
Expand Down
27 changes: 26 additions & 1 deletion server/models/Book.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,31 @@ const Logger = require('../Logger')
* @property {string} title
*/

/**
* @typedef AudioFileObject
* @property {number} index
* @property {string} ino
* @property {{filename:string, ext:string, path:string, relPath:string, size:number, mtimeMs:number, ctimeMs:number, birthtimeMs:number}} metadata
* @property {number} addedAt
* @property {number} updatedAt
* @property {number} trackNumFromMeta
* @property {number} discNumFromMeta
* @property {number} trackNumFromFilename
* @property {number} discNumFromFilename
* @property {boolean} manuallyVerified
* @property {string} format
* @property {number} duration
* @property {number} bitRate
* @property {string} language
* @property {string} codec
* @property {string} timeBase
* @property {number} channels
* @property {string} channelLayout
* @property {ChapterObject[]} chapters
* @property {Object} metaTags
* @property {string} mimeType
*/

class Book extends Model {
constructor(values, options) {
super(values, options)
Expand Down Expand Up @@ -52,7 +77,7 @@ class Book extends Model {
this.duration
/** @type {string[]} */
this.narrators
/** @type {Object} */
/** @type {AudioFileObject[]} */
this.audioFiles
/** @type {EBookFileObject} */
this.ebookFile
Expand Down
7 changes: 7 additions & 0 deletions server/models/LibraryItem.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ const Logger = require('../Logger')
const oldLibraryItem = require('../objects/LibraryItem')
const libraryFilters = require('../utils/queries/libraryFilters')
const { areEquivalent } = require('../utils/index')
const Book = require('./Book')
const Podcast = require('./Podcast')

/**
* @typedef LibraryFileObject
Expand Down Expand Up @@ -791,6 +793,11 @@ class LibraryItem extends Model {
return this.getOldLibraryItem(libraryItem)
}

/**
*
* @param {import('sequelize').FindOptions} options
* @returns {Promise<Book|Podcast>}
*/
getMedia(options) {
if (!this.mediaType) return Promise.resolve(null)
const mixinMethodName = `get${this.sequelize.uppercaseFirst(this.mediaType)}`
Expand Down
9 changes: 7 additions & 2 deletions server/objects/files/AudioFile.js
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ class AudioFile {
channelLayout: this.channelLayout,
chapters: this.chapters,
embeddedCoverArt: this.embeddedCoverArt,
metaTags: this.metaTags ? this.metaTags.toJSON() : {},
metaTags: this.metaTags?.toJSON() || {},
mimeType: this.mimeType
}
}
Expand Down Expand Up @@ -163,11 +163,16 @@ class AudioFile {
return new AudioFile(this.toJSON())
}

/**
*
* @param {AudioFile} scannedAudioFile
* @returns {boolean} true if updates were made
*/
updateFromScan(scannedAudioFile) {
let hasUpdated = false

const newjson = scannedAudioFile.toJSON()
const ignoreKeys = ['manuallyVerified', 'exclude', 'addedAt', 'updatedAt']
const ignoreKeys = ['manuallyVerified', 'ctimeMs', 'addedAt', 'updatedAt']

for (const key in newjson) {
if (key === 'metadata') {
Expand Down
1 change: 1 addition & 0 deletions server/routers/ApiRouter.js
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ class ApiRouter {
constructor(Server) {
this.auth = Server.auth
this.scanner = Server.scanner
this.libraryScanner = Server.libraryScanner
this.playbackSessionManager = Server.playbackSessionManager
this.abMergeManager = Server.abMergeManager
this.backupManager = Server.backupManager
Expand Down
191 changes: 191 additions & 0 deletions server/scanner/AudioFileScanner.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,197 @@
const Path = require('path')
const Logger = require('../Logger')
const prober = require('../utils/prober')
const LibraryItem = require('../models/LibraryItem')
const AudioFile = require('../objects/files/AudioFile')

class AudioFileScanner {
constructor() { }

/**
* Is array of numbers sequential, i.e. 1, 2, 3, 4
* @param {number[]} nums
* @returns {boolean}
*/
isSequential(nums) {
if (!nums?.length) return false
if (nums.length === 1) return true
let prev = nums[0]
for (let i = 1; i < nums.length; i++) {
if (nums[i] - prev > 1) return false
prev = nums[i]
}
return true
}

/**
* Remove
* @param {number[]} nums
* @returns {number[]}
*/
removeDupes(nums) {
if (!nums || !nums.length) return []
if (nums.length === 1) return nums

let nodupes = [nums[0]]
nums.forEach((num) => {
if (num > nodupes[nodupes.length - 1]) nodupes.push(num)
})
return nodupes
}

/**
* Order audio files by track/disc number
* @param {import('../models/Book')} book
* @param {import('../models/Book').AudioFileObject[]} audioFiles
* @returns {import('../models/Book').AudioFileObject[]}
*/
runSmartTrackOrder(book, audioFiles) {
let discsFromFilename = []
let tracksFromFilename = []
let discsFromMeta = []
let tracksFromMeta = []

audioFiles.forEach((af) => {
if (af.discNumFromFilename !== null) discsFromFilename.push(af.discNumFromFilename)
if (af.discNumFromMeta !== null) discsFromMeta.push(af.discNumFromMeta)
if (af.trackNumFromFilename !== null) tracksFromFilename.push(af.trackNumFromFilename)
if (af.trackNumFromMeta !== null) tracksFromMeta.push(af.trackNumFromMeta)
})
discsFromFilename.sort((a, b) => a - b)
discsFromMeta.sort((a, b) => a - b)
tracksFromFilename.sort((a, b) => a - b)
tracksFromMeta.sort((a, b) => a - b)

let discKey = null
if (discsFromMeta.length === audioFiles.length && this.isSequential(discsFromMeta)) {
discKey = 'discNumFromMeta'
} else if (discsFromFilename.length === audioFiles.length && this.isSequential(discsFromFilename)) {
discKey = 'discNumFromFilename'
}

let trackKey = null
tracksFromFilename = this.removeDupes(tracksFromFilename)
tracksFromMeta = this.removeDupes(tracksFromMeta)
if (tracksFromFilename.length > tracksFromMeta.length) {
trackKey = 'trackNumFromFilename'
} else {
trackKey = 'trackNumFromMeta'
}

if (discKey !== null) {
Logger.debug(`[AudioFileScanner] Smart track order for "${book.title}" using disc key ${discKey} and track key ${trackKey}`)
audioFiles.sort((a, b) => {
let Dx = a[discKey] - b[discKey]
if (Dx === 0) Dx = a[trackKey] - b[trackKey]
return Dx
})
} else {
Logger.debug(`[AudioFileScanner] Smart track order for "${book.title}" using track key ${trackKey}`)
audioFiles.sort((a, b) => a[trackKey] - b[trackKey])
}

for (let i = 0; i < audioFiles.length; i++) {
audioFiles[i].index = i + 1
}
return audioFiles
}

/**
* Get track and disc number from audio filename
* @param {{title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} mediaMetadataFromScan
* @param {LibraryItem.LibraryFileObject} audioLibraryFile
* @returns {{trackNumber:number, discNumber:number}}
*/
getTrackAndDiscNumberFromFilename(mediaMetadataFromScan, audioLibraryFile) {
const { title, author, series, publishedYear } = mediaMetadataFromScan
const { filename, path } = audioLibraryFile.metadata
let partbasename = Path.basename(filename, Path.extname(filename))

// Remove title, author, series, and publishedYear from filename if there
if (title) partbasename = partbasename.replace(title, '')
if (author) partbasename = partbasename.replace(author, '')
if (series) partbasename = partbasename.replace(series, '')
if (publishedYear) partbasename = partbasename.replace(publishedYear)

// Look for disc number
let discNumber = null
const discMatch = partbasename.match(/\b(disc|cd) ?(\d\d?)\b/i)
if (discMatch && discMatch.length > 2 && discMatch[2]) {
if (!isNaN(discMatch[2])) {
discNumber = Number(discMatch[2])
}

// Remove disc number from filename
partbasename = partbasename.replace(/\b(disc|cd) ?(\d\d?)\b/i, '')
}

// Look for disc number in folder path e.g. /Book Title/CD01/audiofile.mp3
const pathdir = Path.dirname(path).split('/').pop()
if (pathdir && /^cd\d{1,3}$/i.test(pathdir)) {
const discFromFolder = Number(pathdir.replace(/cd/i, ''))
if (!isNaN(discFromFolder) && discFromFolder !== null) discNumber = discFromFolder
}

const numbersinpath = partbasename.match(/\d{1,4}/g)
const trackNumber = numbersinpath && numbersinpath.length ? parseInt(numbersinpath[0]) : null
return {
trackNumber,
discNumber
}
}

/**
*
* @param {string} mediaType
* @param {LibraryItem.LibraryFileObject} libraryFile
* @param {{title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} mediaMetadataFromScan
* @returns {Promise<AudioFile>}
*/
async scan(mediaType, libraryFile, mediaMetadataFromScan) {
const probeData = await prober.probe(libraryFile.metadata.path)

if (probeData.error) {
Logger.error(`[MediaFileScanner] ${probeData.error} : "${libraryFile.metadata.path}"`)
return null
}

if (!probeData.audioStream) {
Logger.error('[MediaFileScanner] Invalid audio file no audio stream')
return null
}

const audioFile = new AudioFile()
audioFile.trackNumFromMeta = probeData.audioMetaTags.trackNumber
audioFile.discNumFromMeta = probeData.audioMetaTags.discNumber
if (mediaType === 'book') {
const { trackNumber, discNumber } = this.getTrackAndDiscNumberFromFilename(mediaMetadataFromScan, libraryFile)
audioFile.trackNumFromFilename = trackNumber
audioFile.discNumFromFilename = discNumber
}
audioFile.setDataFromProbe(libraryFile, probeData)

return audioFile
}

/**
* Scan LibraryFiles and return AudioFiles
* @param {string} mediaType
* @param {import('./LibraryItemScanData')} libraryItemScanData
* @param {LibraryItem.LibraryFileObject[]} audioLibraryFiles
* @returns {Promise<AudioFile[]>}
*/
async executeMediaFileScans(mediaType, libraryItemScanData, audioLibraryFiles) {
const batchSize = 32
const results = []
for (let batch = 0; batch < audioLibraryFiles.length; batch += batchSize) {
const proms = []
for (let i = batch; i < Math.min(batch + batchSize, audioLibraryFiles.length); i++) {
proms.push(this.scan(mediaType, audioLibraryFiles[i], libraryItemScanData.mediaMetadata))
}
results.push(...await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr)))
}

return results
}
}
module.exports = new AudioFileScanner()
Loading

0 comments on commit 2c8448d

Please sign in to comment.