From 2b3c3cd088db9fe4ac98f19ad0e5b85c048382d2 Mon Sep 17 00:00:00 2001 From: mfcar Date: Sat, 23 Sep 2023 20:38:03 +0100 Subject: [PATCH] Add infra to handle with feed healthy --- client/pages/item/_id/index.vue | 8 +- server/controllers/PodcastController.js | 13 + server/managers/PodcastManager.js | 665 ++++++++++++------------ server/models/Podcast.js | 18 +- server/routers/ApiRouter.js | 1 + server/utils/migrations/dbMigration.js | 6 +- 6 files changed, 379 insertions(+), 332 deletions(-) diff --git a/client/pages/item/_id/index.vue b/client/pages/item/_id/index.vue index 176725b918..cd333cdc8d 100644 --- a/client/pages/item/_id/index.vue +++ b/client/pages/item/_id/index.vue @@ -485,16 +485,16 @@ export default { return this.$toast.error('Podcast does not have an RSS Feed') } this.fetchingRSSFeed = true - var payload = await this.$axios.$post(`/api/podcasts/feed`, { rssFeed: this.mediaMetadata.feedUrl }).catch((error) => { + var payload = await this.$axios.get(`/api/podcasts/${this.libraryItemId}/feed`).catch((error) => { console.error('Failed to get feed', error) this.$toast.error('Failed to get podcast feed') return null }) this.fetchingRSSFeed = false - if (!payload) return + if (!payload || !payload.data) return - console.log('Podcast feed', payload) - const podcastfeed = payload.podcast + console.log('Podcast feed', payload.data) + const podcastfeed = payload.data.podcast if (!podcastfeed.episodes || !podcastfeed.episodes.length) { this.$toast.info('No episodes found in RSS feed') return diff --git a/server/controllers/PodcastController.js b/server/controllers/PodcastController.js index 0e1ebcd3a1..19f16476a9 100644 --- a/server/controllers/PodcastController.js +++ b/server/controllers/PodcastController.js @@ -115,6 +115,19 @@ class PodcastController { res.json({ podcast }) } + async checkPodcastFeed(req, res) { + const libraryItem = req.libraryItem + const podcast = await getPodcastFeed(libraryItem.media.metadata.feedUrl) + + if (!podcast) { + this.podcastManager.setFeedHealthStatus(libraryItem, false) + return res.status(404).send('Podcast RSS feed request failed or invalid response data') + } + + this.podcastManager.setFeedHealthStatus(libraryItem, true) + res.json({ podcast }) + } + async getFeedsFromOPMLText(req, res) { if (!req.body.opmlText) { return res.sendStatus(400) diff --git a/server/managers/PodcastManager.js b/server/managers/PodcastManager.js index 5dec215230..19254e1703 100644 --- a/server/managers/PodcastManager.js +++ b/server/managers/PodcastManager.js @@ -4,9 +4,9 @@ const Database = require('../Database') const fs = require('../libs/fsExtra') -const { getPodcastFeed } = require('../utils/podcastUtils') -const { removeFile, downloadFile } = require('../utils/fileUtils') -const { levenshteinDistance } = require('../utils/index') +const {getPodcastFeed} = require('../utils/podcastUtils') +const {removeFile, downloadFile} = require('../utils/fileUtils') +const {levenshteinDistance} = require('../utils/index') const opmlParser = require('../utils/parsers/parseOPML') const opmlGenerator = require('../utils/generators/opmlGenerator') const prober = require('../utils/prober') @@ -19,374 +19,395 @@ const AudioFile = require('../objects/files/AudioFile') const Task = require("../objects/Task") class PodcastManager { - constructor(watcher, notificationManager, taskManager) { - this.watcher = watcher - this.notificationManager = notificationManager - this.taskManager = taskManager - - this.downloadQueue = [] - this.currentDownload = null - - this.failedCheckMap = {} - this.MaxFailedEpisodeChecks = 24 - } - - getEpisodeDownloadsInQueue(libraryItemId) { - return this.downloadQueue.filter(d => d.libraryItemId === libraryItemId) - } - - clearDownloadQueue(libraryItemId = null) { - if (!this.downloadQueue.length) return - - if (!libraryItemId) { - Logger.info(`[PodcastManager] Clearing all downloads in queue (${this.downloadQueue.length})`) - this.downloadQueue = [] - } else { - var itemDownloads = this.getEpisodeDownloadsInQueue(libraryItemId) - Logger.info(`[PodcastManager] Clearing downloads in queue for item "${libraryItemId}" (${itemDownloads.length})`) - this.downloadQueue = this.downloadQueue.filter(d => d.libraryItemId !== libraryItemId) + constructor(watcher, notificationManager, taskManager) { + this.watcher = watcher + this.notificationManager = notificationManager + this.taskManager = taskManager + + this.downloadQueue = [] + this.currentDownload = null + + this.failedCheckMap = {} + this.MaxFailedEpisodeChecks = 24 } - } - - async downloadPodcastEpisodes(libraryItem, episodesToDownload, isAutoDownload) { - let index = Math.max(...libraryItem.media.episodes.filter(ep => ep.index == null || isNaN(ep.index)).map(ep => Number(ep.index))) + 1 - for (const ep of episodesToDownload) { - const newPe = new PodcastEpisode() - newPe.setData(ep, index++) - newPe.libraryItemId = libraryItem.id - newPe.podcastId = libraryItem.media.id - const newPeDl = new PodcastEpisodeDownload() - newPeDl.setData(newPe, libraryItem, isAutoDownload, libraryItem.libraryId) - this.startPodcastEpisodeDownload(newPeDl) + + getEpisodeDownloadsInQueue(libraryItemId) { + return this.downloadQueue.filter(d => d.libraryItemId === libraryItemId) } - } - - async startPodcastEpisodeDownload(podcastEpisodeDownload) { - SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) - if (this.currentDownload) { - this.downloadQueue.push(podcastEpisodeDownload) - SocketAuthority.emitter('episode_download_queued', podcastEpisodeDownload.toJSONForClient()) - return + + clearDownloadQueue(libraryItemId = null) { + if (!this.downloadQueue.length) return + + if (!libraryItemId) { + Logger.info(`[PodcastManager] Clearing all downloads in queue (${this.downloadQueue.length})`) + this.downloadQueue = [] + } else { + var itemDownloads = this.getEpisodeDownloadsInQueue(libraryItemId) + Logger.info(`[PodcastManager] Clearing downloads in queue for item "${libraryItemId}" (${itemDownloads.length})`) + this.downloadQueue = this.downloadQueue.filter(d => d.libraryItemId !== libraryItemId) + } } - const task = new Task() - const taskDescription = `Downloading episode "${podcastEpisodeDownload.podcastEpisode.title}".` - const taskData = { - libraryId: podcastEpisodeDownload.libraryId, - libraryItemId: podcastEpisodeDownload.libraryItemId, + async downloadPodcastEpisodes(libraryItem, episodesToDownload, isAutoDownload) { + let index = Math.max(...libraryItem.media.episodes.filter(ep => ep.index == null || isNaN(ep.index)).map(ep => Number(ep.index))) + 1 + for (const ep of episodesToDownload) { + const newPe = new PodcastEpisode() + newPe.setData(ep, index++) + newPe.libraryItemId = libraryItem.id + newPe.podcastId = libraryItem.media.id + const newPeDl = new PodcastEpisodeDownload() + newPeDl.setData(newPe, libraryItem, isAutoDownload, libraryItem.libraryId) + this.startPodcastEpisodeDownload(newPeDl) + } } - task.setData('download-podcast-episode', 'Downloading Episode', taskDescription, false, taskData) - this.taskManager.addTask(task) - SocketAuthority.emitter('episode_download_started', podcastEpisodeDownload.toJSONForClient()) - this.currentDownload = podcastEpisodeDownload + async startPodcastEpisodeDownload(podcastEpisodeDownload) { + SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) + if (this.currentDownload) { + this.downloadQueue.push(podcastEpisodeDownload) + SocketAuthority.emitter('episode_download_queued', podcastEpisodeDownload.toJSONForClient()) + return + } - // If this file already exists then append the episode id to the filename - // e.g. "/tagesschau 20 Uhr.mp3" becomes "/tagesschau 20 Uhr (ep_asdfasdf).mp3" - // this handles podcasts where every title is the same (ref https://github.com/advplyr/audiobookshelf/issues/1802) - if (await fs.pathExists(this.currentDownload.targetPath)) { - this.currentDownload.appendEpisodeId = true - } + const task = new Task() + const taskDescription = `Downloading episode "${podcastEpisodeDownload.podcastEpisode.title}".` + const taskData = { + libraryId: podcastEpisodeDownload.libraryId, + libraryItemId: podcastEpisodeDownload.libraryItemId, + } + task.setData('download-podcast-episode', 'Downloading Episode', taskDescription, false, taskData) + this.taskManager.addTask(task) - // Ignores all added files to this dir - this.watcher.addIgnoreDir(this.currentDownload.libraryItem.path) + SocketAuthority.emitter('episode_download_started', podcastEpisodeDownload.toJSONForClient()) + this.currentDownload = podcastEpisodeDownload - // Make sure podcast library item folder exists - if (!(await fs.pathExists(this.currentDownload.libraryItem.path))) { - Logger.warn(`[PodcastManager] Podcast episode download: Podcast folder no longer exists at "${this.currentDownload.libraryItem.path}" - Creating it`) - await fs.mkdir(this.currentDownload.libraryItem.path) - } + // If this file already exists then append the episode id to the filename + // e.g. "/tagesschau 20 Uhr.mp3" becomes "/tagesschau 20 Uhr (ep_asdfasdf).mp3" + // this handles podcasts where every title is the same (ref https://github.com/advplyr/audiobookshelf/issues/1802) + if (await fs.pathExists(this.currentDownload.targetPath)) { + this.currentDownload.appendEpisodeId = true + } - let success = false - if (this.currentDownload.urlFileExtension === 'mp3') { - // Download episode and tag it - success = await ffmpegHelpers.downloadPodcastEpisode(this.currentDownload).catch((error) => { - Logger.error(`[PodcastManager] Podcast Episode download failed`, error) - return false - }) - } else { - // Download episode only - success = await downloadFile(this.currentDownload.url, this.currentDownload.targetPath).then(() => true).catch((error) => { - Logger.error(`[PodcastManager] Podcast Episode download failed`, error) - return false - }) - } + // Ignores all added files to this dir + this.watcher.addIgnoreDir(this.currentDownload.libraryItem.path) - if (success) { - success = await this.scanAddPodcastEpisodeAudioFile() - if (!success) { - await fs.remove(this.currentDownload.targetPath) - this.currentDownload.setFinished(false) - task.setFailed('Failed to download episode') - } else { - Logger.info(`[PodcastManager] Successfully downloaded podcast episode "${this.currentDownload.podcastEpisode.title}"`) - this.currentDownload.setFinished(true) - task.setFinished() - } - } else { - task.setFailed('Failed to download episode') - this.currentDownload.setFinished(false) - } + // Make sure podcast library item folder exists + if (!(await fs.pathExists(this.currentDownload.libraryItem.path))) { + Logger.warn(`[PodcastManager] Podcast episode download: Podcast folder no longer exists at "${this.currentDownload.libraryItem.path}" - Creating it`) + await fs.mkdir(this.currentDownload.libraryItem.path) + } - this.taskManager.taskFinished(task) + let success = false + if (this.currentDownload.urlFileExtension === 'mp3') { + // Download episode and tag it + success = await ffmpegHelpers.downloadPodcastEpisode(this.currentDownload).catch((error) => { + Logger.error(`[PodcastManager] Podcast Episode download failed`, error) + return false + }) + } else { + // Download episode only + success = await downloadFile(this.currentDownload.url, this.currentDownload.targetPath).then(() => true).catch((error) => { + Logger.error(`[PodcastManager] Podcast Episode download failed`, error) + return false + }) + } - SocketAuthority.emitter('episode_download_finished', this.currentDownload.toJSONForClient()) - SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) + if (success) { + success = await this.scanAddPodcastEpisodeAudioFile() + if (!success) { + await fs.remove(this.currentDownload.targetPath) + this.currentDownload.setFinished(false) + task.setFailed('Failed to download episode') + } else { + Logger.info(`[PodcastManager] Successfully downloaded podcast episode "${this.currentDownload.podcastEpisode.title}"`) + this.currentDownload.setFinished(true) + task.setFinished() + } + } else { + task.setFailed('Failed to download episode') + this.currentDownload.setFinished(false) + } - this.watcher.removeIgnoreDir(this.currentDownload.libraryItem.path) - this.currentDownload = null - if (this.downloadQueue.length) { - this.startPodcastEpisodeDownload(this.downloadQueue.shift()) - } - } + this.taskManager.taskFinished(task) - async scanAddPodcastEpisodeAudioFile() { - const libraryFile = await this.getLibraryFile(this.currentDownload.targetPath, this.currentDownload.targetRelPath) + SocketAuthority.emitter('episode_download_finished', this.currentDownload.toJSONForClient()) + SocketAuthority.emitter('episode_download_queue_updated', this.getDownloadQueueDetails()) - const audioFile = await this.probeAudioFile(libraryFile) - if (!audioFile) { - return false + this.watcher.removeIgnoreDir(this.currentDownload.libraryItem.path) + this.currentDownload = null + if (this.downloadQueue.length) { + this.startPodcastEpisodeDownload(this.downloadQueue.shift()) + } } - const libraryItem = await Database.libraryItemModel.getOldById(this.currentDownload.libraryItem.id) - if (!libraryItem) { - Logger.error(`[PodcastManager] Podcast Episode finished but library item was not found ${this.currentDownload.libraryItem.id}`) - return false - } + async scanAddPodcastEpisodeAudioFile() { + const libraryFile = await this.getLibraryFile(this.currentDownload.targetPath, this.currentDownload.targetRelPath) - const podcastEpisode = this.currentDownload.podcastEpisode - podcastEpisode.audioFile = audioFile + const audioFile = await this.probeAudioFile(libraryFile) + if (!audioFile) { + return false + } - if (audioFile.chapters?.length) { - podcastEpisode.chapters = audioFile.chapters.map(ch => ({ ...ch })) - } + const libraryItem = await Database.libraryItemModel.getOldById(this.currentDownload.libraryItem.id) + if (!libraryItem) { + Logger.error(`[PodcastManager] Podcast Episode finished but library item was not found ${this.currentDownload.libraryItem.id}`) + return false + } - libraryItem.media.addPodcastEpisode(podcastEpisode) - if (libraryItem.isInvalid) { - // First episode added to an empty podcast - libraryItem.isInvalid = false - } - libraryItem.libraryFiles.push(libraryFile) - - if (this.currentDownload.isAutoDownload) { - // Check setting maxEpisodesToKeep and remove episode if necessary - if (libraryItem.media.maxEpisodesToKeep && libraryItem.media.episodesWithPubDate.length > libraryItem.media.maxEpisodesToKeep) { - Logger.info(`[PodcastManager] # of episodes (${libraryItem.media.episodesWithPubDate.length}) exceeds max episodes to keep (${libraryItem.media.maxEpisodesToKeep})`) - await this.removeOldestEpisode(libraryItem, podcastEpisode.id) - } - } + const podcastEpisode = this.currentDownload.podcastEpisode + podcastEpisode.audioFile = audioFile - libraryItem.updatedAt = Date.now() - await Database.updateLibraryItem(libraryItem) - SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) - const podcastEpisodeExpanded = podcastEpisode.toJSONExpanded() - podcastEpisodeExpanded.libraryItem = libraryItem.toJSONExpanded() - SocketAuthority.emitter('episode_added', podcastEpisodeExpanded) + if (audioFile.chapters?.length) { + podcastEpisode.chapters = audioFile.chapters.map(ch => ({...ch})) + } - if (this.currentDownload.isAutoDownload) { // Notifications only for auto downloaded episodes - this.notificationManager.onPodcastEpisodeDownloaded(libraryItem, podcastEpisode) - } + libraryItem.media.addPodcastEpisode(podcastEpisode) + if (libraryItem.isInvalid) { + // First episode added to an empty podcast + libraryItem.isInvalid = false + } + libraryItem.libraryFiles.push(libraryFile) + + if (this.currentDownload.isAutoDownload) { + // Check setting maxEpisodesToKeep and remove episode if necessary + if (libraryItem.media.maxEpisodesToKeep && libraryItem.media.episodesWithPubDate.length > libraryItem.media.maxEpisodesToKeep) { + Logger.info(`[PodcastManager] # of episodes (${libraryItem.media.episodesWithPubDate.length}) exceeds max episodes to keep (${libraryItem.media.maxEpisodesToKeep})`) + await this.removeOldestEpisode(libraryItem, podcastEpisode.id) + } + } + + libraryItem.updatedAt = Date.now() + await Database.updateLibraryItem(libraryItem) + SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) + const podcastEpisodeExpanded = podcastEpisode.toJSONExpanded() + podcastEpisodeExpanded.libraryItem = libraryItem.toJSONExpanded() + SocketAuthority.emitter('episode_added', podcastEpisodeExpanded) + + if (this.currentDownload.isAutoDownload) { // Notifications only for auto downloaded episodes + this.notificationManager.onPodcastEpisodeDownloaded(libraryItem, podcastEpisode) + } - return true - } - - async removeOldestEpisode(libraryItem, episodeIdJustDownloaded) { - var smallestPublishedAt = 0 - var oldestEpisode = null - libraryItem.media.episodesWithPubDate.filter(ep => ep.id !== episodeIdJustDownloaded).forEach((ep) => { - if (!smallestPublishedAt || ep.publishedAt < smallestPublishedAt) { - smallestPublishedAt = ep.publishedAt - oldestEpisode = ep - } - }) - // TODO: Should we check for open playback sessions for this episode? - // TODO: remove all user progress for this episode - if (oldestEpisode && oldestEpisode.audioFile) { - Logger.info(`[PodcastManager] Deleting oldest episode "${oldestEpisode.title}"`) - const successfullyDeleted = await removeFile(oldestEpisode.audioFile.metadata.path) - if (successfullyDeleted) { - libraryItem.media.removeEpisode(oldestEpisode.id) - libraryItem.removeLibraryFile(oldestEpisode.audioFile.ino) return true - } else { - Logger.warn(`[PodcastManager] Failed to remove oldest episode "${oldestEpisode.title}"`) - } - } - return false - } - - async getLibraryFile(path, relPath) { - var newLibFile = new LibraryFile() - await newLibFile.setDataFromPath(path, relPath) - return newLibFile - } - - async probeAudioFile(libraryFile) { - const path = libraryFile.metadata.path - const mediaProbeData = await prober.probe(path) - if (mediaProbeData.error) { - Logger.error(`[PodcastManager] Podcast Episode downloaded but failed to probe "${path}"`, mediaProbeData.error) - return false } - const newAudioFile = new AudioFile() - newAudioFile.setDataFromProbe(libraryFile, mediaProbeData) - newAudioFile.index = 1 - return newAudioFile - } - - // Returns false if auto download episodes was disabled (disabled if reaches max failed checks) - async runEpisodeCheck(libraryItem) { - const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) - const latestEpisodePublishedAt = libraryItem.media.latestEpisodePublished - Logger.info(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" | Last check: ${lastEpisodeCheckDate} | ${latestEpisodePublishedAt ? `Latest episode pubDate: ${new Date(latestEpisodePublishedAt)}` : 'No latest episode'}`) - - // Use latest episode pubDate if exists OR fallback to using lastEpisodeCheckDate - // lastEpisodeCheckDate will be the current time when adding a new podcast - const dateToCheckForEpisodesAfter = latestEpisodePublishedAt || lastEpisodeCheckDate - Logger.debug(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" checking for episodes after ${new Date(dateToCheckForEpisodesAfter)}`) - - var newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, dateToCheckForEpisodesAfter, libraryItem.media.maxNewEpisodesToDownload) - Logger.debug(`[PodcastManager] runEpisodeCheck: ${newEpisodes ? newEpisodes.length : 'N/A'} episodes found`) - - if (!newEpisodes) { // Failed - // Allow up to MaxFailedEpisodeChecks failed attempts before disabling auto download - if (!this.failedCheckMap[libraryItem.id]) this.failedCheckMap[libraryItem.id] = 0 - this.failedCheckMap[libraryItem.id]++ - if (this.failedCheckMap[libraryItem.id] >= this.MaxFailedEpisodeChecks) { - Logger.error(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}" - disabling auto download`) - libraryItem.media.autoDownloadEpisodes = false - delete this.failedCheckMap[libraryItem.id] - } else { - Logger.warn(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}"`) - } - } else if (newEpisodes.length) { - delete this.failedCheckMap[libraryItem.id] - Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) - this.downloadPodcastEpisodes(libraryItem, newEpisodes, true) - } else { - delete this.failedCheckMap[libraryItem.id] - Logger.debug(`[PodcastManager] No new episodes for "${libraryItem.media.metadata.title}"`) + + async removeOldestEpisode(libraryItem, episodeIdJustDownloaded) { + var smallestPublishedAt = 0 + var oldestEpisode = null + libraryItem.media.episodesWithPubDate.filter(ep => ep.id !== episodeIdJustDownloaded).forEach((ep) => { + if (!smallestPublishedAt || ep.publishedAt < smallestPublishedAt) { + smallestPublishedAt = ep.publishedAt + oldestEpisode = ep + } + }) + // TODO: Should we check for open playback sessions for this episode? + // TODO: remove all user progress for this episode + if (oldestEpisode && oldestEpisode.audioFile) { + Logger.info(`[PodcastManager] Deleting oldest episode "${oldestEpisode.title}"`) + const successfullyDeleted = await removeFile(oldestEpisode.audioFile.metadata.path) + if (successfullyDeleted) { + libraryItem.media.removeEpisode(oldestEpisode.id) + libraryItem.removeLibraryFile(oldestEpisode.audioFile.ino) + return true + } else { + Logger.warn(`[PodcastManager] Failed to remove oldest episode "${oldestEpisode.title}"`) + } + } + return false } - libraryItem.media.lastEpisodeCheck = Date.now() - libraryItem.updatedAt = Date.now() - await Database.updateLibraryItem(libraryItem) - SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) - return libraryItem.media.autoDownloadEpisodes - } - - async checkPodcastForNewEpisodes(podcastLibraryItem, dateToCheckForEpisodesAfter, maxNewEpisodes = 3) { - if (!podcastLibraryItem.media.metadata.feedUrl) { - Logger.error(`[PodcastManager] checkPodcastForNewEpisodes no feed url for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`) - return false + async getLibraryFile(path, relPath) { + var newLibFile = new LibraryFile() + await newLibFile.setDataFromPath(path, relPath) + return newLibFile } - var feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl) - if (!feed || !feed.episodes) { - Logger.error(`[PodcastManager] checkPodcastForNewEpisodes invalid feed payload for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`, feed) - return false + + async probeAudioFile(libraryFile) { + const path = libraryFile.metadata.path + const mediaProbeData = await prober.probe(path) + if (mediaProbeData.error) { + Logger.error(`[PodcastManager] Podcast Episode downloaded but failed to probe "${path}"`, mediaProbeData.error) + return false + } + const newAudioFile = new AudioFile() + newAudioFile.setDataFromProbe(libraryFile, mediaProbeData) + newAudioFile.index = 1 + return newAudioFile } - // Filter new and not already has - var newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url)) + // Returns false if auto download episodes was disabled (disabled if reaches max failed checks) + async runEpisodeCheck(libraryItem) { + const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) + const latestEpisodePublishedAt = libraryItem.media.latestEpisodePublished + Logger.info(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" | Last check: ${lastEpisodeCheckDate} | ${latestEpisodePublishedAt ? `Latest episode pubDate: ${new Date(latestEpisodePublishedAt)}` : 'No latest episode'}`) + + // Use latest episode pubDate if exists OR fallback to using lastEpisodeCheckDate + // lastEpisodeCheckDate will be the current time when adding a new podcast + const dateToCheckForEpisodesAfter = latestEpisodePublishedAt || lastEpisodeCheckDate + Logger.debug(`[PodcastManager] runEpisodeCheck: "${libraryItem.media.metadata.title}" checking for episodes after ${new Date(dateToCheckForEpisodesAfter)}`) + + let newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, dateToCheckForEpisodesAfter, libraryItem.media.maxNewEpisodesToDownload) + Logger.debug(`[PodcastManager] runEpisodeCheck: ${newEpisodes ? newEpisodes.length : 'N/A'} episodes found`) + + if (!newEpisodes) { // Failed + // Allow up to MaxFailedEpisodeChecks failed attempts before disabling auto download + if (!this.failedCheckMap[libraryItem.id]) this.failedCheckMap[libraryItem.id] = 0 + this.failedCheckMap[libraryItem.id]++ + if (this.failedCheckMap[libraryItem.id] >= this.MaxFailedEpisodeChecks) { + Logger.error(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}" - disabling auto download`) + libraryItem.media.autoDownloadEpisodes = false + delete this.failedCheckMap[libraryItem.id] + } else { + Logger.warn(`[PodcastManager] runEpisodeCheck ${this.failedCheckMap[libraryItem.id]} failed attempts at checking episodes for "${libraryItem.media.metadata.title}"`) + } + libraryItem.media.metadata.feedHealthy = false + } else if (newEpisodes.length) { + delete this.failedCheckMap[libraryItem.id] + Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) + this.downloadPodcastEpisodes(libraryItem, newEpisodes, true) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } else { + delete this.failedCheckMap[libraryItem.id] + Logger.debug(`[PodcastManager] No new episodes for "${libraryItem.media.metadata.title}"`) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } - if (maxNewEpisodes > 0) { - newEpisodes = newEpisodes.slice(0, maxNewEpisodes) + libraryItem.media.lastEpisodeCheck = Date.now() + libraryItem.updatedAt = Date.now() + await Database.updateLibraryItem(libraryItem) + SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) + return libraryItem.media.autoDownloadEpisodes } - return newEpisodes - } - - async checkAndDownloadNewEpisodes(libraryItem, maxEpisodesToDownload) { - const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) - Logger.info(`[PodcastManager] checkAndDownloadNewEpisodes for "${libraryItem.media.metadata.title}" - Last episode check: ${lastEpisodeCheckDate}`) - var newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, libraryItem.media.lastEpisodeCheck, maxEpisodesToDownload) - if (newEpisodes.length) { - Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) - this.downloadPodcastEpisodes(libraryItem, newEpisodes, false) - } else { - Logger.info(`[PodcastManager] No new episodes found for podcast "${libraryItem.media.metadata.title}"`) + async checkPodcastForNewEpisodes(podcastLibraryItem, dateToCheckForEpisodesAfter, maxNewEpisodes = 3) { + if (!podcastLibraryItem.media.metadata.feedUrl) { + Logger.error(`[PodcastManager] checkPodcastForNewEpisodes no feed url for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id})`) + return false + } + var feed = await getPodcastFeed(podcastLibraryItem.media.metadata.feedUrl) + if (!feed || !feed.episodes) { + Logger.error(`[PodcastManager] checkPodcastForNewEpisodes invalid feed payload for ${podcastLibraryItem.media.metadata.title} (ID: ${podcastLibraryItem.id}, URL: ${podcastLibraryItem.media.metadata.feedUrl})`, feed) + return false + } + + // Filter new and not already has + var newEpisodes = feed.episodes.filter(ep => ep.publishedAt > dateToCheckForEpisodesAfter && !podcastLibraryItem.media.checkHasEpisodeByFeedUrl(ep.enclosure.url)) + + if (maxNewEpisodes > 0) { + newEpisodes = newEpisodes.slice(0, maxNewEpisodes) + } + + return newEpisodes } - libraryItem.media.lastEpisodeCheck = Date.now() - libraryItem.updatedAt = Date.now() - await Database.updateLibraryItem(libraryItem) - SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) + async checkAndDownloadNewEpisodes(libraryItem, maxEpisodesToDownload) { + const lastEpisodeCheckDate = new Date(libraryItem.media.lastEpisodeCheck || 0) + Logger.info(`[PodcastManager] checkAndDownloadNewEpisodes for "${libraryItem.media.metadata.title}" - Last episode check: ${lastEpisodeCheckDate}`) + let newEpisodes = await this.checkPodcastForNewEpisodes(libraryItem, libraryItem.media.lastEpisodeCheck, maxEpisodesToDownload) + if (!newEpisodes) { + libraryItem.media.metadata.feedHealthy = false + } else if (newEpisodes.length) { + Logger.info(`[PodcastManager] Found ${newEpisodes.length} new episodes for podcast "${libraryItem.media.metadata.title}" - starting download`) + this.downloadPodcastEpisodes(libraryItem, newEpisodes, false) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } else { + Logger.info(`[PodcastManager] No new episodes found for podcast "${libraryItem.media.metadata.title}"`) + libraryItem.media.metadata.lastSuccessfulFetchAt = Date.now() + libraryItem.media.metadata.feedHealthy = true + } - return newEpisodes - } + libraryItem.media.lastEpisodeCheck = Date.now() + libraryItem.updatedAt = Date.now() + await Database.updateLibraryItem(libraryItem) + SocketAuthority.emitter('item_updated', libraryItem.toJSONExpanded()) - async findEpisode(rssFeedUrl, searchTitle) { - const feed = await getPodcastFeed(rssFeedUrl).catch(() => { - return null - }) - if (!feed || !feed.episodes) { - return null + return newEpisodes } - const matches = [] - feed.episodes.forEach(ep => { - if (!ep.title) return + setFeedHealthStatus(libraryItem, isHealthy) { + libraryItem.media.metadata.feedHealthy = isHealthy + if (isHealthy) { + libraryItem.media.lastSuccessfulFetchAt = Date.now() + } + libraryItem.updatedAt = Date.now() + Database.updateLibraryItem(libraryItem) + } - const epTitle = ep.title.toLowerCase().trim() - if (epTitle === searchTitle) { - matches.push({ - episode: ep, - levenshtein: 0 + async findEpisode(rssFeedUrl, searchTitle) { + const feed = await getPodcastFeed(rssFeedUrl).catch(() => { + return null }) - } else { - const levenshtein = levenshteinDistance(searchTitle, epTitle, true) - if (levenshtein <= 6 && epTitle.length > levenshtein) { - matches.push({ - episode: ep, - levenshtein - }) + if (!feed || !feed.episodes) { + return null } - } - }) - return matches.sort((a, b) => a.levenshtein - b.levenshtein) - } - - async getOPMLFeeds(opmlText) { - var extractedFeeds = opmlParser.parse(opmlText) - if (!extractedFeeds || !extractedFeeds.length) { - Logger.error('[PodcastManager] getOPMLFeeds: No RSS feeds found in OPML') - return { - error: 'No RSS feeds found in OPML' - } + + const matches = [] + feed.episodes.forEach(ep => { + if (!ep.title) return + + const epTitle = ep.title.toLowerCase().trim() + if (epTitle === searchTitle) { + matches.push({ + episode: ep, + levenshtein: 0 + }) + } else { + const levenshtein = levenshteinDistance(searchTitle, epTitle, true) + if (levenshtein <= 6 && epTitle.length > levenshtein) { + matches.push({ + episode: ep, + levenshtein + }) + } + } + }) + return matches.sort((a, b) => a.levenshtein - b.levenshtein) } - var rssFeedData = [] + async getOPMLFeeds(opmlText) { + var extractedFeeds = opmlParser.parse(opmlText) + if (!extractedFeeds || !extractedFeeds.length) { + Logger.error('[PodcastManager] getOPMLFeeds: No RSS feeds found in OPML') + return { + error: 'No RSS feeds found in OPML' + } + } + + var rssFeedData = [] - for (let feed of extractedFeeds) { - var feedData = await getPodcastFeed(feed.feedUrl, true) - if (feedData) { - feedData.metadata.feedUrl = feed.feedUrl - rssFeedData.push(feedData) - } + for (let feed of extractedFeeds) { + var feedData = await getPodcastFeed(feed.feedUrl, true) + if (feedData) { + feedData.metadata.feedUrl = feed.feedUrl + rssFeedData.push(feedData) + } + } + + return { + feeds: rssFeedData + } } - return { - feeds: rssFeedData + /** + * OPML file string for podcasts in a library + * @param {import('../models/Podcast')[]} podcasts + * @returns {string} XML string + */ + generateOPMLFileText(podcasts) { + return opmlGenerator.generate(podcasts) } - } - - /** - * OPML file string for podcasts in a library - * @param {import('../models/Podcast')[]} podcasts - * @returns {string} XML string - */ - generateOPMLFileText(podcasts) { - return opmlGenerator.generate(podcasts) - } - - getDownloadQueueDetails(libraryId = null) { - let _currentDownload = this.currentDownload - if (libraryId && _currentDownload?.libraryId !== libraryId) _currentDownload = null - - return { - currentDownload: _currentDownload?.toJSONForClient(), - queue: this.downloadQueue.filter(item => !libraryId || item.libraryId === libraryId).map(item => item.toJSONForClient()) + + getDownloadQueueDetails(libraryId = null) { + let _currentDownload = this.currentDownload + if (libraryId && _currentDownload?.libraryId !== libraryId) _currentDownload = null + + return { + currentDownload: _currentDownload?.toJSONForClient(), + queue: this.downloadQueue.filter(item => !libraryId || item.libraryId === libraryId).map(item => item.toJSONForClient()) + } } - } } + module.exports = PodcastManager diff --git a/server/models/Podcast.js b/server/models/Podcast.js index 60311bfde7..c2ab917d33 100644 --- a/server/models/Podcast.js +++ b/server/models/Podcast.js @@ -50,6 +50,10 @@ class Podcast extends Model { this.createdAt /** @type {Date} */ this.updatedAt + /** @type {Date} */ + this.lastSuccessfulFetchAt + /** @type {boolean} */ + this.feedHealthy } static getOldPodcast(libraryItemExpanded) { @@ -71,7 +75,9 @@ class Podcast extends Model { itunesArtistId: podcastExpanded.itunesArtistId, explicit: podcastExpanded.explicit, language: podcastExpanded.language, - type: podcastExpanded.podcastType + type: podcastExpanded.podcastType, + lastSuccessfulFetchAt: podcastExpanded.lastSuccessfulFetchAt?.valueOf() || null, + feedHealthy: !!podcastExpanded.feedHealthy }, coverPath: podcastExpanded.coverPath, tags: podcastExpanded.tags, @@ -108,7 +114,9 @@ class Podcast extends Model { maxNewEpisodesToDownload: oldPodcast.maxNewEpisodesToDownload, coverPath: oldPodcast.coverPath, tags: oldPodcast.tags, - genres: oldPodcastMetadata.genres + genres: oldPodcastMetadata.genres, + lastSuccessfulFetchAt: oldPodcastMetadata.lastSuccessfulFetchAt, + feedHealthy: !!oldPodcastMetadata.feedHealthy } } @@ -144,7 +152,9 @@ class Podcast extends Model { maxNewEpisodesToDownload: DataTypes.INTEGER, coverPath: DataTypes.STRING, tags: DataTypes.JSON, - genres: DataTypes.JSON + genres: DataTypes.JSON, + lastSuccessfulFetchAt: DataTypes.DATE, + feedHealthy: DataTypes.BOOLEAN }, { sequelize, modelName: 'podcast' @@ -152,4 +162,4 @@ class Podcast extends Model { } } -module.exports = Podcast \ No newline at end of file +module.exports = Podcast diff --git a/server/routers/ApiRouter.js b/server/routers/ApiRouter.js index 74d8aa566e..67566cbfb9 100644 --- a/server/routers/ApiRouter.js +++ b/server/routers/ApiRouter.js @@ -226,6 +226,7 @@ class ApiRouter { // this.router.post('/podcasts', PodcastController.create.bind(this)) this.router.post('/podcasts/feed', PodcastController.getPodcastFeed.bind(this)) + this.router.get('/podcasts/:id/feed', PodcastController.middleware.bind(this), PodcastController.checkPodcastFeed.bind(this)) this.router.post('/podcasts/opml', PodcastController.getFeedsFromOPMLText.bind(this)) this.router.get('/podcasts/:id/checknew', PodcastController.middleware.bind(this), PodcastController.checkNewEpisodes.bind(this)) this.router.get('/podcasts/:id/downloads', PodcastController.middleware.bind(this), PodcastController.getEpisodeDownloads.bind(this)) diff --git a/server/utils/migrations/dbMigration.js b/server/utils/migrations/dbMigration.js index 3d38cca6ad..dd13608892 100644 --- a/server/utils/migrations/dbMigration.js +++ b/server/utils/migrations/dbMigration.js @@ -198,7 +198,9 @@ function migratePodcast(oldLibraryItem, LibraryItem) { updatedAt: LibraryItem.updatedAt, coverPath: oldPodcast.coverPath, tags: oldPodcast.tags, - genres: oldPodcastMetadata.genres + genres: oldPodcastMetadata.genres, + lastSuccessfulFetchAt: oldPodcastMetadata.lastSuccessfulFetchAt || null, + feedHealthy: !!oldPodcastMetadata.feedHealthy || null } _newRecords.podcast = Podcast oldDbIdMap.podcasts[oldLibraryItem.id] = Podcast.id @@ -1708,4 +1710,4 @@ module.exports.migrationPatch2 = async (ctx) => { Logger.error(`[dbMigration] Migration from 2.3.3+ column creation failed`, error) throw new Error('Migration 2.3.3+ failed ' + error) } -} \ No newline at end of file +}