Merge pull request #1821 from 0xi4o/bug/web-scrapper-ui

Fix: Error Handling in Web Scraper
FlowiseAI · Feb 27, 2024 · bdb93f7 · bdb93f7
2 parents 2d9bf58 + 07ce50c
commit bdb93f7
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 12 deletions.
diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts
@@ -1245,18 +1245,22 @@ export class App {
         // ----------------------------------------
 
         this.app.get('/api/v1/fetch-links', async (req: Request, res: Response) => {
-            const url = decodeURIComponent(req.query.url as string)
-            const relativeLinksMethod = req.query.relativeLinksMethod as string
-            if (!relativeLinksMethod) {
-                return res.status(500).send('Please choose a Relative Links Method in Additional Parameters.')
-            }
+            try {
+                const url = decodeURIComponent(req.query.url as string)
+                const relativeLinksMethod = req.query.relativeLinksMethod as string
+                if (!relativeLinksMethod) {
+                    return res.status(500).send('Please choose a Relative Links Method in Additional Parameters.')
+                }
 
-            const limit = parseInt(req.query.limit as string)
-            if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`)
-            const links: string[] = relativeLinksMethod === 'webCrawl' ? await webCrawl(url, limit) : await xmlScrape(url, limit)
-            if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`)
+                const limit = parseInt(req.query.limit as string)
+                if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`)
+                const links: string[] = relativeLinksMethod === 'webCrawl' ? await webCrawl(url, limit) : await xmlScrape(url, limit)
+                if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`)
 
-            res.json({ status: 'OK', links })
+                res.json({ status: 'OK', links })
+            } catch (e: any) {
+                return res.status(500).send('Could not fetch links from the URL.')
+            }
         })
 
         // ----------------------------------------

diff --git a/packages/ui/src/ui-component/dialog/ManageScrapedLinksDialog.js b/packages/ui/src/ui-component/dialog/ManageScrapedLinksDialog.js
@@ -16,7 +16,7 @@ import {
     Stack,
     Typography
 } from '@mui/material'
-import { IconTrash, IconX } from '@tabler/icons'
+import { IconEraser, IconTrash, IconX } from '@tabler/icons'
 import PerfectScrollbar from 'react-perfect-scrollbar'
 
 import { BackdropLoader } from 'ui-component/loading/BackdropLoader'
@@ -113,6 +113,10 @@ const ManageScrapedLinksDialog = ({ show, dialogProps, onCancel, onSave }) => {
         setSelectedLinks(links)
     }
 
+    const handleRemoveAllLinks = () => {
+        setSelectedLinks([])
+    }
+
     const handleSaveLinks = () => {
         onSave(url, selectedLinks)
     }
@@ -145,6 +149,7 @@ const ManageScrapedLinksDialog = ({ show, dialogProps, onCancel, onSave }) => {
                             />
                         </FormControl>
                         <Button
+                            disabled={!url}
                             sx={{ borderRadius: '12px', mt: 1, display: 'flex', flexShrink: 0 }}
                             size='small'
                             variant='contained'
@@ -154,7 +159,21 @@ const ManageScrapedLinksDialog = ({ show, dialogProps, onCancel, onSave }) => {
                         </Button>
                     </Stack>
                 </Box>
-                <Typography sx={{ mb: 2, fontWeight: 500 }}>Scraped Links</Typography>
+                <Box sx={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', mb: 1.5 }}>
+                    <Typography sx={{ fontWeight: 500 }}>Scraped Links</Typography>
+                    {selectedLinks.length > 0 ? (
+                        <StyledButton
+                            sx={{ height: 'max-content', width: 'max-content' }}
+                            variant='outlined'
+                            color='error'
+                            title='Clear All Links'
+                            onClick={handleRemoveAllLinks}
+                            startIcon={<IconEraser />}
+                        >
+                            Clear All
+                        </StyledButton>
+                    ) : null}
+                </Box>
                 <>
                     {loading && <BackdropLoader open={loading} />}
                     {selectedLinks.length > 0 ? (