-
Notifications
You must be signed in to change notification settings - Fork 0
/
CustomCrawlDecisionMaker.cs
37 lines (35 loc) · 1.74 KB
/
CustomCrawlDecisionMaker.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
using System;
using Abot2.Core;
using Abot2.Poco;
using System.Linq;
namespace websitechangenotifier
{
public class CustomCrawlDecisionMaker : CrawlDecisionMaker
{
public override CrawlDecision ShouldCrawlPage(PageToCrawl pageToCrawl, CrawlContext crawlContext)
{
CrawlDecision decision = base.ShouldCrawlPage(pageToCrawl, crawlContext);
if (decision.Allow && !pageToCrawl.IsRoot)
{
if (pageToCrawl.Uri.AbsoluteUri.EndsWith("/parents/letters/") //https://www.kingsleighprimary.co.uk/parents/letters/
|| pageToCrawl.Uri.AbsoluteUri.EndsWith("/blog/") //https://www.kingsleighprimary.co.uk/blog/
|| pageToCrawl.Uri.AbsoluteUri.Contains("/events/?kps-month=202")
|| pageToCrawl.Uri.AbsoluteUri.EndsWith("/parents/")
|| pageToCrawl.Uri.AbsoluteUri.EndsWith("/parents/newsletters/") //https://www.kingsleighprimary.co.uk/parents/newsletters/
|| pageToCrawl.Uri.AbsoluteUri.EndsWith("/blog-category/reception/")//https://www.kingsleighprimary.co.uk/blog-category/reception/
|| pageToCrawl.Uri.AbsoluteUri.EndsWith(".pdf")//Allow any upload file
|| pageToCrawl.Uri.AbsoluteUri.EndsWith("/policies/") //https://www.kingsleighprimary.co.uk/policies/
|| pageToCrawl.Uri.AbsoluteUri.EndsWith("/classes/reception/") //https://www.kingsleighprimary.co.uk/classes/reception/
)
{
//Only include these
}
else
{
decision = new CrawlDecision { Allow = false, Reason = "Ignoring intentionally" };
}
}
return decision;
}
}
}