Pull latest Main code into my branch

CustomCrawlDecisionMaker.cs

@@ -0,0 +1,37 @@
+using System;
+using Abot2.Core;
+using Abot2.Poco;
+using System.Linq;
+
+namespace websitechangenotifier
+{
+    public class CustomCrawlDecisionMaker : CrawlDecisionMaker
+    {
+        public override CrawlDecision ShouldCrawlPage(PageToCrawl pageToCrawl, CrawlContext crawlContext)
+        {
+            CrawlDecision decision = base.ShouldCrawlPage(pageToCrawl, crawlContext);
+            if (decision.Allow && !pageToCrawl.IsRoot)
+            {
+                if (pageToCrawl.Uri.AbsoluteUri.EndsWith("/parents/letters/") //https://www.kingsleighprimary.co.uk/parents/letters/
+                || pageToCrawl.Uri.AbsoluteUri.EndsWith("/blog/") //https://www.kingsleighprimary.co.uk/blog/
+                || pageToCrawl.Uri.AbsoluteUri.Contains("/events/?kps-month=202")
+                || pageToCrawl.Uri.AbsoluteUri.EndsWith("/parents/")
+                || pageToCrawl.Uri.AbsoluteUri.EndsWith("/parents/newsletters/") //https://www.kingsleighprimary.co.uk/parents/newsletters/
+                || pageToCrawl.Uri.AbsoluteUri.EndsWith("/blog-category/reception/")//https://www.kingsleighprimary.co.uk/blog-category/reception/
+                || pageToCrawl.Uri.AbsoluteUri.EndsWith(".pdf")//Allow any upload file
+               || pageToCrawl.Uri.AbsoluteUri.EndsWith("/policies/") //https://www.kingsleighprimary.co.uk/policies/
+               || pageToCrawl.Uri.AbsoluteUri.EndsWith("/classes/reception/") //https://www.kingsleighprimary.co.uk/classes/reception/
+                 )
+                {
+                    //Only include these
+                }
+                else
+                {
+                    decision = new CrawlDecision { Allow = false, Reason = "Ignoring intentionally" };
+                }
+            }
+
+            return decision;
+        }
+    }
+}

EmailHelpers.cs

@@ -1,5 +1,7 @@
+using System;
 using System.Net;
 using System.Net.Mail;
+using Polly;
 
 namespace websitechangenotifier
 {
@@ -8,16 +10,21 @@ public class EmailHelpers
         public void SendEmail(string subject, string body)
         {
             SmtpClient smtp = new SmtpClient();
-            {
-                smtp.DeliveryMethod = SmtpDeliveryMethod.Network;
-                smtp.UseDefaultCredentials = false;
-                smtp.EnableSsl = true;
-                smtp.Host = "smtp.gmail.com";
-                smtp.Port = 587;
-                smtp.Credentials = new NetworkCredential("mosoftwareenterprisesemails@gmail.com", "twzszxlpuqmgrliv");
-                // send the email
-                smtp.Send("noreply@mosoftwareenterprises.co.uk", "bigmansbro@gmail.com", subject, body);
-            }
+            smtp.DeliveryMethod = SmtpDeliveryMethod.Network;
+            smtp.UseDefaultCredentials = false;
+            smtp.EnableSsl = true;
+            smtp.Host = "smtp.gmail.com";
+            smtp.Port = 587;
+            smtp.Credentials = new NetworkCredential("mosoftwareenterprisesemails@gmail.com", "kctboibbaivumdev");
+
+            Policy
+              .Handle<Exception>()
+              .WaitAndRetry(new[]
+              {
+    TimeSpan.FromSeconds(1),
+    TimeSpan.FromSeconds(2),
+    TimeSpan.FromSeconds(3)
+              }).Execute(() => smtp.Send("mosoftwareenterprisesemails+noreply@gmail.com", "bigmansbro@gmail.com", subject, body));
         }
     }
 }

Program.cs

@@ -1,15 +1,12 @@
 using System;
-using System.Net.Http;
-using System.Threading.Tasks;
-using Abot2.Core;
 using Abot2.Crawler;
-using Abot2.Poco;
 using Serilog;
-using Serilog.Formatting.Json;
 using System.Linq;
 using System.Collections.Generic;
-using System.Text;
 using System.Diagnostics;
+using System.Threading.Tasks;
+using Abot2.Poco;
+using System.Text;
 
 namespace websitechangenotifier
 {
@@ -65,9 +62,10 @@ private static PoliteWebCrawler SetupCrawler()
             {
                 MaxPagesToCrawl = 0,
                 MaxLinksPerPage = 0,
-                MinCrawlDelayPerDomainMilliSeconds = 3000,
+                MinCrawlDelayPerDomainMilliSeconds = 1500,
             };
-            var crawler = new PoliteWebCrawler(config);
+
+            var crawler = new PoliteWebCrawler(config,new CustomCrawlDecisionMaker(), null, null, null, null, null, null,null);
 
             crawler.PageCrawlCompleted += Crawler_PageCrawlCompleted;
             crawler.PageCrawlDisallowed += Crawler_PageCrawlDisallowed;