Checks for broken links in HTML pages

LinkChecker.linq

<Query Kind="Program">
  <Reference>&lt;RuntimeDirectory&gt;\System.Net.Http.dll</Reference>
  <NuGetReference>HtmlAgilityPack</NuGetReference>
  <Namespace>HtmlAgilityPack</Namespace>
  <Namespace>System.Net.Http</Namespace>
  <Namespace>System.Threading.Tasks</Namespace>
  <Namespace>System.Collections.Concurrent</Namespace>
</Query>

void Main()
{
    var directory = "INPUT DIRECTORY HERE";
    var excludeLinks = new HashSet<string>();
    var client = new HttpClient();
    var badLinks = new ConcurrentDictionary<string, HashSet<string>>();   
    var files = Directory.EnumerateFiles(directory, "*.html", SearchOption.AllDirectories).ToList();
    
    Parallel.ForEach(files, html =>
    {
        var document = new HtmlDocument();
        document.Load(html);

        foreach (var anchor in document.DocumentNode.SelectNodes("//a"))
        {
            var href = anchor.GetAttributeValue("href", string.Empty);
            if (string.IsNullOrEmpty(href) || !href.StartsWith("http") || excludeLinks.Contains(href))
            {
                continue;
            }
            
            try
            {	        
                var result = client.SendAsync(new HttpRequestMessage(HttpMethod.Head, new Uri(href))).Result;

                if (result.StatusCode == System.Net.HttpStatusCode.NotFound)
                {
                    HashSet<string> links;
                    if (!badLinks.TryGetValue(html, out links))
                    {
                        links = new HashSet<string> { href };
                        badLinks.TryAdd(html, links);
                    }
                    else
                    {
                        links.Add(href);
                    }
                }
            }
            catch (Exception)
            {
                HashSet<string> links;
                if (!badLinks.TryGetValue(html, out links))
                {
                    links = new HashSet<string> { href };
                    badLinks.TryAdd(html, links);
                }
                else
                {
                    links.Add(href);
                }
            }         
        }

    });
    
    badLinks.Dump();
}