example crawler using HtmlAgilityPack

DemoCrawler.cs

public static class OtoDomCrawler
{
    public static async Task<List<Page>> Search(string url)
    {
        var offers = new List<Page>();
        while (url != null)
        {
            var html = (await new HtmlWeb().LoadFromWebAsync(url)).DocumentNode;
            offers.AddRange(GetOffers(html));
            url = html.SelectSingleNode("//a[@data-dir='next']")?.GetAttributeValue("href", null);
        }
        return offers;
    }
    private static IEnumerable<Page> GetOffers(HtmlNode html)
    {
        foreach (var offer in html.SelectNodes("//*[@class='offer-item-details']"))
        {
            var title = offer.SelectSingleNode(".//*[@class='offer-item-title']");
            var link = title?.AncestorsAndSelf("a").FirstOrDefault();
            yield return new Page
            {
                Link = link?.GetAttributeValue("href", null)?.Split("#")?.FirstOrDefault(),
                Title = title?.InnerText
            };
        }
    }
}