starwars wiki infobox dates

findLinksAndMissingLinks.cs

async Task Main()
{
	/*
		private readonly FilterDefinition<BsonDocument> _containsYearValueFilter = Builders<BsonDocument>.Filter.Regex("Data.Links.Content", new BsonRegularExpression(new Regex(".*\\s+(ABY|BBY)")));
		private readonly FilterDefinition<BsonDocument> _containsYearLinkFilter = Builders<BsonDocument>.Filter.Regex("Data.Links.Href", new BsonRegularExpression(new Regex("_(ABY|BBY)")));
	*/
	var mongoClient = new MongoClient("mongodb://admin:password@localhost:27017");
	var starWarsDb = mongoClient.GetDatabase("starwars");
	var resultsDb = mongoClient.GetDatabase("results");


	var collectionsCursor = starWarsDb.ListCollectionNames();
	var collectionNames = await collectionsCursor.ToListAsync();

	var withDateLinks = Builders<Record>.Filter.And(
		Builders<Record>.Filter.Regex("Data.Links.Content", new BsonRegularExpression(new Regex("\\d.*\\s+(ABY|BBY)"))),
		Builders<Record>.Filter.Regex("Data.Links.Href", new BsonRegularExpression(new Regex("_(ABY|BBY)")))
	);

	var withoutDateLinks = Builders<Record>.Filter.Not(
		Builders<Record>.Filter.And(
			Builders<Record>.Filter.Regex("Data.Links.Content", new BsonRegularExpression(new Regex("\\d.*\\s+(ABY|BBY)"))),
			Builders<Record>.Filter.Regex("Data.Links.Href", new BsonRegularExpression(new Regex("_(ABY|BBY)")))
		)
	);

	var collections = new List<CollectionDates>();

	foreach (var collectionName in collectionNames)
	{
		var withLinks = await (await starWarsDb.GetCollection<Record>(collectionName).FindAsync(withDateLinks)).ToListAsync();
		var withoutLinks = await (await starWarsDb.GetCollection<Record>(collectionName).FindAsync(withoutDateLinks)).ToListAsync();

		collections.Add(new CollectionDates { Collection = collectionName, WithDateLinks = withLinks, WithoutDateLinks = withoutLinks });
	}

	collections.Where(m => m.WithDateLinks.Count > 0).OrderBy(x => x.WithoutDateLinks.Count);
		// .Dump();

	// go to each page with missing links and do a quick regex check for any dates in the Article Text

	var datesCollection = resultsDb.GetCollection<ArticleResult>("articles");

	using (var context = AngleSharp.BrowsingContext.New())
	{
		using (var httpClient = new HttpClient())
		{
			foreach (var collection in collections)
			{
				if (collection.WithDateLinks.Any())
				{
					foreach (var item in collection.WithoutDateLinks)
					{
						var response = await httpClient.GetAsync($"https://starwars.fandom.com/api.php?action=parse&pageid={item.PageId}&format=json");

						if (response.IsSuccessStatusCode)
						{
							using (var json = await System.Text.Json.JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync()))
							{
								var html = json.RootElement.GetProperty("parse").GetProperty("text").GetProperty("*").ToString().Replace("\\", string.Empty);
								
								using(var doc = await context.OpenAsync(r => r.Content(html)))
								{
									// We dont want to query date links in infoboxes
									// but, they shouldnt actually have any anyway, considering this collection said they dont have them
									
									foreach(var infobox in doc.QuerySelectorAll(".portable-infobox"))
									{
										infobox.Remove();
									}
									
									var dateLinks = doc.QuerySelectorAll("a").Where(a =>
									{
										var href = a.GetAttribute("href");
										return href.Contains("_BBY") || href.Contains("_ABY");
									})
									.ToList();
									
									if (dateLinks.Any())
									{
										var excerpts = dateLinks.Select(p => p.Parent).Distinct().Select(element => element.TextContent).ToList();
										var article = new ArticleResult() { Excerpts = excerpts, PageId = item.PageId, PageUrl = item.PageUrl, TemplateUrl = item.TemplateUrl };
										datesCollection.InsertOne(article.Dump());
									}
								}
							}
						}
					}
				}
			}
		}
	}
}

[BsonIgnoreExtraElements]
public class CollectionDates
{
	public string Collection { get; set; }
	public List<Record> WithDateLinks { get; set; }
	public List<Record> WithoutDateLinks { get; set; }
	public List<ArticleResult> DatesInArticle { get; set; }
}

public class ArticleResult
{
	[BsonElement]
	public string TemplateUrl { get; set; }
	
	[BsonElement]
	public List<string> Excerpts { get; set; }

	[BsonElement]
	public string PageUrl { get; set; }

	[BsonElement]
	public int PageId { get; set; }
}

[BsonIgnoreExtraElements]
public class Record
{
	[BsonIgnore]
	public Hyperlinq Link => new Hyperlinq(PageUrl);

	[BsonElement]
	public string PageUrl { get; set; }

	[BsonElement, BsonId]
	public int PageId { get; set; }

	[BsonElement]
	public Data[] Data { get; set; }
	
	[BsonElement]
	public string TemplateUrl { get; set; }
}

public class Data
{
	[BsonElement]
	public string Label { get; set; }

	[BsonElement]
	public string[] Values { get; set; }

	[BsonElement]
	public Link[] Links { get; set; }
}

public class Link
{
	[BsonElement]
	public string? Content { get; set; }

	[BsonElement]
	public string? Href { get; set; }
}