process.relationships.cs

const string WikiFragment = "/wiki/";
const string RelationshipsSection = "_Relationships_";

async Task Main()
{
	Util.AutoScrollResults = true;
	
	List<Loaded> allFiles = new DirectoryInfo(@"C:\Projects\github\pjmagee\starwars-data")		
		.EnumerateFiles("*.json", new EnumerationOptions { RecurseSubdirectories = true })
		.AsParallel()
		.WithExecutionMode(ParallelExecutionMode.ForceParallelism)
		.Select(file =>
		{
			using (var stream = file.OpenRead())
			{
				using(var doc = JsonDocument.Parse(stream))
				{
					string pageUrl = doc.RootElement.GetProperty("PageUrl").GetString()!;
					string thisUrl = HttpUtility.UrlDecode(pageUrl.Split(WikiFragment).Last() ?? string.Empty);
					return new Loaded { File = file, Json = doc.RootElement.Clone(), Url = thisUrl };
				}
			}
		})
		.ToList();

	await Parallel.ForEachAsync(allFiles, new ParallelOptions { MaxDegreeOfParallelism = -1, CancellationToken = this.QueryCancelToken }, async (file, token) => 
	{
		await ProcessMentions(file, allFiles, token);
	});
}

static async ValueTask ProcessMentions(Loaded file, List<Loaded> files, CancellationToken token)
{
	foreach (var f in files.Where(other => other.Json.ToString().Contains(file.Url)))
	{
		file.Mentions.Add(f);		
	}
	
	if(file.Mentions.Any())
		file.Mentions.Dump(1);
	
	/*
	using (var stream = loaded.File.OpenWrite())
	{
		using (var jsonWriter = new Utf8JsonWriter(stream, new() { Indented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, SkipValidation = true }))
		{
			jsonWriter.WriteStartObject();

			foreach (var item in loaded.Json.EnumerateObject().Where(o => !o.NameEquals(RelationshipsSection)))
			{
				item.WriteTo(jsonWriter);
			}

			jsonWriter.WritePropertyName(RelationshipsSection);
				jsonWriter.WriteStartObject();
				
					jsonWriter.WritePropertyName("Mentions");

					jsonWriter.WriteStartArray();
						foreach (var other in loaded.Mentions)
						{
							jsonWriter.WriteStartObject();
							jsonWriter.WriteNumber("PageId", other.Json.GetProperty("PageId").GetInt32());
							jsonWriter.WriteString("PageTitle", other.Json.GetProperty("PageTitle").GetString());
							jsonWriter.WriteString("PageUrl", other.Json.GetProperty("PageUrl").GetString());
							jsonWriter.WriteString("Template", other.Json.GetProperty("Template").GetProperty("Links").EnumerateArray().Select(l => l.GetProperty("Href").GetString().Split(':').Last()).First());
							jsonWriter.WriteEndObject();
						}			
					jsonWriter.WriteEndArray();

				jsonWriter.WriteEndObject();
			
			
			jsonWriter.WriteEndObject();
		}
		
		await stream.FlushAsync();
	}*/
	
	file.Processed = true;	
	
	var processed = files.Count(f => f.Processed);
	var allFiles = files.Count;
	
	$"{processed} / {allFiles}".Dump();
}

public async Task OldMethod()
{
	SemaphoreSlim sync = new SemaphoreSlim(1);

	var thisFiles = new DirectoryInfo(@"C:\Projects\github\pjmagee\starwars-data").EnumerateFiles("*.json", new EnumerationOptions { RecurseSubdirectories = true });

	ConcurrentDictionary<string, ConcurrentBag<JsonElement>> allRelationships = new ConcurrentDictionary<string, ConcurrentBag<JsonElement>>();

	await Parallel.ForEachAsync(thisFiles, new ParallelOptions() { MaxDegreeOfParallelism = 1 }, async (thisFile, thisCancellationToken) =>
	{
		ConcurrentBag<JsonElement> thisRelationships = new ConcurrentBag<JsonElement>();

		using (var thisStream = thisFile.OpenRead())
		{
			using (var thisDocument = await JsonDocument.ParseAsync(thisStream))
			{
				string pageUrl = thisDocument.RootElement.GetProperty("PageUrl").GetString()!;
				string thisUrl = HttpUtility.UrlDecode(pageUrl.Split(WikiFragment).Last() ?? string.Empty);

				var otherFiles = new DirectoryInfo(@"C:\Projects\github\pjmagee\starwars-data").EnumerateFiles("*.json", new EnumerationOptions { RecurseSubdirectories = true }).Where(f => f.FullName != thisFile.FullName);

				await Parallel.ForEachAsync(otherFiles, async (otherFile, cancellationToken) =>
				{
					using (var otherStream = otherFile.OpenRead())
					{
						using (var other = await JsonDocument.ParseAsync(otherStream))
						{
							if (other.RootElement.EnumerateObject().Any(otherSection => ThisRelatesToOther(thisUrl, otherSection)))
							{
								thisRelationships.Add(other.RootElement.Clone());
							}
						}
					}
				});

				using (var transformedStream = File.OpenWrite($"C:\\temp\\{thisFile.Name}"))
				{
					using (var jsonWriter = new Utf8JsonWriter(transformedStream, new() { Indented = true }))
					{
						jsonWriter.WriteStartObject();

						foreach (var item in thisDocument.RootElement.EnumerateObject())
						{
							item.WriteTo(jsonWriter);
						}

						jsonWriter.WritePropertyName("__Relationships__");
						jsonWriter.WriteStartObject();
						jsonWriter.WritePropertyName("MentionedBy");

						jsonWriter.WriteStartArray();
						foreach (var item in thisRelationships)
						{
							jsonWriter.WriteStartObject();
							jsonWriter.WriteNumber("PageId", item.GetProperty("PageId").GetInt32());
							jsonWriter.WriteString("PageTitle", item.GetProperty("PageTitle").GetString());
							jsonWriter.WriteString("PageUrl", item.GetProperty("PageUrl").GetString());
							jsonWriter.WriteString("Template", item.GetProperty("Template").GetProperty("Links").EnumerateArray().Select(l => l.GetProperty("Href").GetString().Split(':').Last()).First());
							jsonWriter.WriteEndObject();
						}
						jsonWriter.WriteEndArray();

						jsonWriter.WriteEndObject();
					}
				}

				allRelationships.TryAdd(pageUrl, thisRelationships);
			}
		}

		try
		{
			await sync.WaitAsync();
			await File.WriteAllTextAsync("C:\\temp\\relationships.json", System.Text.Json.JsonSerializer.Serialize(allRelationships, new JsonSerializerOptions() { WriteIndented = true }));
		}
		finally
		{
			sync.Release();
		}
	});
}

public class Loaded
{
	public FileInfo File { get; set; }
	public JsonElement Json { get; set; }
	public string Url { get; set; }
	public bool Processed { get; set; }
	public List<Loaded> Mentions { get; set; } = new List<Loaded>();
}

static bool ThisRelatesToOther(string thisUrl, JsonProperty otherSection)
{	
	return otherSection.Value.ValueKind == JsonValueKind.Object &&
		   otherSection.Value.GetProperty("Links").EnumerateArray()
				   .Any(le => LinksEquals(thisUrl, le.GetProperty("Href").GetString()?.Split(WikiFragment).Last() ?? string.Empty));
}

static bool LinksEquals(string thisUrl, string otherUrl) => string.Equals(thisUrl, otherUrl, StringComparison.OrdinalIgnoreCase);