Skip to content

Instantly share code, notes, and snippets.

@arsinclair
Created December 20, 2020 13:35
Show Gist options
  • Save arsinclair/d4986d795034c83d2f72d10e2af87bde to your computer and use it in GitHub Desktop.
Save arsinclair/d4986d795034c83d2f72d10e2af87bde to your computer and use it in GitHub Desktop.
Program.cs - OmniSharp Errors
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace json_reducer
{
class Program
{
static void Main(string[] args)
{
reducePerAssetURL();
reducePerFileCRC();
}
private static void reducePerAssetURL()
{
List<string> assetFiles = Directory.EnumerateFiles("./data/Asset/").ToList();
Dictionary<string, string> uniqueAssets = new Dictionary<string, string>(); // url : assetId
Dictionary<string, string> transitionMap = new Dictionary<string, string>(); // replaceAssetIdFrom : replaceAssetIdTo
List<string> duplicateJSONToDelete = new List<string>();
// Build a Duplicates Map
foreach (var assetFile in assetFiles)
{
JObject json = JObject.Parse(File.ReadAllText(assetFile));
string url = ((JObject)json["file"])["url"].ToString();
string id = json["id"].ToString();
if (id == "8522cf53-f780-5638-a197-6f4d9750da7a")
{
Console.WriteLine("Pedro!");
}
if (uniqueAssets.ContainsKey(url))
{
transitionMap.Add(id, uniqueAssets[url]);
duplicateJSONToDelete.Add(assetFile);
}
else
{
uniqueAssets.Add(url, id);
}
}
// Replace IDs that point to duplicate asset files
List<string> subDirectories = Directory.GetDirectories("./data").Where(x => x.Contains("Asset") == false && x.Contains("DescriptionTextNode") == false).ToList();
foreach (var directory in subDirectories)
{
Console.WriteLine($"Analyzing {directory}");
var files = Directory.EnumerateFiles(directory).ToList();
var total = files.Count;
var counter = 0;
foreach (var filePath in files)
{
counter++;
Console.WriteLine($"[{directory}] File: {counter}/{total}");
string content = File.ReadAllText(filePath);
if (filePath.Contains("0add5e0c-419c-555a-be2d-cbc5abcba062"))
{
Console.WriteLine("Julio!");
}
bool toBeOverwritten = false;
foreach (var assetId in transitionMap.Keys)
{
if (content.Contains(assetId))
{
toBeOverwritten = true;
content = Regex.Replace(content, assetId, transitionMap[assetId]);
}
}
if (toBeOverwritten)
{
File.WriteAllText(filePath, content);
}
}
}
// Delete duplicate asset files
foreach (var duplicateFile in duplicateJSONToDelete)
{
File.Delete(duplicateFile);
}
}
private static void reducePerFileCRC()
{
List<string> subDirectories = Directory.GetDirectories("./data").Where(x => x.Contains("Audio") || x.Contains("Video")).ToList();
foreach (var directory in subDirectories)
{
var files = Directory.EnumerateFiles(directory).ToList();
}
List<string> files = Directory.GetFiles("./data/", "*.*", SearchOption.AllDirectories).ToList();
Dictionary<string, string> hashed = new Dictionary<string, string>();
Dictionary<string, int> duplications = new Dictionary<string, int>();
foreach (var file in files)
{
var content = File.ReadAllText(file);
var hasheueu = CreateMD5(content);
if (hashed.Keys.Contains(hasheueu) == false)
{
hashed.Add(hasheueu, file);
}
else
{
if (duplications.Keys.Contains(file))
{
duplications[file] = duplications[file] + 1;
}
else
{
duplications.Add(file, 1);
}
}
}
foreach (var dupe in duplications)
{
Console.WriteLine($"Dupe: {dupe.Key}, {dupe.Value}");
}
}
public static string CreateMD5(string input)
{
using (MD5 md5 = MD5.Create())
{
byte[] inputBytes = Encoding.ASCII.GetBytes(input);
byte[] hashBytes = md5.ComputeHash(inputBytes);
StringBuilder sb = new StringBuilder();
for (int i = 0; i < hashBytes.Length; i++)
{
sb.Append(hashBytes[i].ToString("X2"));
}
return sb.ToString();
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment