-
-
Save arsinclair/d4986d795034c83d2f72d10e2af87bde to your computer and use it in GitHub Desktop.
Program.cs - OmniSharp Errors
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
using System.Collections.Generic; | |
using System.IO; | |
using System.Linq; | |
using System.Security.Cryptography; | |
using System.Text; | |
using System.Text.RegularExpressions; | |
using Newtonsoft.Json; | |
using Newtonsoft.Json.Linq; | |
namespace json_reducer | |
{ | |
class Program | |
{ | |
static void Main(string[] args) | |
{ | |
reducePerAssetURL(); | |
reducePerFileCRC(); | |
} | |
private static void reducePerAssetURL() | |
{ | |
List<string> assetFiles = Directory.EnumerateFiles("./data/Asset/").ToList(); | |
Dictionary<string, string> uniqueAssets = new Dictionary<string, string>(); // url : assetId | |
Dictionary<string, string> transitionMap = new Dictionary<string, string>(); // replaceAssetIdFrom : replaceAssetIdTo | |
List<string> duplicateJSONToDelete = new List<string>(); | |
// Build a Duplicates Map | |
foreach (var assetFile in assetFiles) | |
{ | |
JObject json = JObject.Parse(File.ReadAllText(assetFile)); | |
string url = ((JObject)json["file"])["url"].ToString(); | |
string id = json["id"].ToString(); | |
if (id == "8522cf53-f780-5638-a197-6f4d9750da7a") | |
{ | |
Console.WriteLine("Pedro!"); | |
} | |
if (uniqueAssets.ContainsKey(url)) | |
{ | |
transitionMap.Add(id, uniqueAssets[url]); | |
duplicateJSONToDelete.Add(assetFile); | |
} | |
else | |
{ | |
uniqueAssets.Add(url, id); | |
} | |
} | |
// Replace IDs that point to duplicate asset files | |
List<string> subDirectories = Directory.GetDirectories("./data").Where(x => x.Contains("Asset") == false && x.Contains("DescriptionTextNode") == false).ToList(); | |
foreach (var directory in subDirectories) | |
{ | |
Console.WriteLine($"Analyzing {directory}"); | |
var files = Directory.EnumerateFiles(directory).ToList(); | |
var total = files.Count; | |
var counter = 0; | |
foreach (var filePath in files) | |
{ | |
counter++; | |
Console.WriteLine($"[{directory}] File: {counter}/{total}"); | |
string content = File.ReadAllText(filePath); | |
if (filePath.Contains("0add5e0c-419c-555a-be2d-cbc5abcba062")) | |
{ | |
Console.WriteLine("Julio!"); | |
} | |
bool toBeOverwritten = false; | |
foreach (var assetId in transitionMap.Keys) | |
{ | |
if (content.Contains(assetId)) | |
{ | |
toBeOverwritten = true; | |
content = Regex.Replace(content, assetId, transitionMap[assetId]); | |
} | |
} | |
if (toBeOverwritten) | |
{ | |
File.WriteAllText(filePath, content); | |
} | |
} | |
} | |
// Delete duplicate asset files | |
foreach (var duplicateFile in duplicateJSONToDelete) | |
{ | |
File.Delete(duplicateFile); | |
} | |
} | |
private static void reducePerFileCRC() | |
{ | |
List<string> subDirectories = Directory.GetDirectories("./data").Where(x => x.Contains("Audio") || x.Contains("Video")).ToList(); | |
foreach (var directory in subDirectories) | |
{ | |
var files = Directory.EnumerateFiles(directory).ToList(); | |
} | |
List<string> files = Directory.GetFiles("./data/", "*.*", SearchOption.AllDirectories).ToList(); | |
Dictionary<string, string> hashed = new Dictionary<string, string>(); | |
Dictionary<string, int> duplications = new Dictionary<string, int>(); | |
foreach (var file in files) | |
{ | |
var content = File.ReadAllText(file); | |
var hasheueu = CreateMD5(content); | |
if (hashed.Keys.Contains(hasheueu) == false) | |
{ | |
hashed.Add(hasheueu, file); | |
} | |
else | |
{ | |
if (duplications.Keys.Contains(file)) | |
{ | |
duplications[file] = duplications[file] + 1; | |
} | |
else | |
{ | |
duplications.Add(file, 1); | |
} | |
} | |
} | |
foreach (var dupe in duplications) | |
{ | |
Console.WriteLine($"Dupe: {dupe.Key}, {dupe.Value}"); | |
} | |
} | |
public static string CreateMD5(string input) | |
{ | |
using (MD5 md5 = MD5.Create()) | |
{ | |
byte[] inputBytes = Encoding.ASCII.GetBytes(input); | |
byte[] hashBytes = md5.ComputeHash(inputBytes); | |
StringBuilder sb = new StringBuilder(); | |
for (int i = 0; i < hashBytes.Length; i++) | |
{ | |
sb.Append(hashBytes[i].ToString("X2")); | |
} | |
return sb.ToString(); | |
} | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment