Everquest Landmark downloader

Downloader.cs

using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using REsideUtility;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace REsideUtility
{
    public delegate void FullDownloadJobComplete(string jobstart);


    class DownloadJob
    {
        public string name;
        public string url;
        public string output;
        public bool decompress;
        public string status;
        public string jobtag;

    }

    public class Downloader
    {

        static ConcurrentQueue<DownloadJob> DownloadQueue = new ConcurrentQueue<DownloadJob>();
        static List<DownloadJob> DownloadingQueue = new List<DownloadJob>();
        public static ConcurrentDictionary<string, FullDownloadJobComplete> JobCallbacks = new ConcurrentDictionary<string, FullDownloadJobComplete>();

        static Thread DownloadWorker = null;



        public static void Download(string filename, string url, string outputfilename, string job)
        {
            DoDownloadJob(new DownloadJob()
            {
                url = url,
                output = outputfilename,
                decompress = false,
                name = filename,
                status = filename + " Has Not Started",
                jobtag = job,
            });
        }

        public static void DownloadAndDecompress(string filename, string url, string outputfilename, string job)
        {
            DoDownloadJob(new DownloadJob()
            {
                url = url,
                output = outputfilename,
                decompress = true,
                name = filename,
                status = filename + " Has Not Started",
                jobtag = job
            });



        }

        private static void DoDownloadJob(DownloadJob job)
        {
            if (DownloadWorker == null)
            {
                DownloadWorker = new Thread(new ThreadStart(DownloadThreadWork));
                DownloadWorker.Start();
            }

            DownloadQueue.Enqueue(job);
        }



        public static int DownloadJobsToComplete()
        {
            return DownloadQueue.Count;
        }

        public static string[] ReportDownloadStatus()
        {
            List<string> s = new List<string>();
            s.Add("Files In Queue: " + DownloadQueue.Count);
            Parallel.ForEach(DownloadingQueue, j =>
            {
                if (j == null) return;
                s.Add(j.status);
            });

            return s.ToArray();
        }


        public static JObject GetJobjectFromManifest(string Manifest)
        {

            WebClient c = new WebClient();
            string d = c.DownloadString(Manifest + ".txt");

            System.Xml.XmlDocument doc = new System.Xml.XmlDocument();
            doc.LoadXml(d);

            string jsonText = JsonConvert.SerializeXmlNode(doc, Formatting.Indented);
            return JObject.Parse(jsonText);
        }

        const int MaxFilesToDownload = 1;
        static int FilesBeingDownloaded = 0;

        public static List<string> JobErrors = new List<string>();

        static object CountLock = new object();
        public static void DownloadThreadWork()
        {
            WebClient cl = new WebClient();

            cl.Headers.Add("user-agent", "Quicksilver Player/1.0.3.183 (Windows; PlanetSide 2)");

            cl.DownloadProgressChanged += new DownloadProgressChangedEventHandler(cl_DownloadProgressChanged);
            cl.DownloadDataCompleted += new DownloadDataCompletedEventHandler(cl_DownloadDataCompleted);


            while (true)
            {

                lock (CountLock)
                {
                    if (FilesBeingDownloaded < MaxFilesToDownload)
                    {
                        DownloadJob job;
                        if (DownloadQueue.TryDequeue(out job))
                        {

                            try
                            {
                                cl.DownloadDataAsync(new Uri(job.url), job);
                            }
                            catch (WebException e)
                            {

                            }

                            FilesBeingDownloaded++;
                        }
                        DownloadingQueue.Add(job);
                    }
                }

                Thread.Sleep(100);
            }

        }

        static void cl_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e)
        {
            DownloadJob job = (DownloadJob)e.UserState;
            if (e.Cancelled)
            {
                job.status = job.name + " Cancelled";
                Console.WriteLine("Job was Cancelled!");

                return;
            }
            if (e.Error != null)
            {
                job.status = job.name + " ERROR";
                Console.WriteLine("Error: " + e.Error);

                lock (CountLock)
                {

                    JobErrors.Add(string.Format("File {0} at {1} had error {2}", job.name, job.url, e.Error.Message));

                    DownloadingQueue.Remove(job);

                    FilesBeingDownloaded--;

                }

                return;
            }


            byte[] data = e.Result;
            if (job.decompress)
            {
                job.status = job.name + " is Decompressing";
                data = SevenZip.Compression.LZMA.SevenZipHelper.Decompress(data);
            }

            using (BinaryWriter wr = new BinaryWriter(File.Open(job.output + job.name, FileMode.OpenOrCreate)))
            {
                job.status = job.name + " is Writing";
                if (data == null)
                {

                    job.status = job.name + " NULL DATA";
                    return;
                }

                wr.Write(data);
            }
            job.status = job.name + " is Waiting on lock";

            lock (CountLock)
            {
                int jobsLeft = DownloadQueue.Where(s => s.jobtag == job.jobtag).Count();

                if (jobsLeft == 0)
                {
                    JobCallbacks[job.jobtag](job.jobtag);
                    FullDownloadJobComplete d;
                    JobCallbacks.TryRemove(job.jobtag, out d);

                }

                FilesBeingDownloaded--;

                DownloadingQueue.Remove(job);
            }

            job.status = job.name + " Done";

            Console.WriteLine(job.name + " is Done");
        }

        static void cl_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e)
        {
            DownloadJob j = (DownloadJob)e.UserState;
            j.status = string.Format("Downloading {0}: {1}% ({2}/{3})", j.name, e.ProgressPercentage, e.BytesReceived, e.TotalBytesToReceive);

        }
    }
}

EQNDownloader.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.IO;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json;
using System.Collections.Concurrent;
using System.Threading;
using System.Threading.Tasks;




namespace REsideUtility
{
   
    public static class EQNDownloader
    {
     
       

        static void EmptyJobCompleteCallback(string time)
        {
            return;
        }

        public static void DownloadPS2(JObject thisManifest, JObject lastManifest, string outputfolder)
        {
            DownloadPS2Job(thisManifest, lastManifest, outputfolder, EmptyJobCompleteCallback);
        }

        public static string DownloadPS2Job(JObject thisManifest, JObject lastManifest, string outputfolder, FullDownloadJobComplete jobComplete)
        {
            string jobStart = DateTime.Now.ToString();
            DownloadPS2Job(thisManifest, lastManifest, outputfolder, jobComplete, jobStart);
            return jobStart;
        }

        public static void DownloadPS2Job(JObject thisManifest, JObject lastManifest, string outputfolder, FullDownloadJobComplete jobComplete, string jobName)
        {
         
            

            if (!Directory.Exists("output/")) Directory.CreateDirectory("output/");
            if (!Directory.Exists("output/PS2Install")) Directory.CreateDirectory("output/PS2Install");
            if (!Directory.Exists("output/PS2Install/" + outputfolder)) Directory.CreateDirectory("output/PS2Install/" + outputfolder);

            string of = "output/PS2Install/" + outputfolder + "/";


            JToken folders = thisManifest["digest"]["folder"];

            string downloadURL = (string)thisManifest["digest"]["@defaultServerFolder"];

            Downloader.JobCallbacks[jobName] = jobComplete;

            if (lastManifest == null) UpdateFilesInFolder(downloadURL,"", of, folders, null, jobName);
            else
            {
                JToken oldFolders = lastManifest["digest"]["folder"];
                UpdateFilesInFolder(downloadURL, "", of, folders, oldFolders, jobName);
            }
        }

        private static void UpdateFilesInFolder(string downloadurl,string remoteFolder, string outFolder, JToken thisFolder, JToken lastFolder, string job)
        {

            if (!Directory.Exists(outFolder)) Directory.CreateDirectory(outFolder);
            //Sometimes there is a null child
            if (thisFolder == null) return;
            //If the folder is an array of files (not a folder structure), go through each of the folders and parse them
            if (thisFolder.Type == JTokenType.Array)
            {
                foreach (var token in thisFolder)
                {
                    UpdateFilesInFolder(downloadurl, remoteFolder, outFolder, token, lastFolder, job);
                }
                return;
            }


            JToken t = null;
            if (thisFolder["file"] != null)
            {
                if (thisFolder["file"].Type == JTokenType.Array)
                {
                    foreach (var file in thisFolder["file"])
                    {
                        if (lastFolder != null)
                        {
                            t = lastFolder.SelectToken(file.Path.Replace("digest.folder.", ""));

                        }
                        UpdateFile(downloadurl, remoteFolder, outFolder, file, t, job);
                    }

                }
                else
                {
                    if (lastFolder != null)
                    {
                        t = lastFolder.SelectToken(thisFolder.Path.Replace("digest.folder.", ""));
                        t = t["file"];

                    }
                    UpdateFile(downloadurl, remoteFolder, outFolder, thisFolder["file"], t, job);
                }
            }

            if (thisFolder["folder"] != null)
            {
                if (thisFolder["folder"].Type == JTokenType.Array)
                {
                    foreach (JToken jsonfolder in thisFolder["folder"])
                    {
                        if (jsonfolder["@name"] == null)
                        {
                            UpdateFilesInFolder(downloadurl,remoteFolder, outFolder, jsonfolder, lastFolder, job);
                        }
                        else
                        {
                            string name = (string)jsonfolder["@name"];
                            UpdateFilesInFolder(downloadurl, remoteFolder + name + "/", outFolder + name + "/", jsonfolder, lastFolder, job);
                        }
                    }
                }
                else
                {
                    thisFolder = thisFolder["folder"];
                    if (thisFolder["@name"] == null)
                    {
                        UpdateFilesInFolder(downloadurl, remoteFolder, outFolder, thisFolder["folder"], lastFolder, job);
                    }
                    else
                    {

                        string name = (string)thisFolder["@name"];
                        UpdateFilesInFolder(downloadurl, remoteFolder + name + "/", outFolder + name + "/", thisFolder["folder"], lastFolder, job);
                    }
                }
            }

        }

        private static void UpdateFile(string downloadurl, string remoteFolder, string outfolder, JToken file, JToken oldFile, string job)
        {
            if (file["@delete"] != null) return;
             string filename = (string)file["@name"];


            //If oldFile is null, that means this is a new file.
            if (oldFile != null)
            {
                string oldFilename = (string)oldFile["@name"];

                if (oldFilename == filename)
                {
                    return;
                }
            }

            

            string downloadURL = downloadurl + "/" + remoteFolder + filename + ".zs";

            if (file["@uncompressedSize"] == null || file["@compressedSize"] == null)
            {
                return; //If there is no size the file doesn't exist on the server
            }
            else
            {
                int uncompressedsize = (int)file["@uncompressedSize"];
                int compressedsize = (int)file["@compressedSize"];

                if (uncompressedsize > compressedsize)
                {
                    Downloader.DownloadAndDecompress(filename, downloadURL, outfolder, job);
                }
                else
                {
                    Downloader.Download(filename, downloadURL, outfolder, job);
                }
            }

            Console.WriteLine("Downloading: " + filename);


        }



    }
}

Main.cs

static void Main(string[] args)
{
    
    string[] manifests = new string[]
    {                        
        "http://manifest.patch.station.sony.com/patch/eqnext/test/digest/play/test64-cdn.soe",
        "http://manifest.patch.station.sony.com/patch/eqnext/test/digest/play/shared-cdn.soe",
        "http://manifest.patch.station.sony.com/patch/eqnext/test/digest/common/shared-cdn.soe",
        "http://manifest.patch.station.sony.com/patch/eqnext/test/digest/common/test-cdn.soe",
    };


    foreach (string manifest in manifests)
    {
        JObject thisManifest = Downloader.GetJobjectFromManifest(manifest);
        
        
        EQNDownloader.DownloadPS2Job(thisManifest, null, "EQNLandmark", dateTime =>
        {
            Console.WriteLine("Jobs done");
        }, "EQNLandmarkDownload");
    
    }   
    
}