Skip to content

Instantly share code, notes, and snippets.

@RoyAwesome

RoyAwesome/Downloader.cs

Last active August 29, 2015 13:55
Show Gist options
  • Save RoyAwesome/8749139 to your computer and use it in GitHub Desktop.
Save RoyAwesome/8749139 to your computer and use it in GitHub Desktop.
Download ZIP
Everquest Landmark downloader
Raw
Downloader.cs
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using REsideUtility;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace REsideUtility
{
public delegate void FullDownloadJobComplete(string jobstart);
class DownloadJob
{
public string name;
public string url;
public string output;
public bool decompress;
public string status;
public string jobtag;
}
public class Downloader
{
static ConcurrentQueue<DownloadJob> DownloadQueue = new ConcurrentQueue<DownloadJob>();
static List<DownloadJob> DownloadingQueue = new List<DownloadJob>();
public static ConcurrentDictionary<string, FullDownloadJobComplete> JobCallbacks = new ConcurrentDictionary<string, FullDownloadJobComplete>();
static Thread DownloadWorker = null;
public static void Download(string filename, string url, string outputfilename, string job)
{
DoDownloadJob(new DownloadJob()
{
url = url,
output = outputfilename,
decompress = false,
name = filename,
status = filename + " Has Not Started",
jobtag = job,
});
}
public static void DownloadAndDecompress(string filename, string url, string outputfilename, string job)
{
DoDownloadJob(new DownloadJob()
{
url = url,
output = outputfilename,
decompress = true,
name = filename,
status = filename + " Has Not Started",
jobtag = job
});
}
private static void DoDownloadJob(DownloadJob job)
{
if (DownloadWorker == null)
{
DownloadWorker = new Thread(new ThreadStart(DownloadThreadWork));
DownloadWorker.Start();
}
DownloadQueue.Enqueue(job);
}
public static int DownloadJobsToComplete()
{
return DownloadQueue.Count;
}
public static string[] ReportDownloadStatus()
{
List<string> s = new List<string>();
s.Add("Files In Queue: " + DownloadQueue.Count);
Parallel.ForEach(DownloadingQueue, j =>
{
if (j == null) return;
s.Add(j.status);
});
return s.ToArray();
}
public static JObject GetJobjectFromManifest(string Manifest)
{
WebClient c = new WebClient();
string d = c.DownloadString(Manifest + ".txt");
System.Xml.XmlDocument doc = new System.Xml.XmlDocument();
doc.LoadXml(d);
string jsonText = JsonConvert.SerializeXmlNode(doc, Formatting.Indented);
return JObject.Parse(jsonText);
}
const int MaxFilesToDownload = 1;
static int FilesBeingDownloaded = 0;
public static List<string> JobErrors = new List<string>();
static object CountLock = new object();
public static void DownloadThreadWork()
{
WebClient cl = new WebClient();
cl.Headers.Add("user-agent", "Quicksilver Player/1.0.3.183 (Windows; PlanetSide 2)");
cl.DownloadProgressChanged += new DownloadProgressChangedEventHandler(cl_DownloadProgressChanged);
cl.DownloadDataCompleted += new DownloadDataCompletedEventHandler(cl_DownloadDataCompleted);
while (true)
{
lock (CountLock)
{
if (FilesBeingDownloaded < MaxFilesToDownload)
{
DownloadJob job;
if (DownloadQueue.TryDequeue(out job))
{
try
{
cl.DownloadDataAsync(new Uri(job.url), job);
}
catch (WebException e)
{
}
FilesBeingDownloaded++;
}
DownloadingQueue.Add(job);
}
}
Thread.Sleep(100);
}
}
static void cl_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e)
{
DownloadJob job = (DownloadJob)e.UserState;
if (e.Cancelled)
{
job.status = job.name + " Cancelled";
Console.WriteLine("Job was Cancelled!");
return;
}
if (e.Error != null)
{
job.status = job.name + " ERROR";
Console.WriteLine("Error: " + e.Error);
lock (CountLock)
{
JobErrors.Add(string.Format("File {0} at {1} had error {2}", job.name, job.url, e.Error.Message));
DownloadingQueue.Remove(job);
FilesBeingDownloaded--;
}
return;
}
byte[] data = e.Result;
if (job.decompress)
{
job.status = job.name + " is Decompressing";
data = SevenZip.Compression.LZMA.SevenZipHelper.Decompress(data);
}
using (BinaryWriter wr = new BinaryWriter(File.Open(job.output + job.name, FileMode.OpenOrCreate)))
{
job.status = job.name + " is Writing";
if (data == null)
{
job.status = job.name + " NULL DATA";
return;
}
wr.Write(data);
}
job.status = job.name + " is Waiting on lock";
lock (CountLock)
{
int jobsLeft = DownloadQueue.Where(s => s.jobtag == job.jobtag).Count();
if (jobsLeft == 0)
{
JobCallbacks[job.jobtag](job.jobtag);
FullDownloadJobComplete d;
JobCallbacks.TryRemove(job.jobtag, out d);
}
FilesBeingDownloaded--;
DownloadingQueue.Remove(job);
}
job.status = job.name + " Done";
Console.WriteLine(job.name + " is Done");
}
static void cl_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e)
{
DownloadJob j = (DownloadJob)e.UserState;
j.status = string.Format("Downloading {0}: {1}% ({2}/{3})", j.name, e.ProgressPercentage, e.BytesReceived, e.TotalBytesToReceive);
}
}
}
Raw
EQNDownloader.cs
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.IO;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json;
using System.Collections.Concurrent;
using System.Threading;
using System.Threading.Tasks;
namespace REsideUtility
{
public static class EQNDownloader
{
static void EmptyJobCompleteCallback(string time)
{
return;
}
public static void DownloadPS2(JObject thisManifest, JObject lastManifest, string outputfolder)
{
DownloadPS2Job(thisManifest, lastManifest, outputfolder, EmptyJobCompleteCallback);
}
public static string DownloadPS2Job(JObject thisManifest, JObject lastManifest, string outputfolder, FullDownloadJobComplete jobComplete)
{
string jobStart = DateTime.Now.ToString();
DownloadPS2Job(thisManifest, lastManifest, outputfolder, jobComplete, jobStart);
return jobStart;
}
public static void DownloadPS2Job(JObject thisManifest, JObject lastManifest, string outputfolder, FullDownloadJobComplete jobComplete, string jobName)
{
if (!Directory.Exists("output/")) Directory.CreateDirectory("output/");
if (!Directory.Exists("output/PS2Install")) Directory.CreateDirectory("output/PS2Install");
if (!Directory.Exists("output/PS2Install/" + outputfolder)) Directory.CreateDirectory("output/PS2Install/" + outputfolder);
string of = "output/PS2Install/" + outputfolder + "/";
JToken folders = thisManifest["digest"]["folder"];
string downloadURL = (string)thisManifest["digest"]["@defaultServerFolder"];
Downloader.JobCallbacks[jobName] = jobComplete;
if (lastManifest == null) UpdateFilesInFolder(downloadURL,"", of, folders, null, jobName);
else
{
JToken oldFolders = lastManifest["digest"]["folder"];
UpdateFilesInFolder(downloadURL, "", of, folders, oldFolders, jobName);
}
}
private static void UpdateFilesInFolder(string downloadurl,string remoteFolder, string outFolder, JToken thisFolder, JToken lastFolder, string job)
{
if (!Directory.Exists(outFolder)) Directory.CreateDirectory(outFolder);
//Sometimes there is a null child
if (thisFolder == null) return;
//If the folder is an array of files (not a folder structure), go through each of the folders and parse them
if (thisFolder.Type == JTokenType.Array)
{
foreach (var token in thisFolder)
{
UpdateFilesInFolder(downloadurl, remoteFolder, outFolder, token, lastFolder, job);
}
return;
}
JToken t = null;
if (thisFolder["file"] != null)
{
if (thisFolder["file"].Type == JTokenType.Array)
{
foreach (var file in thisFolder["file"])
{
if (lastFolder != null)
{
t = lastFolder.SelectToken(file.Path.Replace("digest.folder.", ""));
}
UpdateFile(downloadurl, remoteFolder, outFolder, file, t, job);
}
}
else
{
if (lastFolder != null)
{
t = lastFolder.SelectToken(thisFolder.Path.Replace("digest.folder.", ""));
t = t["file"];
}
UpdateFile(downloadurl, remoteFolder, outFolder, thisFolder["file"], t, job);
}
}
if (thisFolder["folder"] != null)
{
if (thisFolder["folder"].Type == JTokenType.Array)
{
foreach (JToken jsonfolder in thisFolder["folder"])
{
if (jsonfolder["@name"] == null)
{
UpdateFilesInFolder(downloadurl,remoteFolder, outFolder, jsonfolder, lastFolder, job);
}
else
{
string name = (string)jsonfolder["@name"];
UpdateFilesInFolder(downloadurl, remoteFolder + name + "/", outFolder + name + "/", jsonfolder, lastFolder, job);
}
}
}
else
{
thisFolder = thisFolder["folder"];
if (thisFolder["@name"] == null)
{
UpdateFilesInFolder(downloadurl, remoteFolder, outFolder, thisFolder["folder"], lastFolder, job);
}
else
{
string name = (string)thisFolder["@name"];
UpdateFilesInFolder(downloadurl, remoteFolder + name + "/", outFolder + name + "/", thisFolder["folder"], lastFolder, job);
}
}
}
}
private static void UpdateFile(string downloadurl, string remoteFolder, string outfolder, JToken file, JToken oldFile, string job)
{
if (file["@delete"] != null) return;
string filename = (string)file["@name"];
//If oldFile is null, that means this is a new file.
if (oldFile != null)
{
string oldFilename = (string)oldFile["@name"];
if (oldFilename == filename)
{
return;
}
}
string downloadURL = downloadurl + "/" + remoteFolder + filename + ".zs";
if (file["@uncompressedSize"] == null || file["@compressedSize"] == null)
{
return; //If there is no size the file doesn't exist on the server
}
else
{
int uncompressedsize = (int)file["@uncompressedSize"];
int compressedsize = (int)file["@compressedSize"];
if (uncompressedsize > compressedsize)
{
Downloader.DownloadAndDecompress(filename, downloadURL, outfolder, job);
}
else
{
Downloader.Download(filename, downloadURL, outfolder, job);
}
}
Console.WriteLine("Downloading: " + filename);
}
}
}
Raw
Main.cs
static void Main(string[] args)
{
string[] manifests = new string[]
{
"http://manifest.patch.station.sony.com/patch/eqnext/test/digest/play/test64-cdn.soe",
"http://manifest.patch.station.sony.com/patch/eqnext/test/digest/play/shared-cdn.soe",
"http://manifest.patch.station.sony.com/patch/eqnext/test/digest/common/shared-cdn.soe",
"http://manifest.patch.station.sony.com/patch/eqnext/test/digest/common/test-cdn.soe",
};
foreach (string manifest in manifests)
{
JObject thisManifest = Downloader.GetJobjectFromManifest(manifest);
EQNDownloader.DownloadPS2Job(thisManifest, null, "EQNLandmark", dateTime =>
{
Console.WriteLine("Jobs done");
}, "EQNLandmarkDownload");
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment