Skip to content

Instantly share code, notes, and snippets.

@yutopio
Last active August 29, 2015 13:55
Show Gist options
  • Save yutopio/8701146 to your computer and use it in GitHub Desktop.
Save yutopio/8701146 to your computer and use it in GitHub Desktop.
Splits a large file into smaller files, and merges them back into the original one.
using System;
using System.IO;
class Program
{
const int SplitSize = 1024 * 1024 * 1024; // Splits into 1GB files
const int BufferSize = 1024 * 1024; // using 1MB size buffer
static void Main(string[] args)
{
if (args.Length == 1)
{
// Split(args[0]);
// Merge(args[0]);
}
else if (args.Length == 2)
{
var result = Compare(args[0], args[1]);
Console.WriteLine(result ? "Identical" : "Different");
}
}
static void Split(string name)
{
var input = new FileStream(name, FileMode.Open, FileAccess.Read, FileShare.None);
Console.WriteLine("Split input file: {0}", name);
var size = (input.Length / SplitSize) + (input.Length % SplitSize == 0 ? 0 : 1);
Console.WriteLine("Generates {0} files ({1} bytes each)", size, SplitSize);
var buf = new byte[BufferSize];
var placeholder = new string('0', (int)Math.Floor(Math.Log10(size)) + 1);
for (var i = 0; i < size; )
{
var output = new FileStream(name + "." + (++i).ToString(placeholder),
FileMode.CreateNew, FileAccess.Write, FileShare.None);
Console.WriteLine("=== {0}/{1} ===", i, size);
for (var j = 0; j < SplitSize; )
{
var read = input.Read(buf, 0, Math.Min(BufferSize, SplitSize - j));
if (read == 0) break;
output.Write(buf, 0, read);
j += read;
Console.Write('.');
}
output.Close();
Console.WriteLine();
}
input.Close();
Console.WriteLine("End");
}
static void Merge(string name)
{
var f = new FileInfo(name);
var t = f.Name.LastIndexOf('.');
var len = f.Name.Length - t - 1;
if (len >= 9)
{
Console.WriteLine("Too many split?");
return;
}
var max = (int)Math.Pow(10, len);
var placeholder = new string('0', len);
var baseName = name.Substring(0, name.Length - (len + 1));
var output = new FileStream(baseName,
FileMode.CreateNew, FileAccess.Write, FileShare.None);
Console.WriteLine("Merge output file: {0}", baseName);
var buf = new byte[BufferSize];
for (var i = 1; i < max; i++)
{
name = baseName + "." + i.ToString(placeholder);
if (!File.Exists(name)) break;
var input = new FileStream(name, FileMode.Open, FileAccess.Read, FileShare.None);
Console.WriteLine("=== {0} ===", i);
for (var j = 0; j < input.Length; )
{
var read = input.Read(buf, 0, BufferSize);
if (read == 0) break;
output.Write(buf, 0, read);
j += read;
Console.Write('.');
}
input.Close();
Console.WriteLine();
}
output.Close();
Console.WriteLine("End");
}
static bool Compare(string f1, string f2)
{
var files = new[] { f1, f2 };
Console.WriteLine("Comparing files:\n{0}\n{1}", f1, f2);
var f = new FileStream[2];
var buf = new byte[2][];
for (var i = 0; i < 2; i++)
{
f[i] = new FileStream(files[i], FileMode.Open, FileAccess.Read, FileShare.None);
buf[i] = new byte[BufferSize];
}
try
{
if (f[0].Length != f[1].Length) return false;
else
for (var j = 0L; j < f[0].Length; )
{
var read = 0;
for (var i = 0; i < 2; i++)
{
read = 0;
while (read < BufferSize)
{
var now = f[i].Read(buf[i], read, BufferSize - read);
if (now == 0) break;
read += now;
}
}
j += read;
for (var i = 0; i < read; i++)
if (buf[0][i] != buf[1][i])
return false;
Console.Write('.');
}
return true;
}
finally
{
Console.WriteLine();
f[0].Close();
f[1].Close();
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment