Last active
January 5, 2017 11:30
-
-
Save rofr/1df18968833c88f263ad to your computer and use it in GitHub Desktop.
OrigoDB simplified - complete engine in 50 lines of code. 2000 writes per second, 3 million reads
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Num threads: 10 | |
| Start: 2015-08-20 01:42:10 | |
| Reads: 133758452 | |
| Writes: 66050 | |
| Reads/s: 4458615,06666667 | |
| Writes/s: 2201,66666666667 | |
| Total: 133824502 | |
| TPS: 4460816,73333333 | |
| Stop time: 2015-08-20 01:42:50 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| public class OrigoDbLite<T> where T : new() | |
| { | |
| private readonly FileStream _journal; | |
| private readonly IFormatter _formatter; | |
| private readonly T _system; | |
| private readonly ReaderWriterLockSlim _lock; | |
| public void ExecuteWrite(WriteTransaction<T> writeTransaction) | |
| { | |
| lock (_journal) | |
| { | |
| _formatter.Serialize(_journal, writeTransaction); | |
| _journal.Flush(); | |
| _lock.EnterWriteLock(); | |
| writeTransaction.Apply(_system); | |
| _lock.ExitWriteLock(); | |
| } | |
| } | |
| public R ExecuteRead<R>(ReadTransaction<T, R> readTransaction) | |
| { | |
| _lock.EnterReadLock(); | |
| var result = readTransaction.Apply(_system); | |
| _lock.ExitReadLock(); | |
| return result; | |
| } | |
| public OrigoDbLite(string journalFile) | |
| { | |
| _lock = new ReaderWriterLockSlim(); | |
| _formatter = new BinaryFormatter(); | |
| if (File.Exists(journalFile)) _system = Load(journalFile); | |
| else _system = new T(); | |
| _journal = File.Create(journalFile, 4096, FileOptions.WriteThrough); | |
| //_journal = File.Open(journalFile, FileMode.Append,FileAccess.Write, FileShare.None); | |
| } | |
| /// <summary> | |
| /// Replay transactions from a file | |
| /// </summary> | |
| private T Load(string file) | |
| { | |
| var state = new T(); | |
| var stream = File.OpenRead(file); | |
| while (stream.Position < stream.Length) | |
| { | |
| var writeTransaction = (WriteTransaction<T>) _formatter.Deserialize(stream); | |
| writeTransaction.Apply(state); | |
| } | |
| stream.Close(); | |
| return state; | |
| } | |
| public void Close() | |
| { | |
| lock (_journal) _journal.Close(); | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [Serializable] | |
| public abstract class WriteTransaction<T> | |
| { | |
| public abstract void Apply(T system); | |
| } | |
| public abstract class ReadTransaction<T,R> | |
| { | |
| public abstract R Apply(T system); | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ///Three custom transactions for a Dictionary<int,int>() | |
| // Get, Remove and Set | |
| public class Get : ReadTransaction<Dictionary<int, int>, int> | |
| { | |
| public readonly int Key; | |
| public Get(int key) | |
| { | |
| Key = key; | |
| } | |
| public override int Apply(Dictionary<int, int> system) | |
| { | |
| int result; | |
| system.TryGetValue(Key, out result); | |
| return result; | |
| } | |
| } | |
| [Serializable] | |
| public class Remove : WriteTransaction<Dictionary<int, int>> | |
| { | |
| public readonly int Key; | |
| public Remove(int key) | |
| { | |
| Key = key; | |
| } | |
| public override void Apply(Dictionary<int, int> system) | |
| { | |
| system.Remove(Key); | |
| } | |
| } | |
| [Serializable] | |
| public class Set : WriteTransaction<Dictionary<int, int>> | |
| { | |
| public readonly int Key; | |
| public readonly int Value; | |
| public Set(int key, int value) | |
| { | |
| Key = key; | |
| Value = value; | |
| } | |
| public override void Apply(Dictionary<int, int> system) | |
| { | |
| system[Key] = Value; | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| public class Program | |
| { | |
| private static OrigoDbLite<Dictionary<int, int>> engine; | |
| public static void Main(string[] args) | |
| { | |
| int numThreads = 10; | |
| Console.WriteLine("Num threads: " + numThreads); | |
| if (File.Exists("journal.dat")) File.Delete("journal.dat"); | |
| engine = new OrigoDbLite<Dictionary<int, int>>("journal.dat"); | |
| Console.WriteLine("Start: " + DateTime.Now); | |
| float readWriteRatio = 2000; | |
| var duration = TimeSpan.FromMinutes(0.5); | |
| var tasks = Enumerable.Repeat(42, numThreads) | |
| .Select( | |
| _ => Task.Factory.StartNew(r => RandomTransactions(readWriteRatio, duration), 0)).ToArray(); | |
| Task.WaitAll(tasks); | |
| int[] readsWrites = tasks.Select(t => t.Result).Aggregate(new[] {0, 0}, (sums, result) => | |
| { | |
| sums[0] += result[0]; | |
| sums[1] += result[1]; | |
| return sums; | |
| }); | |
| double readsPerSecond = readsWrites[0]/duration.TotalSeconds; | |
| double writesPerSecond = readsWrites[1]/duration.TotalSeconds; | |
| Console.WriteLine("Reads: {0}", readsWrites[0]); | |
| Console.WriteLine("Writes: {0}", readsWrites[1]); | |
| Console.WriteLine("Reads/s: {0}", readsPerSecond); | |
| Console.WriteLine("Writes/s: {0}", writesPerSecond); | |
| Console.WriteLine("Total: {0}", readsWrites.Sum()); | |
| Console.WriteLine("TPS: {0}", readsWrites.Sum() * 1.0 / duration.TotalSeconds); | |
| Console.WriteLine("Stop time: {0}", DateTime.Now); | |
| } | |
| /// <summary> | |
| /// Execute a mix of reads and writes during a fixed period of time | |
| /// </summary> | |
| /// <param name="readWriteRatio"></param> | |
| /// <returns></returns> | |
| private static int[] RandomTransactions(float readWriteRatio, TimeSpan duration) | |
| { | |
| //normalize: | |
| float readWeight = readWriteRatio / (readWriteRatio + 1); | |
| const int READ = 0; | |
| const int WRITE = 1; | |
| var stopWatch = new Stopwatch(); | |
| int[] result = {0, 0}; | |
| var rnd = new Random(); | |
| stopWatch.Start(); | |
| while (stopWatch.Elapsed < duration) | |
| { | |
| if (rnd.NextDouble() < readWeight) | |
| { | |
| result[READ]++; | |
| engine.ExecuteRead(new Get(rnd.Next())); | |
| } | |
| else | |
| { | |
| result[WRITE]++; | |
| engine.ExecuteWrite(RandomWriteTransaction(rnd)); | |
| } | |
| } | |
| return result; | |
| } | |
| private static WriteTransaction<Dictionary<int, int>> RandomWriteTransaction(Random r) | |
| { | |
| if (r.NextDouble() < 0.5) return new Set(r.Next(), r.Next()); | |
| return new Remove(r.Next()); | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment