Skip to content

Instantly share code, notes, and snippets.

@AndyAyersMS

AndyAyersMS/NuGet.Config

Last active January 13, 2016 04:44
Show Gist options
  • Save AndyAyersMS/2fd78ecd186cd1afc167 to your computer and use it in GitHub Desktop.
Save AndyAyersMS/2fd78ecd186cd1afc167 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
benchmarks.xml
<?xml version="1.0" encoding="UTF-8"?>
<benchmark-system>
<benchmark-root-directory>c:\tf\CodeGen\src\QA\CLR\CQPerfSuite\benchmarks</benchmark-root-directory>
<benchmark-suite>
<name>Roslyn</name>
<benchmark>
<name>Roslyn</name>
<directory>.\RoslynSep13\SourcesSnapShot</directory>
<executable>..\CompilerBinaries\rcsc.exe</executable>
<args>/nologo /optimize+ /noconfig @Repro.rsp</args>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>System.XML</name>
<benchmark>
<name>System.XML</name>
<directory>.\System.XML\x86ret</directory>
<executable>SimpleTest.exe</executable>
<args>2000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>SharpChess</name>
<benchmark>
<name>SharpChess</name>
<directory>.\SharpChess</directory>
<executable>SharpChess2.exe</executable>
<args>6 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>V8</name>
<benchmark>
<name>v8-richards</name>
<directory>.\v8\richards</directory>
<executable>richards.exe</executable>
<args>20000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>v8-crypto</name>
<directory>.\v8\crypto</directory>
<executable>crypto.exe</executable>
<args>200 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>v8-deltablue</name>
<directory>.\v8\deltablue</directory>
<executable>deltablue.exe</executable>
<args>1000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>BIO</name>
<benchmark>
<name>bio-mums</name>
<directory>.\bio\test</directory>
<executable>MumUtil.exe</executable>
<args>/v /o:mum.out ..\TestData\NC_000913.bd.fasta ..\TestData\NC_000913.bd_10xWithError_1.fa &gt; out 2&gt;&amp;1</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>Fractals</name>
<benchmark>
<name>Fractals</name>
<directory>.\Fractals</directory>
<executable>FractalPerf.exe</executable>
<args>&gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>Spec</name>
<benchmark>
<name>bzip2</name>
<directory>.\bzip2</directory>
<executable>bzip2.exe</executable>
<args>mscorlib.il &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>Crafty</name>
<directory>.\crafty</directory>
<executable>crafty.exe</executable>
<args>&lt; crafty.in &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>Mono</name>
<benchmark>
<name>mono-binarytrees</name>
<directory>.\mono</directory>
<executable>binarytrees.csharp.exe</executable>
<args>20 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>mono-chameneos-redux</name>
<directory>.\mono</directory>
<executable>chameneosredux.csharp.exe</executable>
<args>6000000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>mono-fannkuch-redux</name>
<directory>.\mono</directory>
<executable>fannkuchredux.csharp-3.exe</executable>
<args>12 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>mono-fasta</name>
<directory>.\mono</directory>
<executable>fasta.csharp-2.exe</executable>
<args>25000000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>mono-fasta-redux</name>
<directory>.\mono</directory>
<executable>fastaredux.csharp.exe</executable>
<args>25000000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>mono-knucleotide</name>
<directory>.\mono</directory>
<executable>knucleotide.csharp-4.exe</executable>
<args>0 &lt; knucleotide-input25000000.txt &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>mono-nbody</name>
<directory>.\mono</directory>
<executable>nbody.csharp-3.exe</executable>
<args>50000000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>mono-regexdna</name>
<directory>.\mono</directory>
<executable>regexdna.csharp-6.exe</executable>
<args>0 &lt; regexdna-input2500000.txt &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>mono-revcomp</name>
<directory>.\mono</directory>
<executable>revcomp.csharp.exe</executable>
<args>0 &lt; revcomp-input25000000.txt &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>mono-spectralnorm</name>
<directory>.\mono</directory>
<executable>spectralnorm.csharp-2.exe</executable>
<args>5500 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>mono-threadring</name>
<directory>.\mono</directory>
<executable>threadring.csharp.exe</executable>
<args>2000000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>mono-pi-digits</name>
<directory>.\mono</directory>
<executable>pi-digits.exe</executable>
<args>10000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>BenchStone</name>
<benchmark>
<name>BenchI-8queens</name>
<directory>.\benchstone\BenchI</directory>
<executable>8queens.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-ackerman</name>
<directory>.\benchstone\BenchI</directory>
<executable>ackerman.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-addarra2</name>
<directory>.\benchstone\BenchI</directory>
<executable>addarra2.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-addarray</name>
<directory>.\benchstone\BenchI</directory>
<executable>addarray.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-array1</name>
<directory>.\benchstone\BenchI</directory>
<executable>array1.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-array2</name>
<directory>.\benchstone\BenchI</directory>
<executable>array2.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-benche</name>
<directory>.\benchstone\BenchI</directory>
<executable>benche.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-binserch</name>
<directory>.\benchstone\BenchI</directory>
<executable>binserch.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-bubsort</name>
<directory>.\benchstone\BenchI</directory>
<executable>bubsort.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-bubsort2</name>
<directory>.\benchstone\BenchI</directory>
<executable>bubsort2.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-csieve</name>
<directory>.\benchstone\BenchI</directory>
<executable>csieve.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-dhrysto</name>
<directory>.\benchstone\BenchI</directory>
<executable>dhrysto.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-fib</name>
<directory>.\benchstone\BenchI</directory>
<executable>fib.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-heapsort</name>
<directory>.\benchstone\BenchI</directory>
<executable>heapsort.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-iniarray</name>
<directory>.\benchstone\BenchI</directory>
<executable>iniarray.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-logicarr</name>
<directory>.\benchstone\BenchI</directory>
<executable>logicarr.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-midpoint</name>
<directory>.\benchstone\BenchI</directory>
<executable>midpoint.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-mulmtx</name>
<directory>.\benchstone\BenchI</directory>
<executable>mulmtx.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-permutat</name>
<directory>.\benchstone\BenchI</directory>
<executable>permutat.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-pi</name>
<directory>.\benchstone\BenchI</directory>
<executable>pi.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-puzzle</name>
<directory>.\benchstone\BenchI</directory>
<executable>puzzle.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-quicksrt</name>
<directory>.\benchstone\BenchI</directory>
<executable>quicksrt.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-shellsrt</name>
<directory>.\benchstone\BenchI</directory>
<executable>shellsrt.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-sq_mtx</name>
<directory>.\benchstone\BenchI</directory>
<executable>sq_mtx.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-treesort</name>
<directory>.\benchstone\BenchI</directory>
<executable>treesort.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-tree_ins</name>
<directory>.\benchstone\BenchI</directory>
<executable>tree_ins.exe</executable>
</benchmark>
<benchmark>
<name>BenchI-xpos_mtx</name>
<directory>.\benchstone\BenchI</directory>
<executable>xpos_mtx.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-adams_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>adams_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-benchmk2</name>
<directory>.\benchstone\BenchF</directory>
<executable>benchmk2.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-benchmrk</name>
<directory>.\benchstone\BenchF</directory>
<executable>benchmrk.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-bisect_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>bisect_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-dmath</name>
<directory>.\benchstone\BenchF</directory>
<executable>dmath.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-fft</name>
<directory>.\benchstone\BenchF</directory>
<executable>fft.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-inprod_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>inprod_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-inv_mt_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>inv_mt_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-lloops</name>
<directory>.\benchstone\BenchF</directory>
<executable>lloops.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-lorenz</name>
<directory>.\benchstone\BenchF</directory>
<executable>lorenz.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-matinv4</name>
<directory>.\benchstone\BenchF</directory>
<executable>matinv4.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-newt_e_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>newt_e_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-newt_r_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>newt_r_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-regula_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>regula_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-romber_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>romber_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-secant_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>secant_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-simpsn_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>simpsn_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-sq_mtx_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>sq_mtx_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-trap_d_d</name>
<directory>.\benchstone\BenchF</directory>
<executable>trap_d_d.exe</executable>
</benchmark>
<benchmark>
<name>BenchF-whetsto</name>
<directory>.\benchstone\BenchF</directory>
<executable>whetsto.exe</executable>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>ByteMark</name>
<benchmark>
<name>ByteMark-assignjagged</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\assignjagged.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-assignrect</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe &gt; out</executable>
<args>-ccommands\assignrect.dat</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-bitfield</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\bitfield.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-emfclass</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\emfclass.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-emfstruct</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\emfstruct.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-four</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\four.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-huff</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\huff.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-idea</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\idea.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-lu</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\lu.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-nnetjagged</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\nnetjagged.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-nnetrect</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\nnetrect.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-numsortjagged</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\numsortjagged.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-numsortrect</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\numsortrect.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>ByteMark-stringsort</name>
<directory>.\bytemark</directory>
<executable>bytemark.exe</executable>
<args>-ccommands\stringsort.dat &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>CoreMark</name>
<benchmark>
<name>CoreMark</name>
<directory>.\CoreMark</directory>
<executable>coremark.exe</executable>
<args>0 0 0x66 160000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>SciMark</name>
<benchmark>
<name>SciMark</name>
<directory>.\SciMark</directory>
<executable>scimark-csharp.exe</executable>
<args>&gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>FSharpFFT</name>
<benchmark>
<name>FSharpFFT</name>
<directory>.\FSharpFFT</directory>
<executable>FSharpFFT.exe</executable>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>TrueTypeBench</name>
<benchmark>
<name>TrueTypeBench</name>
<directory>.\TrueTypeBench\bin\release</directory>
<executable>truetypebench.exe</executable>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>Micro</name>
<benchmark>
<name>Micro-InlineClassConstructorHelper</name>
<directory>..\micros</directory>
<executable>InlineClassConstructorHelper.exe</executable>
<args>&gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>Json</name>
<benchmark>
<name>Json-Serialize</name>
<directory>.\JSON.Net\Src\Newtonsoft.Json.BenchmarkConsole\bin\Release\Net45</directory>
<executable>Newtonsoft.Json.BenchmarkConsole.exe</executable>
<args>Serialize-JsonNet-TestClass 1500000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
<benchmark>
<name>Json-Deserialize</name>
<directory>.\JSON.Net\Src\Newtonsoft.Json.BenchmarkConsole\bin\Release\Net45</directory>
<executable>Newtonsoft.Json.BenchmarkConsole.exe</executable>
<args>Deserialize-JsonNet-String 750000 &gt; out</args>
<run-in-shell>true</run-in-shell>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>Rotate</name>
<benchmark>
<name>Rotate</name>
<directory>.\Rotate</directory>
<executable>rotate.exe</executable>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>ConsoleMandel</name>
<benchmark>
<name>ConsoleMandel-8</name>
<directory>.\ConsoleMandel</directory>
<executable>ConsoleMandel.exe</executable>
<args>8 -S -bench 2</args>
</benchmark>
<benchmark>
<name>ConsoleMandel-9</name>
<directory>.\ConsoleMandel</directory>
<executable>ConsoleMandel.exe</executable>
<args>9 -S -bench 2</args>
</benchmark>
<benchmark>
<name>ConsoleMandel-8-SSE</name>
<directory>.\ConsoleMandel</directory>
<executable>ConsoleMandel.exe</executable>
<args>8 -S -bench 2</args>
<useSSE>true</useSSE>
<tags>SIMD,SSE</tags>
</benchmark>
<benchmark>
<name>ConsoleMandel-9-SSE</name>
<directory>.\ConsoleMandel</directory>
<executable>ConsoleMandel.exe</executable>
<args>9 -S -bench 2</args>
<useSSE>true</useSSE>
<tags>SIMD,SSE</tags>
</benchmark>
<benchmark>
<name>ConsoleMandel-8-AVX</name>
<directory>.\ConsoleMandel</directory>
<executable>ConsoleMandel.exe</executable>
<args>8 -S -bench 2</args>
<useAVX>true</useAVX>
<tags>SIMD,AVX</tags>
</benchmark>
<benchmark>
<name>ConsoleMandel-9-AVX</name>
<directory>.\ConsoleMandel</directory>
<executable>ConsoleMandel.exe</executable>
<args>9 -S -bench 2</args>
<useAVX>true</useAVX>
<tags>SIMD,AVX</tags>
</benchmark>
</benchmark-suite>
<benchmark-suite>
<name>RayTracer</name>
<benchmark>
<name>RayTracer</name>
<directory>.\RayTracer</directory>
<executable>RunAuto.cmd</executable>
<args>RayTracer.exe</args>
</benchmark>
<benchmark>
<name>RayTracer-SSE</name>
<directory>.\RayTracer</directory>
<executable>RunAuto.cmd</executable>
<args>RayTracer.exe</args>
<useSSE>true</useSSE>
<tags>SIMD,SSE</tags>
</benchmark>
<benchmark>
<name>RayTracer-AVX</name>
<directory>.\RayTracer</directory>
<executable>RunAuto.cmd</executable>
<args>RayTracer.exe</args>
<useAVX>true</useAVX>
<tags>SIMD,AVX</tags>
</benchmark>
</benchmark-suite>
</benchmark-system>
Raw
NuGet.Config
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<!--To inherit the global NuGet package sources remove the <clear/> line below -->
<clear />
<add key="dotnet-core" value="https://www.myget.org/F/dotnet-core/api/v3/index.json" />
<add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
</packageSources>
</configuration>
Raw
project.json
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"Microsoft.NETCore.Runtime": "1.0.1-beta-*",
"System.IO": "4.0.11-beta-*",
"System.IO.FileSystem.Primitives": "4.0.1-beta-*",
"System.Console": "4.0.0-beta-*",
"System.Runtime": "4.0.21-beta-*",
"System.Diagnostics.Contracts": "4.0.1-beta-*",
"System.Diagnostics.Debug": "4.0.11-beta-*",
"System.Diagnostics.StackTrace": "4.0.1-beta-*",
"System.Diagnostics.Tools": "4.0.1-beta-*",
"System.Diagnostics.Tracing": "4.0.21-beta-*",
"System.Diagnostics.Process": "4.0.0-beta-*",
"System.Xml.XDocument": "4.0.11-beta-*",
"System.Xml.XmlDocument": "4.0.0-*",
"System.Xml.XPath": "4.0.0-*",
"System.Xml.XPath.XmlDocument": "4.0.0-*",
"System.Runtime.Extensions": "4.0.11-beta-*",
},
"frameworks": {
"dnxcore50": { }
}
}
Raw
project.json.new
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"NETStandard.Library": "1.0.0-rc2-23704",
"System.Xml.XDocument": "4.0.0-*",
"System.Xml.XmlDocument": "4.0.0-*",
"System.Xml.XPath": "4.0.0-*",
"System.Xml.XPath.XmlDocument": "4.0.0-*"
},
"frameworks": {
"dnxcore50": { }
}
}
Raw
RunBenchmarks.cs
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Xml;
using System.Xml.XPath;
using System.IO;
namespace BenchmarkConsoleApplication
{
class BenchmarkSuite
{
public string SuiteName;
public List<Benchmark> BenchmarkList;
public BenchmarkSuite
(
string suiteName
)
{
SuiteName = suiteName;
BenchmarkList = new List<Benchmark>();
}
}
class BenchmarkTagSet
{
public string TagName;
public List<Benchmark> BenchmarkList;
public BenchmarkTagSet
(
string tagName
)
{
TagName = tagName;
BenchmarkList = new List<Benchmark>();
}
}
class Benchmark
{
public string Name;
public string SuiteName;
public string Tags;
public string WorkingDirectory;
public string ExePath;
public string ExeArgs;
public bool DoRunInShell;
public bool UseSSE;
public bool UseAVX;
public Benchmark
(
string name,
string suiteName,
string tags,
string workingDirectory,
string exePath,
string exeArgs,
bool doRunInShell,
bool useSSE,
bool useAVX
)
{
Name = name;
SuiteName = suiteName;
Tags = tags;
WorkingDirectory = workingDirectory;
ExePath = exePath;
ExeArgs = exeArgs;
DoRunInShell = doRunInShell;
UseSSE = useSSE;
UseAVX = useAVX;
}
}
class Results
{
public Benchmark Benchmark;
public long[] Times;
public long Minimum;
public long Maximum;
public long Average;
public double StandardDeviation;
public int Failures;
public Results
(
Benchmark benchmark,
int numberOfRuns
)
{
Benchmark = benchmark;
Times = new long[numberOfRuns + 1];
Minimum = 0;
Maximum = 0;
Average = 0;
StandardDeviation = 0.0;
Failures = 0;
}
}
class Controls
{
public bool DoRun;
public bool DoWarmUpRun;
public bool DoVerbose;
public bool DoCoreCLR;
public bool DoListBenchmarks;
public bool DoListBenchmarkSuites;
public bool DoListBenchmarkTagSets;
public int NumberOfRunsPerBenchmark;
public string ComplusVersion;
public string BenchmarksRootDirectory;
public string BenchmarkXmlFileName;
public string SuiteName;
public List<string> IncludeBenchmarkList;
public List<string> ExcludeBenchmarkList;
public List<string> IncludeTagList;
public List<string> ExcludeTagList;
public int SuccessExitCode;
}
class BenchmarkSystem
{
public const bool OptionalField = true;
public Controls Controls = new Controls()
{
NumberOfRunsPerBenchmark = 1,
DoWarmUpRun = false,
DoVerbose = false,
DoCoreCLR = false,
DoListBenchmarks = false,
DoListBenchmarkSuites = false,
DoListBenchmarkTagSets = false,
DoRun = true,
ComplusVersion = "",
SuiteName = "",
BenchmarksRootDirectory = "",
BenchmarkXmlFileName = "benchmarks.xml",
IncludeBenchmarkList = new List<string>(),
ExcludeBenchmarkList = new List<string>(),
IncludeTagList = new List<string>(),
ExcludeTagList = new List<string>(),
SuccessExitCode = 100 // This is by convention with JIT tests/benchmarks. We
// can make it a parameter if we need flexibility.
// Or allow per benchmark override.
};
public Dictionary<string, BenchmarkSuite>
BenchmarkSuiteTable = new Dictionary<string, BenchmarkSuite>();
public Dictionary<string, BenchmarkTagSet>
BenchmarkTagSetTable = new Dictionary<string, BenchmarkTagSet>();
public List<Benchmark> BenchmarkList = new List<Benchmark>();
public List<Benchmark> SelectedBenchmarkList = new List<Benchmark>();
public List<Results> ResultsList = new List<Results>();
public Char[] ListSeparators = new Char[] {',', ' '};
public int Failures = 0;
public int NumberOfBenchmarksRun;
public static int Main(string[] args)
{
Console.WriteLine("RyuJIT Benchmark System");
try
{
BenchmarkSystem benchmarkSystem = new BenchmarkSystem();
benchmarkSystem.ProcessCommandLine(args);
benchmarkSystem.BuildBenchmarksList();
benchmarkSystem.SelectBenchmarks();
benchmarkSystem.RunBenchmarks();
benchmarkSystem.ReportResults();
}
catch (Exception exception)
{
// Need to find portable Environment.Exit()
if (exception.Message == "Exit")
{
return 0;
}
Console.WriteLine("{0}", exception.ToString());
return -1;
}
return 0;
}
public void ProcessCommandLine(string[] args)
{
Controls controls = Controls;
try
{
for(int i = 0; i < args.Length;)
{
string arg = args[i++];
string benchmark;
string[] tags;
switch (arg)
{
case "-n":
arg = args[i++];
controls.NumberOfRunsPerBenchmark = Int32.Parse(arg);
break;
case "-norun":
controls.DoRun = false;
break;
case "-w":
controls.DoWarmUpRun = true;
break;
case "-v":
controls.DoVerbose = true;
break;
case "-r":
arg = args[i++];
controls.BenchmarksRootDirectory = arg;
break;
case "-f":
arg = args[i++];
controls.BenchmarkXmlFileName = arg;
break;
case "-s":
arg = args[i++];
controls.SuiteName = arg;
break;
case "-i":
arg = args[i++];
benchmark = arg;
controls.IncludeBenchmarkList.Add(benchmark);
break;
case "-e":
arg = args[i++];
benchmark = arg;
controls.ExcludeBenchmarkList.Add(benchmark);
break;
case "-list":
controls.DoListBenchmarks = true;
break;
case "-listsuites":
controls.DoListBenchmarkSuites = true;
break;
case "-listtags":
controls.DoListBenchmarkTagSets = true;
break;
case "-tags":
arg = args[i++];
tags = arg.Split(ListSeparators, StringSplitOptions.RemoveEmptyEntries);
controls.IncludeTagList.AddRange(tags);
break;
case "-notags":
arg = args[i++];
tags = arg.Split(ListSeparators, StringSplitOptions.RemoveEmptyEntries);
controls.ExcludeTagList.AddRange(tags);
break;
case "-coreclr":
controls.DoCoreCLR = true;
break;
case "-complus_version":
arg = args[i++];
controls.ComplusVersion = arg;
break;
default:
throw new Exception("invalid argument: " + arg);
}
}
}
catch (Exception)
{
Usage();
}
}
public void Usage()
{
Console.WriteLine("");
Console.WriteLine("Usage: RunBenchmarks [options]");
Console.WriteLine("");
Console.WriteLine(" options: ");
Console.WriteLine("");
Console.WriteLine(" -f <xmlFile> specify benchmark xml file (default benchmarks.xml)");
Console.WriteLine(" -n <number> specify number of runs for each benchmark (default is 1)");
Console.WriteLine(" -w specify that warmup run should be done first");
Console.WriteLine(" -v run in verbose mode");
Console.WriteLine(" -r <rootDir> specify root directory to run from");
Console.WriteLine(" -s <suite> specify a single benchmark suite to run (by name)");
Console.WriteLine(" -i <benchmark> specify benchmark to include by name (multiple -i's allowed)");
Console.WriteLine(" -e <benchmark> specify benchmark to exclude by name (multiple -e's allowed)");
Console.WriteLine(" -list prints a list of the benchmark names and does nothing else");
Console.WriteLine(" -listsuites prints a list of the suite names and does nothing else");
Console.WriteLine(" -listtags prints a list of the tag names and does nothing else");
Console.WriteLine(" -coreclr run benchmarks on CoreCLR (default DesktopCLR)");
Console.WriteLine(" -complus_version <version> run benchmarks on particular DesktopCLR version");
Console.WriteLine(" -norun prints what would be run, but nothing is executed");
Exit(-1);
}
public void AddBenchmark
(
string name,
string suiteName,
string tags,
string workingDirectory,
string exePath,
string exeArgs,
bool doRunInShell,
bool useSSE,
bool useAVX
)
{
BenchmarkSuite benchmarkSuite;
BenchmarkTagSet benchmarkTagSet;
Benchmark benchmark;
benchmark = new Benchmark(name, suiteName, tags,
workingDirectory, exePath, exeArgs, doRunInShell, useSSE, useAVX);
BenchmarkList.Add(benchmark);
if (!BenchmarkSuiteTable.TryGetValue(suiteName, out benchmarkSuite))
{
benchmarkSuite = new BenchmarkSuite(suiteName);
BenchmarkSuiteTable.Add(suiteName, benchmarkSuite);
}
benchmarkSuite.BenchmarkList.Add(benchmark);
String[] tagList = tags.Split(ListSeparators, StringSplitOptions.RemoveEmptyEntries);
foreach (string tag in tagList)
{
if (!BenchmarkTagSetTable.TryGetValue(tag, out benchmarkTagSet))
{
benchmarkTagSet = new BenchmarkTagSet(tag);
BenchmarkTagSetTable.Add(tag, benchmarkTagSet);
}
benchmarkTagSet.BenchmarkList.Add(benchmark);
}
}
public XmlNode SelectSingleNode
(
XmlNode node,
string xpath
)
{
#if DESKTOP
return node.SelectSingleNode(xpath);
#else
return XmlDocumentXPathExtensions.SelectSingleNode(node, xpath);
#endif
}
public string GetField
(
XmlNode node,
string xpath,
bool optional = false
)
{
XmlNode fieldNode = SelectSingleNode(node, xpath);
if (fieldNode == null)
{
if (optional)
{
return "";
}
throw new Exception("missing field: " + xpath);
}
return fieldNode.InnerText;
}
public bool GetBooleanField
(
XmlNode node,
string xpath,
bool optional = false
)
{
string value = GetField(node, xpath, optional);
if (value == "true")
return true;
if (value == "false")
return false;
if (optional)
return false;
throw new Exception("bad boolean value: " + value);
}
public XmlNodeList SelectNodes
(
XmlNode node,
string xpath
)
{
#if DESKTOP
return node.SelectNodes(xpath);
#else
return XmlDocumentXPathExtensions.SelectNodes(node, xpath);
#endif
}
public XmlNodeList GetList
(
XmlNode node,
string xpath
)
{
return SelectNodes(node, xpath);
}
public int Exit (int exitCode)
{
// Need to find portable Environment.Exit()
switch (exitCode)
{
case 0:
case -1:
case -2:
throw new Exception("Exit");
default:
throw new Exception("BadExit");
}
}
public void BuildBenchmarksList()
{
string benchmarksRootDirectory = Controls.BenchmarksRootDirectory;
string benchmarkXmlFileName = Controls.BenchmarkXmlFileName;
string benchmarkXmlFullFileName;
string benchmarkRootDirectoryName;
string benchmarkSuiteName;
string benchmarkDirectoryName;
string benchmarkName;
string benchmarkExecutableName;
string benchmarkArgs;
bool doRunInShell;
bool useSSE;
bool useAVX;
string tags;
benchmarkXmlFullFileName = benchmarkXmlFileName;
if (benchmarksRootDirectory != "")
{
benchmarkXmlFullFileName = benchmarksRootDirectory + "\\" + benchmarkXmlFullFileName;
}
// Load XML description of benchmarks.
XmlDocument benchmarkXml = new XmlDocument();
var xmlFile = new FileStream(benchmarkXmlFullFileName, FileMode.Open, FileAccess.Read);
benchmarkXml.Load(xmlFile);
// Get root directory for benchmark system.
benchmarkRootDirectoryName = GetField(benchmarkXml.DocumentElement, "benchmark-root-directory");
Controls.BenchmarksRootDirectory = benchmarkRootDirectoryName;
// Process each benchmark suite in the list of benchmark suites.
XmlNodeList benchmarkSuiteList = GetList(benchmarkXml.DocumentElement, "benchmark-suite");
foreach (XmlNode benchmarkSuite in benchmarkSuiteList)
{
benchmarkSuiteName = GetField(benchmarkSuite,"name");
//Process each benchmark in benchmark suite.
XmlNodeList benchmarkList = GetList(benchmarkSuite, "benchmark");
foreach (XmlNode benchmark in benchmarkList)
{
benchmarkName = GetField(benchmark, "name");
benchmarkDirectoryName = GetField(benchmark, "directory", OptionalField);
benchmarkExecutableName = GetField(benchmark, "executable");
benchmarkArgs = GetField(benchmark, "args", OptionalField);
useSSE = GetBooleanField(benchmark, "useSSE", OptionalField);
useAVX = GetBooleanField(benchmark, "useAVX", OptionalField);
doRunInShell = GetBooleanField(benchmark, "run-in-shell", OptionalField);
tags = GetField(benchmark, "tags", OptionalField);
AddBenchmark(benchmarkName, benchmarkSuiteName, tags, benchmarkDirectoryName,
benchmarkExecutableName, benchmarkArgs, doRunInShell, useSSE, useAVX);
}
}
if (Controls.DoListBenchmarks)
{
ListBenchmarks();
Exit(-2);
}
if (Controls.DoListBenchmarkSuites)
{
ListBenchmarkSuites();
Exit(-2);
}
if (Controls.DoListBenchmarkTagSets)
{
ListBenchmarkTagSets();
Exit(-2);
}
}
public void ListBenchmarks()
{
Console.WriteLine("Benchmark List");
foreach (Benchmark benchmark in BenchmarkList)
{
Console.WriteLine("{0}", benchmark.Name);
Console.WriteLine(" Suite: {0}", benchmark.SuiteName);
Console.WriteLine(" WorkingDirectory: {0}", benchmark.WorkingDirectory);
Console.WriteLine(" ExePath: {0}", benchmark.ExePath);
Console.WriteLine(" ExeArgs: {0}", benchmark.ExeArgs);
Console.WriteLine(" RunInShell: {0}", benchmark.DoRunInShell);
Console.WriteLine(" UseSSE: {0}", benchmark.UseSSE);
Console.WriteLine(" UseAVX: {0}", benchmark.UseAVX);
Console.WriteLine(" Tags: {0}", benchmark.Tags);
}
}
public void ListBenchmarkSuites()
{
Console.WriteLine("Benchmark Suite List");
var benchmarkSuiteList = BenchmarkSuiteTable.Keys;
foreach (var suiteName in benchmarkSuiteList)
{
Console.WriteLine("{0}", suiteName);
}
}
public void ListBenchmarkTagSets()
{
Console.WriteLine("Benchmark TagSet List");
var benchmarkTagSetList = BenchmarkTagSetTable.Keys;
foreach (var tagName in benchmarkTagSetList)
{
Console.WriteLine("{0}", tagName);
}
}
public void SelectBenchmarks()
{
List<Benchmark> benchmarkList = BenchmarkList;
List<string> includeBenchmarkList = Controls.IncludeBenchmarkList;
List<string> excludeBenchmarkList = Controls.ExcludeBenchmarkList;
List<string> includeTagList = Controls.IncludeTagList;
List<string> excludeTagList = Controls.ExcludeTagList;
string suiteName = Controls.SuiteName;
if (suiteName != "")
{
BenchmarkSuite benchmarkSuite = null;
if (!BenchmarkSuiteTable.TryGetValue(suiteName, out benchmarkSuite))
{
throw new Exception("bad suite name: " + suiteName);
}
benchmarkList = benchmarkSuite.BenchmarkList;
}
foreach (Benchmark benchmark in benchmarkList)
{
string benchmarkName = benchmark.Name;
bool include = true;
if (includeBenchmarkList.Count > 0)
{
include = false;
if (includeBenchmarkList.Contains(benchmarkName))
{
include = true;
}
}
if (include && (excludeBenchmarkList.Count > 0))
{
if (excludeBenchmarkList.Contains(benchmarkName))
{
include = false;
}
}
if (include && (excludeTagList.Count > 0))
{
foreach (string tag in excludeTagList)
{
BenchmarkTagSet benchmarkTagSet = null;
if (!BenchmarkTagSetTable.TryGetValue(tag, out benchmarkTagSet))
{
throw new Exception("bad tag: " + tag);
}
List<Benchmark> excludeTagBenchmarkList = benchmarkTagSet.BenchmarkList;
if (excludeTagBenchmarkList.Contains(benchmark))
{
include = false;
}
}
}
if (include)
{
SelectedBenchmarkList.Add(benchmark);
}
}
}
public Results RunBenchmark(Benchmark benchmark)
{
bool doRun = Controls.DoRun;
int numberOfRuns = Controls.NumberOfRunsPerBenchmark;
bool doWarmUpRun = Controls.DoWarmUpRun;
bool doCoreCLR = Controls.DoCoreCLR;
bool doVerbose = Controls.DoVerbose;
string complusVersion = Controls.ComplusVersion;
string benchmarksRootDirectory = Controls.BenchmarksRootDirectory;
string benchmarkDirectory = benchmarksRootDirectory + "\\" + benchmark.WorkingDirectory;
bool doRunInShell = benchmark.DoRunInShell;
bool useSSE = benchmark.UseSSE;
bool useAVX = benchmark.UseAVX;
int successExitCode = Controls.SuccessExitCode;
int failures = 0;
Results results = new Results(benchmark, numberOfRuns);
if (!doRun)
{
return results;
}
string workingDirectory = benchmarkDirectory;
string fileName = workingDirectory + "\\" + benchmark.ExePath;
string args = benchmark.ExeArgs;
if (doCoreCLR)
{
args = fileName + " " + args;
fileName = "corerun.exe";
}
if (doRunInShell)
{
args = "/C " + benchmark.ExePath + " " + args;
fileName = "cmd.exe";
}
if (doVerbose)
{
Console.WriteLine("Running benchmark {0} ...", benchmark.Name);
Console.WriteLine("Invoking: {0} {1}", fileName, args);
}
for (int run = (doWarmUpRun) ? 0 : 1; run <= numberOfRuns; run++)
{
ProcessStartInfo startInfo = new ProcessStartInfo()
{
FileName = fileName,
Arguments = args,
WorkingDirectory = workingDirectory,
UseShellExecute = false
};
if (complusVersion != "")
{
startInfo.Environment["COMPLUS_version"] = complusVersion;
startInfo.Environment["COMPLUS_defaultVersion"] = complusVersion;
}
if (useSSE)
{
startInfo.Environment["COMPLUS_FeatureSIMD"] = "1";
startInfo.Environment["COMPLUS_EnableAVX"] = "0";
}
if (useAVX)
{
startInfo.Environment["COMPLUS_FeatureSIMD"] = "1";
startInfo.Environment["COMPLUS_EnableAVX"] = "1";
}
startInfo.Environment["COMPLUS_gcConcurrent"] = "0";
startInfo.Environment["COMPLUS_gcServer"] = "0";
startInfo.Environment["COMPLUS_NoGuiOnAssert"] = "1";
startInfo.Environment["COMPLUS_BreakOnUncaughtException"] = "1";
var clockTime = Stopwatch.StartNew();
int exitCode = 0;
try
{
using (var proc = Process.Start(startInfo))
{
proc.EnableRaisingEvents = true;
proc.WaitForExit();
exitCode = proc.ExitCode;
}
this.NumberOfBenchmarksRun++;
}
catch (Exception exception)
{
Console.WriteLine("Could not launch test {0} exception: {1}",
startInfo.FileName, exception);
}
clockTime.Stop();
long time = clockTime.ElapsedMilliseconds;
if (exitCode != successExitCode)
{
failures++;
time = 0;
}
results.Times[run] = time;
if (doVerbose)
{
Console.Write("Iteration benchmark {0} ", benchmark.Name);
if (exitCode == successExitCode)
{
Console.Write("elapsed time {0}ms", time);
}
else
{
Console.Write("FAILED");
}
if (run == 0)
{
Console.Write(" (warmup)");
}
Console.WriteLine("");
}
}
long sum = 0;
long minimum = results.Times[1];
long maximum = minimum;
for (int run = 1; run <= numberOfRuns; run++)
{
long time = results.Times[run];
sum += time;
minimum = Math.Min(minimum, time);
maximum = Math.Max(maximum, time);
}
long average = sum / (long)numberOfRuns;
double standardDeviation = 0.0;
if (numberOfRuns > 1)
{
double s = 0.0;
double a = (double)average;
double n = (double)numberOfRuns;
for (int run = 1; run <= numberOfRuns; run++)
{
double time = (double) results.Times[run];
double t = (time - a);
s += (t * t);
}
double variance = s / n;
if (a == 0.0)
{
standardDeviation = 0.0;
}
else
{
standardDeviation = 100.0 * (Math.Sqrt(variance) / a); // stddev as a percentage
standardDeviation = Math.Round(standardDeviation, 2, MidpointRounding.AwayFromZero);
}
}
results.Average = average;
results.Minimum = minimum;
results.Maximum = maximum;
results.StandardDeviation = standardDeviation;
results.Failures = failures;
return results;
}
public void RunBenchmarks()
{
bool doVerbose = Controls.DoVerbose;
if (doVerbose)
{
Console.WriteLine("Run benchmarks ...");
}
foreach (Benchmark benchmark in SelectedBenchmarkList)
{
Results results = RunBenchmark(benchmark);
Failures += results.Failures;
ResultsList.Add(results);
}
}
public void ReportResults()
{
bool doVerbose = Controls.DoVerbose;
int numberOfBenchmarksRun = this.NumberOfBenchmarksRun;
int numberOfFailures = this.Failures;
int numberOfPasses = numberOfBenchmarksRun - numberOfFailures;
int numberOfRunsPerBenchmark = Controls.NumberOfRunsPerBenchmark;
int numberOfFailuresPerBenchmark = 0;
if (doVerbose)
{
Console.WriteLine("Report on benchmarks ...");
}
Console.WriteLine("Benchmark,Minimum(ms),Maximum(ms),Average(ms),StdDev(%),Passed/Failed(#)");
foreach (Results results in ResultsList)
{
string name = results.Benchmark.Name;
Console.Write("{0},", name);
long minimum = results.Minimum;
long maximum = results.Maximum;
long average = results.Average;
double standardDeviation = results.StandardDeviation;
Console.Write("{0},{1},{2},{3}", minimum, maximum, average, standardDeviation);
numberOfFailuresPerBenchmark = results.Failures;
if (numberOfFailuresPerBenchmark > 0)
{
Console.Write(",FAILED({0})", numberOfFailuresPerBenchmark);
}
else
{
Console.Write(",PASSED({0})", numberOfRunsPerBenchmark);
}
Console.WriteLine("");
}
Console.WriteLine("TOTAL BENCHMARKS({0}), PASSED({1}), FAILED({2})",
numberOfBenchmarksRun, numberOfPasses, numberOfFailures);
}
}
}
@AndyAyersMS
Copy link
Author

screenshot from 2016-01-12 15 19 50

Working...

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment