SourceFrom file size Encoding inconsitency

readme.md

• install SourceLink (via run chocho install sourcelink) and be sure it's in your path
• copy the test.csx to a solution where the 1.1.0 Microsoft.CodeAnalysis.CSharp package is installed
• start the VS development console (msbuild has to be in the path), run csi test.csx

Example output:

v:\example> csi .\CompilationCrazy.csx

SourceLink 1.1.0
876aa6db6eb96d35bbdee9f1586d9f94ff90d928 v:\example\precompilation\obj\huge.cs
d8f12548c91048de8323c43651382c65d2d909b2 v:\example\obj\hugebom.cs
2bdda5eb6c481e15d60b97250e28f11b3d5c736a v:\example\obj\small.cs
e1071dc724170d52d308084a9eb07e467b558ab5 v:\example\obj\smallbom.cs
obj/test-csc.pdb has 4 source files

roslyn compilation success True
SourceLink 1.1.0
f272be7ddd0653c8472336fe65dfa7ffd4ae1b2e v:\stackexchange\precompilation\obj\huge.cs
d8f12548c91048de8323c43651382c65d2d909b2 v:\stackexchange\precompilation\obj\hugebom.cs
2bdda5eb6c481e15d60b97250e28f11b3d5c736a v:\example\obj\small.cs
e1071dc724170d52d308084a9eb07e467b558ab5 v:\example\obj\smallbom.cs
obj/test-roslyn.pdb has 4 source files

notice how the huge.cs file has a different hash in the two PDBs, whereas the small.cs hash is the same

test.csx

#r "packages/Microsoft.CodeAnalysis.CSharp.1.1.0/lib/net45/Microsoft.CodeAnalysis.CSharp.dll"
#r "packages/Microsoft.CodeAnalysis.Common.1.1.0/lib/net45/Microsoft.CodeAnalysis.dll"
using System.IO;
using System.Net;
using System.Linq;
using System.Text;
using System.Runtime.InteropServices;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.Text;
using Microsoft.CodeAnalysis.Emit;

Directory.CreateDirectory("obj");
var smallFile = "obj/small.cs";
var hugeFile = "obj/huge.cs";
var smallFileBom = "obj/smallbom.cs";
var hugeFileBom = "obj/hugebom.cs";
var cscFile = "obj/test-csc.dll";
var apiFile = "obj/test-roslyn.dll";
var apiPdbFile = Path.ChangeExtension(apiFile, "pdb");

private void CreateClass(string path, int numMembers, Encoding encoding)
{
    var utf8WithoutBom = encoding;
    using (var st = File.Create(path))
    using (var wr = new StreamWriter(st, encoding))
    {
        wr.Write($@"
using System;
namespace Test
{{
    public class {Path.GetFileNameWithoutExtension(st.Name)}
    {{
");
        while (numMembers > 0)
        {
            switch (numMembers % 6)
            {
                case 1:
                    wr.WriteLine($"internal static void Foo{numMembers}() => Console.WriteLine(\"HALP\");");
                    break;
                case 2:
                    wr.WriteLine($@"public class Nested{numMembers} {{
    public string Foo {{ get; set; }}
    private Nested{numMembers}() {{ }}
}}
public Nested{numMembers} N{numMembers} {{ get; set; }}
");
                    break;
                case 3:
                    wr.WriteLine($"public delegate bool Delegate{numMembers}(object o);");
                    wr.WriteLine($"public Delegate{numMembers} D{numMembers} {{ get; set; }} = x => false;");
                    break;

                default:
                    wr.WriteLine($"public string Bar{numMembers} {{ get; set; }} = \"{numMembers} remaining\";");
                    break;

            }
            numMembers--;
        }
        wr.Write(@"
    }
}");
    }
}

var noBom = new System.Text.UTF8Encoding(false);
var bom = new System.Text.UTF8Encoding(true);
CreateClass(smallFile, 10, noBom);
CreateClass(smallFileBom, 10, bom);
CreateClass(hugeFile, 1000, noBom);
CreateClass(hugeFileBom, 1000, bom);

private void Exec(string command, string commandArgs = null)
{
    var p = new Process
    {
        StartInfo = new ProcessStartInfo
        {
            FileName = command,
            Arguments = commandArgs,
            RedirectStandardError = true,
            RedirectStandardOutput = true,
            UseShellExecute = false,
        }
    };
    p.OutputDataReceived += (_, args) => Console.WriteLine(args.Data);
    p.ErrorDataReceived += (_, args) => Console.Error.WriteLine(args.Data);
    p.Start();
    p.BeginOutputReadLine();
    p.BeginErrorReadLine();
    p.WaitForExit();
}

var cscLine = 
   $@"/nologo /noconfig /nowarn:1701,1702 /nostdlib+ /errorreport:prompt /warn:4 /define:DEBUG;TRACE /highentropyva-
    /reference:C:\Windows\Microsoft.NET\Framework\v2.0.50727\mscorlib.dll
    /reference:C:\Windows\Microsoft.NET\Framework\v2.0.50727\System.dll
    /reference:C:\Windows\Microsoft.NET\Framework\v2.0.50727\System.Configuration.dll
    /reference:..\StackOverflow\StackOverflow\bin\StackExchange.Network.dll
    /debug+ /debug:full /filealign:512 /optimize-
    /target:library /utf8output {smallFile} {smallFileBom} {hugeFile} {hugeFileBom}";

Exec("csc.exe", cscLine + $" /out:{cscFile}");

// get it via `choco install sourcelink`
// http://stackoverflow.com/questions/17120215/how-does-visual-studio-know-if-the-source-file-matches-the-original-version
Exec("sourcelink", $"checksums -p {Path.ChangeExtension(cscFile, "pdb")}");

var sdkDirectory = RuntimeEnvironment.GetRuntimeDirectory();
var cscArgs = CSharpCommandLineParser.Default.Parse((cscLine + $" /out:{apiFile}").Split(new [] {' ', '\n','\r'}, StringSplitOptions.RemoveEmptyEntries), Environment.CurrentDirectory, sdkDirectory);

private SyntaxTree ReadSourceTree(CommandLineSourceFile source, CSharpCommandLineArguments args)
{
    using (var file = new FileStream(source.Path, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1))
    {
        var sourceText = SourceText.From(file, bom);
        return SyntaxFactory.ParseSyntaxTree(sourceText, args.ParseOptions, source.Path);
    }
}

var compilation = CSharpCompilation.Create(
    options: cscArgs.CompilationOptions.WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default),
    references: cscArgs.MetadataReferences.Select(x => (MetadataReference)MetadataReference.CreateFromFile(x.Reference, x.Properties)),
    syntaxTrees: cscArgs.SourceFiles.Select(x => ReadSourceTree(x, cscArgs)),
    assemblyName: cscArgs.CompilationName);

using (var pe = File.Create(apiFile))
using (var pdb = File.Create(apiPdbFile))
using (var res = compilation.CreateDefaultWin32Resources(true, cscArgs.NoWin32Manifest, null, null))
{
    var result = compilation.Emit(peStream: pe, pdbStream: pdb, win32Resources: res, options: cscArgs.EmitOptions);
    foreach(var d in result.Diagnostics) Console.WriteLine(d);
    Console.WriteLine($"roslyn compilation success {result.Success}");
}
Exec("sourcelink", $"checksums -p {apiPdbFile}");