Advanced Source Generator API Example

.gitignore

bin/
obj/
.vs/

Examples.cs

using Microsoft.CodeAnalysis;
using System;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;

namespace SourceGeneratorApiExample
{
    // I'm just using inheritence here for laziness; imagine this is realy the interface and the method below is added to the initialize context.
    // Or maybe it's a whole new API, no idea.
    abstract class SampleGenerator
    {
        public abstract void Initialize();

        // Imagine you have a method roughly like this on InitializeGeneratorContext.
        public void RegisterAdvancedGeneration(SourceGeneratorStep<GeneratorRunResult> advancedGeneration) { }

        // This is my placeholder method in all samples that represents where user logic goes. Note it's static -- we can expect users
        // to write static methods for this.
        protected static GeneratorRunResult ActualGenerationLogic()
        {
            return default;
        }
    }

    class SyntaxXmlGeneratorLikeWeHaveToday : SampleGenerator
    {
        public override void Initialize()
        {
            // Our SyntaxXml generator simply consumes Syntax.xml, and outputs some generated files. Simple case.
            // Observe this generator is never given access to the compilation, so you can't write a bug.
            // ProcessAdditionalFilesAtOnce finds all the files that match the glob, and hands those to the method all at once.
            // Nothing prevents you from writing * if you really want to.
            RegisterAdvancedGeneration(
                Steps.ProcessAdditionalFilesAtOnce(glob: "Syntax.xml", GenerateOutputFilesFromSyntaxXmlFile));
        }

        private static GeneratorRunResult GenerateOutputFilesFromSyntaxXmlFile(ImmutableArray<AdditionalText> additionalFiles, CancellationToken cancellationToken)
        {
            // The .Single() here is just because our glob can't match more than one thing.
            var syntaxXml = additionalFiles.Single();

            // TODO: generate stuff using syntaxXml
            return ActualGenerationLogic();
        }
    }

    class ResxGenerator : SampleGenerator
    {
        public override void Initialize()
        {
            // For a resx each input file is independent, so we can process them one at a time; a change to one doesn't involve a change to the others
            // This is different from the AtOnce which means you have to reprocess as a group.
            SourceGeneratorStep<ImmutableArray<GeneratorRunResult>> processEachResx =
                Steps.ProcessAdditionalFilesOneAtATime("*.resx", GenerateFromResx);

            // We processed n additional files that could each have produced multiple generated files. We can easily flatten that down to a single list
            // of files. Practically, I presume we either add a little "flatten" method or just make an overload to regsiter that does this for you.
            var flattenList = Steps.Map(processEachResx, (inputs, ct) => new GeneratorRunResult(/* inputs.SelectMany...*/));

            RegisterAdvancedGeneration(flattenList);
        }

        private static GeneratorRunResult GenerateFromResx(AdditionalText additionalText, CancellationToken cancellationToken)
        {
            return ActualGenerationLogic();
        }
    }

    class GeneratorThatLightsUpOnACertainTypeInTheCompilation : SampleGenerator
    {
        public override void Initialize()
        {
            // This I think is a common case: imagine we want to light up if a certain type in your framework. Write a step that extracts just the
            // boolean -- then we can rerun this test, and reuse results if the boolean is still the same.
            SourceGeneratorStep<bool> typeExistsInCompilation = Steps.ProcessCompilation(DoesTypeExistInCompilation);

            // Once we know that fact, it's just depending on an additional files from there. In this example, we still have to consume all additional
            // files at once. The cool part though is if a compilation edit happens and tyepExistsInCompilation came up with the same answer
            // we can cache. This uses a variant of ProcessAdditionalFilesAtOnce that also lets you pass in an extra parameter --
            // we could alternatively have done this as a Step.Combine() which processes the magic files, but sometimes this might be easier, or you
            // can optimize your processing first.
            RegisterAdvancedGeneration(Steps.ProcessAdditionalFilesAtOnce("*.magicfiles", ProcessMagicFiles, typeExistsInCompilation));
        }

        private static bool DoesTypeExistInCompilation(Compilation compilation, CancellationToken arg2)
        {
            return compilation.GetTypeByMetadataName("Something") != null;
        }

        private static GeneratorRunResult ProcessMagicFiles(ImmutableArray<AdditionalText> additionalFiles, bool typeExistsInCompilation, CancellationToken cancellationToken)
        {
            return ActualGenerationLogic();
        }
    }

    class SyntaxXmlGeneratorThatAlsoProcessesSyntaxKind : SampleGenerator
    {
        public override void Initialize()
        {
            // This is an example of a fancy case. Imagine our Syntax.xml generator that has these problems:
            //
            // 1. The regular nodes depend only on the Syntax.xml.
            // 2. The tests depends also on something from the compilation (our syntaxkind enum). We know we don't have to
            //    generate the nodes if the compilation changes, we do need to generate the tests if the kinds in the compilation change.
            // 3. We don't want to regenerate if anything else in the compilation changes.
            // 3. Parsing the XML file is not cheap. But the representation is relatively cheap. (NB this isn't really true, but whatever.)

            // Step 1: when the additional file changes, we'll parse into some cheap representaton; this we can implicitly cache.
            var parseStep = Steps.ProcessAdditionalFilesAtOnce("Syntax.xml", ParseXml);

            // Step 2: emit the nodes directly from that
            RegisterAdvancedGeneration(Steps.Map(parseStep, GenerateNodesFromParsedRepresentation));

            // Step 3: for the second bit, we also need to extract syntax kinds. The idea here is by making this a step, we can
            // potentially do caching at the generator driver layer: if the syntax kinds extracted are the same as the previous step, we don't have to run
            // downstream steps.
            var syntaxKindExtractionStep = Steps.ProcessCompilation(FetchKindsFromCompilation);

            // Step 4: output the generation of the tests that consumes both the kids and also parsed inputs. This way if a change to the kinds
            // happens, we rerun this but we don't have to reparse a file that didn't change.
            // Note: we can let you call RegisterAdvancedGeneration multiple times, in this case we just collect them up.
            RegisterAdvancedGeneration(Steps.Combine(parseStep, syntaxKindExtractionStep, GenerateTests));
        }

        private class ParsedSyntaxXml { } 
        private class CachedKinds { }

        private ParsedSyntaxXml ParseXml(ImmutableArray<AdditionalText> additionalTexts, CancellationToken cancellationToken)
        {
            return new ParsedSyntaxXml();
        }

        private static GeneratorRunResult GenerateNodesFromParsedRepresentation(ParsedSyntaxXml parsedInput, CancellationToken arg2)
        {
            return ActualGenerationLogic();
        }

        private static CachedKinds FetchKindsFromCompilation(Compilation compilation, CancellationToken arg2)
        {
            throw new NotImplementedException();
        }

        private static GeneratorRunResult GenerateTests(ParsedSyntaxXml parsedSyntaxXml, CachedKinds cachedKinds, CancellationToken arg3)
        {
            return ActualGenerationLogic();
        }
    }

    class GeneratorThatDependsOnNothing : SampleGenerator
    {
        public override void Initialize()
        {
            // When you just want to jam in a constant file, no questions asked. Very cheap -- never reran!
            RegisterAdvancedGeneration(Steps.Constant(new GeneratorRunResult()));
        }
    }
}

ProposedApi.cs

using System;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;

namespace Microsoft.CodeAnalysis
{
    public abstract class SourceGeneratorStep<T>
    {
        internal abstract bool UsesCompilation { get; }
        internal abstract bool UsesAdditionalFile(string name);

        internal abstract T Run(Compilation compilation, ImmutableArray<AdditionalText> additionalFiles, CancellationToken cancellationToken);
    }

    /// <summary>
    /// This is just a factory class to allow for type inference.
    /// </summary>
    public static class Steps
    {
        public static SourceGeneratorStep<TOutput> ProcessCompilation<TOutput>(Func<Compilation, CancellationToken, TOutput> processCompilation) =>
            new ProcessCompilationStep<TOutput>(processCompilation);

        public static SourceGeneratorStep<TOutput> ProcessAdditionalFilesAtOnce<TOutput>(string glob, Func<ImmutableArray<AdditionalText>, CancellationToken, TOutput> processAdditionalFiles) =>
            new ProcessAdditionalFilesAtOnceStep<TOutput>(glob, processAdditionalFiles);

        public static SourceGeneratorStep<TOutput> ProcessAdditionalFilesAtOnce<TInput, TOutput>(string glob, Func<ImmutableArray<AdditionalText>, TInput, CancellationToken, TOutput> processAdditionalFiles, SourceGeneratorStep<TInput> additionalInput) =>
            new ProcessAdditionalFilesAtOnceStep<TInput, TOutput>(glob, processAdditionalFiles, additionalInput);

        public static SourceGeneratorStep<ImmutableArray<TOutput>> ProcessAdditionalFilesOneAtATime<TOutput>(string glob, Func<AdditionalText, CancellationToken, TOutput> processAdditionalFiles) =>
            new ProcessAdditionalFilesOneAtATimeStep<TOutput>(glob, processAdditionalFiles);

        public static SourceGeneratorStep<TOutput> Combine<TInput1, TInput2, TOutput>(SourceGeneratorStep<TInput1> input1, SourceGeneratorStep<TInput2> input2, Func<TInput1, TInput2, CancellationToken, TOutput> combine) =>
            new CombineStep<TInput1, TInput2, TOutput>(input1, input2, combine);

        public static SourceGeneratorStep<TOutput> Map<TInput, TOutput>(SourceGeneratorStep<TInput> input, Func<TInput, CancellationToken, TOutput> map) =>
            new MapStep<TInput, TOutput>(input, map);

        public static SourceGeneratorStep<TOutput> Constant<TOutput>(TOutput value)
            => new ConstantStep<TOutput>(value);
    }

    internal class ConstantStep<TOutput> : SourceGeneratorStep<TOutput>
    {
        private TOutput value;

        public ConstantStep(TOutput value)
        {
            this.value = value;
        }

        internal override bool UsesCompilation => false;

        internal override TOutput Run(Compilation compilation, ImmutableArray<AdditionalText> additionalFiles, CancellationToken cancellationToken)
        {
            return value;
        }

        internal override bool UsesAdditionalFile(string name) => false;
    }

    internal class ProcessCompilationStep<TOutput> : SourceGeneratorStep<TOutput>
    {
        private readonly Func<Compilation, CancellationToken, TOutput> _processCompilation;

        internal ProcessCompilationStep(Func<Compilation, CancellationToken, TOutput> processCompilation)
        {
            _processCompilation = processCompilation;
        }

        internal override bool UsesCompilation => true;
        internal override bool UsesAdditionalFile(string name) => false;

        internal override TOutput Run(Compilation compilation, ImmutableArray<AdditionalText> additionalFiles, CancellationToken cancellationToken)
        {
            return _processCompilation(compilation, cancellationToken);
        }
    }

    internal class ProcessAdditionalFilesAtOnceStep<TOutput> : SourceGeneratorStep<TOutput>
    {
        private readonly string _glob;
        private readonly Func<ImmutableArray<AdditionalText>, CancellationToken, TOutput> _processAdditionalFiles;

        internal ProcessAdditionalFilesAtOnceStep(string glob, Func<ImmutableArray<AdditionalText>, CancellationToken, TOutput> processAdditionalFiles)
        {
            _glob = glob;
            _processAdditionalFiles = processAdditionalFiles;
        }

        internal override bool UsesAdditionalFile(string name)
        {
            // TODO: glob
            return true;
        }

        internal override bool UsesCompilation => false;

        internal override TOutput Run(Compilation compilation, ImmutableArray<AdditionalText> additionalFiles, CancellationToken cancellationToken)
        {
            return _processAdditionalFiles(additionalFiles, cancellationToken);
        }
    }

    internal class ProcessAdditionalFilesAtOnceStep<TInput, TOutput> : SourceGeneratorStep<TOutput>
    {
        private readonly string _glob;
        private readonly Func<ImmutableArray<AdditionalText>, TInput, CancellationToken, TOutput> _processAdditionalFiles;
        private readonly SourceGeneratorStep<TInput> _additionalInput;

        internal ProcessAdditionalFilesAtOnceStep(string glob, Func<ImmutableArray<AdditionalText>, TInput, CancellationToken, TOutput> processAdditionalFiles, SourceGeneratorStep<TInput> additionalInput)
        {
            _glob = glob;
            _processAdditionalFiles = processAdditionalFiles;
            _additionalInput = additionalInput;
        }

        internal override bool UsesAdditionalFile(string name)
        {
            if (_additionalInput.UsesAdditionalFile(name))
            {
                return true;
            }

            // TODO: glob
            return true;
        }

        internal override bool UsesCompilation => _additionalInput.UsesCompilation;

        internal override TOutput Run(Compilation compilation, ImmutableArray<AdditionalText> additionalFiles, CancellationToken cancellationToken)
        {
            return _processAdditionalFiles(
                additionalFiles,
                _additionalInput.Run(compilation, additionalFiles, cancellationToken),
                cancellationToken);
        }
    }

    internal class ProcessAdditionalFilesOneAtATimeStep<TOutput> : SourceGeneratorStep<ImmutableArray<TOutput>>
    {
        private readonly string _glob;
        private readonly Func<AdditionalText, CancellationToken, TOutput> _processAdditionalFile;

        internal ProcessAdditionalFilesOneAtATimeStep(string glob, Func<AdditionalText, CancellationToken, TOutput> processAdditionalFile)
        {
            _glob = glob;
            _processAdditionalFile = processAdditionalFile;
        }

        internal override bool UsesAdditionalFile(string name)
        {
            // TODO: glob
            return true;
        }

        internal override bool UsesCompilation => false;

        internal override ImmutableArray<TOutput> Run(Compilation compilation, ImmutableArray<AdditionalText> additionalFiles, CancellationToken cancellationToken)
        {
            // TODO: the cache here would look at an additional-file-by-additional file basis.
            return additionalFiles.Select(a => _processAdditionalFile(a, cancellationToken)).ToImmutableArray();
        }
    }

    internal class CombineStep<TInput1, TInput2, TOutput> : SourceGeneratorStep<TOutput>
    {
        private readonly SourceGeneratorStep<TInput1> _input1;
        private readonly SourceGeneratorStep<TInput2> _input2;
        private readonly Func<TInput1, TInput2, CancellationToken, TOutput> _combine;

        public CombineStep(SourceGeneratorStep<TInput1> input1, SourceGeneratorStep<TInput2> input2, Func<TInput1, TInput2, CancellationToken, TOutput> combine)
        {
            _input1 = input1;
            _input2 = input2;
            _combine = combine;
        }

        internal override bool UsesAdditionalFile(string name) => _input1.UsesAdditionalFile(name) || _input2.UsesAdditionalFile(name);

        internal override bool UsesCompilation => _input1.UsesCompilation || _input2.UsesCompilation;

        internal override TOutput Run(Compilation compilation, ImmutableArray<AdditionalText> additionalFiles, CancellationToken cancellationToken)
        {
            return _combine(
                _input1.Run(compilation, additionalFiles, cancellationToken),
                _input2.Run(compilation, additionalFiles, cancellationToken),
                cancellationToken);
        }
    }

    internal class MapStep<TInput, TOutput> : SourceGeneratorStep<TOutput>
    {
        private readonly SourceGeneratorStep<TInput> _input;
        private readonly Func<TInput, CancellationToken, TOutput> _map;

        public MapStep(SourceGeneratorStep<TInput> input, Func<TInput, CancellationToken, TOutput> map)
        {
            _input = input;
            _map = map;
        }

        internal override bool UsesAdditionalFile(string name) => _input.UsesAdditionalFile(name);
        internal override bool UsesCompilation => _input.UsesCompilation;

        internal override TOutput Run(Compilation compilation, ImmutableArray<AdditionalText> additionalFiles, CancellationToken cancellationToken)
        {
            return _map(_input.Run(compilation, additionalFiles, cancellationToken), cancellationToken);
        }
    }
}

SourceGeneratorApiExample.csproj

<Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
    <TargetFramework>netstandard2.0</TargetFramework>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="Microsoft.CodeAnalysis" Version="3.8.0" />
  </ItemGroup>

</Project>

SourceGeneratorApiExample.sln


Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.30803.129
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SourceGeneratorApiExample", "SourceGeneratorApiExample.csproj", "{26EB335F-9137-4840-88A3-C477FB6FB99A}"
EndProject
Global
	GlobalSection(SolutionConfigurationPlatforms) = preSolution
		Debug|Any CPU = Debug|Any CPU
		Release|Any CPU = Release|Any CPU
	EndGlobalSection
	GlobalSection(ProjectConfigurationPlatforms) = postSolution
		{26EB335F-9137-4840-88A3-C477FB6FB99A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
		{26EB335F-9137-4840-88A3-C477FB6FB99A}.Debug|Any CPU.Build.0 = Debug|Any CPU
		{26EB335F-9137-4840-88A3-C477FB6FB99A}.Release|Any CPU.ActiveCfg = Release|Any CPU
		{26EB335F-9137-4840-88A3-C477FB6FB99A}.Release|Any CPU.Build.0 = Release|Any CPU
	EndGlobalSection
	GlobalSection(SolutionProperties) = preSolution
		HideSolutionNode = FALSE
	EndGlobalSection
	GlobalSection(ExtensibilityGlobals) = postSolution
		SolutionGuid = {5B3E352F-C246-4F99-B175-617C33A271B8}
	EndGlobalSection
EndGlobal