gistfile1.cs

WindsorContainer container;

void Main()
{
    CompositionRoot ();
    Test ();
}

void Test ()
{
    var foo = container.Resolve<IFoo> ();
    var myFirestarter = container.Resolve<IFirestarter<int>> (); 
    var myWaterstarterYouTroubleStarter = container.Resolve<IWaterstarter<Nestling<int>>> (); 
}
    
void CompositionRoot()
{
    container = new WindsorContainer ();
    container.Register (
        Component.For (typeof (IFoo)).ImplementedBy (typeof (ConcreteFoo)),
        Component.For (typeof (IFirestarter<>)).ImplementedBy (typeof (ConcreteFirestarter<>)),
        
        // This here is the trick, tell Windsor how to map the generic arguments
        Component.For (typeof (IWaterstarter<>)).ImplementedBy (typeof (ConcreteWaterstarter<>), new NestedGenericStrategy ())
        );
}

public class NestedGenericStrategy : IGenericImplementationMatchingStrategy
{
    public Type[] GetGenericArguments (ComponentModel model, CreationContext context)
    {
        var nestlingType = context.RequestedType.GetGenericArguments ().Single ();
        var outerType = nestlingType.GetGenericArguments ().Single ();
        return new [] { outerType };
    }
}

// Define other methods and classes here
public interface IFoo { }
public class ConcreteFoo : IFoo { } 

// Define other methods and classes here
public interface IFirestarter<T> { }
public class ConcreteFirestarter<T> : IFirestarter<T> { } 

public interface IWaterstarter<T> { }
public class Nestling<T> { }
public class ConcreteWaterstarter<T> : IWaterstarter<Nestling<T>> { }