LLM fallback article

stream_completion.go

func (agent *LLMAgent) StreamCompletion(ctx context.Context, request *CompletionRequest, tokens chan<- Token) error {
    modelsToTry := agent.getModelsToTry()

    for i, model := range modelsToTry {
        hasReceivedFirstToken, err := agent.tryStreamSingleModel(ctx, request, model, tokens, agent.getModelTimeout())

        if err == nil {
            return nil // Success
        }

        // Only retry if we haven't started streaming and have more models to try
        if !hasReceivedFirstToken && i < len(modelsToTry)-1 {
            continue // Try next model
        }

        return err
    }
}