CNCF claim check

README.md

I was triggered in local chat that some project with name CNCF is done by community and 203K developers contribute to it. So do not trust strangers on the internet and just check claims. Also this is fun scripting project

Program.fs

open Octokit
open LibGit2Sharp
open System.IO
open System.Linq

let github_token = "changeme"

let rec queryRepos (github:GitHubClient) page items =
    task {
        let searchRequest = SearchRepositoriesRequest(Topic = "cncf", Page = page, SortField = RepoSearchSort.Stars)
        let! repos = github.Search.SearchRepo(searchRequest)
        let newItems = items |> Seq.append repos.Items
        let! x = if (repos.TotalCount <= (page - 1) * 100) || (page = 11 (* GitHub does not allow more then 1000 results *)) then
                    task { return newItems }
                 else
                    (queryRepos github (page+1) newItems)
        return x
    }

let print_repos filename =
    let tokenAuth = new Octokit.Credentials(github_token);
    let github = new GitHubClient(new ProductHeaderValue("CNCFChecker"), Credentials = tokenAuth);

    let w = task {
        let! repos = queryRepos github 0 []
        let lines = repos |> Seq.map (fun x -> sprintf "%s %s %s" x.FullName x.CloneUrl x.DefaultBranch)
        File.WriteAllLines(filename, lines)
        //repos |> Seq.iter (fun x -> printfn "%s %s %s" x.FullName x.CloneUrl x.DefaultBranch)
    }
    w.Wait()

type RepoData(name: string, repoUrl: string, defaultBranch: string) =
    member val Name: string = name with get, set
    member val RepoUrl: string = repoUrl with get, set
    member val DefaultBranch: string = defaultBranch with get, set

let line2RepoData (line: string) =
    let parts = line.Split(' ')
    RepoData(parts[0], parts[1], parts[2])

let load_repos filename =
    let lines = File.ReadLines(filename)
    let repos = lines |> Seq.map line2RepoData
    repos

let repo_temp (repo: RepoData) = "temp/" + repo.Name

let cloneRepo repo =
    let deployerPath = repo_temp repo
    if Directory.Exists deployerPath then () else 
        let clone_options = new CloneOptions(BranchName = repo.DefaultBranch, Checkout = false) 
        let cloned_path = Repository.Clone(repo.RepoUrl, deployerPath, clone_options)
        //printfn "Cloned to %s" cloned_path
        ()
    
let collect_usage_stats repo =
    let repo_path = repo_temp repo
    let gitrepo = new Repository(repo_path)
    let commits = gitrepo.Commits
    commits.GroupBy(fun x -> x.Author.Email).Select(fun x -> (x.Key, x.Count())).ToArray()

let analyze_repo (repo: RepoData) = 
    let print_usage line =
        let (email, count) = line
        printfn "%s %s %d" repo.Name email count

    try
        try
            cloneRepo repo
            let usage = collect_usage_stats repo
            usage |> Seq.iter print_usage
        with
          | :? LibGit2SharpException -> printfn "Failed to checkout %s" repo.Name
    with
          | ex -> raise (System.Exception (sprintf "Failed to analyze %s with url %s" repo.Name repo.RepoUrl, ex))

print_repos "repos.txt"
let repos = load_repos "repos.txt"
repos |> Seq.iter analyze_repo

//let first = repos |> Seq.head
//cloneRepo first
//print_usage first