msdn.fs

msdn.fs

open FSharp.Data
open System
open OfficeOpenXml
open System.IO
open OfficeOpenXml.FormulaParsing.Excel.Functions.Math
open Polly
open System.Net

type HomePage = HtmlProvider<"./Samples/Home.html">
type Products = JsonProvider<"./Samples/Products.json">
type Language = JsonProvider<"./Samples/Language.json">
type ItemList = JsonProvider<"./Samples/ItemList.json">
type ItemDetails = JsonProvider<"./Samples/ItemDetails.json">
type PipelineItem<'t> = 
    | Complete
    | Next of 't
    static member mapNext fn this = 
        match this with 
        | Next t -> Next (fn t)
        | Complete -> Complete
    static member mapComplete fn this = 
        match this with 
        | Next _ -> ()
        | Complete -> fn();
        this

let CommonFormHeaders = [
    ("Referer", "http://msdn.itellyou.cn/");
    ("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")]

let msdnPost url body = async {
    let! result = Http.AsyncRequestString(url=url, 
                                          headers=CommonFormHeaders, 
                                          body=FormValues(body), 
                                          httpMethod="POST")
    return result
}


let retryPolicy = Policy.Handle<WebException>().RetryAsync(3)

let retry f = async {
    try
        let! result = f
                      |> Async.StartAsTask 
                      |> fun f -> fun () -> f
                      |> retryPolicy.ExecuteAsync 
                      |> Async.AwaitTask
        return Ok result
    with 
        | e -> return Error e.Message
}

let getItemList productId lang = async {
    let! response = msdnPost "http://msdn.itellyou.cn/Category/GetList"
                             [("id", productId);
                             ("lang", lang);
                             ("filter", "true")] |> retry

    return response
           |> Result.map ItemList.Parse
           |> Result.map (fun x -> x.Result)
}

let getItemDetails itemId = async {
    let! response = msdnPost "http://msdn.itellyou.cn/Category/GetProduct"
                             [("id", itemId)] |> retry
    return response
           |> Result.map ItemDetails.Parse
           |> Result.map (fun x -> x.Result)
}

let getProductsByCategory categoryId = async {
    let! response = msdnPost "http://msdn.itellyou.cn/Category/Index"
                             [("id", categoryId)] |> retry
    return response
           |> Result.map Products.Parse
}

let getLanguageForProduct productId = async {
    let! response = msdnPost "http://msdn.itellyou.cn/Category/GetLang"
                             [("id", productId)] |> retry
    return response 
           |> Result.map Language.Parse
           |> Result.map (fun x -> x.Result)
}

let getCategories url = async {
    let! sample = HomePage.AsyncLoad(url)
    return sample.Html.CssSelect("#accordion .panel")
           |> Seq.map (fun x -> x.CssSelect("a").Head)
           |> Seq.map (fun x -> (x.AttributeValue("data-menuid"), x.InnerText()))
}

let excelPipeline = MailboxProcessor<PipelineItem<ItemDetails.Result>>.Start(fun ps ->
    let excel = new ExcelPackage(File.OpenWrite("msdn.xlsx"))
    let worksheet = excel.Workbook.Worksheets.Add("msdn")
    let cells = worksheet.Cells
    let rec writeRow row = async {
        let! result = ps.Receive()
        match result with 
        | Next item -> 
            cells.[row, 1].Value <- item.FileName
            cells.[row, 2].Value <- item.Size
            cells.[row, 3].Value <- item.DownLoad
            cells.[row, 4].Value <- item.PostDateString
            cells.[row, 5].Value <- item.Sha1
            printfn "%s(%s)" item.FileName item.Size
            return! writeRow (row + 1)
        | Complete ->
            excel.Save()
            excel.Dispose()
            printfn "done excel"
    }
    writeRow 1
)

let itemListPipeline = MailboxProcessor<PipelineItem<ItemList.Result>>.Start(fun ps -> 
    let rec loop() = async {
        let! result = ps.Receive()
        match result with 
        | Next item -> 
            let! itemDetails = getItemDetails(item.Id.ToString())
            itemDetails
            |> Result.map (fun x -> Next x)
            |> Result.map excelPipeline.Post
            |> Result.mapError (printfn "%A")
            |> ignore
            return! loop()
        | Complete -> Complete |> excelPipeline.Post
    }
    loop()
)

let langPipeline = MailboxProcessor<PipelineItem<string * string>>.Start(fun ps -> 
    let rec loop() = async {
        let! result = ps.Receive()
        match result with 
        | Next (productId, langId) -> 
            let! itemList = getItemList productId langId
            itemList
            |> Result.map (Seq.map (fun x -> Next x))
            |> Result.map (Seq.iter itemListPipeline.Post)
            |> ignore
            return! loop()
        | Complete -> Complete |> itemListPipeline.Post
    }
    loop()
)

let productPipeline = MailboxProcessor<PipelineItem<Products.Root>>.Start(fun ps ->
    let rec loop() = async {
        let! result = ps.Receive()
        match result with 
        | Next product -> 
            let! itemList = getLanguageForProduct(product.Id.ToString())
            itemList
            |> Result.map (Seq.map (fun x -> Next (product.Id.ToString(), x.Id.ToString())))
            |> Result.map (Seq.iter langPipeline.Post)
            |> ignore
            return! loop()
        | Complete -> Complete |> langPipeline.Post
    }
    loop()
)

let categoryPipeline = MailboxProcessor<PipelineItem<string * string>>.Start(fun ps ->
    let rec loop() = async {
        let! result = ps.Receive()
        match result with 
        | Next category -> 
            let! products = getProductsByCategory (fst category)
            products 
            |> Result.map (Seq.map (fun x -> Next x))
            |> Result.map (Seq.iter productPipeline.Post)
            |> ignore
            return! loop()
        | Complete -> Complete |> productPipeline.Post
    }
    loop()
)

let asyncMainTest = async {
    let! categories = getCategories "http://msdn.itellyou.cn/"
    let! products = getProductsByCategory "aff8a80f-2dee-4bba-80ec-611ac56d3849"
    let! langs = getLanguageForProduct "1c238a4b-49e6-40e1-90d4-e0ddaf5f6276"
    let! itemList = getItemList "1c238a4b-49e6-40e1-90d4-e0ddaf5f6276" "c5fe5be2-1c54-49a0-80fb-bfab286484eb"
    let! itemDetails = getItemDetails "663bff89-f9d1-4243-b497-ca61f669125c"
    printfn "%A" itemDetails
}

let asyncMain = async {
    let! categories = getCategories "http://msdn.itellyou.cn/"
    for category in categories do 
        let! products = getProductsByCategory (fst category)
        match products with 
        | Ok products -> 
            for product in products do
                let! langs = getLanguageForProduct(product.Id.ToString())
                match langs with 
                | Ok langs ->
                    for lang in langs do 
                        let! itemList = getItemList (product.Id.ToString()) (lang.Id.ToString())
                        match itemList with 
                        | Ok itemList -> 
                            for item in itemList do
                                let! itemDetails = getItemDetails (item.Id.ToString())
                                match itemDetails with 
                                | Ok itemDetails -> 
                                    Next itemDetails |> excelPipeline.Post
                                | Error msg -> printfn "%A" msg
                        | Error msg -> printfn "%A" msg
                | Error msg -> printfn "%A" msg
        | Error msg -> printfn "%A" msg
    Complete |> excelPipeline.Post
}

let asyncMainAllPipe = async {
    let! categories = getCategories "http://msdn.itellyou.cn/"
    categories
    |> Seq.map (fun x -> Next x)
    |> Seq.iter categoryPipeline.Post
    printfn "done main"
    Complete |> categoryPipeline.Post    
}

[<EntryPoint>]
let main argv = 
    try
        asyncMainAllPipe |> Async.RunSynchronously
    with 
        | e -> printfn "%A" e
    Console.ReadLine() |> ignore
    0