HtmlLinks.fsx

#r "<path to nuget>/.nuget/packages/fsharp.data/3.3.2/lib/netstandard2.0/FSharp.Data.dll" 

open FSharp.Data
open System


 type Comment =
    {
        Url : Uri
        Tag : string
    }

type MetaLink =
    {
        Url : Uri
        Name : string
        Comment : Comment option
    }

let descendantsByClass (document : HtmlDocument) elementType classType =
    document.Descendants [elementType]
    |> Seq.toArray
    |> Array.choose (fun x -> 
        x.TryGetAttribute("class")
        |> Option.bind (fun c -> 
            if c.Value() = classType then 
                Some x
            else
                None
            )
    )

let nodeDescendantsByClass (document : HtmlNode) elementType classType =
    document.Descendants [elementType]
    |> Seq.toArray
    |> Array.choose (fun x -> 
        x.TryGetAttribute("class")
        |> Option.bind (fun c -> 
            if c.Value() = classType then 
                Some x
            else
                None
            )
    )

let spaceUri = new Uri("https://www.space.com/")

let metaLinks (uri : Uri) =
    let htmlDocument = HtmlDocument.Load(uri.ToString())

    htmlDocument.Descendants ["a"]
    |> Seq.toArray
    |> Array.choose (fun x ->
        match x.TryGetAttribute("href") |> Option.map (fun a -> a.Value()) with
        | Some link ->
            {
                Url = 
                    if link.ToLower().StartsWith("http") then
                        new Uri(link)
                    else
                        new Uri(uri, link)
                Name = x.InnerText()
                Comment = None
            } |> Some
        | None ->
            None
    )

metaLinks spaceUri 
|> Array.iter (fun x -> 
    printfn "%A" x
)