SiteMapParser

module SitemapParser

open FSharp.Data
open System.IO
open System.IO.Compression
open System.Net.Http

type SiteMapIndexProvider = XmlProvider< """<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>location</loc><lastmod>2000-01-01T00:00:00+00:00</lastmod></sitemap><sitemap><loc>location</loc><lastmod>2000-01-01T00:00:00+00:00</lastmod></sitemap></sitemapindex>""" >

type SiteMapProvider = XmlProvider< """<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>location</loc><lastmod>2000-01-01T00:00:00+00:00</lastmod><changefreq>monthly</changefreq><priority>0.0</priority></url><url><loc>location</loc><lastmod>2000-01-01T00:00:00+00:00</lastmod><changefreq>monthly</changefreq><priority>0.0</priority></url></urlset>""" >

let private getSiteMap (path : string) = 
  async { 
    use client = new HttpClient()
    let! stream = client.GetStreamAsync path |> Async.AwaitTask
    use reader = 
      match path with
      | path when path.EndsWith ".gz" -> new StreamReader(new GZipStream(stream, CompressionMode.Decompress))
      | _ -> new StreamReader(stream)
    return reader.ReadToEnd()
  }
  |> Async.RunSynchronously

let siteMaps path = 
  match getSiteMap path with
  | xml when xml.Contains "sitemapindex" -> 
    let index = SiteMapIndexProvider.Parse xml
    seq { 
      for siteMap in index.Sitemaps do
        yield getSiteMap siteMap.Loc |> SiteMapProvider.Parse
    }
  | xml -> seq { yield SiteMapProvider.Parse xml }

let urls path = 
  seq { 
    for siteMap in siteMaps path do
      for url in siteMap.Urls do
        yield url
  }