Skip to content

Instantly share code, notes, and snippets.

@igorkulman
Last active August 29, 2015 13:58
Show Gist options
  • Save igorkulman/10019990 to your computer and use it in GitHub Desktop.
Save igorkulman/10019990 to your computer and use it in GitHub Desktop.
Mobile app stores crawler in F#
type AppStoreData = JsonProvider<"http://itunes.apple.com/search?term=LEMA&entity=software&country=CZ">
let searchAppStore (term:string) (country:string) =
let data = http (sprintf "http://itunes.apple.com/search?term=%s&entity=software&country=%s" term country)
let res = AppStoreData.Parse data
res.Results
|> Seq.map (fun x-> {Name=x.TrackName; Package=x.BundleId; IconUrl=x.ArtworkUrl60; StoreUrl = ""})
let http (url : string) =
let wc = new WebClient()
wc.Headers.Add("user-agent","Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36")
let html = wc.DownloadString url
html
let searchAppStore term =
let data = http (sprintf "https://play.google.com/store/search?q=%s&c=apps&num=100" term)
let doc = new HtmlDocument()
doc.LoadHtml(data)
seq {
for div in doc.DocumentNode.SelectNodes("//div") do
if (div.Attributes.Contains("class") && div.Attributes.["class"].Value="card no-rationale square-cover apps small") then
let a = div.Descendants("a") |> Seq.filter (fun x->x.Attributes.Contains("class") && x.Attributes.["class"].Value="title") |> Seq.head
let img = div.Descendants("img") |> Seq.filter( fun x->x.Attributes.Contains("class") && x.Attributes.["class"].Value="cover-image") |> Seq.head
yield { Name = a.Attributes.["title"].Value; Package = div.Attributes.["data-docid"].Value; IconUrl = img.Attributes.["src"].Value; StoreUrl="https://play.google.com"+a.Attributes.["href"].Value}
}
type App = { Name: string; Package: string; IconUrl : string; StoreUrl: string}
type AppStoreData = XmlProvider<"http://marketplaceedgeservice.windowsphone.com/v8/catalog/apps?os=8.0.10521.0&cc=CZ&lang=en-US&chunkSize=50&q=igor%20kulman">
let searchAppStore term country=
let data = http (sprintf "http://marketplaceedgeservice.windowsphone.com/v8/catalog/apps?os=8.0.10521.0&cc=%s&lang=en-US&chunkSize=50&q=%s" country term)
let res = AppStoreData.Parse data
res.Entries
|> Seq.map (fun x-> {Name=x.Title.Value; Package=x.Id.Replace("urn:uuid:",""); StoreUrl=(sprintf "http://windowsphone.com/s?appid=%s" (x.Id.Replace("urn:uuid:",""))); IconUrl = (sprintf "http://cdn.marketplaceimages.windowsphone.com/v8/images/%s&imagetype=icon_small" (x.Id.Replace("urn:uuid:","")))})
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment