Last active
August 29, 2015 13:58
-
-
Save igorkulman/10019990 to your computer and use it in GitHub Desktop.
Mobile app stores crawler in F#
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
type AppStoreData = JsonProvider<"http://itunes.apple.com/search?term=LEMA&entity=software&country=CZ"> | |
let searchAppStore (term:string) (country:string) = | |
let data = http (sprintf "http://itunes.apple.com/search?term=%s&entity=software&country=%s" term country) | |
let res = AppStoreData.Parse data | |
res.Results | |
|> Seq.map (fun x-> {Name=x.TrackName; Package=x.BundleId; IconUrl=x.ArtworkUrl60; StoreUrl = ""}) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
let http (url : string) = | |
let wc = new WebClient() | |
wc.Headers.Add("user-agent","Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36") | |
let html = wc.DownloadString url | |
html |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
let searchAppStore term = | |
let data = http (sprintf "https://play.google.com/store/search?q=%s&c=apps&num=100" term) | |
let doc = new HtmlDocument() | |
doc.LoadHtml(data) | |
seq { | |
for div in doc.DocumentNode.SelectNodes("//div") do | |
if (div.Attributes.Contains("class") && div.Attributes.["class"].Value="card no-rationale square-cover apps small") then | |
let a = div.Descendants("a") |> Seq.filter (fun x->x.Attributes.Contains("class") && x.Attributes.["class"].Value="title") |> Seq.head | |
let img = div.Descendants("img") |> Seq.filter( fun x->x.Attributes.Contains("class") && x.Attributes.["class"].Value="cover-image") |> Seq.head | |
yield { Name = a.Attributes.["title"].Value; Package = div.Attributes.["data-docid"].Value; IconUrl = img.Attributes.["src"].Value; StoreUrl="https://play.google.com"+a.Attributes.["href"].Value} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
type App = { Name: string; Package: string; IconUrl : string; StoreUrl: string} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
type AppStoreData = XmlProvider<"http://marketplaceedgeservice.windowsphone.com/v8/catalog/apps?os=8.0.10521.0&cc=CZ&lang=en-US&chunkSize=50&q=igor%20kulman"> | |
let searchAppStore term country= | |
let data = http (sprintf "http://marketplaceedgeservice.windowsphone.com/v8/catalog/apps?os=8.0.10521.0&cc=%s&lang=en-US&chunkSize=50&q=%s" country term) | |
let res = AppStoreData.Parse data | |
res.Entries | |
|> Seq.map (fun x-> {Name=x.Title.Value; Package=x.Id.Replace("urn:uuid:",""); StoreUrl=(sprintf "http://windowsphone.com/s?appid=%s" (x.Id.Replace("urn:uuid:",""))); IconUrl = (sprintf "http://cdn.marketplaceimages.windowsphone.com/v8/images/%s&imagetype=icon_small" (x.Id.Replace("urn:uuid:","")))}) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment