Created
July 17, 2015 00:21
-
-
Save nvbn/b805e535e3bea9cee796 to your computer and use it in GitHub Desktop.
Flights parser
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(ns parsy.background.core | |
(:require-macros [cljs.core.async.macros :refer [go go-loop]]) | |
(:require [cljs.core.async :refer [chan <! >! merge]] | |
[cljs-time.core :refer [days plus after?]] | |
[cljs-time.format :refer [formatter parse unparse]])) | |
(enable-console-print!) | |
; Map of tab-id => chan | |
(def waiting (atom {})) | |
; Puts received message to waiting channel: | |
(.. js/chrome -runtime -onMessage (addListener | |
#(go (let [result (js->clj %1 :keywordize-keys true) | |
{:keys [tab]} (js->clj %2 :keywordize-keys true) | |
waiter (get @waiting (:id tab))] | |
(>! waiter result))))) | |
(defn run-script | |
[{:keys [id]}] | |
(let [result (chan)] | |
(.. js/chrome -tabs (executeScript id #js {:file "content/main.js"})) | |
(swap! waiting assoc id result) | |
result)) | |
(defn open-tab | |
[url] | |
(let [done (chan)] | |
(.. js/chrome -tabs (create #js {:url url} | |
#(go (>! done %)))) | |
done)) | |
(defn make-url | |
[id-from id-to date] | |
(str "https://avia.yandex.ru/search/?fromId=" id-from | |
"&toId=" id-to | |
"&when=" date | |
"&return_date=&oneway=2&adult_seats=1&children_seats=0&infant_seats=0&klass=economy")) | |
(defn days-range | |
[date-from date-to] | |
(let [input-format (formatter "MM.dd") | |
date-from (parse input-format date-from) | |
date-to (parse input-format date-to) | |
get-range (fn get-range [start] | |
(if (after? start date-to) | |
nil | |
(cons start (lazy-seq (get-range (plus start (days 1)))))))] | |
(get-range date-from))) | |
(defn get-flights | |
[id-from id-to date] | |
(go (let [search-format (formatter "dd+MMM") | |
tab (<! (open-tab (make-url id-from id-to (unparse search-format date)))) | |
{:keys [id]} (js->clj tab :keywordize-keys true) | |
{:keys [flights]} (<! (run-script tab))] | |
(.. js/chrome -tabs (remove id)) | |
(map #(assoc % :date date) flights)))) | |
(defn concat-flights | |
[flights] | |
(go-loop [[flight & flights] flights | |
result []] | |
(if flights | |
(recur flights (concat result (<! flight))) | |
result))) | |
(defn present | |
[prices] | |
(let [present-format (formatter "MM.dd")] | |
(go (->> (<! prices) | |
(map #(update % :date (fn [date] (unparse present-format date)))) | |
clj->js | |
(.table js/console))))) | |
(defn run | |
[id-from id-to date-from date-to] | |
(->> (days-range date-from date-to) | |
(map #(get-flights id-from id-to %)) | |
concat-flights | |
present)) | |
(set! (.-run js/window) run) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(ns parsy.content.core | |
(:require-macros [cljs.core.async.macros :refer [go-loop go]]) | |
(:require [cljs.core.async :refer [timeout <!]] | |
[clojure.string :as string] | |
[dommy.core :refer-macros [sel1 sel]])) | |
(enable-console-print!) | |
(defn text | |
[line selector] | |
(when-let [el (sel1 line selector)] | |
(string/replace (.-textContent el) #" " ""))) | |
(defn price | |
[val] | |
(js/parseInt (string/replace val #"[^\d]" "") 10)) | |
(defn arrival | |
[val] | |
(string/join "" (concat (take 5 val) [" "] (drop 5 val)))) | |
(defn get-flights | |
[] | |
(for [line (sel ".flight_list__forward")] | |
{:company (text line ".flight_list__company-names") | |
:departure (text line ".flight_list__departure-time") | |
:duration (text line ".flight_list__flight-duration") | |
:arrival (arrival (text line ".flight_list__arrival-time")) | |
:price (price (text line ".price_kb"))})) | |
(defn ready? | |
[] | |
(sel1 ".flight_list")) | |
(go-loop [] | |
(if (ready?) | |
(.. js/chrome -runtime (sendMessage #js {:status :ok | |
:flights (clj->js (get-flights))})) | |
(do (<! (timeout 500)) | |
(recur)))) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "parsy", | |
"version": "0.1", | |
"background": { | |
"scripts": [ | |
"background/main.js" | |
], | |
"persistent": false | |
}, | |
"permissions": [ | |
"tabs", | |
"http://*/*", | |
"https://*/*" | |
], | |
"manifest_version": 2 | |
} |
@franciscocpg It's code from article - https://nvbn.github.io/2015/07/17/flight-prices/
For running it you need to create cljs project, compile cljs to js and load it as a chrome extension.
Also yandex avia was changed a bit, so it's not working anymore.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
How can I compile/run this code?