Scraper for Google Play reviews

google_play_scraper-reviews.clj

(ns google-play-scraper.reviews
    (:require
      [clojure.string :as s]
      [org.httpkit.client :as http]
      [clojure.data.json :as json]
      [net.cgrand.enlive-html :as html]
      [clj-time.format :as f]))

(defn reviews-req [app-id page]
  (http/post "https://play.google.com/store/getreviews"
             {:form-params {"pageNum"         page
                            "id"              app-id
                            "reviewSortOrder" 0
                            "hl"              "en"
                            "reviewType"      0
                            "xhr"             1}}))

(defn review-id [review]
  (some-> (html/select review [:div.review-header])
          (first)
          (get-in [:attrs :data-reviewid])
          (s/trim)))

(defn user-name [review]
  (some-> (html/select review [:span.author-name])
          (first) :content (first) (s/trim)))

(defn user-image [review]
  (some-> (html/select review [:span.responsive-img-hdpi :span])
          (first) (get-in [:attrs :style])
          ((fn [style] (re-find #"url\((.*)\)" style)))
          (last)))

(defn review-date [review]
  (some->> (html/select review [:span.review-date])
           (first) :content (first) (s/trim)
           (f/parse (f/formatter "MMMM d, yyyy"))))

(defn review-url [review]
  (some-> (html/select review [:div.review-info :.reviews-permalink])
          (first) (get-in [:attrs :href])
          ((fn [path] (str "https://play.google.com" path)))))

(defn review-score [review]
  (some-> (html/select review [:.star-rating-non-editable-container])
          (first) (get-in [:attrs :aria-label])
          ((fn [stars-str] (re-find #"\d" stars-str)))
          (Long.)))

(defn review-title [review]
  (some-> (html/select review [:span.review-title])
          (first) :content (first) (s/trim)))

(defn review-text [review]
  (some-> (html/select review [:div.review-body])
          (first) :content (nth 2) (s/trim)))

(defn reply-date [review]
  (some->> (html/select review [:.developer-reply :span.review-date])
           (first) :content (first) (s/trim)
           (f/parse (f/formatter "MMMM d, yyyy"))))

(defn reply-text [review]
  (some-> (html/select review [:.developer-reply])
          (first) :content (nth 2) (s/trim)))

(defn parse-review [review]
  (->> {:review/id         (review-id review)
        :review/user-name  (user-name review)
        :review/user-image (user-image review)
        :review/date       (review-date review)
        :review/url        (review-url review)
        :review/score      (review-score review)
        :review/title      (review-title review)
        :review/text       (review-text review)
        :review/reply-date (reply-date review)
        :review/reply-text (reply-text review)}
       (filter val)
       (into {})))

(defn raw-reviews [resp]
  (-> resp
      (subs 4)
      (json/read-str)
      (get-in [0 2])
      (html/html-snippet)
      (html/select {[:div.single-review] [:div.developer-reply]})))

(defn reviews [app-id page]
  (let [{:keys [status body] :as resp} @(reviews-req app-id page)]
    (case status
      200 (map parse-review (raw-reviews body))
      503 (throw (ex-info "Rate limit reached" {}))
      (throw (ex-info "Unknown error" resp)))))

This API endpoint does not seems to be working anymore since Sept. 5th 2019