application.clj

(ns clustermap.application
  (:require [parenjin.application :as app]
            [parenjin.job :as job]
            [clustermap.config :as conf]
            [clustermap.connectors :as conn]
            [clomponents.control :as clomp]))


(def app-spec {:enjins {
                        :ping {:model :clustermap.datasets.ping.dataset/dataset}

                        :api-cache {:model :clustermap.datasets.api-cache.dataset/dataset
                                    :params {:prefix #app/ref :app-prefix}
                                    :connectors {:postgresql :postgresql}}

                        :iso3166-country-codes {:model :clustermap.datasets.iso3166-country-codes.dataset/dataset
                                                :connectors {:postgresql :postgresql}}

                        :exchange-rates {:model :clustermap.datasets.exchange-rates.dataset/dataset
                                         :params {:prefix #app/ref :app-prefix}
                                         :connectors {:postgresql :postgresql}}

                        :boundaryline {:model :clustermap.datasets.boundaryline.dataset/dataset
                                       :params {:prefix ""}
                                       :connectors {:postgresql :postgresql
                                                    :cassandra :cassandra
                                                    :elasticsearch :elasticsearch}}

                        :geo-sponsors {:model :clustermap.datasets.geo-sponsors.dataset/dataset
                                       :params {:prefix #app/ref :app-prefix}
                                       :connectors {:postgresql :postgresql
                                                    :elasticsearch :elasticsearch}}

                        :ons-postcode-directory {:model :clustermap.datasets.ons-postcode-directory.dataset/dataset
                                                 :params {:prefix ""}
                                                 :connectors {:postgresql :postgresql
                                                              :cassandra :cassandra}
                                                 :enjins {:boundaryline :boundaryline}
                                                 :webservices :none}

                        :os-boundaryline {:model :clustermap.datasets.os-boundaryline.dataset/dataset
                                          :params {:prefix ""}
                                          :connectors {:postgresql :postgresql}
                                          :enjins {:boundaryline :boundaryline
                                                   :ons-postcode-directory :ons-postcode-directory}}

                        :ons-2011-census {:model :clustermap.datasets.ons-2011-census.dataset/dataset
                                          :params {:prefix ""}
                                          :connectors {:postgresql :postgresql}
                                          :enjins {:boundaryline :boundaryline}}

                        :nuts-boundaryline {:model :clustermap.datasets.nuts-boundaries.dataset/dataset
                                            :params {:prefix ""}
                                            :connectors {:postgresql :postgresql}
                                            :enjins {:boundaryline :boundaryline
                                                     :ons-postcode-directory :ons-postcode-directory}}

                        :euroglobalmap {:model :clustermap.datasets.euroglobalmap.dataset/dataset
                                        :params {:prefix ""}
                                        :connectors {:postgresql :postgresql}
                                        :enjins {:boundaryline :boundaryline}}

                        :geocommons {:model :clustermap.datasets.geocommons.dataset/dataset
                                     :params {:prefix ""}
                                     :connectors {:postgresql :postgresql}
                                     :enjins {:boundaryline :boundaryline}}

                        :companies-house-free {:model :clustermap.datasets.companies-house-free.dataset/dataset
                                               :params {:prefix ""}
                                               :connectors {:postgresql :postgresql
                                                            :elasticsearch :elasticsearch}
                                               :enjins {:ons-postcode-directory :ons-postcode-directory
                                                        :boundaryline :boundaryline}
                                               :webservices :none}

                        :normalized-registry {:model :clustermap.datasets.normalized-registry.dataset/dataset
                                              :params {:prefix #app/ref :app-prefix}
                                              :connectors {:postgresql :postgresql
                                                           :elasticsearch :elasticsearch
                                                           :nominatim-geocoding :nominatim-geocoding}
                                              :enjins {:boundaryline :boundaryline
                                                       :companies :companies
                                                       :recordmatch :recordmatch
                                                       :api-cache :api-cache
                                                       :iso3166-country-codes :iso3166-country-codes
                                                       :exchange-rates :exchange-rates}}

                        :creditsafe-uk-companies {:model :clustermap.datasets.creditsafe-companies-house.dataset/dataset
                                                  :params {:prefix "" :version ""}
                                                  :connectors {:cassandra :cassandra
                                                               :postgresql :postgresql
                                                               :elasticsearch :elasticsearch}
                                                  :enjins {:ons-postcode-directory :ons-postcode-directory
                                                           :boundaryline :boundaryline
                                                           :companies :companies
                                                           :normalized-registry :normalized-registry}}

                        :creditsafe-marketing {:model :clustermap.datasets.creditsafe-marketing.dataset/dataset
                                               :params {:prefix #app/ref :app-prefix}
                                               :connectors {:postgresql :postgresql
                                                            :elasticsearch :elasticsearch}
                                               :enjins {:boundaryline :boundaryline
                                                        :normalized-registry :normalized-registry}}

                        :twitter {:model :clustermap.datasets.twitter.dataset/dataset
                                  :params {:prefix #app/ref :app-prefix}
                                  :connectors {:twitter :twitter
                                               :postgresql :postgresql}}

                        :clustermap-webapps {:model :clustermap.datasets.clustermap-webapp.dataset/dataset
                                             :params {:prefix ""
                                                      :base-root-dir "../clustermap2-webapps"
                                                      :default-root-dir "../clustermap2-webapps/__default__"}
                                             :connectors {:postgresql :postgresql}}

                        :companies {:model :clustermap.datasets.companies.dataset/dataset
                                    :params {:prefix #app/ref :app-prefix}
                                    :connectors {:postgresql :postgresql
                                                 :elasticsearch :elasticsearch
                                                 :cassandra :cassandra}
                                    :enjins {:ons-postcode-directory :ons-postcode-directory
                                             :creditsafe-uk-companies :creditsafe-uk-companies
                                             :twitter :twitter
                                             :tenant-config :clustermap-webapps
                                             :boundaryline :boundaryline
                                             :recordmatch :recordmatch}}

                        :crunchbase {:model :clustermap.datasets.crunchbase.dataset/dataset
                                     :params {:prefix #app/ref :app-prefix}
                                     :connectors {:postgresql :postgresql
                                                  :crunchbase :crunchbase
                                                  :google-geocoding :google-geocoding}
                                     :enjins {:api-cache :api-cache
                                              :companies :companies
                                              :iso3166-country-codes :iso3166-country-codes
                                              :boundaryline :boundaryline}}

                        :angellist {:model :clustermap.datasets.angellist.dataset/dataset
                                    :params {:prefix #app/ref :app-prefix}
                                    :connectors {:postgresql :postgresql
                                                 :angellist :angellist
                                                 :google-geocoding :google-geocoding}
                                    :enjins {:api-cache :api-cache
                                             :companies :companies
                                             :boundaryline :boundaryline}}

                        :seeddb {:model :clustermap.datasets.seeddb.dataset/dataset
                                 :params {:prefix #app/ref :app-prefix}
                                 :connectors {:postgresql :postgresql
                                              :google-geocoding :google-geocoding}
                                 :enjins {:api-cache :api-cache
                                          :companies :companies}}

                        :startup-companies {:model :clustermap.datasets.startup-companies.dataset/dataset
                                            :params {:prefix #app/ref :app-prefix}
                                            :connectors {:postgresql :postgresql}
                                            :enjins {:companies :companies
                                                     :crunchbase :crunchbase
                                                     :angellist :angellist
                                                     :boundaryline :boundaryline}}

                        ;;;;;;;;;;; woe

                        :west-of-england {:model :clustermap.datasets.west-of-england.dataset/dataset
                                          :params {:prefix #app/fix-ref [:app-prefix "woe"]
                                                   :version ""}
                                          :connectors {:postgresql :postgresql}
                                          :enjins {:creditsafe-uk-companies :creditsafe-uk-companies
                                                   :companies :companies
                                                   :boundaryline :boundaryline
                                                   :ons-postcode-directory :ons-postcode-directory}}

                        ;;;;;;;;;;; BVCA

                        :bvca {:model :clustermap.datasets.bvca.dataset/dataset
                               :params {:prefix #app/fix-ref [:app-prefix "bvca"]}
                               :connectors {:postgresql :postgresql
                                            :elasticsearch :elasticsearch}
                               :enjins {:creditsafe-uk-companies :creditsafe-uk-companies
                                        :companies :companies
                                        :boundaryline :boundaryline
                                        :ons-postcode-directory :ons-postcode-directory}}

                        ;;;;;;;;;;; EUStartupHubs

                        :recordmatch {:model :clustermap.datasets.recordmatch.dataset/dataset
                                      :params {:prefix #app/ref :app-prefix}
                                      :connectors {:postgresql :postgresql
                                                   :elasticsearch :elasticsearch}}

                        :eustartuphubs {:model :clustermap.datasets.eustartuphubs.dataset/dataset
                                        :params {:prefix #app/ref :app-prefix}
                                        :connectors {:postgresql :postgresql
                                                     :elasticsearch :elasticsearch}

                                        :enjins {:companies :companies
                                                 :recordmatch :recordmatch
                                                 :crunchbase :crunchbase
                                                 :angellist :angellist
                                                 :seeddb :seeddb
                                                 :creditsafe-marketing :creditsafe-marketing
                                                 :creditsafe-uk-companies :creditsafe-uk-companies
                                                 :normalized-registry :normalized-registry
                                                 :startup-companies :startup-companies
                                                 :exchange-rates :exchange-rates}}

                        :camclustermap {:model :clustermap.datasets.camclustermap.dataset/dataset
                                        :params {:prefix #app/ref :app-prefix}
                                        :connectors {:postgresql :postgresql
                                                     :elasticsearch :elasticsearch}

                                        :enjins {:boundaryline :boundaryline
                                                 :ons-postcode-directory :ons-postcode-directory
                                                 :companies :companies
                                                 :creditsafe-uk-companies :creditsafe-uk-companies}}

                        :medcity {:model :clustermap.datasets.medcity.dataset/dataset
                                  :params {:prefix #app/ref :app-prefix}
                                  :connectors {:postgresql :postgresql
                                               :elasticsearch :elasticsearch}

                                  :enjins {:companies :companies
                                           :creditsafe-uk-companies :creditsafe-uk-companies}}

                        ;;;;;;;;;;;;; uk company explorer

                        ;; :ukcoexplorer {:model :clustermap.datasets.eu-startup-data.dataset/dataset
                        ;;                :params {:prefix #app/fix-ref [:app-prefix "ukcoexplorer"]}
                        ;;                :connectors {:postgresql :postgresql
                        ;;                             :elasticsearch :elasticsearch}
                        ;;                :enjins {:clustermap-webapps :clustermap-webapps
                        ;;                         :creditsafe-uk-companies :creditsafe-uk-companies
                        ;;                         :companies :companies
                        ;;                         :boundaryline :boundaryline
                        ;;                         :ons-postcode-directory :ons-postcode-directory}}

                        ;; :greatlondontechcensus {:model :clustermap.datasets.eu-startup-data.dataset/dataset
                        ;;                         :params {:prefix #app/fix-ref [:app-prefix "greatlondontechcensus"]}
                        ;;                         :connectors {:postgresql :postgresql
                        ;;                                      :elasticsearch :elasticsearch}
                        ;;                         :enjins {:clustermap-webapps :clustermap-webapps
                        ;;                                  :creditsafe-uk-companies :creditsafe-uk-companies
                        ;;                                  :companies :companies
                        ;;                                  :boundaryline :boundaryline
                        ;;                                  :ons-postcode-directory :ons-postcode-directory}}


                        }
               :jobs {
                      ;; :streaming [[:twitter :stream]]

                      :update [
                               [:west-of-england :update]
                               [:bvca :update]
                               ]

                      :reindex [
                                [:west-of-england :reindex]
                                ]
                      }
               })

(defn app
  []
  (app/create-application (clustermap.connectors/registry) app-spec))

(defn webstart
  []
  (let [webc (:web (clustermap.connectors/registry))]

    (clomp/destroy webc)
    (clomp/create webc {:routes [(-> (app)
                                     (app/create-webservice (= :development (:environment (conf/config)))))]})))

(defn runjob
  [job-name]
  (let [app (app)
        job-id (-> job-name name keyword)
        jobs (app/job app job-id)]
    (job/run-jobs-serial jobs)))

(defn runservice
  [job-name]
  (let [app (app)
        job-id (-> job-name name keyword)
        jobs (app/job app job-id)]
    (job/run-jobs-parallel jobs)
    (job/join-jobs jobs)))