github stars

stars.clj

#!/usr/bin/env bb
(require '[babashka.http-client :as http])
(require '[cheshire.core :as json]) ;;

(defn build-headers
  [auth-token]
  {"Content-Type" "application/json"
   "Accept" "application/json"
   "Authorization" (str "bearer " auth-token)})


(def auth-token (System/getenv "GITHUBAPI_AUTH_TOKEN"))


(def user-repos-count-query
   "query { viewer { login  
          starredRepositories(last:10) { 
          totalCount      
    }}}")


(defn run-graphql-query [query auth-token]
  (-> (http/post "https://api.github.com/graphql"
                 {:headers (build-headers auth-token) 
                  :body (json/generate-string {:query query })})
      :body
      (json/parse-string true)))


(defn get-total-count
  [auth-token]
  (let [resp (run-graphql-query user-repos-count-query auth-token)]
   (get-in resp [:data :viewer :starredRepositories :totalCount])))


(defn fetch-stars
  [per_page auth-token]
  (let [totalPages
        (/ (get-total-count auth-token) per_page)]
   (doseq [pageNum (range totalPages)]
    (spit 
     (str "./stars" pageNum ".json") 
      (:body 
       (http/get (str "https://api.github.com/users/usametov/starred?per_page=" per_page "&page=" pageNum) 
                 {:headers (build-headers auth-token)}))))))


(fetch-stars 100 auth-token)

topics.clj

#!/usr/bin/env bb
(require '[babashka.http-client :as http])

(def topics
  ["bug-bounty" "bugbounty" "bugbountytips" "api-pentest" "api-security" "api-sec"
"apisec" "api-hacking" "hacktoberfest" "security" "c2" "osint" "recon" "discovery"
"emails" "information-gathering" "reconnaissance" "redteam" "cobalt-strike" "subdomain-enumeration"
"evil-twin"])

(def auth-token (System/getenv "GITHUBAPI_AUTH_TOKEN"))
(def page-size 30)

(defn build-query
  [topic min-stars min-date page]
   {"pushed:>" min-date  
    "stars:>" min-stars  
    :q (str "topic:" topic)
    :page page})

(defn build-headers
  [auth-token]
  {"Content-Type" "application/json"
   "Accept" "application/json"
   "Authorization" (str "bearer " auth-token)})

(def headers (build-headers auth-token))

(defn get-total-count
  [topic]
  (:total_count 
    (json/parse-string 
     (:body 
       (http/get 
           "https://api.github.com/search/repositories"
           {:query-params (build-query topic 50 "2023-06-01" 0)
            :headers headers})) true)))

(doseq [topic topics]
  (let [total-count (get-total-count topic)
        page-count (quot total-count page-size)]
    (doseq [page (range 1 (inc page-count))]
      (Thread/sleep 1000)
      (spit (str "./topics/" topic page ".json") 
      (:body 
       (http/get 
           "https://api.github.com/search/repositories"
           {:query-params (build-query topic 50 "2023-06-01" page)
            :headers headers}))))))