WoW Auction Scraper

main.go

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"github.com/mxplusb/azeroth"
	"go.uber.org/ratelimit"
	"net/http"
	"os"
	"os/user"
	"path/filepath"
)

var realm string
var initDB bool
var auctionInit azeroth.AuctionAPI
var data azeroth.AuctionFile
var targetItem azeroth.Item
var target string

func init() {
	flag.StringVar(&realm, "realm", "", "Realm to query.")
	flag.BoolVar(&initDB, "init", false, "Update DB. This will take awhile.")
	flag.Parse()
}

func main() {
	if initDB {
		fmt.Println("Making new DB.")
		NewDB()
		fmt.Println("done.")
		os.Exit(0)
	}

	if realm == "" {
		fmt.Println("Gotta add a realm bro. Can't leave it blank.")
		os.Exit(1)
	}

	if target == "" {
		fmt.Println("Gotta pick an item to search for mate. Can't leave it blank.")
		os.Exit(0)
	}

	cl := NewClient()
	initReq := NewRequest("https://us.api.battle.net" + fmt.Sprintf("%s%s", azeroth.AUCTION_ENDPOINT, realm))
	resp, err := cl.Do(initReq)
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close()

	if err := json.NewDecoder(resp.Body).Decode(&auctionInit); err != nil {
		panic(err)
	}

	req := NewRequest(auctionInit.Files[0].URL)
	dataResp, err := cl.Do(req)
	if err != nil {
		panic(err)
	}

	if err := json.NewDecoder(dataResp.Body).Decode(&data); err != nil {
		panic(err)
	}

	usr, err := user.Current()
	if err != nil {
		panic(err)
	}

	db, err := gorm.Open("sqlite3", filepath.Join(usr.HomeDir, "auctioneer.db"))
	if err != nil {
		fmt.Println("Can't connect to the DB. Try and run it with -init to rebuilt the local item DB.")
		panic(err)
	}

	db.Where(&azeroth.Item{Name: target}).First(&targetItem)

	for idx := range data.Auctions {
		if data.Auctions[idx].Item == targetItem.ItemID {
			fmt.Println()
		}
	}
}

func NewClient() *http.Client {
	return http.DefaultClient
}

func NewRequest(url string) *http.Request {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		panic(err)
	}
	values := req.URL.Query()
	values.Add("apikey", os.Getenv("WAREHOUSE_API_KEY"))
	req.URL.RawQuery = values.Encode()
	return req
}

func NewDB() {
	usr, err := user.Current()
	if err != nil {
		panic(err)
	}

	rl := ratelimit.New(95)
	ch := make(chan int, 10)

	fmt.Printf("Made DB at %s\n", filepath.Join(usr.HomeDir, "auctioneer.db"))
	db, err := gorm.Open("sqlite3", filepath.Join(usr.HomeDir, "auctioneer.db"))
	if err != nil {
		panic(err)
	}
	defer db.Close()

	db.DropTableIfExists(azeroth.Item{})
	db.CreateTable(azeroth.Item{})

	for i := 0; i < 10; i++ {
		go getItemWorker(ch, rl, db)
	}

	for i := 0; i < 250000; i++ {
		ch <- i
	}

	close(ch)
}

func getItemWorker(ch <-chan int, rl ratelimit.Limiter, db *gorm.DB) {
	cl := NewClient()
	for item := range ch {
		rl.Take()
		req := NewRequest("https://us.api.battle.net" + fmt.Sprintf("%s%d", azeroth.ITEM_ENDPOINT, item))
		resp, err := cl.Do(req)
		if err != nil {
			fmt.Println(err)
			continue
		}

		if resp.StatusCode == 404 {
			continue
		}

		var itemData azeroth.Item
		if err := json.NewDecoder(resp.Body).Decode(&itemData); err != nil {
			fmt.Println(err)
			continue
		}

		resp.Body.Close()

		db.NewRecord(itemData)
		db.Create(&itemData)

		fmt.Printf("Saved %s.\n", itemData.Name)
	}
}