Vk seeker

structs.go

package vk

import "errors"


var (
	ErrVkTooManyRequestsPerSecond = errors.New("Vk error, code: 6, msg: Too many requests per second")
	ErrVkUndefinedError           = errors.New("Vk error undefined")
)

const (
	SeekUserFilterRelationNotMarried         = 1 << iota
	SeekUserFilterRelationInRelationship
	SeekUserFilterRelationEngaged
	SeekUserFilterRelationMarried
	SeekUserFilterRelationComplicated
	SeekUserFilterRelationActivelySearching
	SeekUserFilterRelationInLove
	SeekUserFilterRelationCivilMarried
)

const (
	SexAny     = iota
	SexFemale
	SexMale
)

type SeekUserKeyCriteria struct {
	City              int
	Country           int
	MinAge            int
	MaxAge            int
	RelationsBitMask  int
	Sex               int
}

type UserProfile struct {
	UID         int    `json:"uid"`
	FirstName   string `json:"first_name"`
	LastName    string `json:"last_name"`
	CanSeeAudio int    `json:"can_see_audio"`
	CanWritePrivateMessage int `json:"can_write_private_message"`

	LastSeen struct {
		Time     int `json:"time"`
		Platform int `json:"platform"`
	} `json:"last_seen"`

	FollowersCount int    `json:"followers_count"`
	HomeTown       string `json:"home_town,omitempty"`

	Personal struct {
		Langs      []string `json:"langs"`
		Religion   string   `json:"religion"`
		InspiredBy string   `json:"inspired_by"`
		PeopleMain int      `json:"people_main"`
		LifeMain   int      `json:"life_main"`
		Smoking    int      `json:"smoking"`
		Alcohol    int      `json:"alcohol"`
	} `json:"personal,omitempty"`
}

type UserExtendedProfile struct {

	UserProfile

	Counters struct {
		Albums        int `json:"albums"`
		Videos        int `json:"videos"`
		Audios        int `json:"audios"`
		Notes         int `json:"notes"`
		Photos        int `json:"photos"`
		Groups        int `json:"groups"`
		Gifts         int `json:"gifts"`
		Friends       int `json:"friends"`
		OnlineFriends int `json:"online_friends"`
		UserPhotos    int `json:"user_photos"`
		Followers     int `json:"followers"`
		Subscriptions int `json:"subscriptions"`
		Pages         int `json:"pages"`
	} `json:"counters"`
}

type CommonFilter struct {

	Criteria SeekUserKeyCriteria

	ProfilePredicate func(profile UserProfile) bool

	ExtProfilePredicate func(profile UserExtendedProfile) bool
}

util.go

package util

import "time"

func CopyMapStrStr(src map[string]string) (map[string]string) {

	dst := make(map[string]string)
	for k,v := range src {
		dst[k] = v
	}

	return dst
}


func RetryIfError(unit func() error) {

	var err error

	for i := 0; i < 5; i++{

		if err = unit(); err == nil {

			return
		}

		time.Sleep(500 * time.Millisecond)
	}

	PanicIfError(err)
}
func PanicIfError(err error) {

	if err != nil {

		panic(err)
	}
}

vk-seeker.go

package main

import (
	"fmt"
	"io/ioutil"
)

import (
	"errors"
	"strconv"
	"strings"
	"regexp"
	"vk-seeker/vk"
	"vk-seeker/util"
)



func main() {

	communities, err := readCommunitiesIdsFromFile("/home/mentat/Golang/src/vk-seeker/communities.txt")

	util.PanicIfError(err)

	var filter = vk.CommonFilter {

		Criteria: vk.SeekUserKeyCriteria{

			City              : 314,
			Country           : 2,
			MinAge            : 20,
			MaxAge            : 22,
			RelationsBitMask  : vk.SeekUserFilterRelationNotMarried|vk.SeekUserFilterRelationActivelySearching,
			Sex               : vk.SexFemale,
		},

		ProfilePredicate: acceptProfile,
		ExtProfilePredicate: acceptExtendedProfile,
	}

	var users = vk.ProcessCommunities(communities, filter)

	for _, user := range users {

		fmt.Println("http://vk.com/id"+strconv.Itoa(user.UID))
	}
}


func acceptProfile(profile vk.UserProfile) bool {

	if profile.CanSeeAudio == 0 {

		return false
	}

	if profile.CanWritePrivateMessage == 0 {

		return false
	}

	switch profile.Personal.LifeMain {

	case 1,2,6,7: {
		return false
	}
	}

	switch profile.Personal.PeopleMain {

	case 3,4,5: {
		return false
	}
	}

	switch profile.Personal.LifeMain {

	case 1,2: {
		return false
	}
	}

	switch profile.Personal.Alcohol {

	case 1,2: {
		return false
	}
	}

	if profile.FollowersCount > 500 {

		return false
	}

	return true
}
func acceptExtendedProfile(profile vk.UserExtendedProfile) bool {

	if profile.Counters.Friends > 250 {

		return false
	}

	if profile.Counters.Friends < 40 {

		return false
	}

	return true
}

func readCommunitiesIdsFromFile(filename string) ([]string, error) {

	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	lines := strings.Split(string(content), "\n")

	for _, str := range lines {

		if match, err := regexp.MatchString("[0-9]+", str); !match || err != nil {

			return nil, errors.New("Invalid community id: " + str)
		}
	}

	return lines, nil
}

vk.go

package vk

import (
	"net/url"
	"net/http"
	"io/ioutil"
	"encoding/json"
	"errors"
	"strconv"
	"vk-seeker/util"
	"github.com/tidwall/gjson"
)


// url to get access token
// https://oauth.vk.com/authorize?client_id=5987387&display=page&scope=offline&response_type=token&v=5.63

const API_METHOD_URL = "https://api.vk.com/method/"
const API_ACCESS_TOKEN = "609219db1672166ddd4ca729e30e62233f698ec5fb215e3ff9b9a8c0981e0ca232297b2840555fff8bee2" //todo

const API_USERS_FETCH_FIELDS_PARAM = "can_write_private_message,counters,followers_count,home_town,personal,can_see_audio,last_seen"


func ProcessCommunities(communities []string, filter CommonFilter) []UserExtendedProfile {

	var userIds = getUsersIdsInCommunitiesWithCriteria(communities, filter.Criteria)

	var userIdPacks = mergeIdsToVkPackSizedCsvStrings(userIds)

	var profiles = getUsersProfilesFromIdPacks(userIdPacks)

	var profilesToProcess []UserProfile
	for _, profile := range profiles {

		if filter.ProfilePredicate(profile) {

			profilesToProcess = append(profilesToProcess, profile)
		}
	}

	var filteredProfiles []UserExtendedProfile
	for _, profile := range profilesToProcess {

		var extProfile = getUserExtendedProfileWithRetry(strconv.Itoa(profile.UID))

		if filter.ExtProfilePredicate(extProfile) {

			filteredProfiles = append(filteredProfiles, extProfile)
		}
	}

	return filteredProfiles
}

func getUserExtendedProfileWithRetry(userId string) UserExtendedProfile {

	var err error
	var profile UserExtendedProfile

	util.RetryIfError(func () error {

		profile, err = getUserExtendedProfile(userId)

		if err != nil {
			return err
		}

		return nil
	})

	return profile
}

func getUsersProfilesFromIdPacks(userIdPacks []string) []UserProfile {

	var profiles []UserProfile
	for _,pack := range userIdPacks {

		util.RetryIfError(func () error {

			var users, err = getUsersProfiles(pack)

			if err != nil {
				return err
			}

			profiles = append(profiles, users...)
			return nil
		})
	}

	return profiles
}

func getUsersIdsInCommunitiesWithCriteria(communities []string, criteria SeekUserKeyCriteria) ([]string){

	var allUsersIds []string
	for _,community := range communities {

		util.RetryIfError(func () error {

			var users, err = getUsersIdsInCommunityWithCriteria(community, criteria)

			if err != nil {
				return err
			}

			allUsersIds = append(allUsersIds, users...)
			return nil
		})
	}

	return allUsersIds
}

func mergeIdsToVkPackSizedCsvStrings(allUsersIds []string) []string {

	//Vk restriction, it allows only less then 1000 users per request
	return mergeIdsToFixedSizedCsvStrings(1000, allUsersIds)
}
func mergeIdsToFixedSizedCsvStrings(sliceMaxSize int, allUsersIds []string) []string {

	var userIdsStrings []string
	var currIdsString = ""

	for i, id := range allUsersIds {

		//VK restriction, it allows only 1000 ids per profiles request
		if i != 0 && i % sliceMaxSize == 0 {

			userIdsStrings = append(userIdsStrings, currIdsString)
			currIdsString = ""
		}

		if currIdsString == "" {
			currIdsString += id
		} else {
			currIdsString += ","+id
		}
	}

	userIdsStrings = append(userIdsStrings, currIdsString)

	return userIdsStrings
}

func getUserExtendedProfile(userId string) (UserExtendedProfile, error) {

	var jsonResponse, err = request("users.get", map[string]string{"user_ids": userId, "fields": API_USERS_FETCH_FIELDS_PARAM})

	if err = getVkErrorIfItWas(jsonResponse); err != nil {

		return UserExtendedProfile{}, err
	}

	var profile = struct {
		Profile []UserExtendedProfile `json:"response"`
	}{}

	err = json.Unmarshal([]byte(jsonResponse), &profile)

	if err != nil {

		return UserExtendedProfile{}, err
	}

	return profile.Profile[0], nil
}

func getUsersProfiles(csvUserIds string) ([]UserProfile, error){

	var jsonResponse, err = request("users.get", map[string]string{"user_ids": csvUserIds, "fields": API_USERS_FETCH_FIELDS_PARAM})

	if err = getVkErrorIfItWas(jsonResponse); err != nil {

		return nil, err
	}

	var profiles = struct {
		Profiles []UserProfile `json:"response"`
	}{}

	err = json.Unmarshal([]byte(jsonResponse), &profiles)

	if err != nil {

		return nil, err
	}

	return profiles.Profiles, nil
}

func getUsersIdsInCommunityWithCriteria(communityId string, criteria SeekUserKeyCriteria) ([]string,error) {

	var allUsers = []string{}

	var mapParams = convertCriteriaToParamMap(communityId, criteria)

	for i := range mapParams {

		var users, err = getUsersInCommunityWithParams(mapParams[i])

		if err != nil {

			return users, err
		}

		allUsers = append(allUsers, users...)
	}

	return allUsers, nil
}

func getUsersInCommunityWithParams(params map[string]string) ([]string,error) {

	var jsonResponse, err = request("users.search", params)

	var users = []string{}

	if err != nil {
		return nil, err
	}

	result := gjson.Get(jsonResponse, "response")

	if err = getVkErrorIfItWas(jsonResponse); err != nil {

		return nil, err
	}

	if !result.Exists() {

		return nil, errors.New("response from api is not valid")
	}

	result.ForEach(func(key, value gjson.Result) bool{

		if value.Type == gjson.JSON {

			user := value.Map()

			users = append(users, user["uid"].String())
		}

		return true // keep iterating
	})

	return users, nil
}

func convertCriteriaToParamMap(communityId string, criteria SeekUserKeyCriteria) []map[string]string {

	var paramMaps []map[string]string

	commonParams := map[string]string{"group_id": ""+communityId, "count": "1000"}

	if criteria.City > 0 {

		commonParams["city"] = strconv.Itoa(criteria.City)
	}

	if criteria.Country > 0 {

		commonParams["country"] = strconv.Itoa(criteria.Country)
	}

	if criteria.MinAge > 0 {

		commonParams["age_from"] = strconv.Itoa(criteria.MinAge)
	}

	if criteria.MaxAge > 0 {

		commonParams["age_to"] = strconv.Itoa(criteria.MaxAge)
	}

	commonParams["sex"] = strconv.Itoa(int(criteria.Sex))

	relationMasks := [...]int{0,SeekUserFilterRelationNotMarried,
							  SeekUserFilterRelationInRelationship,
							  SeekUserFilterRelationEngaged,
							  SeekUserFilterRelationMarried,
							  SeekUserFilterRelationComplicated,
							  SeekUserFilterRelationActivelySearching,
							  SeekUserFilterRelationInLove,
							  SeekUserFilterRelationCivilMarried}

	for i := range relationMasks {

		if criteria.RelationsBitMask&relationMasks[i] != 0 {

			param := util.CopyMapStrStr(commonParams)
			param["status"] = strconv.Itoa(i)

			paramMaps = append(paramMaps, param)
		}
	}

	if len(paramMaps) == 0 {

		paramMaps = append(paramMaps, commonParams)

	}

	return paramMaps
}

func getVkErrorIfItWas(jsonResponse string) error {

	var errorResponse struct {
		Error struct {
			ErrorCode int    `json:"error_code"`
			ErrorMsg  string `json:"error_msg"`
		} `json:"error"`
	}

	var err = json.Unmarshal([]byte(jsonResponse), &errorResponse)

	if err == nil && errorResponse.Error.ErrorCode > 0 {

		switch errorResponse.Error.ErrorCode {

		case 6 : return ErrVkTooManyRequestsPerSecond
		default: return ErrVkUndefinedError
		}
	}

	return nil
}

func request(methodName string, params map[string]string) (string, error) {

	u, err := url.Parse(API_METHOD_URL + methodName)
	if err != nil {
		return "", err
	}

	q := u.Query()
	for k, v := range params {
		q.Set(k, v)
	}

	q.Set("access_token", API_ACCESS_TOKEN)
	u.RawQuery = q.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		return "", err
	}

	defer resp.Body.Close()
	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	return string(content), nil
}