抓取网易云用户的关注和粉丝并写入到数据库

a

#!/bin/env ruby
require 'json'
require 'sqlite3'

USER_ID = 0

DATABASE_PATH = './a.db'
LIMIT = 100

# @param text [String]
# @return [String]
def encrypt(text)
  process = IO.popen ['node', '/home/bczhc/bundle.js', text]
  read = process.read
  process.close

  fail if $? == nil || $?.exitstatus != 0 || read == nil
  read
end

# @param offset [Integer]
# @param user_id [Integer]
# @return [String]
def fetch_follower(offset, user_id)
  obj = {
    "limit" => LIMIT,
    "offset" => offset,
    "total" => false,
    "userId" => user_id,
  }
  params = encrypt JSON(obj).to_str

  cmd = %{curl --silent -H 'Content-Type: application/x-www-form-urlencoded' -X POST -d '#{params}' 'https://music.163.com/weapi/user/getfolloweds'}
  `#{cmd}`
end

def fetch_following(offset, user_id)
  obj = {
    "limit" => LIMIT,
    "offset" => offset,
    "total" => false,
    "uid" => user_id,
  }
  params = encrypt JSON(obj).to_str

  cmd = %{curl --silent -H 'Content-Type: application/x-www-form-urlencoded' -X POST -d '#{params}' 'https://music.163.com/weapi/user/getfollows/#{user_id}'}
  `#{cmd}`
end

class User
  attr_accessor :id, :name, :follower_num, :following_num, :gender, :signature

  def initialize(id, name, follower_num, following_num, gender, signature)
    @id = id
    @name = name
    @follower_num = follower_num
    @following_num = following_num
    @gender = gender
    @signature = signature
  end

  # @return [User]
  # @param json [JSON]
  def self.from_json(json)
    gender = {
      1 => :male,
      2 => :female,
    }[json["gender"]]

    User.new(
      json["userId"],
      json["nickname"],
      json["followeds"],
      json["follows"],
      gender,
      json["signature"]
    )
  end
end

def fetch_followers_count (user_id)
  response = `curl --silent 'https://music.163.com/user/follows?id=#{user_id}'`
  captures = response.scan /<strong id="fan_count">(\d*)<\/strong>/
  captures[0][0].to_i
end

def fetch_following_count(user_id)
  response = `curl --silent 'https://music.163.com/user/follows?id=#{user_id}'`
  captures = response.scan /id="follow_count">(\d*)<\/strong>/
  captures[0][0].to_i
end

# @return [Hash<Symbol, (User|Integer)>]
def parse_user_json(json)
  user = User.from_json json
  time = json["time"]
  {
    "user" => user,
    "time" => time
  }
end

# @return [Array<Hash<Symbol, (User|Integer)>>]
def fetch_user_followers(user_id)
  count = 0
  total = fetch_followers_count user_id
  return [] if total == 0

  arr = []

  offset = 0
  loop do
    # @type [JSON]
    json = JSON(fetch_follower(offset, user_id))
    fail if json["code"] != 200

    offset += LIMIT

    # @type [JSON]
    followers = json["followeds"]
    followers.each do |user|
      arr.push parse_user_json(user)
      count += 1
      puts "#{count}/#{total}"
    end

    break unless json["more"]

    sleep 0.2
  end

  arr
end

# @return [Array<Hash<Symbol, (User|Integer)>>]
def fetch_user_following(user_id)
  count = 0
  total = fetch_following_count user_id
  return [] if total == 0

  arr = []

  offset = 0
  loop do
    # @type [JSON]
    json = JSON(fetch_following(offset, user_id))
    fail if json["code"] != 200

    offset += LIMIT

    # @type [JSON]
    following = json["follow"]
    following.each do |user|
      arr.push parse_user_json(user)
      count += 1
      puts "#{count}/#{total}"
    end

    break unless json["more"]

    sleep 0.2
  end

  arr
end

def gender_convert(gender)
  type = gender.class
  if type == Symbol
    return 1 if gender == :male
    return 2 if gender == :female
    throw "Undefined gender"
  end
  if type == Integer
    return :male if gender == 1
    return :female if gender == 2
    throw "Undefined gender"
  end
end

class Type
  FOLLOWING = 1
  FOLLOWED = 2
end

class Relation
  attr_accessor :user, :target_user, :type, :time

  # @param user_id [Integer]
  # @param target_user [User]
  # @param type [Integer]
  # @param time [Integer]
  def initialize(user_id, target_user, type, time)
    @user = user_id
    @target_user = target_user
    @type = type
    @time = time
  end
end

# @return [Array<Relation>]
def fetch_user_relations(user_id)
  following = fetch_user_following user_id
  followed = fetch_user_followers user_id

  relations = []

  following.each do |obj|
    # @type [User]
    target_user = obj['user']
    time = obj['time']
    relations.push Relation.new(user_id, target_user, Type::FOLLOWING, time)
  end

  followed.each do |obj|
    # @type [User]
    target_user = obj['user']
    time = obj['time']
    relations.push Relation.new(user_id, target_user, Type::FOLLOWED, time)
  end

  relations
end

class Database
  # @param path [String]
  def initialize(path)
    @database = SQLite3::Database.new path

    @database.execute <<-EOF
CREATE TABLE IF NOT EXISTS user
(
    id            INTEGER PRIMARY KEY,
    name          TEXT    NOT NULL,
    followers_num INTEGER NOT NULL,
    following_num INTEGER NOT NULL,
    -- 1: male, 2: female
    gender        INTEGER,
    signature     TEXT
)
    EOF

    @database.execute <<-EOF
CREATE TABLE IF NOT EXISTS relation
(
    user        INTEGER NOT NULL,
    target_user INTEGER NOT NULL,
    -- 1: following, 2: followed by
    type        INTEGER NOT NULL,
    "time"      INTEGER NOT NULL
)
    EOF

    @insert_user_stmt = @database.prepare <<-EOF
INSERT INTO user (id, name, followers_num, following_num, gender, signature)
VALUES (?, ?, ?, ?, ?, ?)
    EOF

    @insert_relation_stmt = @database.prepare <<-EOF
INSERT INTO relation (user, target_user, type, "time")
VALUES (?, ?, ?, ?)
    EOF
  end

  # @param user [User]
  def insert_user(user)
    binds = [
      user.id,
      user.name,
      user.follower_num,
      user.following_num,
      gender_convert(user.gender),
      user.signature
    ]
    @insert_user_stmt.execute binds
  end

  # @param relation [Relation]
  def insert_relation(relation)
    binds = [
      relation.user,
      relation.target_user.id,
      relation.type,
      relation.time
    ]
    @insert_relation_stmt.execute binds
  end

  def begin_transaction
    @database.execute 'BEGIN TRANSACTION'
  end

  def commit
    @database.execute 'COMMIT'
  end
end

# @type [Array<Relation>]
all_relations = []
# @type [Array<User>]
all_users = []

relations = fetch_user_relations USER_ID
relations.each do |x|
  all_relations.push x
  all_users.push x.target_user
end

# all_users.dup.filter { |x| x.following_num <= 10000 && x.follower_num <= 10000 }.each do |user|
#   relations = fetch_user_relations user.id
#   relations.each do |x|
#     all_relations.push x
#     all_users.push x.target_user
#   end
# end

tmp = Hash.new
all_users.each do |user|
  unless tmp.has_key? user.id
    tmp[user.id] = user
  end
end
all_users.clear
tmp.each_value { |x| all_users.push x }

puts "Total users: #{all_users.size}"
puts "Total relations: #{all_relations.size}"

database = Database.new DATABASE_PATH
database.begin_transaction
relations.each { |x| database.insert_relation x }
all_users.each { |x| database.insert_user x }
database.commit

intro.md

要修改的值：USER_ID, DATABASE_PATH, JS_FILE

其中JS_FILE由 https://gist.github.com/bczhc/4e7c6a42a27dc464e16caa1afdfa289a 获取

GEM依赖：

• sqlite3

但是由于网页端API的限制，粉丝数好像最多只能拿到一千条。没办法了。