maimainet_musicscore.py

#!python2
#coding:utf8

#Usage:
#Install Python 2.7
#Execute `pip install requests lxml` in Powershell/cmd/Terminal
#Supply your own Sega ID & password in `username_password.txt`
#Run the script
#If errors occur, just re-run the script

import requests,json,os
from lxml import html

def check_if_ok(result):
	if not result.ok:
		print result.url,result.status_code,result.reason
		exit()

def post(session_requests,post_url,post_data,referer_url):
	useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
	headers={'user-agent':useragent,'referer':referer_url}
	result=session_requests.post(post_url,data=post_data,headers=headers)
	check_if_ok(result)
	return result

def get(session_requests,get_url,referer_url):
	useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
	headers={'user-agent':useragent,'referer':referer_url}
	result=session_requests.get(get_url,headers=headers)
	check_if_ok(result)
	return result



session_requests=requests.session()

#login
print 'login'
username,password=open('username_password.txt').read().split('\n')
login_page_url='https://maimai-net.com/maimai-mobile/'
login_post_url='https://maimai-net.com/maimai-mobile/index/submit/'
login_post_data={
	'segaId':username,
	'passWd':password,
	'x':0,
	'y':0
}
post(session_requests,login_post_url,login_post_data,login_page_url)

#musicscore
difficulties=['easy','basic','advanced','expert','master','remaster','utage']
baseurl='https://maimai-net.com/maimai-mobile/music/%sGenre/'
for i in difficulties:
	print i
	if os.path.exists('music_scores_%s.json'%i): continue
	response=get(session_requests,baseurl%i,'https://maimai-net.com/maimai-mobile/music/')
	response.encoding='utf8'
	content=response.text
	tree=html.fromstring(content)
	data_diff=tree.xpath('/html/body/div[@class="data_%s"]'%i)[0]
	accordion=data_diff.xpath('div[@id="accordion"]')[0]
	last_played_date=[x.strip() for x in accordion.xpath('ul/li/div[1]/text()')]
	play_count=[0 if x==u'---回' else int(x[:-1]) for x in accordion.xpath('ul/li/table/tr[1]/td[2]/text()')]
	high_score=[0 if x==u'---' else int(x.replace(',','')) for x in accordion.xpath('ul/li/table/tr[2]/td[2]/text()')]
	high_sync_rate=[0 if x==u'---%' else float(x[:-1]) for x in accordion.xpath('ul/li/table/tr[3]/td[2]/text()')]
	music_id=accordion.xpath('ul/li/div[2]/form/input[@name="musicId"]/@value')
	keys=['last_played_date','play_count','high_score','high_sync_rate','music_id']
	if i=='utage':
		music_diff=accordion.xpath('ul/li/div[2]/form/input[@name="musicDiff"]/@value')
		keys.append('music_diff')
	music_score_data=[{x:eval(x+'[y]') for x in keys} for y in range(len(music_id))]
	h3=accordion.xpath('h3')
	for j in range(len(h3)):
		title=h3[j].text.strip()
		achievement_rate=0
		badges=[]
		played_flag=title==''
		if played_flag:
			title=h3[j].xpath('div[1]/text()')[0]
			achievement_rate=float(h3[j].xpath('div[2]/table/tr/td[1]/text()')[1].strip()[:-1])
			badges=[x.split('icon_')[1].split('.png')[0] for x in h3[j].xpath('div[2]/table/tr/td[2]/img/@src')]
		music_score_data[j].update({x:eval(x) for x in ['title','achievement_rate','badges']})

	open('music_scores_%s.json'%i,'w').write(json.dumps(music_score_data,indent=4,ensure_ascii=False).encode('utf8'))

maimainet_musicscore_ranking.py

#!python2
#coding:utf8

#Usage:
#Install Python 2.7
#Execute `pip install requests lxml` in Powershell/cmd/Terminal
#Supply your own Sega ID & password in `username_password.txt`
#Run the script
#If errors occur, just re-run the script

import requests,json,os,glob,re
from lxml import html

def check_if_ok(result):
	if not result.ok:
		print result.url,result.status_code,result.reason
		exit()

def post(session_requests,post_url,post_data,referer_url):
	useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
	headers={'user-agent':useragent,'referer':referer_url}
	result=session_requests.post(post_url,data=post_data,headers=headers)
	check_if_ok(result)
	return result

def get(session_requests,get_url,referer_url):
	useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
	headers={'user-agent':useragent,'referer':referer_url}
	result=session_requests.get(get_url,headers=headers)
	check_if_ok(result)
	return result



session_requests=requests.session()

#login
print 'login'
username,password=open('username_password.txt').read().split('\n')
login_page_url='https://maimai-net.com/maimai-mobile/'
login_post_url='https://maimai-net.com/maimai-mobile/index/submit/'
login_post_data={
	'segaId':username,
	'passWd':password,
	'x':0,
	'y':0
}
post(session_requests,login_post_url,login_post_data,login_page_url)

#musicscore_ranking
charts=['easy','basic','advanced','expert','master','remaster','utage']
baseurl='https://maimai-net.com/maimai-mobile/music/%sGenre/'
for i in charts:
	print i
	if not os.path.exists(i): os.mkdir(i)
	musicscore_response=get(session_requests,baseurl%i,'https://maimai-net.com/maimai-mobile/music/')
	musicscore_response.encoding='utf8'
	musicscore_content=musicscore_response.text
	tree=html.fromstring(musicscore_content)
	data_diff=tree.xpath('/html/body/div[@class="data_%s"]'%i)[0]
	accordion=data_diff.xpath('div[@id="accordion"]')[0]
	keys=['difficulty','musicDiff','musicId']
	data={x: accordion.xpath('ul/li/div[2]/form/input[@name="%s"]/@value'%x) for x in keys}
	ranking_post_data_list=[{y: eval('data["%s"][x]'%y) for y in keys} for x in range(len(data['musicId']))]
	for j in ranking_post_data_list:
		filename_prefix=j['musicId']+'_'+j['musicDiff'] if i=='utage' else j['musicId']
		print filename_prefix
		if glob.glob(i+'/'+filename_prefix+'_*.json')!=[]: continue
		post(session_requests,"https://maimai-net.com/maimai-mobile/music/type/",j,baseurl%i)
		ranking_data={}
		for k in ['friend/','']:
			ranking_type='all' if k=='' else 'friend'
			ranking_response=get(session_requests,'https://maimai-net.com/maimai-mobile/music/ranking/'+k,'https://maimai-net.com/maimai-mobile/music/rankingType')
			ranking_response.encode='utf8'
			ranking_content=ranking_response.text
			tree=html.fromstring(ranking_content)
			music_title=tree.xpath('/html/body/div[@class="status5"]/p[2]/text()')[0]
			invalid_chars=r'\/:*?"<>|'
			music_title=re.sub('[\\%s]'%('\\'.join(invalid_chars)),'_',music_title)
			table=tree.xpath('/html/body/div[@class="status5"]/div[@class="data_body"]/table/tr')
			ranking_data[ranking_type]=[]
			for tr in table:
				place=tr.xpath('td[1]/text()')
				if place==[]: place=int(tr.xpath('td[1]/img/@src')[0].split('/ranking')[1][0])
				else:place=int(place[0])
				username=tr.xpath('td[3]/text()')[0].strip()
				achievement_rate=float(tr.xpath('td[4]/text()')[0].strip()[:-1])
				score=int(tr.xpath('td[5]/text()')[0].strip().replace(',',''))
				ranking_data[ranking_type].append({x: eval(x) for x in ['place','username','achievement_rate','score']})
			open(i+'/%s_%s.json'%(filename_prefix,music_title),'w').write(json.dumps(ranking_data,indent=4,ensure_ascii=False).encode('utf8'))

maimainet_playlog.py

#!python2
#coding:utf8

#Usage:
#Install Python 2.7
#Execute `pip install requests lxml` in Powershell/cmd/Terminal
#Supply your own Sega ID & password in `username_password.txt`
#Run the script
#If errors occur, just re-run the script

import requests,json,os
from lxml import html

def check_if_ok(result):
	if not result.ok:
		print result.url,result.status_code,result.reason
		exit()

def post(session_requests,post_url,post_data,referer_url):
	useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
	headers={'user-agent':useragent,'referer':referer_url}
	result=session_requests.post(post_url,data=post_data,headers=headers)
	check_if_ok(result)
	return result

def get(session_requests,get_url,referer_url):
	useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
	headers={'user-agent':useragent,'referer':referer_url}
	result=session_requests.get(get_url,headers=headers)
	check_if_ok(result)
	return result

def safe_get_xpath(tree,xpath):
	tmp=tree.xpath(xpath)
	if tmp==[]: return ''
	return tmp[0]



session_requests=requests.session()

#login
print 'login'
username,password=open('username_password.txt').read().split('\n')
login_page_url='https://maimai-net.com/maimai-mobile/'
login_post_url='https://maimai-net.com/maimai-mobile/index/submit/'
login_post_data={
	'segaId':username,
	'passWd':password,
	'x':0,
	'y':0
}
login_result=post(session_requests,login_post_url,login_post_data,login_page_url)

#playlog
page=1
print 'playlog page',page
playlog=[]
playlog_page_url='https://maimai-net.com/maimai-mobile/playLog/'
playlog_page_result=get(session_requests,playlog_page_url,'https://maimai-net.com/maimai-mobile/home/')
while True:
	if os.path.exists('playlog_page%d.json'%page):
		page+=1
		continue
	playlog_page_result.encoding='utf8'
	playlog_page_content=playlog_page_result.text
	tree=html.fromstring(playlog_page_content)
	status5=tree.xpath('/html/body/div[@class="status5"]')[0]
	accordion=status5.xpath('div[@id="accordion"]')[0]
	play_count_current_page=int(accordion.xpath('count(*)'))/2
	for i in range(play_count_current_page):
		print 'play'
		play=accordion.xpath('ul[%d]'%(i+1))[0]
		date_time,shop_name=accordion.xpath('h3[%d]/text()'%(i+1))
		special_play_type=safe_get_xpath(play,'li[1]/div[@class="red text_c"]/text()')
		other_players=play.xpath('li[1]/form/button/text()')
		track_count_current_play=int(play.xpath('count(li)'))-1
		tracks=[]
		for j in range(track_count_current_play):
			print 'track'
			track=accordion.xpath("ul[%d]/li[%d]"%(i+1,j+2))[0]
			playIdx=int(track.xpath('div[@class="clearfix text_r"]/form/input')[0].value)
			playlog_detail_post_url='https://maimai-net.com/maimai-mobile/playLog/detail'
			playlog_detail_post_data=dict(playIdx=playIdx)
			playlog_detail_result=post(session_requests,playlog_detail_post_url,playlog_detail_post_data,playlog_page_url)
			playlog_detail_result.encoding='utf8'
			playlog_detail_content=playlog_detail_result.text
			tree=html.fromstring(playlog_detail_content).xpath('/html/body/div[@class="status5"]/div/ul/li')[0]
			track_number,position=tree.xpath('span[1]/text()')[0].rsplit(' ',1)
			difficulty=tree.xpath('span[2]/text()')[0][1:-1]
			title=tree.xpath('div[@class="playdata_music_title"]/text()')[0]
			achievement_rate=float(tree.xpath('div[@class="result_icon_block3 text_c f_l"]/text()')[1].split()[0].split(u'\uff1a')[1][:-1])
			total_stats=[int(x.replace(',','')) for x in tree.xpath('table[1]/tr[2]/*/text()')[1:]]
			next_rank_text=tree.xpath('table[1]/tr[3]/td[2]/text()')[0]
			if next_rank_text==u"−−−−−": next_rank=0
			else: next_rank=int(next_rank_text.replace(',',''))
			max_combo,notecount=map(int,tree.xpath('table[1]/tr[4]/td[2]/text()')[0].split(' / '))
			detailed_stats={y[0]:[int(z.replace(',','')) for z in y[1:]] for y in [x.xpath('td/text()') for x in tree.xpath('table[2]/tr')]}
			if title=='Link':
				if notecount=={'EASY':57,'BASIC':124,'ADVANCE':174,'EXPERT':314,'MASTER':308}[difficulty]: title='Link(JOYPOLIS)'
				else: title='Link(niconico)'
			current_track={x:eval(x) for x in ['track_number','position','difficulty','title','achievement_rate','total_stats','next_rank','max_combo','detailed_stats']}
			tracks.append(current_track)
		play_dict={x:eval(x) for x in ['date_time','shop_name','special_play_type','other_players','tracks']}
		playlog.append(play_dict)
	open('playlog_page%d.json'%page,'w').write(json.dumps(playlog,indent=4,ensure_ascii=False).encode('utf8'))
	if u'次へ' not in playlog_page_content: break
	page+=1
	print 'playlog page',page
	playlog_page_result=get(session_requests,'https://maimai-net.com/maimai-mobile/playLog/next/',playlog_page_url)
	playlog_page_url=playlog_page_result.url