Small script that can retrieve CTF stats for a player or from a CTF game

ctfstats.py

import requests
from pyquery import PyQuery as pq
import pandas as pd
import numpy as np
from tqdm.autonotebook import tqdm

# Note: from the imports, this requires pandas, numpy, tqdm, and pyquery (installable with pip or conda)

# classes to collect stats on
ctfClasses = """Archer
Assassin
Chemist
Dwarf
Elf
Engineer
Heavy
Mage
Medic
Necro
Ninja
Pyro
Scout
Soldier""".split("\n")

def extractCTFText(r):
    """Extracts the ctf section of the webpage
    """
    start = r.text.find("<!-- CTF Statistics -->")
    end = r.text.find("<!-- Party Statistics -->", start)
    if end - start > 0:
        return pq(r.text[start:end])
    else:
        return None

def extractClassText(d, ctfClass):
    """Returns the stats section of a particular class"""
    return d(f"div#ctfAdditionalDetailsDisplay{ctfClass.upper()}").html()

def parsePlaytime(time):
    """Converts the playtime to units of days"""
    d = 0
    for div in time.split(' '):
        if div[-1] == 'y':
            d += int(div[:-1]) * 365
        elif div[-1] == 'd':
            d += int(div[:-1])
        elif div[-1] == 'h':
            d += int(div[:-1]) / 24.0
        elif div[-1] == 'm':
            d += int(div[:-1]) / (24.0 * 60)
        elif div[-1] == 's':
            d += int(div[:-1]) / (24.0 * 3600)
        else:
            d += 0
    
    return d
    
def parseClassText(text):
    """Parses a section of a class
    
    Returns:
    - stats: a pandas Series of the class stats
    """
    labels = [li.text[:-2] for li in pq(text)("li")]
    stats = [span.text for span in pq(text)(".stat-kills")]
    stats[0] = parsePlaytime(stats[0])
    stats = [float(x) for x in stats]
    
    if not "HP Restored" in labels:
        labels.append("HP Restored")
        stats.append(0.0)
    
    if not "Headshots" in labels:
        labels.append("Headshots")
        stats.append(0.0)
    
    labels = [l.lower().replace(' ', '_') for l in labels]
    
    return pd.Series({labels[i]: stats[i] for i in range(min(len(labels), len(stats)))})

def getPlayerStats(player):
    """Retrieves the CTF stats of a particular player
    
    Returns:
    - stats: Dictionary of stats for each class as well as an aggregate (accessible with `stats["Total"]`). Returns `None` if 
    there is no player found for that name
    - response: Raw HTTP response
    
    Usage:
    Get the stats of a player
        ```
            stats, response = getPlayerStats("915")
        ```
    Then view stats breakdown:
        ```
            stats["Elf"] # stats for Elf class
            stats["Elf"]["flags_captured"] # flags captured while Elf
            stats["Total"]["kdr"] # overall KDR (please don't look, it's horrible)
        ```
    """
    r = requests.get(f"http://brawl.com/players/{player}")
    if r.status_code == 200:
        ctf = extractCTFText(r)
        if ctf is None:
            return (None, r)
        stats = {c: parseClassText(extractClassText(ctf, c)) for c in ctfClasses}
        overallStats = sum([stats[c] for c in ctfClasses])
        overallStats["kdr"] = overallStats["kills"] / overallStats["deaths"]
        
        stats["Total"] = overallStats.dropna()
        return (stats, r)
    else:
        return (None, r)

# for later use in filtering games
match_server = "1.ctfmatch.brawl.com"

def extractTable(html, overall = True):
    """Gets the stats table from the page
    
    Parameters:
    - overall: True for overall stats. False for player stats.
    """
    d = pq(html)
    index = 0 if overall else 1
    return d("table")[index]

def parseStatTable(table, detailed=False):
    """Takes the table and converts it to a DataFrame"""
    elem = list(table.iterchildren())
    header = list(elem[0].itertext())[1:-1]
    df = pd.DataFrame(None, columns=header)
    
    if detailed:
        for e in elem[1:]:
            text = list(e.itertext())
            text[2:] = [float(value) for value in text[2:]]
            stat_row = pd.Series({z[0]: z[1] for z in zip(header, text)})
            stat_row["kdr"] = stat_row["kills"] / stat_row["deaths"] if stat_row["deaths"] != 0 else 0.0
            df = df.append(stat_row, ignore_index=True)
    else:
        for e in elem[1:]:
            text = list(e.itertext())
            text[1:] = [float(value) for value in text[1:]]
            stat_row = pd.Series({z[0]: z[1] for z in zip(header, text)})
            stat_row["kdr"] = stat_row["kills"] / stat_row["deaths"] if stat_row["deaths"] != 0 else 0.0
            df = df.append(stat_row, ignore_index=True)
    
    return df

def getMostRecentGame():
    r = requests.get("https://www.brawl.com/MPS/MPSStatsCTF.php")
    d = pq(r.text)
    return int(list(d("a")[0].itertext())[0])

def getMatch(game_id):
    r = requests.get("http://brawl.com/MPS/MPSStatsCTF.php", {"game": game_id})
    overalltable = extractTable(r.text)
    detailedTable = extractTable(r.text, overall=False)
    stats = parseStatTable(overalltable)
    detailedStats = parseStatTable(detailedTable, detailed=True)
    detailedStats = detailedStats[detailedStats.playtime > 0.0]
    
    return (stats, detailedStats)

def getOverallGameStats(game_id):
    """Gets the overall statistics from the game that has id `game_id`
    
    Returns:
    - stats: A pandas DataFrame with the rows corresponding to players and the columns corresponding to particular stats
    
    Usage:
    First get the stats table (e.g. for game 246819)
        ```
            stats = getOverallGameStats(246819)
        ```
    Access a particular player's stats
        ```
            player = "SirLeo"
            sirLeoStats = stats[stats.name == player]
        ```
    View particular properties
        ```
            sirLeoStats["flags_captured"] # captured flags
            sirLeoStats["kdr"] # kdr
        ```
    """
    r = requests.get("http://brawl.com/MPS/MPSStatsCTF.php", {"game": game_id})
    overalltable = extractTable(r.text)
    stats = parseTable(overalltable)
    return stats

def parseGameTable(table):
    """Takes the table and converts it to a DataFrame"""
    elem = list(table.iterchildren())
    header = list(elem[0].itertext())[1:-1]
    df = pd.DataFrame(None, columns=header)
    for e in elem[1:]:
        text = list(e.itertext())
        
        if not "ctfmatch" in text[3]:
            continue
        
        stat_row = pd.Series({z[0]: z[1] for z in zip(header, text)})
        df = df.append(stat_row, ignore_index=True)
    
    return df

def getCompetitiveStats(player, n=-1, show_progress=True, save=True, core_stats=True):
    """Aggregates the n most recent games on the match server by class
    
    Parameters:
    - player: The player's stats to retrieve
    - n: Number of most recent games to sift through. Leave n=-1 for all time stats 
         (caution, may take a long time). Default -1
    - show_progress: Show a progress bar. Default True
    - save: If True, saves stats to player-n.csv
    - core_stats: If True, saves only important stats (i.e. pvp, caps/recovs, medic healing, and playtime)
    
    Returns:
    - df: A pandas DataFrame grouped by class (`kit_type`)
    
    Usage:
    Get 40 most recent games for ItalianPenguin, and saves the core stats to 'ItalianPenguin-40.csv'
        ```
            df = getCompetitiveStats("ItalianPenguin", n=40)
            
        ```
    
    """
    res = None
    r = requests.get("https://www.brawl.com/MPS/MPSStatsCTF.php", {"player": player})
    table = extractTable(r.text)
    df = parseGameTable(table)
    
    important_stats = ["playtime", "kills", "deaths", "kdr", "flags_captured", 
                       "flags_recovered", "flags_stolen", "flags_dropped", "damage_dealt", 
                       "damage_received", "hp_restored"]
    
    if n > 0:
        df = df.tail(n)
    try:
        iter_ = tqdm(df["game_id"]) if show_progress else df["game_id"]
    except KeyError:
        print(f"Bad name {player}")
        return None
    
    for game in iter_:
        try: 
            stats = getMatch(game)[1]
            stats = stats[stats.name == player]
            if res is None:
                res = stats.groupby('kit_type').sum()
            else:
                res = res.append(stats.groupby('kit_type').sum())
        except IndexError:
            continue
    
    res = res.groupby('kit_type').sum()
    res['kdr'] = res['kills'] / res['deaths']
    
    if core_stats:
        res = res[important_stats]
    
    if save:
        label = "all-time" if n < 1 else n
        res.to_csv(f"{player}-{label}.csv")
        
    return res