Crunch posts and do some math

exitpoll.py

from urllib2 import urlopen
from BeautifulSoup import BeautifulStoneSoup as Soup

NUM_PAGES = 73
BASE_URL = "http://www.eveonline.com/ingameboard.asp?a=topic&threadID=1414376&page=%d"

TITLES = {
 1: 'AFK/Busy/Available Indicator (CSM)',
 2: 'Accessibility Focus Group',
 3: 'Account Security Enhancements (CSM)',
 4: 'Add login history in your account managment (CSM)',
 5: 'Add scan angle buttons to scanner window (CSM)',
 6: 'Add the feature of switching characters without relogging. (CSM)',
 7: 'Aggression timer length (CSM)',
 8: 'Aggression timer on gate activation (CSM)',
 9: 'Aggression timer problems (CSM)',
 10: 'Alliance Logos, CSM Intervention Requested (CSM)',
 11: 'Alliances and factional warfare (CSM)',
 12: 'Alliances and factional warfare pt2 (CSM)',
 13: 'Allow remote jumpclone removal (csm)',
 14: 'Api improvements (CSM)',
 15: 'Armaggedon Day 2.0 (CSM)',
 16: 'Assault Frigates 4th Bonus (CSM)',
 17: 'Assembly Array Improvements (CSM)',
 18: 'Autopilot settings for Factional Warfare (CSM)',
 19: 'BPO locking changes (CSM)',
 20: 'Balance self-destruction (CSM)',
 21: 'Battle Recorder (CSM)',
 22: 'Black Ops Improvements (CSM)',
 23: 'Black Ops Improvements Part 2 (CSM)',
 24: 'Boost Nighthawk (CSM)',
 25: 'Boost Warfare Links and Revisit Information Warfare (CSM)',
 26: 'Bountyhunting Improvements (CSM)',
 27: 'Bring Logistics Warp Speed In-Line with T2 Ships (CSM)',
 28: 'ERROR - Implemented - Do not vote for this: Bring back zoom buttons (CSM)',
 29: 'Broadcast "In position" improvement (CSM)',
 30: 'Buff Gallente EWAR (CSM)',
 31: 'Cargo hold of ships in hangar (CSM)',
 32: 'Character lock no resale possible (CSM)',
 33: 'Colonies (CSM)',
 34: 'ERROR - On Sisi - Do not vote for this: Colour Stars By: Systems I can jump to (CSM)',
 35: 'Commit to Fixing the UI (CSM)',
 36: 'Contract API (CSM)',
 37: 'Contract Login Update Notification (CSM)',
 38: 'Contract improvements (CSM)',
 39: 'Contract issues (CSM)',
 40: 'Corporation Bookmarks (CSM)',
 41: 'Corporation and Alliance tool overhaul (CSM)',
 42: 'Courier contract improvements (CSM)',
 43: 'Courier contracts made more user-friendly (CSM)',
 44: 'Covert cynosural fields in high security space (CSM)',
 45: 'Destroyer Improvements (CSM)',
 46: 'Destructible outposts (CSM)',
 47: 'Directscan improvement (CSM)',
 48: 'Display account expiration date on character selection screen (CSM)',
 49: 'Docking games fix (CSM)',
 50: 'Docking in Capital Ships (CSM)',
 51: 'Drone implants (CSM)',
 52: 'Drone improvements (CSM)',
 53: 'Drone protocols (CSM)',
 54: 'Drone quirky behaviour (CSM)',
 55: 'Drone stance interface (CSM)',
 56: 'Dynamic Missions (CSM)',
 57: 'Dynamic Missions Part 2 (CSM)',
 58: 'Dynamic agent qualities (CSM)',
 59: 'Easy overloading toggle (CSM)',
 60: 'ERROR - Implemented - Do not vote for this: End ghost (unpaid account) datacore production (CSM)',
 61: 'Eve Nebula (CSM5)',
 62: 'Eve font changes - Progress requested (CSM)',
 63: 'Events and gathering forums (CSM)',
 64: 'Experimental industry issues (CSM)',
 65: 'FW Complex NPCs and Standings (CSM)',
 66: 'Faction warfare - allied forces (CSM)',
 67: 'Factional Warfare - CCP Inaction Towards Bugs/Exploits (CSM)',
 68: 'Factional Warfare - Complex Spawning Part 2 (CSM)',
 69: 'Factional Warfare - Lack of Development Part 2 (CSM)',
 70: 'Factional Warfare - NPC Review (CSM)',
 71: 'Factional Warfare - Request for Information (CSM)',
 72: 'Factional Warfare \xe2\x80\x93 Focus & Goals (CSM)',
 73: 'Factional warfare - complex spawning (CSM)',
 74: 'ERROR - Duplicate - Vote for #72 instead - Factional warfare - focus and goals (CSM)',
 75: 'Fix kill mails (CSM)',
 76: 'Fix plex keys (CSM)',
 77: 'Forum Censorship (CSM)',
 78: 'Forum fixes (CSM)',
 79: 'Forum option to toggle profanity filter (CSM)',
 80: 'Forum profanity filter (CSM)',
 81: 'Forum reworking (CSM)',
 82: 'Funky POS recommendation (CSM)',
 83: 'General discussion on mining (CSM)',
 84: 'Giving carriers ability to repair drones',
 85: 'Grouping of non-weapon systems (CSM)',
 86: 'Heat Reduction Rigs (CSM)',
 87: 'History of who added/kick character from Corporation',
 88: 'Improve Overload User Interface (CSM)',
 89: 'Improve People & Places search results for Solar Systems. (CSM)',
 90: 'Improve refresh time on loading Corporate Hangar contents. (CSM)',
 91: 'Improvements to the F11 navigation panel (CSM)',
 92: 'In-Game Account Expiration Countdown (CSM)',
 93: 'Increase forum signature file size limitations (CSM)',
 94: 'Ingame Events Menu (CSM)',
 95: 'Issue when Copying and pasting links in Chat (CSM)',
 96: 'Kill rights should be transferable (CSM)',
 97: 'Larger HUD bracket for caps and supercaps (CSM)',
 98: 'Localise the CSM Output (CSM)',
 99: "Looting from wreck you didn't create = looter flagged to (wreck) killer in Empire (CSM)",
 100: 'MOTD for fleet/gang channels (CSM)',
 101: 'Machinima support in Walking in Stations (CSM)',
 102: 'Macro hauler solution (CSM)',
 103: 'Make Advanced Ship Assembly Arrays useful (CSM)',
 104: 'Make Certain Factions More Valid for Mission Running - Part 2 (CSM)',
 105: 'Make More Faction Items Tradable on Market (CSM)',
 106: 'Make guns continue firing at previous target after reload (CSM)',
 107: 'Marks against acc',
 108: 'Mercenary implementation (CSM)',
 109: 'Mining crystals change color of mining laser beam (CSM)',
 110: 'Mission monopoly on standing (CSM)',
 111: 'Missions under review (CSM)',
 112: 'Modifiable corporation orders (CSM)',
 113: 'Modify or Remove Learning Skills (CSM)',
 114: 'Modular Starbase (CSM)',
 115: 'More Control Over Medals (CSM)',
 116: 'More crossfaction ships (CSM)',
 117: 'More orbit range options (CSM)',
 118: 'Move Recover Probes Away From Analyze Button (CSM)',
 119: 'Musical instruments in Ambulation (CSM)',
 120: 'Neutral orcas in high sec wars (CSM)',
 121: 'ORE Faction Control Tower (CSM)',
 122: 'Outpost Dock Radii (CSM)',
 123: 'Overhaul of roles and grantable roles system (CSM)',
 124: 'Personal hangar improvements (CSM)',
 125: 'Personal wallet divisions (CSM)',
 126: 'Planetary Interaction Omnibus',
 127: 'Possibly Practical POS Performance Proposal (CSM)',
 128: 'Precision heavy missiles (CSM)',
 129: 'Prevent cloaked ships in same gang/fleet from decloaking each other',
 130: 'Public POS arrays (CSM)',
 131: 'Redesign sensor booster and ECCM effects (CSM)',
 132: 'Regular revi',
 133: 'Remote jumpclone destruction (CSM)',
 134: 'Remote repair issues (CSM)',
 135: 'Remote repping and aggression (CSM)',
 136: 'Remotely stack items in hangar (CSM)',
 137: 'Remove pause for JC and implants (CSM)',
 138: 'Re',
 139: 'Replace agent avatar with corporate logo in standings ',
 140: 'Retweak Minmatar Ewar Ship Web Abilities (CSM)',
 141: 'Revamp magnetometric sites (CSM)',
 142: 'Review destroyers (CSM)',
 143: 'Revisit the standing matrix (CSM)',
 144: 'Rigs need attention (CSM)',
 145: 'Save and Reuse Scan Probe Patterns (CSM)',
 146: 'Scan-able wrecks&containers for the salvager pr',
 147: 'Scanner change has created an imbalance (CSM)',
 148: 'Scanning Colors (CSM)',
 149: 'Sell multiple items at once (CSM)',
 150: 'Sell order issues (CSM)',
 151: "Semi-permanently show pilot's name in space (CSM)",
 152: 'Send ',
 153: 'Share dividends payout value',
 154: 'Shared Corporation Bookmark',
 155: 'Shares, dividends and stocks (CSM)',
 156: 'Ship Crews (CSM)',
 157: 'Ship fittings visible via Show Info without boarding ship. (CSM)',
 158: 'Ship maintenance bays revisited (CSM)',
 159: 'Show Drone Damage in Logs (CSM)',
 160: 'Show Implants on POD killmails (CSM)',
 161: 'Show bookmarks with mark',
 162: 'Show damaged drones in drone bay (CSM)',
 163: 'Small anchorable structures (CSM)',
 164: 'Smartbomb overhaul (CSM)',
 165: 'Sort ctrl-tab window list by last activation instead of time created (CSM)',
 166: 'Sound Issues (CSM)',
 167: 'Sound engine woes (CSM)',
 168: 'Sources of Mineral Investigation (CSM)',
 169: 'Spanish localization (CSM)',
 170: 'Split the show Effects setting into separate options. (CSM)',
 171: 'Spreading Out Missionrunners (CSM)',
 172: 'Standing recovery (CSM)',
 173: 'Station Owners Unrent Offices (CSM)',
 174: 'Store bookmark folders server-side. (CSM)',
 175: 'Stuff about character tra',
 176: 'Suicide Ganking Part 3 (CSM)',
 177: 'ERROR - On Sisi - Do not vote for this: Targeting from Broadcast History (CSM)',
 178: 'Third bloodline background (CSM)',
 179: 'Titan bridge range (CSM)',
 180: 'Toggle Logging On/Off for each Chat Channel (CSM)',
 181: 'Tracking for Fighters lost in combat',
 182: 'UI Add support for POS Module Status in Overview (CSM)',
 183: 'UI Client remembers passwords between session changes. (CSM)',
 184: 'UI Improvement to "Change Clone"',
 185: 'UI chat channel pilot ',
 186: 'UI combat notifications (CSM)',
 187: 'UI import/UI distance status change (CSM)',
 188: 'UI import/UI electro',
 189: 'UI import/UI hotkeys (CSM)',
 190: 'UI minium size of HUD elements (CSM)',
 191: 'UI personal assets (CSM)',
 192: 'UI skills page issues (CSM)',
 193: 'UI slash commands (CSM)',
 194: 'UI visibility status of modules (CSM)',
 195: 'Unanchoring abandoned, offline POSses (CSM)',
 196: 'Unnerf the bio length (CSM)',
 197: 'User Interface - Big Wins, Fan Favorites and Low Hanging Fruit (CSM)',
 198: 'Visible Aggression Indicat',
 199: 'War-dec mechanics (CSM)',
 200: 'Watch list and broadcasts (CSM)',
 201: 'Whatever happened to the industry expansion (CSM)',
 202: 'Wild 0.0 (CSM)',
 203: 'Wreck outposts (CSM)'}

processed, unprocessed = 0, 0
poll = {}

for index in range(1, NUM_PAGES):
    try:
        data = open('exitpoll.cache/%d.html' % index).read()
    except IOError:
        data = urlopen(BASE_URL % index).read()
        open('exitpoll.cache/%d.html' % index, 'w').write(data)

    try:
        posts = Soup(data).findAll('div', {'id': 'forumSignature'})
    except Exception as e:
        print "bad page:", index, e
        continue

    for post in posts:
        try:
            for piece in [post] if post.string else post.fetchText():
                text = piece.string
                if not text or ',' not in text:
                    continue

                parts = text.split(',')
                if len(parts) == 1:
                    continue

                votes = []
                for part in parts[1:]:
                    try:
                        votes.append(int(part.strip().strip('.!?:')))
                    except ValueError:
                        pass

                author = parts[0]
                try:
                    gotcha = author.split()
                    votes.insert(0, int(gotcha[-1]))
                    author = " ".join(gotcha[:-1])
                except ValueError:
                    pass
                
                if votes:
                    poll[author] = votes

            processed += 1
        except Exception as e:
            print e, post
            bad_post = post
            unprocessed += 1

items = {}
for author, votes in poll.iteritems():
    weight =  min(3, 20.0 / len(votes))
    for vote in votes:
        items.setdefault(vote, 0)
        items[vote] += weight

print "Item #\tVotes\tItem description"
for value, key in sorted(((value, key) for key, value in items.iteritems()), reverse=True):
    print "%s\t%.1f\t%s" % (key, value, TITLES.get(key))

if unprocessed:
    print 'good:', processed, 'bad:', unprocessed, 'ratio: %.2f' % unprocessed / float(processed)