Missing ending fragments

Missing%20ending%20fragments.ipynb

{"nbformat_minor": 0, "cells": [{"execution_count": 2, "cell_type": "code", "source": "import binascii\n\nfrom operator import attrgetter, itemgetter\nfrom moztelemetry import get_pings, get_pings_properties, get_one_ping_per_client, get_clients_history\nfrom collections import defaultdict\n\n%pylab inline", "outputs": [{"output_type": "stream", "name": "stdout", "text": "Populating the interactive namespace from numpy and matplotlib\n"}], "metadata": {"scrolled": true, "collapsed": false, "trusted": true}}, {"execution_count": 3, "cell_type": "code", "source": "sc.defaultParallelism", "outputs": [{"execution_count": 3, "output_type": "execute_result", "data": {"text/plain": "80"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"source": "Get all main pings for a set of recent build-ids:", "cell_type": "markdown", "metadata": {}}, {"execution_count": 4, "cell_type": "code", "source": "pings = get_pings(sc,\n                  app=\"Firefox\",\n                  channel=\"nightly\",\n                  build_id=(\"20150710000000\", \"20150717999999\"),\n                  doc_type=\"main\",\n                  schema=\"v4\")", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"source": "Take only 10% of nightly clients:", "cell_type": "markdown", "metadata": {}}, {"execution_count": 5, "cell_type": "code", "source": "def sample(ping):\n    client_id = ping.get(\"clientId\", None)\n    return client_id and binascii.crc32(ping[\"clientId\"]) % 100 < 10\n\nsampled = pings.filter(sample)", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"source": "Get a subset of fields:", "cell_type": "markdown", "metadata": {}}, {"execution_count": 6, "cell_type": "code", "source": "subset = get_pings_properties(sampled, [\"clientId\",\n                                        \"meta/documentId\",\n                                        \"environment/system/os/name\",\n                                        \"payload/info/reason\",\n                                        \"payload/info/sessionId\",\n                                        \"payload/info/subsessionId\",\n                                        \"payload/info/previousSessionId\",\n                                        \"payload/info/previousSubsessionId\",\n                                        \"payload/info/subsessionCounter\",\n                                        \"payload/info/profileSubsessionCounter\"])", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"source": "Group fragments by client and dedupe by documentId:", "cell_type": "markdown", "metadata": {}}, {"execution_count": 7, "cell_type": "code", "source": "def dedupe_and_sort(group):\n    key, history = group\n    \n    seen = set()\n    result = []\n    \n    for fragment in history:\n        id = fragment[\"meta/documentId\"]\n        if id in seen:\n            continue\n            \n        seen.add(id)\n        result.append(fragment)\n    \n    result.sort(key=itemgetter(\"payload/info/profileSubsessionCounter\"))\n    return result\n\ngrouped = subset.groupBy(lambda x: x[\"clientId\"]).map(dedupe_and_sort).collect()", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"source": "**< Digression>** What's the percentage of clients that have at least one pair of fragments with different documentIds but the same profileSubsessionCounter?", "cell_type": "markdown", "metadata": {}}, {"execution_count": 8, "cell_type": "code", "source": "def duplicate_pssc(grouped):\n    dupes = 0\n    dupe_clients = set()\n\n    for history in grouped:\n        counts = defaultdict(int)\n\n        for fragment in history:\n            key = fragment[\"payload/info/profileSubsessionCounter\"]\n            counts[key] += 1\n\n        for _, v in counts.iteritems():\n            if v > 1:\n                dupes += 1\n                dupe_clients.add(history[0][\"clientId\"])\n                break\n\n    print 100.0*dupes/len(grouped)\n    return dupe_clients\n    \ndupe_clients = duplicate_pssc(grouped)", "outputs": [{"output_type": "stream", "name": "stdout", "text": "3.10894480306\n"}], "metadata": {"scrolled": true, "collapsed": false, "trusted": true}}, {"source": "**< /Digression\\>** Let's remove those clients to be safe.", "cell_type": "markdown", "metadata": {}}, {"execution_count": 9, "cell_type": "code", "source": "dd_grouped = filter(lambda h: h[0][\"clientId\"] not in dupe_clients, grouped)", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"source": "Given the set of chain breaks in consecutive sessions, how many of them are due to missing starting/ending fragments?", "cell_type": "markdown", "metadata": {}}, {"execution_count": 41, "cell_type": "code", "source": "def missing(grouped, cmp, reason=\"\", debug=False):\n    cmp_missing = 0\n    other_missing = 0\n    \n    for history in grouped:    \n        last_session_id = history[-1][\"payload/info/sessionId\"]\n\n        for i in range(1, len(history)):\n            curr_fragment = history[i]\n            current_pss_counter = curr_fragment[\"payload/info/profileSubsessionCounter\"]\n            \n            prev_fragment = history[i - 1]\n            prev_pss_counter = prev_fragment[\"payload/info/profileSubsessionCounter\"]\n\n            # Ignore fragments from the last session as it might not have yet completed\n            if curr_fragment[\"payload/info/sessionId\"] == last_session_id:\n                break\n            \n            # Is a fragment missing? Here we are considering only chain breaks between two consecutive sessions\n            if prev_pss_counter + 1 != current_pss_counter and \\\n               prev_fragment[\"payload/info/sessionId\"] == curr_fragment[\"payload/info/previousSessionId\"]:\n                    \n                # Ignore fake missing fragments\n                if prev_fragment[\"payload/info/reason\"] in (\"aborted-session\", \"shutdown\") and \\\n                   curr_fragment[\"payload/info/subsessionCounter\"] == 1:\n                    continue\n                    \n                if cmp(prev_fragment, curr_fragment):\n                    cmp_missing += 1\n                else:\n                    other_missing += 1            \n\n    total_missing = cmp_missing + other_missing\n    frac = 100.0*cmp_missing/total_missing\n    \n    if debug:\n        print \"CMP {}, Other {}, Total {}\".format(cmp_missing, other_missing, total_missing)\n        print \"{:.2f}% of chain breaks are due to {} fragments\".format(frac, reason)\n    \n    return frac\n\ndef ending_cmp(prev, curr):\n    # Are one or more of the ending fragments missing?\n    return prev[\"payload/info/reason\"] not in (\"aborted-session\", \"shutdown\")\n    \ndef starting_cmp(prev, curr):\n    # Are one or more starting fragments missing?\n    return curr[\"payload/info/subsessionCounter\"] != 1\n\nending = missing(dd_grouped, ending_cmp, \"ending\", debug=True)\nstarting = missing(dd_grouped, starting_cmp, \"starting\", debug=True)", "outputs": [{"output_type": "stream", "name": "stdout", "text": "CMP 29, Other 78, Total 107\n27.10% of chain breaks are due to ending fragments\nCMP 78, Other 29, Total 107\n72.90% of chain breaks are due to starting fragments\n"}], "metadata": {"scrolled": false, "collapsed": false, "trusted": true}}, {"execution_count": null, "cell_type": "code", "source": "", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}], "nbformat": 4, "metadata": {"kernelspec": {"display_name": "Python 2", "name": "python2", "language": "python"}, "language_info": {"mimetype": "text/x-python", "nbconvert_exporter": "python", "version": "2.7.9", "name": "python", "file_extension": ".py", "pygments_lexer": "ipython2", "codemirror_mode": {"version": 2, "name": "ipython"}}}}