Skip to content

Instantly share code, notes, and snippets.

@Uberi

Uberi/e10s-experiment-beta45-withoutaddons-2016-02-17.ipynb

Last active February 24, 2016 17:15
Show Gist options
  • Save Uberi/f8591cf68aab36bbb758 to your computer and use it in GitHub Desktop.
Save Uberi/f8591cf68aab36bbb758 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
e10s-experiment-beta45-withoutaddons-2016-02-17.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### e10s-beta45-withoutaddons: Slow script analysis"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Populating the interactive namespace from numpy and matplotlib\n"
]
}
],
"source": [
"import ujson as json\n",
"import matplotlib.pyplot as plt\n",
"import pandas as pd\n",
"import numpy as np\n",
"import plotly.plotly as py\n",
"import plotly.graph_objs as go\n",
"import IPython\n",
"\n",
"from __future__ import division\n",
"from moztelemetry.spark import get_pings, get_one_ping_per_client, get_pings_properties\n",
"from montecarlino import grouped_permutation_test\n",
"\n",
"%pylab inline\n",
"IPython.core.pylabtools.figsize(16, 7)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This notebook borrows heavily from these previously made notebooks:\n",
"\n",
"* https://github.com/vitillo/e10s_analyses/blob/master/aurora/e10s_experiment.ipynb\n",
"* https://gist.github.com/chutten/4c0c2cefdaf7837f1176"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"16"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sc.defaultParallelism"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Get data from beta 45 e10s experiment 2 (the one with addons vs no addons metadata):"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"dataset = sqlContext.read.load(\"s3://telemetry-parquet/e10s-experiment/[email protected]/generationDate=20160217\", \"parquet\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"1091599"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataset.count()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Count experiment/control samples:"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"235463"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataset.filter(dataset[\"experimentBranch\"] == \"experiment-no-addons\").count()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"256542"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataset.filter(dataset[\"experimentBranch\"] == \"control-no-addons\").count()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def row_2_ping(row):\n",
" ping = {\"payload\": {\"simpleMeasurements\": json.loads(row.simpleMeasurements),\n",
" \"histograms\": json.loads(row.histograms),\n",
" \"keyedHistograms\": json.loads(row.keyedHistograms),\n",
" \"childPayloads\": json.loads(row.childPayloads),\n",
" \"threadHangStats\": json.loads(row.threadHangStats)},\n",
" \"e10s\": True if row.experimentBranch == \"experiment-no-addons\" else False}\n",
" return ping"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"subset = dataset.rdd.filter(lambda r: r.experimentBranch in [\"experiment-no-addons\", \"control-no-addons\"]).map(row_2_ping)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"HISTOGRAMS_URL = \"https://hg.mozilla.org/releases/mozilla-aurora/raw-file/9d3bc275a924/toolkit/components/telemetry/Histograms.json\"\n",
"frame = pd.DataFrame(get_pings_properties(subset, [\"payload/histograms/SLOW_SCRIPT_NOTICE_COUNT\", \"payload/simpleMeasurements/uptime\", \"e10s\"], histograms_url=HISTOGRAMS_URL).collect())"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"e10s 492005\n",
"payload/histograms/SLOW_SCRIPT_NOTICE_COUNT 7983\n",
"payload/simpleMeasurements/uptime 492005\n",
"dtype: int64"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"frame.count()"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def normalize_uptime_hour(frame):\n",
" frame = frame[frame[\"payload/simpleMeasurements/uptime\"] > 0]\n",
" frame = 60 * frame.apply(lambda x: x/frame[\"payload/simpleMeasurements/uptime\"]) # Metric per hour\n",
" frame.drop('payload/simpleMeasurements/uptime', axis=1, inplace=True)\n",
" return frame\n",
"\n",
"def compare_scalars(metric, scalars1, scalars2):\n",
" print \"Median difference in {} is {:.2f}, ({:.2f}, {:.2f}).\".format(metric, np.median(scalars1) - np.median(scalars2), np.median(scalars1), np.median(scalars2))\n",
" print \"The probablity of this effect being purely by chance is {:.2f}.\".format(grouped_permutation_test(median_diff, [scalars1, scalars2], num_samples=10000))\n",
"\n",
"e10s = frame[frame[\"e10s\"] == True]\n",
"e10s = normalize_uptime_hour(e10s)\n",
"\n",
"none10s = frame[frame[\"e10s\"] == False]\n",
"none10s = normalize_uptime_hour(none10s)"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {
"collapsed": false,
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Median difference in payload/histograms/SLOW_SCRIPT_NOTICE_COUNT per hour is 0.12, (0.60, 0.47).\n",
"The probablity of this effect being purely by chance is 0.00.\n"
]
}
],
"source": [
"histogram = \"payload/histograms/SLOW_SCRIPT_NOTICE_COUNT\"\n",
"notices_e10s = e10s[histogram].dropna()\n",
"notices_none10s = none10s[histogram].dropna()\n",
"compare_scalars(histogram + \" per hour\", notices_e10s, notices_none10s)"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {
"collapsed": false,
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"(4792, 3175)"
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"notices_e10s.size, notices_none10s.size"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\"seamless=\"seamless\" src=\"https://plot.ly/~mozilla/378.embed\" height=\"525\" width=\"100%\"></iframe>"
],
"text/plain": [
"<plotly.tools.PlotlyDisplay object>"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"trace1 = go.Histogram(\n",
" x=notices_e10s,\n",
" opacity=0.75,\n",
" name='e10s'\n",
")\n",
"trace2 = go.Histogram(\n",
" x=notices_none10s,\n",
" opacity=0.75,\n",
" name='non-e10s'\n",
")\n",
"data = [trace1, trace2]\n",
"layout = go.Layout(\n",
" barmode='overlay',\n",
" title='User Count vs. SLOW_SCRIPT_NOTICE_COUNT Value',\n",
" xaxis=dict(title='SLOW_SCRIPT_NOTICE_COUNT Value'),\n",
" yaxis=dict(title='User Count')\n",
")\n",
"fig = go.Figure(data=data, layout=layout)\n",
"py.iplot(fig, filename='overlaid-histogram')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The plot doesn't show up on GitHub Gist, so here's a link as well: https://plot.ly/378/~mozilla/"
]
},
{
"cell_type": "code",
"execution_count": 56,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"percentile e10s none10s diff diff%\n",
" 0 0.009942 > 0.009531 0.000411 4.308202\n",
" 1 0.020999 > 0.018132 0.002867 15.814325\n",
" 2 0.030665 > 0.024385 0.006280 25.752175\n",
" 3 0.037412 > 0.030694 0.006718 21.887564\n",
" 4 0.042896 > 0.037095 0.005802 15.639825\n",
" 5 0.055919 > 0.041397 0.014522 35.078526\n",
" 6 0.068466 > 0.046550 0.021916 47.081087\n",
" 7 0.077381 > 0.053463 0.023918 44.738233\n",
" 8 0.085191 > 0.062928 0.022264 35.379762\n",
" 9 0.092362 > 0.072317 0.020045 27.717763\n",
" 10 0.100184 > 0.080776 0.019408 24.026929\n",
" 11 0.106572 > 0.089304 0.017268 19.335670\n",
" 12 0.111524 > 0.096600 0.014924 15.449783\n",
" 13 0.115385 > 0.102215 0.013170 12.884615\n",
" 14 0.118999 > 0.106075 0.012924 12.183834\n",
" 15 0.122449 > 0.110092 0.012357 11.224490\n",
" 16 0.126582 > 0.112570 0.014012 12.447257\n",
" 17 0.131004 > 0.115514 0.015491 13.410378\n",
" 18 0.136674 > 0.119837 0.016837 14.050015\n",
" 19 0.144928 > 0.124740 0.020187 16.183575\n",
" 20 0.153453 > 0.128150 0.025302 19.744158\n",
" 21 0.160905 > 0.133791 0.027114 20.265967\n",
" 22 0.171920 > 0.139393 0.032527 23.334706\n",
" 23 0.183748 > 0.144928 0.038820 26.785816\n",
" 24 0.193548 > 0.157431 0.036118 22.941897\n",
" 25 0.203218 > 0.162602 0.040616 24.979100\n",
" 26 0.212766 > 0.171057 0.041709 24.382794\n",
" 27 0.224555 > 0.178550 0.046005 25.765615\n",
" 28 0.232992 > 0.185026 0.047967 25.924433\n",
" 29 0.243902 > 0.197044 0.046858 23.780488\n",
" 30 0.253861 > 0.206186 0.047675 23.122604\n",
" 31 0.263279 > 0.213523 0.049756 23.302497\n",
" 32 0.274023 > 0.221402 0.052621 23.766928\n",
" 33 0.290578 > 0.231143 0.059435 25.713299\n",
" 34 0.302985 > 0.240000 0.062985 26.243544\n",
" 35 0.315665 > 0.247423 0.068242 27.581330\n",
" 36 0.331273 > 0.258221 0.073051 28.290267\n",
" 37 0.348837 > 0.267857 0.080980 30.232558\n",
" 38 0.368098 > 0.281850 0.086249 30.600922\n",
" 39 0.382166 > 0.294118 0.088048 29.936306\n",
" 40 0.400000 > 0.310240 0.089760 28.932507\n",
" 41 0.416667 > 0.324924 0.091743 28.235266\n",
" 42 0.431655 > 0.338066 0.093588 27.683435\n",
" 43 0.451128 > 0.350877 0.100251 28.571429\n",
" 44 0.468750 > 0.367111 0.101639 27.686323\n",
" 45 0.483871 > 0.382166 0.101705 26.612903\n",
" 46 0.504202 > 0.394737 0.109465 27.731092\n",
" 47 0.526316 > 0.413480 0.112836 27.289215\n",
" 48 0.546582 > 0.430912 0.115670 26.843038\n",
" 49 0.570318 > 0.454037 0.116282 25.610630\n",
" 50 0.595830 > 0.472441 0.123389 26.117304\n",
" 51 0.625000 > 0.487805 0.137195 28.125000\n",
" 52 0.649777 > 0.500000 0.149777 29.955347\n",
" 53 0.681818 > 0.517241 0.164577 31.818182\n",
" 54 0.714286 > 0.540424 0.173861 32.171281\n",
" 55 0.745342 > 0.571429 0.173913 30.434783\n",
" 56 0.779221 > 0.594059 0.185161 31.168831\n",
" 57 0.824728 > 0.617253 0.207475 33.612598\n",
" 58 0.869565 > 0.638298 0.231267 36.231884\n",
" 59 0.915310 > 0.664176 0.251134 37.811370\n",
" 60 0.955291 > 0.694055 0.261236 37.639018\n",
" 61 1.000000 > 0.722892 0.277108 38.333333\n",
" 62 1.041729 > 0.754151 0.287578 38.132702\n",
" 63 1.090909 > 0.783150 0.307760 39.297679\n",
" 64 1.159354 > 0.821918 0.337437 41.054787\n",
" 65 1.224490 > 0.857143 0.367347 42.857143\n",
" 66 1.276596 > 0.882353 0.394243 44.680851\n",
" 67 1.363636 > 0.923077 0.440559 47.727273\n",
" 68 1.463415 > 0.967742 0.495673 51.219512\n",
" 69 1.538462 > 1.028926 0.509535 49.521091\n",
" 70 1.621622 > 1.090909 0.530713 48.648649\n",
" 71 1.714286 > 1.153846 0.560440 48.571429\n",
" 72 1.794533 > 1.209193 0.585340 48.407537\n",
" 73 1.875000 > 1.276596 0.598404 46.875000\n",
" 74 2.013163 > 1.348315 0.664849 49.309623\n",
" 75 2.133347 > 1.428571 0.704776 49.334320\n",
" 76 2.222222 > 1.500000 0.722222 48.148148\n",
" 77 2.333636 > 1.578947 0.754689 47.796970\n",
" 78 2.500000 > 1.666667 0.833333 50.000000\n",
" 79 2.608696 > 1.767367 0.841328 47.603472\n",
" 80 2.727273 > 1.903341 0.823932 43.288729\n",
" 81 2.950820 > 2.068966 0.881854 42.622951\n",
" 82 3.157895 > 2.142857 1.015038 47.368421\n",
" 83 3.371403 > 2.307692 1.063711 46.094133\n",
" 84 3.750000 > 2.500000 1.250000 50.000000\n",
" 85 4.000000 > 2.727273 1.272727 46.666667\n",
" 86 4.285714 > 3.000000 1.285714 42.857143\n",
" 87 4.615385 > 3.211692 1.403692 43.705691\n",
" 88 5.000000 > 3.529412 1.470588 41.666667\n",
" 89 5.454545 > 3.801190 1.653355 43.495715\n",
" 90 6.000000 > 4.119122 1.880878 45.662100\n",
" 91 6.666667 > 4.853950 1.812716 37.345175\n",
" 92 7.500000 > 5.454545 2.045455 37.500000\n",
" 93 8.571429 > 6.000000 2.571429 42.857143\n",
" 94 10.000000 > 6.666667 3.333333 50.000000\n",
" 95 12.000000 > 8.000000 4.000000 50.000000\n",
" 96 15.000000 > 10.000000 5.000000 50.000000\n",
" 97 18.300889 > 12.000000 6.300889 52.507411\n",
" 98 24.000000 > 17.789268 6.210732 34.912800\n",
" 99 30.158824 > 30.000000 0.158824 0.529412\n",
" 100 186.666667 > 146.666667 40.000000 27.272727\n"
]
}
],
"source": [
"print \"percentile e10s none10s diff diff%\"\n",
"for i in range(101):\n",
" experiment, control = np.percentile(notices_e10s, i), np.percentile(notices_none10s, i)\n",
" print \"{:>5} {:>10f} {} {:>10f} {:>10f} {:>10f}\".format(\n",
" i,\n",
" experiment,\n",
" \">\" if experiment > control else \"<=\",\n",
" control,\n",
" experiment - control,\n",
" 100 * (experiment - control) / control\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment