export_regression_bugs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Firefox regression bugs analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import requests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieve the custom status and tracking fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fields_endpoint = 'https://bugzilla.mozilla.org/rest/field/bug'\n",
    "response = requests.get(fields_endpoint)\n",
    "fields = response.json()['fields']\n",
    "\n",
    "status_fields = filter(lambda x: 'cf_status_firefox' in x['name'], fields)\n",
    "tracking_fields = filter(lambda x: 'cf_tracking_firefox' in x['name'], fields)\n",
    "cf_fields = map(lambda x: x['name'],status_fields) + map(lambda x: x['name'],tracking_fields)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare the parameters for the bug request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "endpoint = 'https://bugzilla.mozilla.org/rest/bug'\n",
    "include_fields = [\n",
    "    'id',\n",
    "    'version',\n",
    "    'target_milestone',\n",
    "    'status',\n",
    "    'resolution',\n",
    "    'product',\n",
    "    'platform',\n",
    "    'op_sys',\n",
    "    'keywords',\n",
    "    'is_confirmed',\n",
    "    'creation_time'\n",
    "]\n",
    "\n",
    "params = {\n",
    "    # TODO: find out which products we want to include (hello, etc.)\n",
    "    'product': 'firefox',\n",
    "    'keywords': 'regression',\n",
    "    'creation_time': '2015-11-03',\n",
    "    'exclude_fields': 'cc,cc_detail',\n",
    "    'include_fields': ','.join(include_fields + cf_fields)\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieve the bug list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 21.2 ms, sys: 3.01 ms, total: 24.2 ms\n",
      "Wall time: 7.12 s\n"
     ]
    }
   ],
   "source": [
    "%time response = requests.get(endpoint, params=params)\n",
    "data = response.json()['bugs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from datetime import date, datetime\n",
    "import re\n",
    "\n",
    "# TODO: should I use merge dates rather than release dates?\n",
    "release_dates = {\n",
    "    date(2015, 11, 3) : 42,\n",
    "    date(2015, 12, 15): 43,\n",
    "    date(2016, 1, 26): 44,\n",
    "    date(2016, 3, 8):  45,\n",
    "    date(2016, 4, 26): 46,\n",
    "    date(2016, 6, 7):  47,\n",
    "    date(2016, 8, 2):  48,\n",
    "    date(2016, 9, 13): 49,\n",
    "    date(2016, 11, 8): 50,\n",
    "    date(2017, 1, 24): 51\n",
    "}\n",
    "release_trains = {}\n",
    "for r_date, r_version in release_dates.items():\n",
    "    release_trains[r_date] = {\n",
    "        r_version: 'release',\n",
    "        r_version+1: 'beta',\n",
    "        r_version+2: 'aurora'\n",
    "    }\n",
    "\n",
    "branch_re = re.compile('(\\d{2}) branch')\n",
    "\n",
    "def get_release_cycle(date):\n",
    "    for r_date in sorted(release_dates.keys()):\n",
    "        if date < r_date:\n",
    "            return r_date, release_dates[r_date]\n",
    "    return None\n",
    "\n",
    "def get_release_channel(release_date, branch):\n",
    "    if branch < min(release_trains[release_date].keys()):\n",
    "        return 'old release'\n",
    "    elif branch > max(release_trains[release_date].keys()):\n",
    "        return 'nightly'\n",
    "    try:\n",
    "        return release_trains[release_date][branch]\n",
    "    except KeyError:\n",
    "        print 'error retrieving channel for %s in %s' % (branch, release_trains[release_date])\n",
    "        return 'error'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Transform the keywords list into a string\n",
    "output_data = data[:]\n",
    "for bug in output_data:\n",
    "    bug['keywords'] = \",\".join(bug['keywords'])\n",
    "    \n",
    "    # TODO: figure out how to refine this to take into account release/merge time\n",
    "    creation_time = datetime.strptime(bug['creation_time'][0:10], '%Y-%m-%d').date()\n",
    "    r_date, bug['release_cycle'] = get_release_cycle(creation_time)\n",
    "    \n",
    "    branch = bug['version'].lower()\n",
    "    \n",
    "    if branch == 'trunk':\n",
    "        bug['release_channel'] = 'nightly'\n",
    "    elif branch == 'unspecified':\n",
    "        bug['release_channel'] = branch\n",
    "    else:\n",
    "        match = branch_re.match(branch)\n",
    "        if match:\n",
    "            bug['release_channel'] = get_release_channel(r_date, int(match.group(1)))\n",
    "        else:\n",
    "            bug['release_channel'] = 'unknown'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Retrieve the cf_status and cf_tracking fields available\n",
    "available_cf_field = set()\n",
    "for bug in output_data:\n",
    "    for field_name in bug:\n",
    "        if field_name in cf_fields:\n",
    "            available_cf_field.add(field_name)\n",
    "field_names = include_fields + ['release_channel', 'release_cycle'] + sorted(list(available_cf_field))\n",
    "\n",
    "# Output everything to  a csv file\n",
    "with open('regression_bugs.csv', 'w') as csvfile:\n",
    "    writer = csv.DictWriter(csvfile, fieldnames=field_names)\n",
    "\n",
    "    writer.writeheader()\n",
    "    for bug in output_data:\n",
    "        writer.writerow(bug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}