fail cases

ak.ipynb

gistfile1.txt

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"/Users/mdurant/code/awkward-1.0/\")\n",
    "import awkward1 as ak"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import json\n",
    "\n",
    "def make_chunk(n, fn='out.json'):\n",
    "    \n",
    "    with open(fn, 'w') as f:\n",
    "        data = []\n",
    "        for _ in range(n):\n",
    "            row = {\"data\": {'f': random.random(), \n",
    "                   'i': random.randrange(0, 10), \n",
    "                   'str': random.choice(['fred', 'wilma', 'barney', 'betty']),\n",
    "                   'li': \n",
    "                            [random.choice(list('abcdef')) for _ in range(random.randrange(1, 10))],\n",
    "                   'map': {random.choice(list('abcdef')): random.choice(['fred', 'wilma', 'barney', 'betty'])\n",
    "                    for _ in range(random.randrange(1, 10))},\n",
    "                   }}\n",
    "            # row = [random.choice(list('abcdef')) for _ in range(random.randrange(1, 10))]\n",
    "            data.append(row)\n",
    "        json.dump(data, f)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_chunk(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Array [{data: {f: 0.45, i: 6, ... a: None}}}] type='100 * {\"data\": {\"f\": float6...'>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr = ak.fromjson('out.json')\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numba\n",
    "\n",
    "@numba.njit\n",
    "def len_lists(inarr):\n",
    "    count = 0\n",
    "    for d in inarr:\n",
    "        val = d['data']['map']['a']\n",
    "        if val:\n",
    "            count += 1\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypingError",
     "evalue": "Failed in nopython mode pipeline (step: nopython mode backend)\n\u001b[1m\u001b[1mInvalid use of Function(<class 'bool'>) with argument(s) of type(s): (awkward1.ArrayView(awkward1.NumpyArrayType(array(uint8, 1d, C), none, {\"__array__\": \"char\"}), None, ()))\n * parameterized\n\u001b[1mIn definition 0:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 1:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mIn definition 2:\u001b[0m\n\u001b[1m    TypingError: \u001b[1mInvalid use of Function(<built-in function truth>) with argument(s) of type(s): (awkward1.ArrayView(awkward1.NumpyArrayType(array(uint8, 1d, C), none, {\"__array__\": \"char\"}), None, ()))\n * parameterized\n\u001b[1mIn definition 0:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 1:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mIn definition 2:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 3:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mIn definition 4:\u001b[0m\n\u001b[1m    UnboundLocalError: local variable 'left' referenced before assignment\u001b[0m\n    raised from /Users/mdurant/code/awkward-1.0/awkward1/_connect/_numba/arrayview.py:493\n\u001b[1mIn definition 5:\u001b[0m\n\u001b[1m    UnboundLocalError: local variable 'left' referenced before assignment\u001b[0m\n    raised from /Users/mdurant/code/awkward-1.0/awkward1/_connect/_numba/arrayview.py:493\n\u001b[1mIn definition 6:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 7:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mThis error is usually caused by passing an argument of a type that is unsupported by the named function.\u001b[0m\u001b[0m\u001b[0m\n    raised from /Users/mdurant/conda/envs/py37/lib/python3.7/site-packages/numba/types/functions.py:79\n\u001b[1mIn definition 3:\u001b[0m\n\u001b[1m    TypingError: \u001b[1mInvalid use of Function(<built-in function truth>) with argument(s) of type(s): (awkward1.ArrayView(awkward1.NumpyArrayType(array(uint8, 1d, C), none, {\"__array__\": \"char\"}), None, ()))\n * parameterized\n\u001b[1mIn definition 0:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 1:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mIn definition 2:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 3:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mIn definition 4:\u001b[0m\n\u001b[1m    UnboundLocalError: local variable 'left' referenced before assignment\u001b[0m\n    raised from /Users/mdurant/code/awkward-1.0/awkward1/_connect/_numba/arrayview.py:493\n\u001b[1mIn definition 5:\u001b[0m\n\u001b[1m    UnboundLocalError: local variable 'left' referenced before assignment\u001b[0m\n    raised from /Users/mdurant/code/awkward-1.0/awkward1/_connect/_numba/arrayview.py:493\n\u001b[1mIn definition 6:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 7:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mThis error is usually caused by passing an argument of a type that is unsupported by the named function.\u001b[0m\u001b[0m\u001b[0m\n    raised from /Users/mdurant/conda/envs/py37/lib/python3.7/site-packages/numba/types/functions.py:79\n\u001b[1mThis error is usually caused by passing an argument of a type that is unsupported by the named function.\u001b[0m\u001b[0m\n\u001b[0m\u001b[1m[1] During: lowering \"branch val, 36, 49\" at <ipython-input-11-fbcfedc8e316> (8)\u001b[0m",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypingError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-2c038d8aae72>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlen_lists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/conda/envs/py37/lib/python3.7/site-packages/numba/dispatcher.py\u001b[0m in \u001b[0;36m_compile_for_args\u001b[0;34m(self, *args, **kws)\u001b[0m\n\u001b[1;32m    399\u001b[0m                 \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpatch_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    400\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 401\u001b[0;31m             \u001b[0merror_rewrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'typing'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    402\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mUnsupportedError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    403\u001b[0m             \u001b[0;31m# Something unsupported is present in the user code, add help info\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/conda/envs/py37/lib/python3.7/site-packages/numba/dispatcher.py\u001b[0m in \u001b[0;36merror_rewrite\u001b[0;34m(e, issue_type)\u001b[0m\n\u001b[1;32m    342\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    343\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 344\u001b[0;31m                 \u001b[0mreraise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    345\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    346\u001b[0m         \u001b[0margtypes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/conda/envs/py37/lib/python3.7/site-packages/numba/six.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb)\u001b[0m\n\u001b[1;32m    666\u001b[0m             \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    667\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 668\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    669\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    670\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypingError\u001b[0m: Failed in nopython mode pipeline (step: nopython mode backend)\n\u001b[1m\u001b[1mInvalid use of Function(<class 'bool'>) with argument(s) of type(s): (awkward1.ArrayView(awkward1.NumpyArrayType(array(uint8, 1d, C), none, {\"__array__\": \"char\"}), None, ()))\n * parameterized\n\u001b[1mIn definition 0:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 1:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mIn definition 2:\u001b[0m\n\u001b[1m    TypingError: \u001b[1mInvalid use of Function(<built-in function truth>) with argument(s) of type(s): (awkward1.ArrayView(awkward1.NumpyArrayType(array(uint8, 1d, C), none, {\"__array__\": \"char\"}), None, ()))\n * parameterized\n\u001b[1mIn definition 0:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 1:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mIn definition 2:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 3:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mIn definition 4:\u001b[0m\n\u001b[1m    UnboundLocalError: local variable 'left' referenced before assignment\u001b[0m\n    raised from /Users/mdurant/code/awkward-1.0/awkward1/_connect/_numba/arrayview.py:493\n\u001b[1mIn definition 5:\u001b[0m\n\u001b[1m    UnboundLocalError: local variable 'left' referenced before assignment\u001b[0m\n    raised from /Users/mdurant/code/awkward-1.0/awkward1/_connect/_numba/arrayview.py:493\n\u001b[1mIn definition 6:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 7:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mThis error is usually caused by passing an argument of a type that is unsupported by the named function.\u001b[0m\u001b[0m\u001b[0m\n    raised from /Users/mdurant/conda/envs/py37/lib/python3.7/site-packages/numba/types/functions.py:79\n\u001b[1mIn definition 3:\u001b[0m\n\u001b[1m    TypingError: \u001b[1mInvalid use of Function(<built-in function truth>) with argument(s) of type(s): (awkward1.ArrayView(awkward1.NumpyArrayType(array(uint8, 1d, C), none, {\"__array__\": \"char\"}), None, ()))\n * parameterized\n\u001b[1mIn definition 0:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 1:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mIn definition 2:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 3:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mIn definition 4:\u001b[0m\n\u001b[1m    UnboundLocalError: local variable 'left' referenced before assignment\u001b[0m\n    raised from /Users/mdurant/code/awkward-1.0/awkward1/_connect/_numba/arrayview.py:493\n\u001b[1mIn definition 5:\u001b[0m\n\u001b[1m    UnboundLocalError: local variable 'left' referenced before assignment\u001b[0m\n    raised from /Users/mdurant/code/awkward-1.0/awkward1/_connect/_numba/arrayview.py:493\n\u001b[1mIn definition 6:\u001b[0m\n\u001b[1m    All templates rejected with literals.\u001b[0m\n\u001b[1mIn definition 7:\u001b[0m\n\u001b[1m    All templates rejected without literals.\u001b[0m\n\u001b[1mThis error is usually caused by passing an argument of a type that is unsupported by the named function.\u001b[0m\u001b[0m\u001b[0m\n    raised from /Users/mdurant/conda/envs/py37/lib/python3.7/site-packages/numba/types/functions.py:79\n\u001b[1mThis error is usually caused by passing an argument of a type that is unsupported by the named function.\u001b[0m\u001b[0m\n\u001b[0m\u001b[1m[1] During: lowering \"branch val, 36, 49\" at <ipython-input-11-fbcfedc8e316> (8)\u001b[0m"
     ]
    }
   ],
   "source": [
    "len_lists(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

time1.py

import awkward1 as ak
import pandas as pd
import numpy as np
data = [[1.1, 2.2, 3.3], [], [4.4, 5.5], [6.6, 7.7, 8.8, 9.9]]*100000
array = ak.Array(data)
s = pd.Series(array)

%timeit s.map(len)

    11.4 s ± 138 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)


%timeit list(map(len, data))

    9.15 ms ± 189 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)


s = pd.Series(data)
%timeit s.map(len)

    63.9 ms ± 410 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)


%timeit np.diff(array.layout.offsets)

    386 µs ± 36.7 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)

Here is where map is dispatched to extension types: https://github.com/pandas-dev/pandas/blob/master/pandas/core/base.py#L1139