analyze-stan-headers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "76685b5a-490c-4a79-886f-30c4500a8de9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pathlib\n",
    "import re\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e7783d49-5440-4622-a535-543b69a2120b",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Generate prep.cpp file\n",
    "# g++ -E -std=c++1y -pthread -D_REENTRANT -Wno-sign-compare -Wno-ignored-attributes      -I ./stan/lib/stan_math/lib/tbb_2020.3/include    -O3 -I ./stan/src -I ./stan/lib/rapidjson_1.1.0/ -I ./stan/lib/stan_math/ -I ./stan/lib/stan_math/lib/eigen_3.4.0 -I ./stan/lib/stan_math/lib/boost_1.78.0 -I ./stan/lib/stan_math/lib/sundials_6.1.1/include -I ./stan/lib/stan_math/lib/sundials_6.1.1/src/sundials -fPIC    -DBOOST_DISABLE_ASSERTS          -c -x c++ -o prep.cpp test_models/logistic/logistic.hpp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4d8d0a8d-5c6a-46e7-a2c8-0a7b171f1504",
   "metadata": {},
   "outputs": [],
   "source": [
    "content = pathlib.Path(\"/home/adr/.bridgestan/bridgestan-2.1.1/prep.cpp\").read_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eff870cb-a362-4588-ada7-a60748e3ea69",
   "metadata": {},
   "outputs": [],
   "source": [
    "content = list(content.split(\"\\n\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "73930510-e174-47b3-9676-044616266593",
   "metadata": {},
   "outputs": [],
   "source": [
    "regex = re.compile('\\# ([0-9]*) \\\"([\\<\\>/a-zA-Z0-9\\._\\-]*)\\\"[ 0-9]*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6b2582f1-77a7-472c-bf91-3e236b602cf4",
   "metadata": {},
   "outputs": [],
   "source": [
    "line_counts = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8aab3d93-389c-419e-948d-41b048d9e361",
   "metadata": {},
   "outputs": [],
   "source": [
    "current = None\n",
    "for line in content:\n",
    "    if not line.strip():\n",
    "        continue\n",
    "    match = regex.match(line)\n",
    "    if not match and current:\n",
    "        line_counts[current] += 1\n",
    "        continue\n",
    "    if not match and not current:\n",
    "        assert False\n",
    "    _, current = match.groups()\n",
    "    line_counts.setdefault(current, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eb261fe0-6f1b-4a2b-b055-27d3297cce70",
   "metadata": {},
   "outputs": [],
   "source": [
    "line_counts = pd.Series(line_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "699fbe1b-2cf0-48e5-a894-dc06c2e26766",
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = line_counts.rename(\"count\").to_frame().rename_axis(index=\"file\").reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d0861a7e-c276-4e03-94db-aadb69a00ae6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3322"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of headers (more or less? What are those with zero lines?)\n",
    "len(line_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "09feba5d-a08f-4247-ae7b-a203b90b0bc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2819"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list(name for name, count in line_counts.items() if count > 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "52184360-f82f-42b3-a715-48d97802f32d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "321339"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts[\"count\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8a06fc04-c3d8-422f-8a9b-c18464c2c472",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "path\n",
       "/usr/include/bits/endian.h                             1\n",
       "/usr/include/bits/endianness.h                         1\n",
       "/usr/include/bits/environments.h                       1\n",
       "/usr/include/bits/cpu-set.h                            1\n",
       "/usr/include/bits/floatn-common.h                      1\n",
       "/usr/include/bits/floatn.h                             1\n",
       "/usr/include/bits/flt-eval-method.h                    1\n",
       "/usr/include/bits/fp-fast.h                            1\n",
       "/usr/include/bits/fp-logb.h                            1\n",
       "/usr/include/bits/getopt_core.h                        1\n",
       "/usr/include/bits/getopt_posix.h                       1\n",
       "/usr/include/bits/errno.h                              1\n",
       "/usr/lib/gcc/x86_64-pc-linux-gnu/13.1.1/include       10\n",
       "./stan/lib/stan_math/lib/sundials_6.1.1/include       21\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/unsupported      21\n",
       "./stan/lib/stan_math/lib/tbb_2020.3/include           28\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen           261\n",
       "./stan/lib/stan_math/stan/math/rev                   315\n",
       "./stan/lib/stan_math/stan/math/prim                  950\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost         1555\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of included files by source directory\n",
    "counts[\"path\"] = counts.file.str.split(\"/\").str.slice(0, 7).str.join(\"/\")\n",
    "counts.groupby(\"path\")[\"count\"].count().sort_values().tail(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "68caec11-b60a-4012-ac5c-77cfd5847276",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "path\n",
       "/usr/include/bits/confname.h                  311\n",
       "test_models/logistic/logistic.hpp             409\n",
       "/usr/include/features.h                       449\n",
       "/usr/include/wchar.h                          570\n",
       "./stan/src/stan/io/deserializer.hpp           579\n",
       "/usr/include/pthread.h                        625\n",
       "/usr/include/bits/mathcalls.h                 813\n",
       "./stan/src/stan/model/model_header.hpp       1374\n",
       "/usr/include/stdlib.h                        1389\n",
       "./stan/lib/stan_math/lib/sundials_6.1.1      1796\n",
       "./stan/src/stan/model/indexing               2114\n",
       "/usr/include/libintl.h                       2405\n",
       "/usr/include/errno.h                         3314\n",
       "./stan/lib/stan_math/lib/tbb_2020.3          3377\n",
       "/usr/lib/gcc/x86_64-pc-linux-gnu/13.1.1      5904\n",
       "/usr/include/wctype.h                        6499\n",
       "/usr/include/sys/single_threaded.h           7276\n",
       "./stan/lib/stan_math/stan/math              62864\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0        75391\n",
       "./stan/lib/stan_math/lib/boost_1.78.0      141012\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of lines included from different directories\n",
    "counts[\"path\"] = counts.file.str.split(\"/\").str.slice(0, 6).str.join(\"/\")\n",
    "counts.groupby(\"path\")[\"count\"].sum().sort_values().tail(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "21a1184c-2cc7-423d-b2b2-f9c2c1a2f9eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "path\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/random/detail                      1393\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/src/QR                              1475\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/math/quadrature                    1486\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/src/OrderingMethods                 1525\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/src/LU                              1527\n",
       "./stan/lib/stan_math/stan/math/prim/core/init_threadpool_tbb.hpp               1554\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/fusion/sequence                    1594\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/math/policies                      1740\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/src/SVD                             1888\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/fusion/functional                  1956\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/type_traits/detail                 1963\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/unsupported/Eigen/src                     2045\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/src/SparseLU                        2073\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/function_types/detail              2271\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/lexical_cast/detail                2309\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/units/detail                       2389\n",
       "/usr/include/libintl.h                                                         2405\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/numeric/conversion                 2760\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/src/IterativeLinearSolvers          2887\n",
       "./stan/lib/stan_math/stan/math/prim/functor/hcubature.hpp                      2912\n",
       "/usr/lib/gcc/x86_64-pc-linux-gnu/13.1.1/include/stddef.h                       2947\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/fusion/algorithm                   2968\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/src/Eigenvalues                     2970\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/src/Geometry                        3091\n",
       "./stan/lib/stan_math/stan/math/prim/meta/holder.hpp                            3162\n",
       "/usr/include/errno.h                                                           3314\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/fusion/view                        3334\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/math/tools                         4324\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/mpl/aux_                           4725\n",
       "./stan/lib/stan_math/stan/math/rev/core/init_chainablestack.hpp                4826\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/SparseCore                          5962\n",
       "/usr/include/wctype.h                                                          6499\n",
       "/usr/include/sys/single_threaded.h                                             7276\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/src/SparseCore                      7563\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/Core                                7901\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/function/function_template.hpp     8755\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/numeric/odeint                    10857\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/numeric/ublas                     23036\n",
       "./stan/lib/stan_math/lib/boost_1.78.0/boost/math/special_functions            26065\n",
       "./stan/lib/stan_math/lib/eigen_3.4.0/Eigen/src/Core                           31208\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Same, just a bit more detailed\n",
    "counts[\"path\"] = counts.file.str.split(\"/\").str.slice(0, 9).str.join(\"/\")\n",
    "counts.groupby(\"path\")[\"count\"].sum().sort_values().tail(40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b70074c3-3e59-4912-bc3a-29ff7786d285",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9e966bf-d996-4965-876f-68711a995088",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a86f333-a3e4-4935-811f-2a4471eddfa4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dev",
   "language": "python",
   "name": "dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}