git/yang-zhang.github.io/ds_math/Untitled1.ipynb

proportion-test.ipynb

{
  "cells": [
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "- formula from https://onlinecourses.science.psu.edu/stat414/node/268/\n\n- data from\nhttps://www.kean.edu/~fosborne/bstat/06d2pop.html"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "# https://www.statsmodels.org/dev/generated/statsmodels.stats.proportion.proportions_ztest.html\nimport numpy as np\nfrom statsmodels.stats.proportion import proportions_ztest\nimport scipy\ncounts = np.array([60, 18])\nnobs = np.array([123, 96])\nstat, pval = proportions_ztest(counts, nobs)\nstat, pval",
      "execution_count": 141,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 141,
          "data": {
            "text/plain": "(4.6048528720529776, 4.1275757162227761e-06)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "y1, y2 = counts\nn1, n2 = nobs\np1, p2 = y1/n1, y2/n2",
      "execution_count": 142,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "p1, p2, counts/nobs",
      "execution_count": 143,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 143,
          "data": {
            "text/plain": "(0.48780487804878048, 0.1875, array([ 0.48780488,  0.1875    ]))"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "p1, p2 = counts/nobs",
      "execution_count": 144,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "ph = (y1+y2)/(n1+n2)\nse = np.sqrt(ph*(1-ph) * (1/n1 + 1/n2))\nse",
      "execution_count": 145,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 145,
          "data": {
            "text/plain": "0.065214869267885167"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "z = (p1-p2) / se\nz, np.allclose(z, stat)",
      "execution_count": 146,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 146,
          "data": {
            "text/plain": "(4.6048528720529776, True)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "p = 2*(1-scipy.stats.norm.cdf(abs(z)))\np, np.allclose(p, pval)",
      "execution_count": 147,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 147,
          "data": {
            "text/plain": "(4.1275757163283799e-06, True)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "### confidence interval"
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "formula and data from\nhttps://www.kean.edu/~fosborne/bstat/06d2pop.html"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "confidence=0.99\n(1-confidence)/2",
      "execution_count": 160,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 160,
          "data": {
            "text/plain": "0.0050000000000000044"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "z_critial = scipy.stats.norm.ppf(1-(1-confidence)/2)\nz_critial",
      "execution_count": 161,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 161,
          "data": {
            "text/plain": "2.5758293035489004"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {},
      "cell_type": "markdown",
      "source": "note this se is NOT the same as above"
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "se = np.sqrt(p1*(1-p1)/n1 + p2*(1-p2)/n2)\nse",
      "execution_count": 162,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 162,
          "data": {
            "text/plain": "0.06015168554485361"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "h = z_critial * se\nh",
      "execution_count": 163,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 163,
          "data": {
            "text/plain": "0.15494047428429272"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "p1-p2",
      "execution_count": 164,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 164,
          "data": {
            "text/plain": "0.30030487804878048"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "scrolled": true
      },
      "cell_type": "code",
      "source": "p1-p2-h, p1-p2+h",
      "execution_count": 165,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 165,
          "data": {
            "text/plain": "(0.14536440376448775, 0.45524535233307317)"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "conda-env-anaconda3-py",
      "display_name": "Python [conda env:anaconda3]",
      "language": "python"
    },
    "toc": {
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "base_numbering": 1,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "language_info": {
      "name": "python",
      "version": "3.6.4",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "gist": {
      "id": "",
      "data": {
        "description": "git/yang-zhang.github.io/ds_math/Untitled1.ipynb",
        "public": true
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}