Skip to content

Instantly share code, notes, and snippets.

@tonyfast

tonyfast/meetup-scraper.ipynb

Created March 15, 2016 18:14
Show Gist options
  • Save tonyfast/1a048cb5abe7bec0ef19 to your computer and use it in GitHub Desktop.
Save tonyfast/1a048cb5abe7bec0ef19 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
meetup-scraper.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import requests, IPython, pyquery, pandas, geopy, geopy.geocoders, jinja2, IPython\n",
"%reload_ext jademagic"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"df = None\n",
"for radius in [5,10,25,50,100]:\n",
" response = requests.get(\"https://www.meetup.com/find/groups/\", params={\n",
" 'allMeetups': False,\n",
" 'keywords': 'data science',\n",
" 'radius': radius,\n",
" 'userFreeform': 'Atlanta, GA',\n",
" 'Id': 'z30308',\n",
" 'mcName': 'Atlanta, GA',\n",
" 'eventFilter': 'mysugg',\n",
" })\n",
" q = pyquery.PyQuery(response.content)\n",
" \n",
" _df = pandas.DataFrame({\n",
" 'id': [int(s.attr('data-chapterid')) for s in q('li.groupCard').items()],\n",
" 'url': [s.attr('href') for s in q('.j-groupCard-list [itemprop=\"url\"]').items()],\n",
" 'members': [' '.join(s.text().lstrip('We\\'re').strip().split(' ')[1:]).strip() for s in q('.j-groupCard-list .groupCard--title > p').items()],\n",
" 'headcount': [int(s.text().lstrip('We\\'re').strip().split(' ')[0].replace(',','')) for s in q('.j-groupCard-list .groupCard--title > p').items()],\n",
" })\n",
" _df['radius'] = radius\n",
" if isinstance( df, type(None) ):\n",
" df = _df\n",
" else:\n",
" df = df.append( _df )\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"geodist = df.groupby(df.url).agg({'radius':'min','headcount':'mean','id':'mean'}).sort_values('headcount',ascending=False)\n",
"geodist = geodist.reset_index().set_index('radius')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"application/javascript": [
"\n",
" require(\n",
" [\n",
" \"notebook/js/codecell\",\n",
" \"codemirror/mode/jade/jade\"\n",
" ],\n",
" function(cc){\n",
" cc.CodeCell.options_default.highlight_modes.magic_jade = {\n",
" reg: [\"^%%jade\"]\n",
" }\n",
" }\n",
" );\n",
" "
],
"text/plain": [
"<IPython.core.display.Javascript object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%jade table\n",
"table.table.table-condensed.table-hover.table-striped\n",
" tr\n",
" th Group Name\n",
" th Members\n",
" {% for i,row in geodist.loc[dist].iterrows() %}\n",
" tr\n",
" td {{row['url'].lstrip('https://www.meetup.com/')}}\n",
" td {{row['headcount']}}\n",
" {% endfor %}"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"<table class=\"table table-condensed table-hover table-striped\">\n",
" <tr>\n",
" <th>Group Name</th>\n",
" <th>Members</th>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Atlanta-Society-for-Business-Intelligence/</td>\n",
" <td>1695</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Atlanta-Society-for-Business-Intelligence-ASBI/</td>\n",
" <td>719</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Applied-Data-Science-of-Atlanta/</td>\n",
" <td>392</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Graph-Database-Atlanta/</td>\n",
" <td>323</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Atlanta-Cloudera-Users-Group/</td>\n",
" <td>261</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>DBAtlanta/</td>\n",
" <td>168</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>AtlantaInternetStartups/</td>\n",
" <td>143</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Atlanta-Data-Science/</td>\n",
" <td>105</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>PyData-Atlanta/</td>\n",
" <td>104</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>atlanta-big-data-ML/</td>\n",
" <td>81</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Stacki-Atlanta/</td>\n",
" <td>62</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>BetterCloud-Hackathon-Meetup/</td>\n",
" <td>56</td>\n",
" </tr>\n",
"\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"IPython.display.HTML(jinja2.Template(table).render(dist=5,**globals()))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"<table class=\"table table-condensed table-hover table-striped\">\n",
" <tr>\n",
" <th>Group Name</th>\n",
" <th>Members</th>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Data-Science-ATL/</td>\n",
" <td>2464</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Cloud-Big-Data-and-Data-Science-Group-of-Atlanta/</td>\n",
" <td>575</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Atlanta-BIG-Data-Hadoop-Learning-Group/</td>\n",
" <td>561</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>AC-ATL/</td>\n",
" <td>479</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Atlanta-Analytics-Meetup/</td>\n",
" <td>413</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>generalassemblyatl/</td>\n",
" <td>373</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>ADP-Atlanta-Data-Science-Meetup/</td>\n",
" <td>293</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Atlanta-Apache-Spark-Meetup/</td>\n",
" <td>76</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Atlanta-DAM/</td>\n",
" <td>46</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Digital-Doorways/</td>\n",
" <td>45</td>\n",
" </tr>\n",
"\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"IPython.display.HTML(jinja2.Template(table).render(dist=10,**globals()))"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"<table class=\"table table-condensed table-hover table-striped\">\n",
" <tr>\n",
" <th>Group Name</th>\n",
" <th>Members</th>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>GeorgiaRVInstitute/</td>\n",
" <td>49</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Journal-Club/</td>\n",
" <td>7</td>\n",
" </tr>\n",
"\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"IPython.display.HTML(jinja2.Template(table).render(dist=50,**globals()))"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"<table class=\"table table-condensed table-hover table-striped\">\n",
" <tr>\n",
" <th>Group Name</th>\n",
" <th>Members</th>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Chattanooga-Data-Science/</td>\n",
" <td>141</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>ChaBigData/</td>\n",
" <td>93</td>\n",
" </tr>\n",
"\n",
" <tr>\n",
" <td>Open-Chattanooga/</td>\n",
" <td>34</td>\n",
" </tr>\n",
"\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"IPython.display.HTML(jinja2.Template(table).render(dist=100,**globals()))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.1"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment