Skip to content

Instantly share code, notes, and snippets.

@ricalanis

ricalanis/SearchEngineTest.ipynb

Created February 25, 2018 04:42
Show Gist options
  • Save ricalanis/89a90ca35b7b5385a8b40f6676647ca4 to your computer and use it in GitHub Desktop.
Save ricalanis/89a90ca35b7b5385a8b40f6676647ca4 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
SearchEngineTest.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"\n",
"def bing_search(query):\n",
" url = 'https://api.cognitive.microsoft.com/bing/v5.0/search'\n",
" # query string parameters\n",
" payload = {'q': query}\n",
" # custom headers\n",
" headers = {'Ocp-Apim-Subscription-Key': 'pokemonred'}\n",
" # make GET request\n",
" r = requests.get(url, params=payload, headers=headers)\n",
" # get JSON response\n",
" return r.json()"
]
},
{
"cell_type": "code",
"execution_count": 45,
"metadata": {},
"outputs": [],
"source": [
"j = bing_search('site:www.facebook.com/events/ laredo events art')\n",
"urls = [a[\"displayUrl\"] for a in j[\"webPages\"][\"value\"]]"
]
},
{
"cell_type": "code",
"execution_count": 46,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'_type': 'SearchResponse',\n",
" 'rankingResponse': {'mainline': {'items': [{'answerType': 'WebPages',\n",
" 'resultIndex': 0,\n",
" 'value': {'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.0'}},\n",
" {'answerType': 'WebPages',\n",
" 'resultIndex': 1,\n",
" 'value': {'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.1'}},\n",
" {'answerType': 'WebPages',\n",
" 'resultIndex': 2,\n",
" 'value': {'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.2'}},\n",
" {'answerType': 'WebPages',\n",
" 'resultIndex': 3,\n",
" 'value': {'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.3'}},\n",
" {'answerType': 'WebPages',\n",
" 'resultIndex': 4,\n",
" 'value': {'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.4'}},\n",
" {'answerType': 'WebPages',\n",
" 'resultIndex': 5,\n",
" 'value': {'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.5'}},\n",
" {'answerType': 'WebPages',\n",
" 'resultIndex': 6,\n",
" 'value': {'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.6'}},\n",
" {'answerType': 'WebPages',\n",
" 'resultIndex': 7,\n",
" 'value': {'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.7'}},\n",
" {'answerType': 'WebPages',\n",
" 'resultIndex': 8,\n",
" 'value': {'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.8'}},\n",
" {'answerType': 'WebPages',\n",
" 'resultIndex': 9,\n",
" 'value': {'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.9'}}]}},\n",
" 'webPages': {'totalEstimatedMatches': 7040,\n",
" 'value': [{'dateLastCrawled': '2018-01-26T17:28:00',\n",
" 'displayUrl': 'https://www.facebook.com/events/350689162009906',\n",
" 'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.0',\n",
" 'name': 'Laredo Center for the Arts Annual Dia de los Muertos Festival',\n",
" 'snippet': 'Laredo Center for the Art’s in conjunction with the Mexican Consulate, Laredo Convention and Visitors Bureau, the City of Laredo, Laredo Parks and...',\n",
" 'url': 'https://www.bing.com/cr?IG=030CFE7ACADB4E689CCE6DEFF5BDA30F&CID=0547A6A3551863FF3A9CAD0254B76224&rd=1&h=9nD3drolM6s6C6GIBKkt4zkWF0NIwZlIw5_KpaRI9NU&v=1&r=https%3a%2f%2fwww.facebook.com%2fevents%2f350689162009906&p=DevEx,5075.1'},\n",
" {'dateLastCrawled': '2018-01-27T07:20:00',\n",
" 'displayUrl': 'https://www.facebook.com/events/1592985407617680',\n",
" 'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.1',\n",
" 'name': 'Hecho a Mano - Facebook',\n",
" 'snippet': 'Hecho a Mano is a handmade art market in Laredo, Texas. We invite local artists, crafters and foodies to join us in making a one-night only pop up shop...',\n",
" 'url': 'https://www.bing.com/cr?IG=030CFE7ACADB4E689CCE6DEFF5BDA30F&CID=0547A6A3551863FF3A9CAD0254B76224&rd=1&h=eVv5vrzdM8i58zEm4RXi5IMfffQ0izZqzXt_BMO4XZk&v=1&r=https%3a%2f%2fwww.facebook.com%2fevents%2f1592985407617680%2f&p=DevEx,5092.1'},\n",
" {'displayUrl': 'https://www.facebook.com/events/367434193689352',\n",
" 'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.2',\n",
" 'name': 'Laredo Art Party with Chris Soto - Painting \"Day of the ...',\n",
" 'snippet': 'Events. Events. Calendar. Birthdays. ... Laredo Art Party with Chris Soto - Painting \"Day of the Dead Pumpkins\" Public · Hosted by Laredo Art Party by Chris Soto ...',\n",
" 'url': 'https://www.bing.com/cr?IG=030CFE7ACADB4E689CCE6DEFF5BDA30F&CID=0547A6A3551863FF3A9CAD0254B76224&rd=1&h=Y3vf1aLgIJtdhTkhOM_3UnzHSyufK1W4N9SgDMgZ-Iw&v=1&r=https%3a%2f%2fwww.facebook.com%2fevents%2f367434193689352&p=DevEx,5106.1'},\n",
" {'displayUrl': 'https://www.facebook.com/events/295163097637016',\n",
" 'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.3',\n",
" 'name': 'Laredo Art Party with Chris Soto - Painting \"Pinky Thoughts\"',\n",
" 'snippet': 'Events. Events. Calendar. Birthdays. Discover. Past. Laredo Art Party with Chris Soto - Painting \"Pinky Thoughts\" ... Laredo Art Party with Chris Soto - Painting ...',\n",
" 'url': 'https://www.bing.com/cr?IG=030CFE7ACADB4E689CCE6DEFF5BDA30F&CID=0547A6A3551863FF3A9CAD0254B76224&rd=1&h=TCgWwRR2qKggtXENgcUvez__Ulj8jR5WUp3PES9jGZo&v=1&r=https%3a%2f%2fwww.facebook.com%2fevents%2f295163097637016&p=DevEx,5120.1'},\n",
" {'displayUrl': 'https://www.facebook.com/events/101333637280749',\n",
" 'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.4',\n",
" 'name': 'Laredo Art Party with Chris Soto - Painting \"The Kitty Cat\"',\n",
" 'snippet': 'Join us for the best Painting Party in town happening every Wednesday at 7pm at Laredo Mexican Restaurant conveniently located right next to the...',\n",
" 'url': 'https://www.bing.com/cr?IG=030CFE7ACADB4E689CCE6DEFF5BDA30F&CID=0547A6A3551863FF3A9CAD0254B76224&rd=1&h=1LQR-zAAYCJNLG8T_R_1NYfFdXnN2J7JCiAKYwJgh58&v=1&r=https%3a%2f%2fwww.facebook.com%2fevents%2f101333637280749&p=DevEx,5134.1'},\n",
" {'displayUrl': 'https://www.facebook.com/events/758774757644756',\n",
" 'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.5',\n",
" 'name': 'Mercadito Fest - facebook.com',\n",
" 'snippet': 'Mercadito Fest is a single-day music, food, arts and crafts event in Laredo, Texas, featuring work by over 30 local musicians and artists drawing guests...',\n",
" 'url': 'https://www.bing.com/cr?IG=030CFE7ACADB4E689CCE6DEFF5BDA30F&CID=0547A6A3551863FF3A9CAD0254B76224&rd=1&h=k0y6qTupqAKJ4bMXIK3Yq9NDrT8F0W6eSGFgMJz2TJ8&v=1&r=https%3a%2f%2fwww.facebook.com%2fevents%2f758774757644756&p=DevEx,5148.1'},\n",
" {'dateLastCrawled': '2018-01-28T16:45:00',\n",
" 'displayUrl': 'https://www.facebook.com/events/396459094028237',\n",
" 'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.6',\n",
" 'name': 'WBCA Comedy Jam For George 2017 - facebook.com',\n",
" 'snippet': 'The Funniest Event to Hit Laredo ... The public is invited to support the arts while at the same time enjoy great comedy and the company of good friends.” said ...',\n",
" 'url': 'https://www.bing.com/cr?IG=030CFE7ACADB4E689CCE6DEFF5BDA30F&CID=0547A6A3551863FF3A9CAD0254B76224&rd=1&h=cHZdpjakVHllmcju8QLxN4tpXdL2i5PUD1B-rAh5tE0&v=1&r=https%3a%2f%2fwww.facebook.com%2fevents%2f396459094028237%2f&p=DevEx,5163.1'},\n",
" {'dateLastCrawled': '2018-01-31T03:35:00',\n",
" 'displayUrl': 'https://www.facebook.com/events/266396487199273',\n",
" 'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.7',\n",
" 'name': 'Laredo Art Party with Chris Soto - Painting \"Apple ...',\n",
" 'snippet': 'Laredo Art Party with Chris Soto ... so bring your friends to enjoy our delicious food and drinks while creating your masterpieces of art! This is a 21+ event. ...',\n",
" 'url': 'https://www.bing.com/cr?IG=030CFE7ACADB4E689CCE6DEFF5BDA30F&CID=0547A6A3551863FF3A9CAD0254B76224&rd=1&h=oUdVymReulyWI7ZO-8TUHarAgdNwLh7Aovox72n2XmU&v=1&r=https%3a%2f%2fwww.facebook.com%2fevents%2f266396487199273&p=DevEx,5178.1'},\n",
" {'dateLastCrawled': '2018-02-13T19:10:00',\n",
" 'displayUrl': 'https://www.facebook.com/events/224764124732912',\n",
" 'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.8',\n",
" 'name': 'HECHO A MANO Winter Market - facebook.com',\n",
" 'snippet': 'Share this event with your friends. ... in conjunction with the Laredo Center for the Arts, and Visit Laredo to bring you our monthly Handmade Market and CaminArte!',\n",
" 'url': 'https://www.bing.com/cr?IG=030CFE7ACADB4E689CCE6DEFF5BDA30F&CID=0547A6A3551863FF3A9CAD0254B76224&rd=1&h=36qgdGWfHJZObEyDDA0ecyXfAi00lpAqNwZtLBcm6OU&v=1&r=https%3a%2f%2fwww.facebook.com%2fevents%2f224764124732912&p=DevEx,5193.1'},\n",
" {'dateLastCrawled': '2018-01-28T02:02:00',\n",
" 'displayUrl': 'https://www.facebook.com/events/183304752228370',\n",
" 'id': 'https://api.cognitive.microsoft.com/api/v5/#WebPages.9',\n",
" 'name': 'Yerma Performance - facebook.com',\n",
" 'snippet': 'Laredo Theater Guild International presents Yerma by Federico García ... Performance Art Theatre · Laredo, ... More Events at TAMIU Fine and Performing Arts Center.',\n",
" 'url': 'https://www.bing.com/cr?IG=030CFE7ACADB4E689CCE6DEFF5BDA30F&CID=0547A6A3551863FF3A9CAD0254B76224&rd=1&h=XjH9sczsbXmDxIwf0qV1rFyHwrtyz-wIF-XCtDbyaIo&v=1&r=https%3a%2f%2fwww.facebook.com%2fevents%2f183304752228370%2f&p=DevEx,5208.1'}],\n",
" 'webSearchUrl': 'https://www.bing.com/cr?IG=030CFE7ACADB4E689CCE6DEFF5BDA30F&CID=0547A6A3551863FF3A9CAD0254B76224&rd=1&h=ijNUUU2Td8WSgPtXHkFJQ4tTgViDpg-j-ghsL0uUDP8&v=1&r=https%3a%2f%2fwww.bing.com%2fsearch%3fq%3dsite%253awww.facebook.com%252fevents%252f%2blaredo%2bevents%2bart&p=DevEx,5294.1'}}"
]
},
"execution_count": 46,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"j"
]
},
{
"cell_type": "code",
"execution_count": 47,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['https://www.facebook.com/events/350689162009906',\n",
" 'https://www.facebook.com/events/1592985407617680',\n",
" 'https://www.facebook.com/events/367434193689352',\n",
" 'https://www.facebook.com/events/295163097637016',\n",
" 'https://www.facebook.com/events/101333637280749',\n",
" 'https://www.facebook.com/events/758774757644756',\n",
" 'https://www.facebook.com/events/396459094028237',\n",
" 'https://www.facebook.com/events/266396487199273',\n",
" 'https://www.facebook.com/events/224764124732912',\n",
" 'https://www.facebook.com/events/183304752228370']"
]
},
"execution_count": 47,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"urls"
]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'https://www.bing.com/cr?IG=635D00DC66B34DA8BD8DF305306F7492&CID=12D72DF2BBAF60BB2F0E2653BA0061BB&rd=1&h=eVv5vrzdM8i58zEm4RXi5IMfffQ0izZqzXt_BMO4XZk&v=1&r=https%3a%2f%2fwww.facebook.com%2fevents%2f1592985407617680%2f&p=DevEx,5092.1'"
]
},
"execution_count": 43,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"urls[1]"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
"from bs4 import BeautifulSoup\n",
"import requests"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [
{
"ename": "TypeError",
"evalue": "find() takes at least 1 argument (0 given)",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-36-548df31dd8de>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murls\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m: find() takes at least 1 argument (0 given)"
]
}
],
"source": [
"r = requests.get(urls[0])\n",
"r.content.find()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment