Last active
July 18, 2018 03:30
-
-
Save BlogBlocks/17eb0473f12a1024b587283545c22ead to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Line No. :1 | |
| ROWID :1 | |
| delimiters = ['\n', ' ', ',', '.', '?', '!', ':', 'and_what_else_you_need'] | |
| words = content | |
| for delimiter in delimiters: | |
| new_words = [] | |
| for word in words: | |
| new_words += word.split(delimiter) | |
| words = new_words | |
| Description :words, delimiter, split | |
| -------new entry -------Line No. :2 | |
| ROWID :2 | |
| def insert_info(store): | |
| with sqlite3.connect("misc.db") as db: | |
| #use a text_factory that can interpret 8-bit bytestrings | |
| db.text_factory = str | |
| cursor = db.cursor() | |
| #db.text_factory = str | |
| sql = "insert into storeit (data0, data1, data2) values (?, ?, ?)" | |
| cursor.execute(sql, store) | |
| db.commit() | |
| OR | |
| conn.text_factory = str | |
| Description :text_factory, 8-bit bytestrings, 8-bit | |
| -------new entry -------Line No. :3 | |
| ROWID :3 | |
| import sqlite3 | |
| import sys | |
| conn = sqlite3.connect('snippet.db') | |
| conn.text_factory = str | |
| c = conn.cursor() | |
| count=0 | |
| req=200 | |
| search = raw_input("Search") | |
| #for row in c.execute('SELECT rowid,* FROM tweets WHERE text MATCH %s' % search): | |
| for row in c.execute('SELECT * FROM snippet WHERE keywords MATCH ?', (search,)): | |
| count=count+1 | |
| #print count,"by",(row)[2]," | |
| ",(row)[1]," | |
| " | |
| print count,"-",(row)[1]," -- by",(row)[2]," | |
| " | |
| if count > req: | |
| conn.close() | |
| sys.exit() | |
| Description :text_factory, 8-bit bytestrings, 8-bit | |
| -------new entry -------Line No. :4 | |
| ROWID :4 | |
| with open("Use.txt",'r') as f: | |
| get_all=f.readlines() | |
| ---------------- | |
| with open("file.txt",'w') as f: | |
| for i,line in enumerate(get_all,1): ## STARTS THE NUMBERING FROM 1 (by default it begins with 0) | |
| if i == 2: ## OVERWRITES line:2 | |
| f.writelines("XXXXXXXXXXXXXXXXXXXXXXX | |
| ") | |
| else: | |
| f.writelines(line) | |
| Description :enumerate, edit line, edit, edit line number | |
| -------new entry -------Line No. :5 | |
| ROWID :5 | |
| %%writefile Key.py | |
| def twiter(): | |
| CONSUMER_KEY = 'WWWWWWWWWWWWWWWW' | |
| CONSUMER_SECRET = 'XXXXXXXXXXXXXXXXX' | |
| ACCESS_KEY = 'YYYYYYYYYYYYYYYYYYY' | |
| ACCESS_SECRET = 'ZZZZZZZZZZZZZZZZ' | |
| twir = (CONSUMER_KEY, CONSUMER_SECRET, ACCESS_KEY, ACCESS_SECRET) | |
| return twir | |
| Description :Key, Key.py, twitter api, Twitter API | |
| -------new entry -------Line No. :6 | |
| ROWID :6 | |
| from itertools import tee | |
| count=0 | |
| with open("symmetrymag_tweets.csv") as inf: | |
| for line in inf: | |
| lines = line[39:] | |
| outf = open("symmetrymag.txt", "a") | |
| outf.write(lines) | |
| outf.close() | |
| Description :CSV, clean csv, edit csv, prepare for database | |
| -------new entry -------Line No. :7 | |
| ROWID :7 | |
| search = raw_input("find : ") | |
| file = open("symmetrymag.txt") | |
| lines = file.readlines() | |
| for line in lines: | |
| if search in line:print line | |
| if search == True: | |
| file.close() | |
| exit() | |
| file.close() | |
| Description :search, search text file, textfile, edit textfile | |
| -------new entry -------Line No. :8 | |
| ROWID :8 | |
| import sqlite3 | |
| import time | |
| #account = "TEDTalks.txt" | |
| account = "symmetrymag.txt" | |
| #account = "elonmusk.txt" | |
| #account = "realDonaldTrump.txt" | |
| user = account[:-4] | |
| lines = open(account,"r") | |
| line = lines.readline() | |
| for line in lines: | |
| conn = sqlite3.connect('collection.db') | |
| conn.text_factory = str | |
| c = conn.cursor() | |
| c.execute("INSERT INTO tweets VALUES (?,?)", (line, user)) | |
| conn.commit() | |
| conn.close() | |
| #print line | |
| conn.commit() | |
| conn.close() | |
| Description :insert data, text file to database, text to data | |
| -------new entry -------Line No. :9 | |
| ROWID :9 | |
| import Key | |
| from random import randint | |
| #Twitter API credentials | |
| consumer_key = Key.twiter()[0] | |
| consumer_secret = Key.twiter()[1] | |
| access_key = Key.twiter()[2] | |
| access_secret = Key.twiter()[3] | |
| def get_all_tweets(screen_name): | |
| #Twitter only allows access to a users most recent 3240 tweets with this method | |
| #authorize twitter, initialize tweepy | |
| auth = tweepy.OAuthHandler(consumer_key, consumer_secret) | |
| auth.set_access_token(access_key, access_secret) | |
| api = tweepy.API(auth) | |
| Description :Key , using Key.py, using Key, API key | |
| -------new entry -------Line No. :10 | |
| ROWID :10 | |
| conn = GetSqliteConnection(db_path) | |
| conn.text_factory = lambda x: unicode(x, 'utf-8', 'ignore') | |
| Description :text_factory , utf-8, unicode, lambda | |
| -------new entry -------Line No. :11 | |
| ROWID :11 | |
| %%writefile /home/jack/hidden/Authorize.py | |
| def keys(): | |
| ftp = "ftp.MYsite.com" | |
| username = "Josephine" | |
| password = "WhaWah2525" | |
| login = (ftp,username,password) | |
| return login | |
| ------------ | |
| import sys | |
| sys.path.insert(0, "/home/jack/hidden" | |
| import Authorize | |
| ftp = Authorize.keys()[0] | |
| username = Authorize.keys()[1] | |
| password = Authorize.keys()[2] | |
| print ftp, username, password | |
| Description :Authorize, ftp, hidden passwords, Authorize.keys | |
| -------new entry -------Line No. :12 | |
| ROWID :12 | |
| !sqlite3 ipydb.db "pragma integrity_check;" | |
| Description :verify database, pragma, integrity_check, check database | |
| -------new entry -------Line No. :13 | |
| ROWID :13 | |
| import base64 | |
| string="This is the text above Encoded Base64" | |
| encodedlistvalue=base64.b64encode(string) | |
| string = encodedlistvalue | |
| print encodedlistvalue," | |
| ",base64.b64decode(string) | |
| Description :base64, encoding base64, decoding base64, Encoded Base64 | |
| -------new entry -------Line No. :14 | |
| ROWID :14 | |
| import base64 #encode muliple lines and keep the format | |
| string = | |
| ╲╲╭━━━━━━━╮╱╱ | |
| ╲╭╯╭━╮┈╭━╮╰╮╱ | |
| ╲┃┈┃┈▊┈┃┈▊┈┃╱ | |
| ╲┃┈┗━┛┈┗━┛┈┃╱ | |
| ╱┃┈┏━━━━━┓┈┃╲ | |
| ╱┃┈┃┈┈╭━╮┃┈┃╲ | |
| ╱╰╮╰━━┻━┻╯╭╯╲ | |
| ╱╱╰━━━━━━━╯╲╲ FROM: http://copy.r74n.com/ascii-art | |
| EncodedStringValue=base64.b64encode(string) | |
| string2 = EncodedStringValue | |
| print "Decoded String2 : ",base64.b64decode(string2) | |
| Description :base64, encoding base64, decoding base64, Encoded Base64 | |
| -------new entry -------Line No. :15 | |
| ROWID :15 | |
| import sqlite3 | |
| conn = sqlite3.connect('ipydb.db') | |
| c = conn.cursor()# Never | |
| for row in c.execute('SELECT * FROM python ORDER BY code'): | |
| print"entry :",(row[0]).encode('ascii')," | |
| " | |
| print"keywords :",(row[1]).encode('ascii')," | |
| ----- | |
| "," | |
| " | |
| #data = c.fetchall() | |
| #print data | |
| Description :ascii, encode('ascii'), fetchall, sqlite3 | |
| -------new entry -------Line No. :16 | |
| ROWID :16 | |
| import sqlite3 | |
| import feedparser | |
| import time | |
| import sqlite3 | |
| Dbase = 'bigfeedfts.db' | |
| conn = sqlite3.connect(Dbase) | |
| c = conn.cursor() | |
| c.execute( | |
| CREATE VIRTUAL TABLE IF NOT EXISTS bbctech | |
| USING FTS3(head, feed); | |
| ) | |
| count=0 | |
| while count<35: | |
| count=count+1 | |
| if count==1:feed='http://feeds.bbci.co.uk/news/technology/rss.xml' | |
| if count==2:feed='http://www.cbn.com/cbnnews/us/feed/' | |
| d = feedparser.parse(feed) | |
| for post in d.entries: | |
| aa = `d['feed']['title'],d['feed']['link'],d.entries[0]['link']` | |
| bb = `post.title + ": " + post.link + ""` | |
| conn = sqlite3.connect(Dbase) | |
| c = conn.cursor() | |
| c.execute("INSERT INTO bbctech VALUES (?,?)", (aa,bb)) | |
| conn.commit() | |
| conn.close() | |
| conn = sqlite3.connect(Dbase) | |
| c = conn.cursor()# Never | |
| count=0 | |
| for row in c.execute('SELECT * FROM bbctech ORDER BY rowid DESC'): | |
| row=str(row) | |
| row=row.replace("(u","");row=row.replace('", u"u'," | |
| ") | |
| row=row.replace("/', u'"," ");row=row.replace('"',"") | |
| row=row.replace("', u'"," ");row=row.replace("')"," ") | |
| row=row.replace("'","");row=row.replace(" , uu"," | |
| ") | |
| count=count+1 | |
| print" | |
| Number :",count," ----- | |
| ",(row) | |
| Description :rss, RSS, rss feeds, sqlite3 | |
| -------new entry -------Line No. :17 | |
| ROWID :17 | |
| import sqlite3 | |
| import sys | |
| conn = sqlite3.connect('bigfeedfts.db') | |
| c = conn.cursor() | |
| count=0 | |
| # Limited Amount of Results to 100 | |
| req=100 | |
| term = raw_input("Search Term : ") | |
| for row in c.execute("SELECT * FROM bbctech WHERE feed MATCH ?", (term,)): | |
| row=str(row) | |
| row=row.replace("(u"(u","");row=row.replace("', u'"," "); | |
| row=row.replace("u'"," ");row=row.replace(')", u" ', " | |
| "); | |
| row=row.replace(" http://"," | |
| http://");row=row.replace('")','') | |
| row=row.replace("'","");row=row.replace("#tk.rss_all", "") | |
| count=count+1 | |
| print " | |
| ",count,"----- | |
| ",(row) | |
| if count > req: | |
| conn.close() | |
| sys.exit() | |
| Description :search rss, RSS, rss feeds, search sqlite3 | |
| -------new entry -------Line No. :18 | |
| ROWID :18 | |
| #This is a great search | |
| #Much better that a regular browser search. | |
| #This search keeps a google page that may be scrapedor gleaned or just used as a hyperlink documant. | |
| #Results are fantastic. | |
| from bs4 import BeautifulSoup | |
| import requests | |
| url = u'https://www.google.com/search?num=30&newwindow=1&client=ubuntu&channel=fs&btnG=Search&q=' | |
| query = u'Trump, idiot ' | |
| r = requests.get(url+query) | |
| soup = BeautifulSoup(r.text, 'html.parser') | |
| for list in soup: | |
| print list | |
| text = str(list) | |
| html_str = text | |
| Html_file= open("filename4.html","w") | |
| Html_file.write(html_str) | |
| Html_file.close()#This is a great search | |
| #Much better that a regular browser search. | |
| #This search keeps a google page that may be scrapedor gleaned or just used as a hyperlink documant. | |
| #Results are fantastic. | |
| from bs4 import BeautifulSoup | |
| import requests | |
| url = u'https://www.google.com/search?num=30&newwindow=1&client=ubuntu&channel=fs&btnG=Search&q=' | |
| query = u'Trump, idiot ' | |
| r = requests.get(url+query) | |
| soup = BeautifulSoup(r.text, 'html.parser') | |
| for list in soup: | |
| print list | |
| text = str(list) | |
| html_str = text | |
| Html_file= open("filename4.html","w") | |
| Html_file.write(html_str) | |
| Html_file.close() | |
| ----- | |
| READ THE HTML | |
| from lxml import html | |
| print html.parse('filename4.html').xpath('//body')[0].text_content() | |
| Description :google search, parse html, html, read html | |
| -------new entry -------Line No. :19 | |
| ROWID :19 | |
| from PIL import Image, ImageFont | |
| import GenIm | |
| img = Image.open('tmmpp/HURRICANE_01.png') | |
| position = (340,600) | |
| text= "JackNorthrup_ImageBot" | |
| font = ImageFont.truetype("/home/jack/.fonts/Exo-Black.ttf", 25) | |
| col = (255,255,255,150) | |
| halo_col = (0,0,0) | |
| newim = GenIm.Draw_text_with_halo(img, position, text, font, col, halo_col) | |
| newim.save("tmmpp/HURRICANE_02.png") | |
| !showme tmmpp/HURRICANE_02.png | |
| Description :text on an image, image, text image, text, words on image | |
| -------new entry -------Line No. :20 | |
| ROWID :20 | |
| from itertools import tee | |
| count=0 | |
| with open("realDonaldTrump_tweets.csv") as inf: | |
| # set up iterators | |
| cfg,res = tee(inf) | |
| # advance cfg by four lines | |
| for i in range(4): | |
| next(cfg) | |
| for c,r in zip(cfg, res): | |
| count=count+1 | |
| if "campaign" in c: | |
| #print "Date :",c[21:] | |
| print c[39:] | |
| Description :cvs, remove characters, text edit, clean, clean text | |
| -------new entry -------Line No. :21 | |
| ROWID :21 | |
| import tweepy #https://github.com/tweepy/tweepy | |
| import csv | |
| import sys | |
| sys.path.insert(0,"/home/jack/anaconda2/envs/py27/lib/python2.7/site-packages") | |
| import Key | |
| from random import randint | |
| consumer_key = Key.twiter()[0] | |
| consumer_secret = Key.twiter()[1] | |
| access_key = Key.twiter()[2] | |
| access_secret = Key.twiter()[3] | |
| def get_all_tweets(screen_name): | |
| auth = tweepy.OAuthHandler(consumer_key, consumer_secret) | |
| auth.set_access_token(access_key, access_secret) | |
| api = tweepy.API(auth) | |
| alltweets = [] | |
| new_tweets = api.user_timeline(screen_name = screen_name,count=200) | |
| alltweets.extend(new_tweets) | |
| oldest = alltweets[-1].id - 1 | |
| while len(new_tweets) > 0: | |
| print "getting tweets before %s" % (oldest) | |
| new_tweets = api.user_timeline(screen_name = screen_name,count=200,max_id=oldest) | |
| alltweets.extend(new_tweets) | |
| oldest = alltweets[-1].id - 1 | |
| print (len(alltweets)) | |
| if (len(alltweets)) >200: | |
| outtweets = [[tweet.id_str, tweet.created_at, tweet.text.encode("utf-8")] for tweet in alltweets] | |
| with open('%s_tweets.csv' % screen_name, 'wb') as f: | |
| writer = csv.writer(f) | |
| writer.writerow(["id","created_at","text"]) | |
| writer.writerows(outtweets) | |
| pass | |
| if __name__ == '__main__': | |
| USER = raw_input("User : ") or "CNN" | |
| get_all_tweets(USER) | |
| --- | |
| get all tweets, csv, get many tweets, tweets to csv, tweets.csv | |
| Description :get all tweets, csv, get many tweets, tweets to csv, tweets.csv | |
| -------new entry -------Line No. :22 | |
| ROWID :22 | |
| # TOP ONE IS BEST | |
| textin= open('hashtag.txt', 'r') | |
| lines = textin.readlines() | |
| for line in lines: | |
| time.sleep(1) | |
| print line | |
| --- | |
| import time | |
| textin= open('hashtag.txt', 'r') | |
| lines = textin.read() | |
| time.sleep(1) | |
| print lines, | |
| --- | |
| textin= open('hashtag.txt', 'r') | |
| lines = textin.read().splitlines() | |
| time.sleep(1) | |
| print lines, | |
| --- | |
| read line by line, read(), splitlines(), read().splitlines(), read files | |
| Description :read line by line, read(), splitlines(), read().splitlines(), read files | |
| -------new entry -------Line No. :23 | |
| ROWID :23 | |
| import markovify | |
| f = open("grimm.txt") | |
| text = f.read() | |
| text_model_a = markovify.Text(text) | |
| ebook_b =open('hekel.txt') | |
| text0 = ebook_b.read() | |
| text_model_b = markovify.Text(text0) | |
| for i in range(5): | |
| print(text_model_b.make_short_sentence(140)) | |
| STR0 = (text_model_b.make_short_sentence(140)) | |
| savE = open('savE.txt', 'a') | |
| savE.write(STR0) | |
| savE.close() | |
| # 2. Print five randomly-generated sentences | |
| for i in range(5): | |
| print(text_model_a.make_short_sentence(140)) | |
| STR = (text_model_a.make_short_sentence(140)) | |
| savE = open('savE.txt', 'a') | |
| savE.write(STR) | |
| savE.close() | |
| # 3. Print three randomly-generated sentences of no more than 140 characters | |
| for i in range(5): | |
| print(text_model_a.make_short_sentence(140)) | |
| STR2 = (text_model_a.make_short_sentence(140)) | |
| savE = open('savE.txt', 'a') | |
| savE.write(STR2) | |
| savE.close() | |
| # Combine the models into a single one | |
| both_models = markovify.combine([text_model_a,text_model_b]) | |
| for i in range(5): | |
| print(both_models.make_short_sentence(140)) | |
| STR3 = (both_models.make_short_sentence(140)) | |
| savE = open('savE.txt', 'a') | |
| savE.write(STR3) | |
| savE.close() | |
| --- | |
| markovify , combine , text_model_a , | |
| Description :markovify, combine, text_model_a, markovify, generated sentences | |
| -------new entry -------Line No. :24 | |
| ROWID :24 | |
| Using shutil | |
| from shutil import copyfile | |
| copyfile(src, dst) | |
| ---- | |
| # Python Copy File - Sample Code | |
| from shutil import copyfile | |
| from sys import exit | |
| source = input("Enter source file with full path: ") | |
| target = input("Enter target file with full path: ") | |
| # adding exception handling | |
| try: | |
| copyfile(source, target) | |
| except IOError as e: | |
| print("Unable to copy file. %s" % e) | |
| exit(1) | |
| except: | |
| print("Unexpected error:", sys.exc_info()) | |
| exit(1) | |
| print(" | |
| File copy done! | |
| ") | |
| while True: | |
| print("Do you like to print the file ? (y/n): ") | |
| check = input() | |
| if check == 'n': | |
| break | |
| elif check == 'y': | |
| file = open(target, "r") | |
| print(" | |
| Here follows the file content: | |
| ") | |
| print(file.read()) | |
| file.close() | |
| print() | |
| break | |
| else: | |
| continue | |
| Description :copy, copy file, copy a file, from shutil import copyfile, shutil, copyfile | |
| -------new entry -------Line No. :25 | |
| ROWID :25 | |
| how to turn on the keyboard backlight | |
| USE: xset led on | |
| keyboard backlight | |
| Description :how to turn on backlight, turn on backlight, keyboard backlight, keyboard, backlight | |
| -------new entry -------Line No. :26 | |
| ROWID :26 | |
| import sqlite3 | |
| import os.path | |
| from os import listdir, getcwd | |
| from IPython.core.display import Image | |
| def get_picture_list(rel_path): | |
| abs_path = os.path.join(os.getcwd(),rel_path) | |
| print 'abs_path =', abs_path | |
| dir_files = os.listdir(abs_path) | |
| #print dir_files | |
| return dir_files | |
| picture_list = get_picture_list('snippets') | |
| print picture_list | |
| ------------ | |
| get_picture_list, get list of files, get a file list in memory | |
| files in memeory , file list memory | |
| Description :get_picture_list, get list of files, get a file list in memory | |
| -------new entry -------Line No. :27 | |
| ROWID :27 | |
| import sqlite3 | |
| import sys | |
| conn = sqlite3.connect('pdfs.db') | |
| conn.text_factory = str | |
| c = conn.cursor() | |
| count=0;req=200 | |
| search = raw_input("Search : ") | |
| for row in c.execute('SELECT rowid, text FROM pdfs WHERE text MATCH ?', (search,)): | |
| count=count+1 | |
| print (row)[0],"-",(row)[1], | |
| if count > req: | |
| conn.close() | |
| sys.exit() | |
| -------- | |
| search database, search db, raw_input, SQLite search | |
| Description :search database, search db, raw_input, SQLite search | |
| -------new entry -------Line No. :28 | |
| ROWID :28 | |
| Creating a MySQL database with Python | |
| If you don't have it you may need: | |
| sudo apt-get install libmysqlclient-dev | |
| # Works | |
| import MySQLdb as db | |
| con = db.connect("localhost","root","") | |
| cur = con.cursor() | |
| cur.execute('CREATE DATABASE searchdb01;') | |
| Description :MySQL, Creating a MySQL database, libmysqlclient-dev, searchdb01 | |
| -------new entry -------Line No. :29 | |
| ROWID :29 | |
| # works | |
| import MySQLdb as db | |
| import json | |
| import base64 | |
| con = db.connect("localhost","root","ThinkPadT$#", "searchdb01") | |
| file = ""[mylist.jsn | |
| while 1: | |
| line = file.readline() | |
| if not line: | |
| break | |
| pass # do something | |
| #listname='mylist.json' | |
| #stringlistvalue=json.dumps(listname) | |
| "" | |
| keywords = "" | |
| database, code, python, lesson 1, Oh234 | |
| "" | |
| encodedlistvalue=base64.b64encode(file) | |
| with con: | |
| cur = con.cursor() | |
| cur.execute("CREATE TABLE Code(Id INT PRIMARY KEY AUTO_INCREMENT, Name VARCHAR(2500), Keywords VARCHAR(500))") | |
| #cur.execute("INSERT INTO Code(Name) VALUES('%s')" % (encodedlistvalue)) | |
| cur.execute("INSERT INTO Code(Name, Keywords) VALUES('%s','%s')" % (encodedlistvalue, keywords)) | |
| Description :MySQL, Creating a MySQL database, libmysqlclient-dev, searchdb01 | |
| -------new entry -------Line No. :30 | |
| ROWID :30 | |
| #!/usr/bin/python | |
| import MySQLdb | |
| import sys | |
| import base64 | |
| con = db.connect("localhost","root"," ", "searchdb01") | |
| cur = con.cursor() | |
| # execute the SQL query using execute() method. | |
| cur.execute ("select Id, Name, Keywords from Code") | |
| data = cur.fetchall () | |
| # print the rows | |
| for row in data : | |
| encodedlistvalue=base64.b64decode(row[1]) | |
| print row[0], encodedlistvalue, ' | |
| ', 'Keywords:', row[2], ' | |
| ----------------------------- | |
| ' | |
| # close the cursor object | |
| cur.close () | |
| # close the connection | |
| con.close () | |
| # exit the program | |
| sys.exit() | |
| Description :MySQL, base64 , b64decode, libmysqlclient-dev, searchdb01 | |
| -------new entry -------Line No. :31 | |
| ROWID :31 | |
| import MySQLdb | |
| con = db.connect("localhost","root","ThinkPadT$#", "searchdb01") | |
| param = "lesson" | |
| c = con.cursor() | |
| c.execute("SELECT * FROM Code WHERE Keywords LIKE %s LIMIT 1", ("%" + param + "%",)) | |
| data = c.fetchall() | |
| for row in data : | |
| encodedlistvalue=base64.b64decode(row[1]) | |
| print row[0], encodedlistvalue, ' | |
| ', 'Keywords:', row[2], ' | |
| ----------------------------- | |
| ' | |
| c.close() | |
| Description :MySQL, base64 , b64decode, libmysqlclient-dev, searchdb01 | |
| -------new entry -------Line No. :32 | |
| ROWID :32 | |
| def createdb(dbnew): | |
| import sqlite3 | |
| conn = sqlite3.connect(dbnew) | |
| c = conn.cursor() | |
| query1 = "DROP TABLE IF EXISTS Junk" | |
| query2 = ""CREATE TABLE IF NOT EXISTS Junk( | |
| "language" VARCHAR(32) NOT NULL, | |
| "keywords" VARCHAR(500) default NULL, | |
| "script" VARCHAR(2500) default NULL | |
| ) | |
| "" | |
| c.execute(query1) | |
| c.execute(query2) | |
| dbnew = "newdb.db" | |
| createdb(dbnew) | |
| Description :function to create a database, function, database function | |
| -------new entry -------Line No. :33 | |
| ROWID :33 | |
| import sqlite3 | |
| import os.path | |
| from os import listdir, getcwd | |
| from IPython.core.display import Image | |
| def get_picture_list(rel_path): | |
| abs_path = os.path.join(os.getcwd(),rel_path) | |
| print 'abs_path =', abs_path | |
| dir_files = os.listdir(abs_path) | |
| #print dir_files | |
| return dir_files | |
| picture_list = get_picture_list('snippets') | |
| print picture_list | |
| import sqlite3 | |
| import os.path | |
| from os import listdir, getcwd | |
| from IPython.core.display import Image | |
| def create_or_open_db(db_file): | |
| db_is_new = not os.path.exists(db_file) | |
| conn = sqlite3.connect(db_file) | |
| if db_is_new: | |
| print 'Creating schema' | |
| sql = '''create table if not exists PICTURES( | |
| ID INTEGER PRIMARY KEY AUTOINCREMENT, | |
| PICTURE BLOB, | |
| TYPE TEXT, | |
| FILE_NAME TEXT);''' | |
| conn.execute(sql) # shortcut for conn.cursor().execute(sql) | |
| else: | |
| print 'Schema exists | |
| ' | |
| return conn | |
| def insert_picture(picture_file): | |
| with open(picture_file, 'rb') as input_file: | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| ablob = input_file.read() | |
| base=os.path.basename(picture_file) | |
| afile, ext = os.path.splitext(base) | |
| sql = '''INSERT INTO PICTURES | |
| (PICTURE, TYPE, FILE_NAME) | |
| VALUES(?, ?, ?);''' | |
| c.execute(sql,[sqlite3.Binary(ablob), ext, afile]) | |
| conn.commit() | |
| def loadimages(dbname, path): | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| #conn.execute("DELETE FROM PICTURES") | |
| for fn in picture_list: | |
| picture_file = path+"/"+fn | |
| insert_picture(picture_file) | |
| for r in c.execute("SELECT rowid, FILE_NAME FROM PICTURES"): | |
| print r[0],r[1] | |
| conn.commit() | |
| def get_image(picture_id): | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| c.execute("SELECT PICTURE, TYPE, FILE_NAME FROM PICTURES WHERE id = ?;",(picture_id,)) | |
| #sql = "SELECT PICTURE, TYPE, FILE_NAME FROM PICTURES WHERE id = 19" | |
| param = {'id': picture_id} | |
| #c.execute(sql, param) | |
| ablob, ext, afile = c.fetchone() | |
| filename = afile + ext | |
| with open(filename, 'wb') as output_file: | |
| output_file.write(ablob) | |
| return filename | |
| dbname = "ImageC.db" | |
| db_file = create_or_open_db(dbname) | |
| path = "snippets/" | |
| loadimages(dbname, path) | |
| filename = get_image(16) | |
| print filename | |
| Image(filename=filename) | |
| ----------------- | |
| store, retrieve images,SQLite Databasestore, | |
| retrieve images, from SQLite , Database | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :34 | |
| ROWID :34 | |
| try: | |
| mercury = wikipedia.summary("Mercury") | |
| except wikipedia.exceptions.DisambiguationError as e: | |
| lines = str(e.options) | |
| lines=lines.replace("u'","");lines=lines.replace("', "," | |
| ") | |
| lines=lines.replace("[","");lines=lines.replace("]","") | |
| lines=lines.replace('u"','');lines=lines.replace('"','') | |
| with open("textwik.txt", "w")as f: | |
| f.write(lines) | |
| f.close() | |
| bad_words = ['(disambiguation)', 'All pages'] | |
| with open('textwik.txt') as oldfile, open('usewik.txt', 'w') as newfile: | |
| for line in oldfile: | |
| if not any(bad_word in line for bad_word in bad_words): | |
| newfile.write(line) | |
| --------------- | |
| wikipedia , bad_words, bad words, disambiguation, remove lines from text | |
| Description :wikipedia , bad_words, bad words, disambiguation, remove lines from text | |
| -------new entry -------Line No. :35 | |
| ROWID :35 | |
| %%writefile Image2SQLite.py | |
| import sqlite3 | |
| import os.path | |
| from os import listdir, getcwd | |
| from IPython.core.display import Image | |
| def getImage_list(rel_path): | |
| abs_path = os.path.join(os.getcwd(),rel_path) | |
| print 'abs_path =', abs_path | |
| dir_files = os.listdir(abs_path) | |
| return dir_files | |
| def create_or_open_db(dbname): | |
| db_is_new = not os.path.exists(db_file) | |
| conn = sqlite3.connect(db_file) | |
| if db_is_new: | |
| print 'Creating schema' | |
| sql = '''create table if not exists images( | |
| ID INTEGER PRIMARY KEY AUTOINCREMENT, | |
| image BLOB, | |
| TYPE TEXT, | |
| imagE TEXT);''' | |
| conn.execute(sql) # shortcut for conn.cursor().execute(sql) | |
| else: | |
| print 'Schema exists | |
| ' | |
| conn.commit() | |
| conn.close() | |
| return conn | |
| def insertImage(dbname, imageFile): | |
| with open(imageFile, 'rb') as input_file: | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| ablob = input_file.read() | |
| base=os.path.basename(imageFile) | |
| afile, ext = os.path.splitext(base) | |
| sql = '''INSERT INTO images | |
| (image, TYPE, imagE) | |
| VALUES(?, ?, ?);''' | |
| c.execute(sql,[sqlite3.Binary(ablob), ext, afile]) | |
| conn.commit() | |
| def loadimagE(dbname, path): | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| #conn.execute("DELETE FROM images") | |
| for fn in image_list: | |
| imageFile = path+"/"+fn | |
| insertImage(imageFile) | |
| for r in c.execute("SELECT rowid, imagE FROM images"): | |
| print r[0],r[1] | |
| conn.commit() | |
| conn.close() | |
| def image_id(dbname): | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| rows = c.execute("SELECT rowid, TYPE, imagE FROM images") | |
| for row in rows: | |
| print row[0],row[2]+row[1] | |
| return | |
| def get_image(dbname,image_id): | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| c.execute("SELECT image, TYPE, imagE FROM images WHERE id = ?;",(image_id,)) | |
| #sql = "SELECT image, TYPE, imagE FROM images WHERE id = 19" | |
| param = {'id': image_id} | |
| #c.execute(sql, param) | |
| ablob, ext, afile = c.fetchone() | |
| filename = afile + ext | |
| with open(filename, 'wb') as output_file: | |
| output_file.write(ablob) | |
| return filename | |
| --------------- | |
| USAGE: | |
| import Image2Data | |
| picture_list = Image2Data.get_picture_list('snippets') | |
| print picture_list | |
| import Image2Data | |
| dbname = "ImageE.db" | |
| Image2Data.create_or_open_db(dbname) | |
| #insert one image | |
| import Image2Data | |
| dbname = "ImageD.db" | |
| picture_file = "01.jpg" | |
| Image2Data.insert_picture(dbname, picture_file) | |
| import Image2Data | |
| dbname = "ImageD.db" | |
| path = "snippets" | |
| loadimages(dbname, path) | |
| def image_id(dbname): | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| rows = c.execute("SELECT rowid, TYPE, FILE_NAME FROM PICTURES") | |
| for row in rows: | |
| print row[0],row[2]+row[1] | |
| #list images by id | |
| dbname = "ImageD.db" | |
| image_id(dbname) | |
| #retrieve image by id | |
| filename = get_image(dbname,1) | |
| print filename | |
| Image(filename=filename) | |
| Description :wikipedia , bad_words, bad words, disambiguation, remove lines from text | |
| -------new entry -------Line No. :36 | |
| ROWID :36 | |
| %%writefile Image2SQLite.py | |
| import sqlite3 | |
| import os.path | |
| from os import listdir, getcwd | |
| from IPython.core.display import Image | |
| def getImage_list(rel_path): | |
| abs_path = os.path.join(os.getcwd(),rel_path) | |
| print 'abs_path =', abs_path | |
| dir_files = os.listdir(abs_path) | |
| return dir_files | |
| def create_or_open_db(dbname): | |
| db_is_new = not os.path.exists(db_file) | |
| conn = sqlite3.connect(db_file) | |
| if db_is_new: | |
| print 'Creating schema' | |
| sql = '''create table if not exists images( | |
| ID INTEGER PRIMARY KEY AUTOINCREMENT, | |
| image BLOB, | |
| TYPE TEXT, | |
| imagE TEXT);''' | |
| conn.execute(sql) # shortcut for conn.cursor().execute(sql) | |
| else: | |
| print 'Schema exists | |
| ' | |
| conn.commit() | |
| conn.close() | |
| return conn | |
| def insertImage(dbname, imageFile): | |
| with open(imageFile, 'rb') as input_file: | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| ablob = input_file.read() | |
| base=os.path.basename(imageFile) | |
| afile, ext = os.path.splitext(base) | |
| sql = '''INSERT INTO images | |
| (image, TYPE, imagE) | |
| VALUES(?, ?, ?);''' | |
| c.execute(sql,[sqlite3.Binary(ablob), ext, afile]) | |
| conn.commit() | |
| def loadimagE(dbname, path): | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| #conn.execute("DELETE FROM images") | |
| for fn in image_list: | |
| imageFile = path+"/"+fn | |
| insertImage(imageFile) | |
| for r in c.execute("SELECT rowid, imagE FROM images"): | |
| print r[0],r[1] | |
| conn.commit() | |
| conn.close() | |
| def image_id(dbname): | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| rows = c.execute("SELECT rowid, TYPE, imagE FROM images") | |
| for row in rows: | |
| print row[0],row[2]+row[1] | |
| return | |
| def get_image(dbname,image_id): | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| c.execute("SELECT image, TYPE, imagE FROM images WHERE id = ?;",(image_id,)) | |
| #sql = "SELECT image, TYPE, imagE FROM images WHERE id = 19" | |
| param = {'id': image_id} | |
| #c.execute(sql, param) | |
| ablob, ext, afile = c.fetchone() | |
| filename = afile + ext | |
| with open(filename, 'wb') as output_file: | |
| output_file.write(ablob) | |
| return filename | |
| --------------- | |
| USAGE: | |
| import Image2Data | |
| picture_list = Image2Data.get_picture_list('snippets') | |
| print picture_list | |
| import Image2Data | |
| dbname = "ImageE.db" | |
| Image2Data.create_or_open_db(dbname) | |
| #insert one image | |
| import Image2Data | |
| dbname = "ImageD.db" | |
| picture_file = "01.jpg" | |
| Image2Data.insert_picture(dbname, picture_file) | |
| import Image2Data | |
| dbname = "ImageD.db" | |
| path = "snippets" | |
| loadimages(dbname, path) | |
| def image_id(dbname): | |
| conn = sqlite3.connect(dbname) | |
| c = conn.cursor() | |
| rows = c.execute("SELECT rowid, TYPE, FILE_NAME FROM PICTURES") | |
| for row in rows: | |
| print row[0],row[2]+row[1] | |
| #list images by id | |
| dbname = "ImageD.db" | |
| image_id(dbname) | |
| #retrieve image by id | |
| filename = get_image(dbname,1) | |
| print filename | |
| Image(filename=filename) | |
| ------------ | |
| images to database , image2data , store images, store images as data, SQLite images | |
| Description :images to database , image2data , store images, store images as data, SQLite images | |
| -------new entry -------Line No. :37 | |
| ROWID :37 | |
| # Get number of words in a file | |
| fname = raw_input("Enter file name: ") | |
| num_words = 0 | |
| with open(fname, 'r') as f: | |
| for line in f: | |
| words = line.split() | |
| num_words += len(words) | |
| print "Number of words: ",num_words | |
| Description :Get words file number words file 'Get number of words in a file' | |
| -------new entry -------Line No. :38 | |
| ROWID :38 | |
| from time import sleep | |
| topic = raw_input("Research : ") | |
| fname = topic.replace(" ", "")+".txt" | |
| fname = "Wiki_"+fname | |
| f =open(fname, "w") | |
| f.close() | |
| import wikipedia | |
| rows = wikipedia.search(topic) | |
| for row in rows: | |
| sleep(1) | |
| para = wikipedia.summary(row) | |
| para = ' | |
| '.join((row, para)).encode('utf-8').strip() | |
| enter = open(fname, "a") | |
| enter.write(para) | |
| enter.close() | |
| print para | |
| enter.close() | |
| Description :search summary wikipedia text file | |
| -------new entry -------Line No. :39 | |
| ROWID :39 | |
| # %load SearchFilename.py | |
| ''' | |
| Search a filename for a phrase and how many following lines to display | |
| USAGE: | |
| import SearchFilename | |
| filename = "hek.txt" | |
| length = 4 | |
| SearchFilename.searchfilename(filename, length) | |
| ''' | |
| def searchfilename(filename, length): | |
| f = open(filename, "r") | |
| searchlines = f.readlines() | |
| f.close() | |
| search = str(raw_input("Search Phrase : ")) | |
| for i, line in enumerate(searchlines): | |
| if search in line: | |
| for l in searchlines[i:i+length]: print l, | |
| USAGE: | |
| import SearchFilename | |
| filename = "Automate-the-Boring-Stuff.txt" | |
| # length = how many lines after | |
| length = 4 | |
| SearchFilename.searchfilename(filename, length) | |
| ------ | |
| search text file search file module import searchfile | |
| Description :search text file search file module import searchfile | |
| -------new entry -------Line No. :40 | |
| ROWID :40 | |
| import os | |
| import timeit | |
| def txsearch(): | |
| # Ask to enter string to search | |
| Sstring = raw_input("Search Phrase") | |
| for fname in os.listdir('./'): | |
| # Apply file type filter | |
| if fname.endswith(".txt"): | |
| # Open file for reading | |
| fo = open(fname) | |
| # Read the first line from the file | |
| line = fo.readline() | |
| # Initialize counter for line number | |
| line_no = 1 | |
| # Loop until EOF | |
| while line != '' : | |
| index = line.find(Sstring) | |
| if ( index != -1) : | |
| # Set some parameters no lines longer than 240 characters | |
| # or less than search phrase +30 characters | |
| if len(line)< 240 and len(line)> len(Sstring)+20 : | |
| #print(fname, "[", line_no, ",", index, "] ", line) | |
| #print fname,line[1:-8]," " | |
| print fname,line_no,line | |
| # Read next line | |
| line = fo.readline() | |
| # Increment line counter | |
| line_no += 1 | |
| # Close the files | |
| fo.close() | |
| ------ | |
| search text file search file module import searchfile | |
| Description :search text file search file module import searchfile | |
| -------new entry -------Line No. :41 | |
| ROWID :41 | |
| import sys | |
| import tweepy | |
| import csv | |
| import Key | |
| #pass security information to variables | |
| consumer_key = Key.twiter()[0] | |
| consumer_secret = Key.twiter()[1] | |
| access_key = Key.twiter()[2] | |
| access_secret = Key.twiter()[3] | |
| #use variables to access twitter | |
| auth = tweepy.OAuthHandler(consumer_key, consumer_secret) | |
| auth.set_access_token(access_key, access_secret) | |
| api = tweepy.API(auth) | |
| #create an object called 'customStreamListener' | |
| class CustomStreamListener(tweepy.StreamListener): | |
| def on_status(self, status): | |
| print (status.author.screen_name, status.created_at, status.text) | |
| # Writing status data | |
| with open('OutputStreaming.txt', 'a') as f: | |
| writer = csv.writer(f) | |
| #status.author.screen_name = status.author.screen_name.encode('UTF-8') | |
| #status.text.encode = status.text.encode('UTF-8') | |
| writer.writerow([status.author.screen_name.encode('UTF-8'), status.created_at, status.text.encode('utf8')]) | |
| def on_error(self, status_code): | |
| print >> sys.stderr, 'Encountered error with status code:', status_code | |
| return True # Don't kill the stream | |
| def on_timeout(self): | |
| print >> sys.stderr, 'Timeout...' | |
| return True # Don't kill the stream | |
| def titles(): | |
| # Writing csv titles | |
| with open('OutputStreaming.txt', 'a') as f: | |
| writer = csv.writer(f) | |
| writer.writerow(['Author', 'Date', 'Text']) | |
| def main(): | |
| streamingAPI = tweepy.streaming.Stream(auth, CustomStreamListener()) | |
| #with open("tokens.txt", "r") as f: | |
| with open("tokens2.txt", "r") as f: | |
| tokens = f.readlines() | |
| streamingAPI.filter(track=tokens) | |
| #streamingAPI.filter(track=['Dallas', 'NewYork']) | |
| #status.text.encode('utf-8') | |
| #writer.writerow([unicode(s).encode("utf-8") for s in row]) | |
| #writer.writerow([unicode('Author', 'Date', 'Text').encode("utf-8") for 'Author', 'Date', 'Text' in row]) | |
| Description :csv to text tweepy to text tweepy tweepy streaming twitter import tweepy | |
| -------new entry -------Line No. :42 | |
| ROWID :42 | |
| def tidyt(in_string): | |
| texout = in_string.replace("', u'", " ");texout = texout.replace("\u2018", "") | |
| texout = texout.replace("\u2019 ", "");texout = texout.replace("%", "") | |
| texout = texout.replace("[u'", "");texout = texout.replace(" u'", "") | |
| texout = texout.replace(', u"', ' ');texout = texout.replace('",', ' ') | |
| texout = texout.replace("'", '');texout = texout.replace("Mr. ", 'Mr. ') | |
| texout = texout.replace(",", '');texout = texout.replace(".", '') | |
| ftexout =texout.replace("']", "") | |
| return ftexout | |
| Description :clean text tidy text textout replace string.replace | |
| -------new entry -------Line No. :43 | |
| ROWID :43 | |
| from textblob import TextBlob | |
| import random | |
| import sys | |
| # stdin's read() method just reads in all of standard input as a string; | |
| # use the decode method to convert to ascii (textblob prefers ascii) | |
| text = sys.stdin.read().decode('ascii', errors="replace") | |
| blob = TextBlob(text) | |
| short_sentences = list() | |
| for sentence in blob.sentences: | |
| if len(sentence.words) <= 5: | |
| short_sentences.append(sentence.replace(" | |
| ", " ")) | |
| for item in random.sample(short_sentences, 10): | |
| print item | |
| Description :blob text blob TextBlog stdin Short Sentences | |
| -------new entry -------Line No. :44 | |
| ROWID :44 | |
| Datetime objects ephem | |
| When creating an ephem.Date, you can specify the date as either a date or datetime object from the datetime standard Python module. You can also ask a PyEphem date to convert itself the other direction by calling its datetime() method. | |
| >>> from datetime import date, datetime | |
| >>> print(ephem.Date(datetime(2005, 4, 18, 22, 15))) | |
| 2005/4/18 22:15:00 | |
| >>> d = ephem.Date('2000/12/25 12:41:16') | |
| >>> d.datetime() | |
| datetime.datetime(2000, 12, 25, 12, 41, 15, 999999) | |
| In those last two commands, note that slight round-off error has converted sixteen seconds to 15.999999 seconds! The inevitability of such errors is why PyEphem exposes its own date type instead of returning Python datetime objects automatically. | |
| Tuples | |
| PyEphem can return a date as a six-element tuple giving the year, month, day, hour, minute, and seconds, where the seconds include any fractions of a second. You can also provide such a tuple when creating a PyEphem date. | |
| >>> timetuple = (1984, 5, 30, 12, 23, 45) | |
| >>> print(ephem.Date(timetuple)) | |
| 1984/5/30 12:23:45 | |
| >>> d = ephem.Date('2001/12/14 16:07:57') | |
| >>> print(d.tuple()) | |
| (2001, 12, 14, 16, 7, 57.00000002514571) | |
| Several functions in the Python standard module time will accept the time formatted as one of these six-element tuples. This feature was used in the Time Zones section, above, to convert a PyEphem date into local time. | |
| Triples | |
| There may be occasions where you need to manipulate the year and month but do not need to break the day into hours and minutes. In these cases, you can provide a three-item tuple (a “triple” of values) when creating a PyEphem date, and receive one back by calling the triple() method. | |
| >>> timetriple = (1998, 2, 26.691458333334594) | |
| >>> print(ephem.Date(timetriple)) | |
| 1998/2/26 16:35:42 | |
| >>> d = ephem.Date('1996/4/17 22:37:11.5') | |
| >>> print(d.triple()) | |
| (1996, 4, 17.94249421296263) | |
| Floats | |
| Finally, since a PyEphem date is really just a floating-point number, so you can manually supply the value you want it to have. | |
| >>> print(ephem.Date(37238.1721875)) | |
| 2001/12/14 16:07:57 | |
| >>> d = ephem.Date('2000/12/25 12:41:16') | |
| >>> print('%.6f' % d) | |
| 36884.028657 | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :45 | |
| ROWID :45 | |
| import numpy as np | |
| import matplotlib.pyplot as plt | |
| from jplephem.spk import SPK | |
| import time | |
| ephem = 'de421.bsp' | |
| ephem = 'de405.bsp' | |
| kernel = SPK.open(ephem) | |
| jd_1900_01_01 = 2415020.5004882407 | |
| ntimes = [i*10**n for n in range(5) for i in [1, 2, 5]] | |
| years = [np.zeros(1)] + [np.linspace(0, 100, n) for n in ntimes[1:]] # 100 years | |
| barytup = (0, 3) | |
| earthtup = (3, 399) | |
| # moontup = (3, 301) | |
| microsecs = [] | |
| for y in years: | |
| mics = [] | |
| #for thing in things: | |
| jd = jd_1900_01_01 + y * 365.25 # roughly, it doesn't matter here | |
| tstart = time.clock() | |
| answer = kernel[earthtup].compute(jd) + kernel[barytup].compute(jd) | |
| mics.append(1E+06 * (time.clock() - tstart)) | |
| microsecs.append(mics) | |
| microsecs = np.array(microsecs) | |
| many = [len(y) for y in years] | |
| fig = plt.figure() | |
| ax = plt.subplot(111, xlabel='length of JD object', | |
| ylabel='microseconds', | |
| title='time for jplephem [0,3] and [3,399] with ' + ephem ) | |
| # from here: http://stackoverflow.com/a/14971193/3904031 | |
| for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] + | |
| ax.get_xticklabels() + ax.get_yticklabels()): | |
| item.set_fontsize(item.get_fontsize() + 4) | |
| #for name, mics in zip(names, microsecs): | |
| ax.plot(many, microsecs, lw=2, label='earth') | |
| plt.legend(loc='upper left', shadow=False, fontsize='x-large') | |
| plt.xscale('log') | |
| plt.yscale('log') | |
| plt.ylim(1E+02, 1E+06) | |
| plt.savefig("jplephem speed test " + ephem.split('.')[0]) | |
| plt.show() | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :46 | |
| ROWID :46 | |
| import numpy as np | |
| import matplotlib.pyplot as plt | |
| from skyfield.api import load, JulianDate | |
| import time | |
| ephem = 'de421.bsp' | |
| ephem = 'de405.bsp' | |
| de = load(ephem) | |
| earth = de['earth'] | |
| moon = de['moon'] | |
| earth_barycenter = de['earth barycenter'] | |
| mercury = de['mercury'] | |
| jupiter = de['jupiter barycenter'] | |
| pluto = de['pluto barycenter'] | |
| things = [ earth, moon, earth_barycenter, mercury, jupiter, pluto ] | |
| names = ['earth', 'moon', 'earth barycenter', 'mercury', 'jupiter', 'pluto'] | |
| ntimes = [i*10**n for n in range(5) for i in [1, 2, 5]] | |
| years = [np.zeros(1)] + [np.linspace(0, 100, n) for n in ntimes[1:]] # 100 years | |
| microsecs = [] | |
| for y in years: | |
| from skyfield.api import load | |
| ts = load.timescale() | |
| t = ts.utc(1980, 4, 20) # the new way | |
| jd = ts.tt(jd=2444349.500592) # jd is also supported for tai, tt, tdb | |
| # Depreciated | |
| #jd = JulianDate(utc=(1900 + y, 1, 1)) | |
| mics = [] | |
| for thing in things: | |
| tstart = time.clock() | |
| answer = thing.at(jd).position.km | |
| mics.append(1E+06 * (time.clock() - tstart)) | |
| microsecs.append(mics) | |
| microsecs = np.array(microsecs).T | |
| many = [len(y) for y in years] | |
| fig = plt.figure(figsize=(10, 8)) | |
| ax = plt.subplot(111, xlabel='length of JD object', | |
| ylabel='microseconds', | |
| title='time for thing.at(jd).position.km with ' + ephem ) | |
| for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] + | |
| ax.get_xticklabels() + ax.get_yticklabels()): | |
| item.set_fontsize(item.get_fontsize() + 4) # http://stackoverflow.com/a/14971193/3904031 | |
| for name, mics in zip(names, microsecs): | |
| ax.plot(many, mics, lw=2, label=name) | |
| plt.legend(loc='upper left', shadow=False, fontsize='x-large') | |
| plt.xscale('log') | |
| plt.yscale('log') | |
| plt.savefig("skyfield speed test " + ephem.split('.')[0]) | |
| plt.show() | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :47 | |
| ROWID :47 | |
| # Some helper functions | |
| def norm(x, x0, sigma): | |
| return np.exp(-0.5 * (x - x0) ** 2 / sigma ** 2) | |
| def sigmoid(x, x0, alpha): | |
| return 1. / (1. + np.exp(- (x - x0) / alpha)) | |
| # define the curves | |
| x = np.linspace(0, 1, 100) | |
| y1 = np.sqrt(norm(x, 0.7, 0.05)) + 0.2 * (1.5 - sigmoid(x, 0.8, 0.05)) | |
| y2 = 0.2 * norm(x, 0.5, 0.2) + np.sqrt(norm(x, 0.6, 0.05)) + 0.1 * (1 - sigmoid(x, 0.75, 0.05)) | |
| y3 = 0.05 + 1.4 * norm(x, 0.85, 0.08) | |
| y3[x > 0.85] = 0.05 + 1.4 * norm(x[x > 0.85], 0.85, 0.3) | |
| # draw the curves | |
| ax = pl.axes() | |
| ax.plot(x, y1, c='gray') | |
| ax.plot(x, y2, c='blue') | |
| ax.plot(x, y3, c='red') | |
| ax.text(0.3, -0.1, "Yard") | |
| ax.text(0.5, -0.1, "Steps") | |
| ax.text(0.7, -0.1, "Door") | |
| ax.text(0.9, -0.1, "Inside") | |
| ax.text(0.05, 1.1, "fear that | |
| there's | |
| something | |
| behind me") | |
| ax.plot([0.15, 0.2], [1.0, 0.2], '-k', lw=0.5) | |
| ax.text(0.25, 0.8, "forward | |
| speed") | |
| ax.plot([0.32, 0.35], [0.75, 0.35], '-k', lw=0.5) | |
| ax.text(0.9, 0.4, "embarrassment") | |
| ax.plot([1.0, 0.8], [0.55, 1.05], '-k', lw=0.5) | |
| ax.set_title("Walking back to my | |
| front door at night:") | |
| ax.set_xlim(0, 1) | |
| ax.set_ylim(0, 1.5) | |
| # modify all the axes elements in-place | |
| XKCDify(ax, expand_axes=True) | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :48 | |
| ROWID :48 | |
| np.random.seed(0) | |
| ax = pylab.axes() | |
| x = np.linspace(0, 10, 100) | |
| ax.plot(x, np.sin(x) * np.exp(-0.1 * (x - 5) ** 2), 'b', lw=1, label='damped sine') | |
| ax.plot(x, -np.cos(x) * np.exp(-0.1 * (x - 5) ** 2), 'r', lw=1, label='damped cosine') | |
| ax.set_title('check it out!') | |
| ax.set_xlabel('x label') | |
| ax.set_ylabel('y label') | |
| ax.legend(loc='lower right') | |
| ax.set_xlim(0, 10) | |
| ax.set_ylim(-1.0, 1.0) | |
| #XKCDify the axes -- this operates in-place | |
| XKCDify(ax, xaxis_loc=0.0, yaxis_loc=1.0, | |
| xaxis_arrow='+-', yaxis_arrow='+-', | |
| expand_axes=True) | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :49 | |
| ROWID :49 | |
| "" | |
| XKCD plot generator | |
| ------------------- | |
| Author: Jake Vanderplas | |
| This is a script that will take any matplotlib line diagram, and convert it | |
| to an XKCD-style plot. It will work for plots with line & text elements, | |
| including axes labels and titles (but not axes tick labels). | |
| The idea for this comes from work by Damon McDougall | |
| http://www.mail-archive.com/matplotlib-users@lists.sourceforge.net/msg25499.html | |
| "" | |
| import numpy as np | |
| import pylab as pl | |
| from scipy import interpolate, signal | |
| import matplotlib.font_manager as fm | |
| # We need a special font for the code below. It can be downloaded this way: | |
| import os | |
| import urllib2 | |
| if not os.path.exists('Humor-Sans.ttf'): | |
| fhandle = urllib2.urlopen('http://antiyawn.com/uploads/Humor-Sans-1.0.ttf') | |
| open('Humor-Sans.ttf', 'wb').write(fhandle.read()) | |
| def xkcd_line(x, y, xlim=None, ylim=None, | |
| mag=1.0, f1=30, f2=0.05, f3=15): | |
| "" | |
| Mimic a hand-drawn line from (x, y) data | |
| Parameters | |
| ---------- | |
| x, y : array_like | |
| arrays to be modified | |
| xlim, ylim : data range | |
| the assumed plot range for the modification. If not specified, | |
| they will be guessed from the data | |
| mag : float | |
| magnitude of distortions | |
| f1, f2, f3 : int, float, int | |
| filtering parameters. f1 gives the size of the window, f2 gives | |
| the high-frequency cutoff, f3 gives the size of the filter | |
| Returns | |
| ------- | |
| x, y : ndarrays | |
| The modified lines | |
| "" | |
| x = np.asarray(x) | |
| y = np.asarray(y) | |
| # get limits for rescaling | |
| if xlim is None: | |
| xlim = (x.min(), x.max()) | |
| if ylim is None: | |
| ylim = (y.min(), y.max()) | |
| if xlim[1] == xlim[0]: | |
| xlim = ylim | |
| if ylim[1] == ylim[0]: | |
| ylim = xlim | |
| # scale the data | |
| x_scaled = (x - xlim[0]) * 1. / (xlim[1] - xlim[0]) | |
| y_scaled = (y - ylim[0]) * 1. / (ylim[1] - ylim[0]) | |
| # compute the total distance along the path | |
| dx = x_scaled[1:] - x_scaled[:-1] | |
| dy = y_scaled[1:] - y_scaled[:-1] | |
| dist_tot = np.sum(np.sqrt(dx * dx + dy * dy)) | |
| # number of interpolated points is proportional to the distance | |
| Nu = int(200 * dist_tot) | |
| u = np.arange(-1, Nu + 1) * 1. / (Nu - 1) | |
| # interpolate curve at sampled points | |
| k = min(3, len(x) - 1) | |
| res = interpolate.splprep([x_scaled, y_scaled], s=0, k=k) | |
| x_int, y_int = interpolate.splev(u, res[0]) | |
| # we'll perturb perpendicular to the drawn line | |
| dx = x_int[2:] - x_int[:-2] | |
| dy = y_int[2:] - y_int[:-2] | |
| dist = np.sqrt(dx * dx + dy * dy) | |
| # create a filtered perturbation | |
| coeffs = mag * np.random.normal(0, 0.01, len(x_int) - 2) | |
| b = signal.firwin(f1, f2 * dist_tot, window=('kaiser', f3)) | |
| response = signal.lfilter(b, 1, coeffs) | |
| x_int[1:-1] += response * dy / dist | |
| y_int[1:-1] += response * dx / dist | |
| # un-scale data | |
| x_int = x_int[1:-1] * (xlim[1] - xlim[0]) + xlim[0] | |
| y_int = y_int[1:-1] * (ylim[1] - ylim[0]) + ylim[0] | |
| return x_int, y_int | |
| def XKCDify(ax, mag=1.0, | |
| f1=50, f2=0.01, f3=15, | |
| bgcolor='w', | |
| xaxis_loc=None, | |
| yaxis_loc=None, | |
| xaxis_arrow='+', | |
| yaxis_arrow='+', | |
| ax_extend=0.1, | |
| expand_axes=False): | |
| ""Make axis look hand-drawn | |
| This adjusts all lines, text, legends, and axes in the figure to look | |
| like xkcd plots. Other plot elements are not modified. | |
| Parameters | |
| ---------- | |
| ax : Axes instance | |
| the axes to be modified. | |
| mag : float | |
| the magnitude of the distortion | |
| f1, f2, f3 : int, float, int | |
| filtering parameters. f1 gives the size of the window, f2 gives | |
| the high-frequency cutoff, f3 gives the size of the filter | |
| xaxis_loc, yaxis_log : float | |
| The locations to draw the x and y axes. If not specified, they | |
| will be drawn from the bottom left of the plot | |
| xaxis_arrow, yaxis_arrow : str | |
| where to draw arrows on the x/y axes. Options are '+', '-', '+-', or '' | |
| ax_extend : float | |
| How far (fractionally) to extend the drawn axes beyond the original | |
| axes limits | |
| expand_axes : bool | |
| if True, then expand axes to fill the figure (useful if there is only | |
| a single axes in the figure) | |
| "" | |
| # Get axes aspect | |
| ext = ax.get_window_extent().extents | |
| aspect = (ext[3] - ext[1]) / (ext[2] - ext[0]) | |
| xlim = ax.get_xlim() | |
| ylim = ax.get_ylim() | |
| xspan = xlim[1] - xlim[0] | |
| yspan = ylim[1] - xlim[0] | |
| xax_lim = (xlim[0] - ax_extend * xspan, | |
| xlim[1] + ax_extend * xspan) | |
| yax_lim = (ylim[0] - ax_extend * yspan, | |
| ylim[1] + ax_extend * yspan) | |
| if xaxis_loc is None: | |
| xaxis_loc = ylim[0] | |
| if yaxis_loc is None: | |
| yaxis_loc = xlim[0] | |
| # Draw axes | |
| xaxis = pl.Line2D([xax_lim[0], xax_lim[1]], [xaxis_loc, xaxis_loc], | |
| linestyle='-', color='k') | |
| yaxis = pl.Line2D([yaxis_loc, yaxis_loc], [yax_lim[0], yax_lim[1]], | |
| linestyle='-', color='k') | |
| # Label axes3, 0.5, 'hello', fontsize=14) | |
| ax.text(xax_lim[1], xaxis_loc - 0.02 * yspan, ax.get_xlabel(), | |
| fontsize=14, ha='right', va='top', rotation=12) | |
| ax.text(yaxis_loc - 0.02 * xspan, yax_lim[1], ax.get_ylabel(), | |
| fontsize=14, ha='right', va='top', rotation=78) | |
| ax.set_xlabel('') | |
| ax.set_ylabel('') | |
| # Add title | |
| ax.text(0.5 * (xax_lim[1] + xax_lim[0]), yax_lim[1], | |
| ax.get_title(), | |
| ha='center', va='bottom', fontsize=16) | |
| ax.set_title('') | |
| Nlines = len(ax.lines) | |
| lines = [xaxis, yaxis] + [ax.lines.pop(0) for i in range(Nlines)] | |
| for line in lines: | |
| x, y = line.get_data() | |
| x_int, y_int = xkcd_line(x, y, xlim, ylim, | |
| mag, f1, f2, f3) | |
| # create foreground and background line | |
| lw = line.get_linewidth() | |
| line.set_linewidth(2 * lw) | |
| line.set_data(x_int, y_int) | |
| # don't add background line for axes | |
| if (line is not xaxis) and (line is not yaxis): | |
| line_bg = pl.Line2D(x_int, y_int, color=bgcolor, | |
| linewidth=8 * lw) | |
| ax.add_line(line_bg) | |
| ax.add_line(line) | |
| # Draw arrow-heads at the end of axes lines | |
| arr1 = 0.03 * np.array([-1, 0, -1]) | |
| arr2 = 0.02 * np.array([-1, 0, 1]) | |
| arr1[::2] += np.random.normal(0, 0.005, 2) | |
| arr2[::2] += np.random.normal(0, 0.005, 2) | |
| x, y = xaxis.get_data() | |
| if '+' in str(xaxis_arrow): | |
| ax.plot(x[-1] + arr1 * xspan * aspect, | |
| y[-1] + arr2 * yspan, | |
| color='k', lw=2) | |
| if '-' in str(xaxis_arrow): | |
| ax.plot(x[0] - arr1 * xspan * aspect, | |
| y[0] - arr2 * yspan, | |
| color='k', lw=2) | |
| x, y = yaxis.get_data() | |
| if '+' in str(yaxis_arrow): | |
| ax.plot(x[-1] + arr2 * xspan * aspect, | |
| y[-1] + arr1 * yspan, | |
| color='k', lw=2) | |
| if '-' in str(yaxis_arrow): | |
| ax.plot(x[0] - arr2 * xspan * aspect, | |
| y[0] - arr1 * yspan, | |
| color='k', lw=2) | |
| # Change all the fonts to humor-sans. | |
| prop = fm.FontProperties(fname='Humor-Sans.ttf', size=16) | |
| for text in ax.texts: | |
| text.set_fontproperties(prop) | |
| # modify legend | |
| leg = ax.get_legend() | |
| if leg is not None: | |
| leg.set_frame_on(False) | |
| for child in leg.get_children(): | |
| if isinstance(child, pl.Line2D): | |
| x, y = child.get_data() | |
| child.set_data(xkcd_line(x, y, mag=10, f1=100, f2=0.001)) | |
| child.set_linewidth(2 * child.get_linewidth()) | |
| if isinstance(child, pl.Text): | |
| child.set_fontproperties(prop) | |
| # Set the axis limits | |
| ax.set_xlim(xax_lim[0] - 0.1 * xspan, | |
| xax_lim[1] + 0.1 * xspan) | |
| ax.set_ylim(yax_lim[0] - 0.1 * yspan, | |
| yax_lim[1] + 0.1 * yspan) | |
| # adjust the axes | |
| ax.set_xticks([]) | |
| ax.set_yticks([]) | |
| if expand_axes: | |
| ax.figure.set_facecolor(bgcolor) | |
| ax.set_axis_off() | |
| ax.set_position([0, 0, 1, 1]) | |
| return ax | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :50 | |
| ROWID :50 | |
| %%writefile SatInfo.py | |
| from itertools import izip, islice | |
| from time import sleep | |
| def satinfo(): | |
| search_string = raw_input("Load : ") | |
| with open('visual.txt', 'r') as infile, open('visual.tmp', 'w') as outfile: | |
| for line in infile: | |
| if search_string in line: | |
| outfile.writelines([line, next(infile), next(infile)]) | |
| from time import sleep | |
| with open('visual.tmp', 'r') as f: | |
| while True: | |
| name = f.readline() | |
| one = f.readline() | |
| two = f.readline() | |
| sleep(1) | |
| return name, one, two | |
| def prntlist(): | |
| from time import sleep | |
| with open('visual.txt', 'r') as f: | |
| while True: | |
| next_n_lines = list(islice(f, 3)) | |
| if not next_n_lines: | |
| break | |
| sleep(.5) | |
| print next_n_lines[0], | |
| def reuse(): | |
| from time import sleep | |
| with open('visual.tmp', 'r') as f: | |
| while True: | |
| name = f.readline() | |
| one = f.readline() | |
| two = f.readline() | |
| sleep(1) | |
| #print name, one, two | |
| f.close() | |
| return name, one, two | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :51 | |
| ROWID :51 | |
| Join All Files Starting With Wiki_ | |
| import glob | |
| import os | |
| for f in glob.glob("Wiki_*.txt"): | |
| os.system("cat "+f+" >> ALL_WIKI.txt") | |
| ----------- | |
| Count All Words in a File | |
| fname = raw_input("Enter file name: ") | |
| num_words = 0 | |
| with open(fname, 'r') as f: | |
| for line in f: | |
| words = line.split() | |
| num_words += len(words) | |
| print "Number of words: ",num_words | |
| ------------ | |
| Remove All Blank Lines | |
| from Txmanip import RemoveBlank | |
| origFile = "ALL_WIKI.txt" | |
| saveAS = "ALL_WIKI_GOOD.txt" | |
| RemoveBlank.removeblank(origFile, saveAS) | |
| ------------ | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :52 | |
| ROWID :52 | |
| # %load SearchFilename.py | |
| ''' | |
| Search a filename for a phrase and how many following lines to display | |
| USAGE: | |
| import SearchFilename | |
| filename = "hek.txt" | |
| length = 4 | |
| SearchFilename.searchfilename(filename, length) | |
| ''' | |
| def searchfilename(filename, length): | |
| f = open(filename, "r") | |
| searchlines = f.readlines() | |
| f.close() | |
| search = str(raw_input("Search Phrase : ")) | |
| for i, line in enumerate(searchlines): | |
| if search in line: | |
| for l in searchlines[i:i+length]: print l, | |
| #USAGE: | |
| import SearchFilename | |
| filename = "ALL_WIKI_GOOD.txt" | |
| # length = how many lines after | |
| length = 7 | |
| SearchFilename.searchfilename(filename, length) | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :53 | |
| ROWID :53 | |
| import json | |
| import sys | |
| from time import sleep | |
| import sqlite3 | |
| import csv | |
| title = "ftp2.ipynb" | |
| f= open(title,"w") | |
| f.close() | |
| conn = sqlite3.connect('ftp.db') | |
| conn.text_factory=str | |
| c = conn.cursor() | |
| with open(title, 'a') as outfile: | |
| for row in c.execute('SELECT text, title FROM ipynb'): | |
| row0 = row[1] | |
| #row0.write(outfile) | |
| #outfile = csv.writer(row0) | |
| outfile.write(row0) | |
| conn.close() | |
| f.close() | |
| jupyter to database from database to jupyter notebook | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :54 | |
| ROWID :54 | |
| import json | |
| import sys | |
| from time import sleep | |
| import sqlite3 | |
| conn = sqlite3.connect('ftp.db') | |
| conn.text_factory=str | |
| c = conn.cursor() | |
| for row in c.execute('SELECT text, title FROM ipynb'): | |
| sleep(.5) | |
| #row0 = base64.b64decode(row[0]) | |
| #print row0, row[1] | |
| print row[1] | |
| jupyter to view database from database to jupyter notebook | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :55 | |
| ROWID :55 | |
| import base64 | |
| from time import sleep | |
| title = "FTP_and_Editor.ipynb" | |
| import sqlite3 | |
| conn = sqlite3.connect('ftp.db') | |
| conn.text_factory=str | |
| c = conn.cursor() | |
| with open(title, 'r') as f: | |
| lines = f.readlines() | |
| for line in lines: | |
| encodedlistvalue=base64.b64encode(line) | |
| c.execute("INSERT INTO ipynb VALUES (?,?)", (encodedlistvalue, line)) | |
| conn.commit() | |
| #print encodedlistvalue | |
| conn.close() | |
| f.close() | |
| jupyter to view database from database to jupyter notebook | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :56 | |
| ROWID :56 | |
| import sqlite3 | |
| conn = sqlite3.connect('ftp.db') | |
| c = conn.cursor() | |
| conn.text_factory=str | |
| c.execute("" | |
| CREATE TABLE ipynb (text, title); | |
| "") | |
| conn.commit() | |
| conn.close() | |
| jupyter to view database from database to jupyter notebook | |
| Description :store, retrieve images, from SQLite Database | |
| -------new entry -------Line No. :57 | |
| ROWID :57 | |
| import sqlite3 | |
| import sys | |
| conn = sqlite3.connect('notebooks.db') | |
| conn.text_factory = str | |
| c = conn.cursor() | |
| count=0;req=200 | |
| id1 = raw_input("Starting ID : ") | |
| id2 = raw_input("How Many Rows : ") | |
| # id1 is start id2 is how many lines | |
| for row in c.execute('SELECT rowid, * from ipynb LIMIT ? OFFSET ?', (id2, id1)): | |
| count=count+1 | |
| #print row[0],row[1]," | |
| ",row[2] | |
| print row[2] | |
| if count > req: | |
| conn.close() | |
| sys.exit() | |
| Description :print span of rows, line numbers print by id | |
| -------new entry -------Line No. :58 | |
| ROWID :58 | |
| import base64 | |
| from time import sleep | |
| import sqlite3 | |
| conn = sqlite3.connect('notebooks.db') | |
| conn.text_factory=str | |
| c = conn.cursor() | |
| line =" " | |
| title = "index" | |
| c.execute("INSERT INTO ipynb VALUES (?,?)", (title, line)) | |
| conn.commit() | |
| #print encodedlistvalue | |
| conn.close() | |
| f.close() | |
| Description :create index column | |
| -------new entry -------Line No. :59 | |
| ROWID :59 | |
| import json | |
| import sys | |
| from time import sleep | |
| import sqlite3 | |
| import csv | |
| conn = sqlite3.connect('notebooks.db') | |
| conn.text_factory=str | |
| c = conn.cursor() | |
| #c.execute("DELETE title, line FROM ipynb where rowid MATCH '362113' "): | |
| #c.execute("delete from ipynb where rowid=362113;"): | |
| c.execute("DELETE FROM ipynb WHERE rowid = ?", (362113,)) | |
| conn.commit() | |
| conn.close() | |
| Description :delete sqlite by rowid delete id rowid ROWID | |
| -------new entry -------Line No. :60 | |
| ROWID :60 | |
| import sqlite3 | |
| connection = sqlite3.connect('notebooks.db') | |
| connection.row_factory = sqlite3.Row | |
| cursor = connection.execute('select * from ipynb') | |
| # instead of cursor.description: | |
| row = cursor.fetchone() | |
| names = row.keys() | |
| print names | |
| Description :get column names, lost column names, unknown colums | |
| -------new entry -------Line No. :61 | |
| ROWID :61 | |
| # clustering dataset | |
| from sklearn.cluster import KMeans | |
| from sklearn import metrics | |
| import numpy as np | |
| import matplotlib.pyplot as plt | |
| x1 = np.random.randint(0,20, size=50) | |
| x2 = np.random.randint(0,20, size=50) | |
| # create new plot and data | |
| plt.plot() | |
| X = np.array(list(zip(x1, x2))).reshape(len(x1), 2) | |
| colors = ['b', 'g', 'c'] | |
| markers = ['o', 'v', 's'] | |
| # KMeans algorithm | |
| K = 3 | |
| kmeans_model = KMeans(n_clusters=K).fit(X) | |
| print(kmeans_model.cluster_centers_) | |
| centers = np.array(kmeans_model.cluster_centers_) | |
| plt.plot() | |
| plt.title('k means centroids') | |
| for i, l in enumerate(kmeans_model.labels_): | |
| plt.plot(x1[i], x2[i], color=colors[l], marker=markers[l],ls='None') | |
| plt.xlim([-1, 21]) | |
| plt.ylim([-1, 21]) | |
| plt.scatter(centers[:,0], centers[:,1], marker="X", color='r') | |
| plt.show() | |
| Description :clusters, centers of clusters, find centers clusters, find cluster centers | |
| -------new entry -------Line No. :62 | |
| ROWID :62 | |
| # clustering dataset | |
| from sklearn.cluster import KMeans | |
| from sklearn import metrics | |
| import numpy as np | |
| import matplotlib.pyplot as plt | |
| x1 = np.random.randint(1,9, size=30) | |
| x2 = np.random.randint(1,9, size=30) | |
| plt.plot() | |
| plt.xlim([0, 10]) | |
| plt.ylim([0, 10]) | |
| plt.title('Dataset') | |
| plt.scatter(x1, x2) | |
| plt.show() | |
| # create new plot and data | |
| plt.plot() | |
| X = np.array(list(zip(x1, x2))).reshape(len(x1), 2) | |
| colors = ['b', 'g', 'r'] | |
| markers = ['o', 'v', 's'] | |
| # KMeans algorithm | |
| K = 3 | |
| kmeans_model = KMeans(n_clusters=K).fit(X) | |
| plt.plot() | |
| for i, l in enumerate(kmeans_model.labels_): | |
| plt.plot(x1[i], x2[i], color=colors[l], marker=markers[l],ls='None') | |
| plt.xlim([0, 10]) | |
| plt.ylim([0, 10]) | |
| plt.show() | |
| Description :clusters, locating clusters, finding clusters, cluster | |
| -------new entry -------Line No. :63 | |
| ROWID :63 | |
| from gtts import gTTS | |
| import os | |
| tts = gTTS(text='Good morning. Good morning', lang='en') | |
| tts.save("good.mp3") | |
| os.system("mpg321 good.mp3") | |
| Description :text to speach, gTTS, system read mp3, txt to mp3 | |
| -------new entry -------Line No. :64 | |
| ROWID :64 | |
| import os | |
| import re | |
| from pygame import mixer | |
| import datetime | |
| import time | |
| from gtts import gTTS | |
| mp3_nameold='111' | |
| mp3_name = "1.mp3" | |
| mixer.init() | |
| f = open("test001.txt","r") | |
| ss = f.readline() | |
| while ss: | |
| split_regex = re.compile(r'[.|!|?|…]') | |
| sentences = filter(lambda t: t, [t.strip() for t in split_regex.split(ss)]) | |
| for x in sentences: | |
| if(x!=""): | |
| print(x) | |
| tts=gTTS(text=x, lang='en') | |
| tts.save(mp3_name) | |
| mixer.music.load(mp3_name) | |
| mixer.music.play() | |
| while mixer.music.get_busy(): | |
| time.sleep(0.1) | |
| mp3_nameold=mp3_name | |
| now_time = datetime.datetime.now() | |
| mp3_name = now_time.strftime("%d%m%Y%I%M%S")+".mp3" | |
| ss = f.readline() | |
| f.close | |
| mixer.music.load('1.mp3') | |
| mixer.stop | |
| mixer.quit | |
| if(os.path.exists(mp3_nameold)): | |
| os.remove(mp3_nameold) | |
| Description :text to speech, gTTS, system read mp3, txt to mp3 | |
| -------new entry -------Line No. :65 | |
| ROWID :65 | |
| # USAGE example: | |
| # import UnCamel | |
| # x = "TimeToUncamelCaseTextUsingFunctions" | |
| # UnCamel.uncamel(x) | |
| def uncamel(x): | |
| return reduce(lambda a,b: a + ((b.upper() == b and (len(a) and a[-1].upper() != a[-1])) | |
| and (' ' + b) or b), x, '') | |
| def uncam(x): | |
| import re | |
| return re.sub("([a-z])([A-Z])","\g<1> \g<2>",x) | |
| ------- | |
| def uncamel(x): | |
| return reduce(lambda a,b: a + ((b.upper() == b and | |
| (len(a) and a[-1].upper() != a[-1])) and | |
| (' ' + b) or b), x, '') | |
| x = "LetUsTryToUncamelCaseTextUsingFunctions" | |
| uncamel(x) | |
| Description :UnCamel, import UnCamel, uncamel(x) | |
| -------new entry -------Line No. :66 | |
| ROWID :66 | |
| import os | |
| import os.path | |
| title = "ipynb.list" | |
| f= open(title,"w") | |
| f.close() | |
| count=0 | |
| for dirpath, dirnames, filenames in os.walk("/home/jack/"): | |
| for filename in [f for f in filenames if f.endswith(".ipynb")]: | |
| count=count+1 | |
| Path = os.path.join(dirpath, filename) | |
| with open(title, 'a') as outfile: | |
| path = Path+" | |
| " | |
| outfile.write(path) | |
| Description :walk directory save files to text file save dir to file | |
| -------new entry -------Line No. :67 | |
| ROWID :67 | |
| titlelist = "ipynb.list" | |
| titles = open(titlelist,"r") | |
| for title in titles.readlines(): | |
| filename = os.path.basename(title) | |
| description = filename | |
| description = description.replace("-", " ") | |
| description = description.replace("_", " ") | |
| description = description.replace(".ipynb", " ") | |
| description = re.sub("([a-z])([A-Z])","\g<1> \g<2>",description) | |
| title = title.replace(" | |
| ", "") | |
| print title, description | |
| Description :create descriptions file list read by path walk directory walk directories | |
| -------new entry -------Line No. :68 | |
| ROWID :68 | |
| bad_words = ['.ipynb_checkpoints', 'checkpoint', '/.ipynb'] | |
| with open('ipynb.list') as oldfile, open('ipynb-clean.list', 'w') as newfile: | |
| for line in oldfile: | |
| if not any(bad_word in line for bad_word in bad_words): | |
| newfile.write(line) | |
| Description :clean a file remove lines with strings, remove str from lines, bad_words = ['.ipynb_checkpoints', 'checkpoint', | |
| -------new entry -------Line No. :69 | |
| ROWID :69 | |
| import sqlite3 | |
| import re | |
| import sys | |
| import time | |
| database = "test.db" | |
| #database = "junk2.db" | |
| conn = sqlite3.connect(database) | |
| conn.text_factory = lambda x: unicode(x, "utf-8", "ignore") | |
| c = conn.cursor() | |
| c.execute("" | |
| CREATE VIRTUAL TABLE IF NOT EXISTS ipynb | |
| USING FTS4(file, content, description); | |
| "") | |
| conn.commit() | |
| conn.close() | |
| conn = sqlite3.connect(database) | |
| c = conn.cursor() | |
| count=1 | |
| titlelist = "ipynb-clean.list" | |
| titles = open(titlelist,"r") | |
| for title in titles.readlines(): | |
| filename = os.path.basename(title) | |
| # Use for debug print filename,":" | |
| description = filename | |
| description = description.replace("-", " ") | |
| description = description.replace("_", " ") | |
| description = description.replace(".ipynb", " ") | |
| description = re.sub("([a-z])([A-Z])","\g<1> \g<2>",description) | |
| title = title.replace(" | |
| ", "") | |
| dt=time.ctime(os.path.getctime(title)) | |
| dt=str(dt) | |
| #dt = dt.replace(" ","") | |
| description = description+"Date :"+dt | |
| suf = title.replace("/home/jack/","") | |
| suf = suf.replace(".ipynb","_") | |
| suf = suf.replace("/","_") | |
| filename = suf+filename | |
| with open(title, "rb") as input_file: | |
| ablob = input_file.read() | |
| content = sqlite3.Binary(ablob) | |
| c.execute("INSERT INTO ipynb (file, content, description) VALUES(?, ?, ?)", | |
| (filename, content, description)) | |
| conn.commit() | |
| line = file | |
| #line ="Good-mouse-sizing-and-cropping.ipynb" | |
| index = "Index" | |
| c.execute("INSERT INTO ipynb VALUES (?,?,?)", (index, filename, description)) | |
| conn.commit() | |
| count=count+1 | |
| print count,filename, description | |
| c.close() | |
| conn.close() | |
| Description :BEST best create a database of jupyter notebooks notebook to database SQLite notebook 2 database | |
| -------new entry -------Line No. :70 | |
| ROWID :70 | |
| import sqlite3 | |
| #database = "FTS4_IPYNB_indexed.db" | |
| database = "test.db" | |
| #database = "/home/jack/Desktop/text_stuff/notebooks.db" | |
| conn = sqlite3.connect(database) | |
| conn.text_factory=str | |
| c = conn.cursor() | |
| c.execute("SELECT COUNT(*) from ipynb") | |
| (number_of_rows,)=c.fetchone() | |
| print (number_of_rows,) | |
| Description :count rows in sqlite database count rows SQLite | |
| -------new entry -------Line No. :71 | |
| ROWID :71 | |
| import json | |
| import sys | |
| from time import sleep | |
| import sqlite3 | |
| import csv | |
| database = "test.db" | |
| conn = sqlite3.connect(database) | |
| conn.text_factory=str | |
| c = conn.cursor() | |
| ROWID = | |
| c.execute("DELETE FROM ipynb WHERE rowid = ?", (ROWID,)) | |
| conn.commit() | |
| conn.close() | |
| Description :delete rows in sqlite database DELETE rows SQLite | |
| -------new entry -------Line No. :72 | |
| ROWID :72 | |
| import sqlite3 | |
| #database = "FTS4_IPYNB.db" | |
| database = "FTS4_IPYNB_indexed.db" | |
| conn = sqlite3.connect(database) | |
| c = conn.cursor() | |
| conn.text_factory=str | |
| c.execute("" | |
| CREATE VIRTUAL TABLE IF NOT EXISTS ipynb | |
| USING FTS4(file, content, description); | |
| "") | |
| conn.commit() | |
| conn.close() | |
| conn = sqlite3.connect(database) | |
| c = conn.cursor() | |
| count=0 | |
| while count<19: | |
| count=count+1 | |
| if count==1:PATH = "/home/jack/Desktop/deep-dream-generator/notebooks/" | |
| if count==2:PATH = "/home/jack/Desktop/text_stuff/" | |
| if count==3:PATH = "/home/jack/Desktop/imagebot/" | |
| if count==4:PATH = "/home/jack/Desktop/Snippet_Warehouse/" | |
| if count==5:PATH = "/home/jack/Desktop/gitjupyter/" | |
| if count==6:PATH = "/home/jack/Desktop/jack_watch/" | |
| if count==7:PATH = "/home/jack/Desktop/jack_watch/nltk/" | |
| if count==8:PATH = "/home/jack/Desktop/jack_watch/Python-Lectures/" | |
| if count==9:PATH = "/home/jack/Desktop/jack_watch/jupyter_examples-master/" | |
| if count==10:PATH = "/home/jack/Desktop/Books/numerical-python-book-code/" | |
| if count==11:PATH = "/home/jack/Desktop/Books/pydata-book/" | |
| if count==12:PATH = "/home/jack/Desktop/Ruby/" | |
| if count==13:PATH = "/home/jack/Desktop/alice/ChatterBot/" | |
| if count==14:PATH = "/home/jack/Desktop/deep-dream-generator/LOCAL-notebooks/" | |
| if count==15:PATH = "/home/jack/Desktop/numpy-array-filters/" | |
| if count==16:PATH = "/home/jack/Desktop/pycode/" | |
| if count==17:PATH = "/home/jack/Desktop/pycode/vpython2/TrigonometryBot/" | |
| if count==18:PATH = "/home/jack/Desktop/temp/args_csv_Twython_ImageBot/" | |
| if count==19:PATH = "/home/jack/python3-starter/notebooks/" | |
| for file in os.listdir(PATH): | |
| if file.endswith(".ipynb"): | |
| filename = PATH+file | |
| filein = PATH | |
| filein = filein.replace("/home/jack/", "") | |
| filein = filein.replace("/", "_") | |
| filein = filein+file | |
| description = filein | |
| description = description.replace("_", " ") | |
| description = description.replace("-", " ") | |
| description = description.replace("/", " ") | |
| description = description+" | |
| "+PATH+file+" | |
| "+file | |
| with open(filename, "rb") as input_file: | |
| ablob = input_file.read() | |
| content = sqlite3.Binary(ablob) | |
| c.execute("INSERT INTO ipynb (file, content, description) VALUES(?, ?, ?)", | |
| (filein, content, description)) | |
| print os.path.join(PATH, file, filein) | |
| conn.commit() | |
| line = file | |
| #line ="Good-mouse-sizing-and-cropping.ipynb" | |
| title = "index" | |
| c.execute("INSERT INTO ipynb VALUES (?,?,?)", (title, file, description)) | |
| conn.commit() | |
| c.close() | |
| conn.close() | |
| Description :SQLite inserting files by directory walk - adding files to database by extension | |
| -------new entry -------Line No. :73 | |
| ROWID :73 | |
| from urllib2 import Request, urlopen | |
| import json | |
| URL = 'http://api.apixu.com/v1/forecast.json?key=a5e6cd9fdf4a4b4fabe55704170811&q=Manila&days=7' | |
| request=Request(URL) | |
| response = urlopen(request) | |
| forecast = response.read() | |
| print forecast | |
| Description :urllib2 import Request, urlopen write to json save url json to file | |
| -------new entry -------Line No. :74 | |
| ROWID :74 | |
| data =[{'state': 'Florida', | |
| 'shortname': 'FL', | |
| 'info': { | |
| 'governor': 'Rick Scott' | |
| }, | |
| 'counties': [{'name': 'Dade', 'population': 12345}, | |
| {'name': 'Broward', 'population': 40000}, | |
| {'name': 'Palm Beach', 'population': 60000}]}, | |
| {'state': 'Ohio', | |
| 'shortname': 'OH', | |
| 'info': { | |
| 'governor': 'John Kasich' | |
| }, | |
| 'counties': [{'name': 'Summit', 'population': 1234}, | |
| {'name': 'Cuyahoga', 'population': 1337}]}] | |
| from pandas.io.json import json_normalize | |
| result = json_normalize(data, 'counties', ['state', 'shortname', ['info', 'governor']]) | |
| result | |
| Description :json_normalize, normalize, pandas, json | |
| -------new entry -------Line No. :75 | |
| ROWID :75 | |
| %%writefile weather.json | |
| {"location":{"name":"Paris","region":"Ile-de-France","country":"France","lat":48.87,"lon":2.33,"tz_id":"Europe/Paris","localtime_epoch":1510121183,"localtime":"2017-11-08 7:06"},"current":{"last_updated_epoch":1510120819,"last_updated":"2017-11-08 07:00","temp_c":2.0,"temp_f":35.6,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":1.3,"wind_kph":2.2,"wind_degree":220,"wind_dir":"SW","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":87,"cloud":75,"feelslike_c":2.0,"feelslike_f":35.6,"vis_km":10.0,"vis_miles":6.0},"forecast":{"forecastday":[{"date":"2017-11-08","date_epoch":1510099200,"day":{"maxtemp_c":9.7,"maxtemp_f":49.5,"mintemp_c":7.7,"mintemp_f":45.9,"avgtemp_c":7.3,"avgtemp_f":45.1,"maxwind_mph":3.8,"maxwind_kph":6.1,"totalprecip_mm":0.0,"totalprecip_in":0.0,"avgvis_km":18.8,"avgvis_miles":11.0,"avghumidity":59.0,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"uv":0.9},"astro":{"sunrise":"07:49 AM","sunset":"05:20 PM","moonrise":"09:31 PM","moonset":"12:19 PM"},"hour":[{"time_epoch":1510095600,"time":"2017-11-08 00:00","temp_c":6.4,"temp_f":43.5,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":187,"wind_dir":"S","pressure_mb":1018.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":49,"cloud":5,"feelslike_c":6.0,"feelslike_f":42.8,"windchill_c":6.0,"windchill_f":42.8,"heatindex_c":6.4,"heatindex_f":43.5,"dewpoint_c":-3.4,"dewpoint_f":25.9,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.9,"vis_miles":12.0},{"time_epoch":1510099200,"time":"2017-11-08 01:00","temp_c":6.1,"temp_f":43.0,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":177,"wind_dir":"S","pressure_mb":1018.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":50,"cloud":5,"feelslike_c":5.6,"feelslike_f":42.1,"windchill_c":5.6,"windchill_f":42.1,"heatindex_c":6.1,"heatindex_f":43.0,"dewpoint_c":-3.5,"dewpoint_f":25.7,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.9,"vis_miles":12.0},{"time_epoch":1510102800,"time":"2017-11-08 02:00","temp_c":5.9,"temp_f":42.6,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":3.1,"wind_kph":5.0,"wind_degree":187,"wind_dir":"S","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":51,"cloud":6,"feelslike_c":5.2,"feelslike_f":41.4,"windchill_c":5.2,"windchill_f":41.4,"heatindex_c":5.9,"heatindex_f":42.6,"dewpoint_c":-3.5,"dewpoint_f":25.7,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.4,"vis_miles":12.0},{"time_epoch":1510106400,"time":"2017-11-08 03:00","temp_c":5.7,"temp_f":42.3,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":3.4,"wind_kph":5.4,"wind_degree":198,"wind_dir":"SSW","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":51,"cloud":8,"feelslike_c":4.7,"feelslike_f":40.5,"windchill_c":4.7,"windchill_f":40.5,"heatindex_c":5.7,"heatindex_f":42.3,"dewpoint_c":-3.6,"dewpoint_f":25.5,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.8,"vis_miles":11.0},{"time_epoch":1510110000,"time":"2017-11-08 04:00","temp_c":5.5,"temp_f":41.9,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":3.8,"wind_kph":6.1,"wind_degree":209,"wind_dir":"SSW","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":52,"cloud":9,"feelslike_c":4.3,"feelslike_f":39.7,"windchill_c":4.3,"windchill_f":39.7,"heatindex_c":5.5,"heatindex_f":41.9,"dewpoint_c":-3.6,"dewpoint_f":25.5,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.3,"vis_miles":11.0},{"time_epoch":1510113600,"time":"2017-11-08 05:00","temp_c":5.4,"temp_f":41.7,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":3.4,"wind_kph":5.4,"wind_degree":198,"wind_dir":"SSW","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":54,"cloud":20,"feelslike_c":4.5,"feelslike_f":40.1,"windchill_c":4.5,"windchill_f":40.1,"heatindex_c":5.4,"heatindex_f":41.7,"dewpoint_c":-3.1,"dewpoint_f":26.4,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.4,"vis_miles":11.0},{"time_epoch":1510117200,"time":"2017-11-08 06:00","temp_c":5.3,"temp_f":41.5,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":2.9,"wind_kph":4.7,"wind_degree":187,"wind_dir":"S","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":57,"cloud":31,"feelslike_c":4.6,"feelslike_f":40.3,"windchill_c":4.6,"windchill_f":40.3,"heatindex_c":5.3,"heatindex_f":41.5,"dewpoint_c":-2.6,"dewpoint_f":27.3,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.6,"vis_miles":11.0},{"time_epoch":1510120800,"time":"2017-11-08 07:00","temp_c":5.2,"temp_f":41.4,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":2.5,"wind_kph":4.0,"wind_degree":176,"wind_dir":"S","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":59,"cloud":42,"feelslike_c":4.8,"feelslike_f":40.6,"windchill_c":4.8,"windchill_f":40.6,"heatindex_c":5.2,"heatindex_f":41.4,"dewpoint_c":-2.1,"dewpoint_f":28.2,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.7,"vis_miles":11.0},{"time_epoch":1510124400,"time":"2017-11-08 08:00","temp_c":5.6,"temp_f":42.1,"is_day":1,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/day/122.png","code":1009},"wind_mph":2.5,"wind_kph":4.0,"wind_degree":168,"wind_dir":"SSE","pressure_mb":1018.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":60,"cloud":57,"feelslike_c":5.3,"feelslike_f":41.5,"windchill_c":5.3,"windchill_f":41.5,"heatindex_c":5.6,"heatindex_f":42.1,"dewpoint_c":-1.5,"dewpoint_f":29.3,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.5,"vis_miles":11.0},{"time_epoch":1510128000,"time":"2017-11-08 09:00","temp_c":5.9,"temp_f":42.6,"is_day":1,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/day/116.png","code":1003},"wind_mph":2.2,"wind_kph":3.6,"wind_degree":160,"wind_dir":"SSE","pressure_mb":1018.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":61,"cloud":73,"feelslike_c":5.7,"feelslike_f":42.3,"windchill_c":5.7,"windchill_f":42.3,"heatindex_c":5.9,"heatindex_f":42.6,"dewpoint_c":-0.9,"dewpoint_f":30.4,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.2,"vis_miles":11.0},{"time_epoch":1510131600,"time":"2017-11-08 10:00","temp_c":6.3,"temp_f":43.3,"is_day":1,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/day/122.png","code":1009},"wind_mph":2.2,"wind_kph":3.6,"wind_degree":151,"wind_dir":"SSE","pressure_mb":1019.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":62,"cloud":88,"feelslike_c":6.2,"feelslike_f":43.2,"windchill_c":6.2,"windchill_f":43.2,"heatindex_c":6.3,"heatindex_f":43.3,"dewpoint_c":-0.3,"dewpoint_f":31.5,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.0,"vis_miles":11.0},{"time_epoch":1510135200,"time":"2017-11-08 11:00","temp_c":7.1,"temp_f":44.8,"is_day":1,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"wind_mph":1.8,"wind_kph":2.9,"wind_degree":117,"wind_dir":"ESE","pressure_mb":1019.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":60,"cloud":84,"feelslike_c":7.1,"feelslike_f":44.8,"windchill_c":7.1,"windchill_f":44.8,"heatindex_c":7.1,"heatindex_f":44.8,"dewpoint_c":-0.1,"dewpoint_f":31.8,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.2,"vis_miles":11.0},{"time_epoch":1510138800,"time":"2017-11-08 12:00","temp_c":8.0,"temp_f":46.4,"is_day":1,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/day/122.png","code":1009},"wind_mph":1.3,"wind_kph":2.2,"wind_degree":82,"wind_dir":"E","pressure_mb":1020.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":58,"cloud":80,"feelslike_c":7.9,"feelslike_f":46.2,"windchill_c":7.9,"windchill_f":46.2,"heatindex_c":8.0,"heatindex_f":46.4,"dewpoint_c":0.2,"dewpoint_f":32.4,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.3,"vis_miles":11.0},{"time_epoch":1510142400,"time":"2017-11-08 13:00","temp_c":8.8,"temp_f":47.8,"is_day":1,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"wind_mph":0.9,"wind_kph":1.4,"wind_degree":47,"wind_dir":"NE","pressure_mb":1020.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":55,"cloud":76,"feelslike_c":8.8,"feelslike_f":47.8,"windchill_c":8.8,"windchill_f":47.8,"heatindex_c":8.8,"heatindex_f":47.8,"dewpoint_c":0.4,"dewpoint_f":32.7,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.5,"vis_miles":11.0},{"time_epoch":1510146000,"time":"2017-11-08 14:00","temp_c":9.1,"temp_f":48.4,"is_day":1,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"wind_mph":1.8,"wind_kph":2.9,"wind_degree":143,"wind_dir":"SE","pressure_mb":1020.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":55,"cloud":75,"feelslike_c":8.9,"feelslike_f":48.0,"windchill_c":8.9,"windchill_f":48.0,"heatindex_c":9.1,"heatindex_f":48.4,"dewpoint_c":0.6,"dewpoint_f":33.1,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.0,"vis_miles":11.0},{"time_epoch":1510149600,"time":"2017-11-08 15:00","temp_c":9.4,"temp_f":48.9,"is_day":1,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":238,"wind_dir":"WSW","pressure_mb":1020.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":55,"cloud":74,"feelslike_c":9.1,"feelslike_f":48.4,"windchill_c":9.1,"windchill_f":48.4,"heatindex_c":9.4,"heatindex_f":48.9,"dewpoint_c":0.8,"dewpoint_f":33.4,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.4,"vis_miles":12.0},{"time_epoch":1510153200,"time":"2017-11-08 16:00","temp_c":9.7,"temp_f":49.5,"is_day":1,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"wind_mph":3.6,"wind_kph":5.8,"wind_degree":334,"wind_dir":"NNW","pressure_mb":1020.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":55,"cloud":74,"feelslike_c":9.2,"feelslike_f":48.6,"windchill_c":9.2,"windchill_f":48.6,"heatindex_c":9.7,"heatindex_f":49.5,"dewpoint_c":1.0,"dewpoint_f":33.8,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.9,"vis_miles":12.0},{"time_epoch":1510156800,"time":"2017-11-08 17:00","temp_c":9.3,"temp_f":48.7,"is_day":1,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/day/122.png","code":1009},"wind_mph":3.4,"wind_kph":5.4,"wind_degree":333,"wind_dir":"NNW","pressure_mb":1021.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":59,"cloud":83,"feelslike_c":8.9,"feelslike_f":48.0,"windchill_c":8.9,"windchill_f":48.0,"heatindex_c":9.3,"heatindex_f":48.7,"dewpoint_c":1.5,"dewpoint_f":34.7,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.6,"vis_miles":12.0},{"time_epoch":1510160400,"time":"2017-11-08 18:00","temp_c":8.8,"temp_f":47.8,"is_day":0,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/night/119.png","code":1006},"wind_mph":2.9,"wind_kph":4.7,"wind_degree":333,"wind_dir":"NNW","pressure_mb":1021.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":63,"cloud":91,"feelslike_c":8.5,"feelslike_f":47.3,"windchill_c":8.5,"windchill_f":47.3,"heatindex_c":8.8,"heatindex_f":47.8,"dewpoint_c":2.1,"dewpoint_f":35.8,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.3,"vis_miles":11.0},{"time_epoch":1510164000,"time":"2017-11-08 19:00","temp_c":8.4,"temp_f":47.1,"is_day":0,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/night/122.png","code":1009},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":332,"wind_dir":"NNW","pressure_mb":1022.0,"pressure_in":30.7,"precip_mm":0.0,"precip_in":0.0,"humidity":67,"cloud":100,"feelslike_c":8.2,"feelslike_f":46.8,"windchill_c":8.2,"windchill_f":46.8,"heatindex_c":8.4,"heatindex_f":47.1,"dewpoint_c":2.6,"dewpoint_f":36.7,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.0,"vis_miles":11.0},{"time_epoch":1510167600,"time":"2017-11-08 20:00","temp_c":8.3,"temp_f":46.9,"is_day":0,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/night/119.png","code":1006},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":335,"wind_dir":"NNW","pressure_mb":1022.0,"pressure_in":30.7,"precip_mm":0.0,"precip_in":0.0,"humidity":69,"cloud":92,"feelslike_c":8.1,"feelslike_f":46.6,"windchill_c":8.1,"windchill_f":46.6,"heatindex_c":8.3,"heatindex_f":46.9,"dewpoint_c":2.9,"dewpoint_f":37.2,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.6,"vis_miles":11.0},{"time_epoch":1510171200,"time":"2017-11-08 21:00","temp_c":8.2,"temp_f":46.8,"is_day":0,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/night/122.png","code":1009},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":337,"wind_dir":"NNW","pressure_mb":1022.0,"pressure_in":30.7,"precip_mm":0.0,"precip_in":0.0,"humidity":71,"cloud":83,"feelslike_c":7.9,"feelslike_f":46.2,"windchill_c":7.9,"windchill_f":46.2,"heatindex_c":8.2,"heatindex_f":46.8,"dewpoint_c":3.2,"dewpoint_f":37.8,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.3,"vis_miles":11.0},{"time_epoch":1510174800,"time":"2017-11-08 22:00","temp_c":8.1,"temp_f":46.6,"is_day":0,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/night/119.png","code":1006},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":340,"wind_dir":"NNW","pressure_mb":1023.0,"pressure_in":30.7,"precip_mm":0.0,"precip_in":0.0,"humidity":73,"cloud":75,"feelslike_c":7.8,"feelslike_f":46.0,"windchill_c":7.8,"windchill_f":46.0,"heatindex_c":8.1,"heatindex_f":46.6,"dewpoint_c":3.5,"dewpoint_f":38.3,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":17.9,"vis_miles":11.0},{"time_epoch":1510178400,"time":"2017-11-08 23:00","temp_c":8.0,"temp_f":46.4,"is_day":0,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/night/122.png","code":1009},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":229,"wind_dir":"SW","pressure_mb":1023.0,"pressure_in":30.7,"precip_mm":0.0,"precip_in":0.0,"humidity":72,"cloud":81,"feelslike_c":7.7,"feelslike_f":45.9,"windchill_c":7.7,"windchill_f":45.9,"heatindex_c":8.0,"heatindex_f":46.4,"dewpoint_c":3.3,"dewpoint_f":37.9,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":17.9,"vis_miles":11.0}]}]}} | |
| Description :json_normalize, weather.json, weather, json normalize, pandas, json | |
| -------new entry -------Line No. :76 | |
| ROWID :76 | |
| %%writefile weather.json | |
| {"location":{"name":"Paris","region":"Ile-de-France","country":"France","lat":48.87,"lon":2.33,"tz_id":"Europe/Paris","localtime_epoch":1510121183,"localtime":"2017-11-08 7:06"},"current":{"last_updated_epoch":1510120819,"last_updated":"2017-11-08 07:00","temp_c":2.0,"temp_f":35.6,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":1.3,"wind_kph":2.2,"wind_degree":220,"wind_dir":"SW","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":87,"cloud":75,"feelslike_c":2.0,"feelslike_f":35.6,"vis_km":10.0,"vis_miles":6.0},"forecast":{"forecastday":[{"date":"2017-11-08","date_epoch":1510099200,"day":{"maxtemp_c":9.7,"maxtemp_f":49.5,"mintemp_c":7.7,"mintemp_f":45.9,"avgtemp_c":7.3,"avgtemp_f":45.1,"maxwind_mph":3.8,"maxwind_kph":6.1,"totalprecip_mm":0.0,"totalprecip_in":0.0,"avgvis_km":18.8,"avgvis_miles":11.0,"avghumidity":59.0,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"uv":0.9},"astro":{"sunrise":"07:49 AM","sunset":"05:20 PM","moonrise":"09:31 PM","moonset":"12:19 PM"},"hour":[{"time_epoch":1510095600,"time":"2017-11-08 00:00","temp_c":6.4,"temp_f":43.5,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":187,"wind_dir":"S","pressure_mb":1018.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":49,"cloud":5,"feelslike_c":6.0,"feelslike_f":42.8,"windchill_c":6.0,"windchill_f":42.8,"heatindex_c":6.4,"heatindex_f":43.5,"dewpoint_c":-3.4,"dewpoint_f":25.9,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.9,"vis_miles":12.0},{"time_epoch":1510099200,"time":"2017-11-08 01:00","temp_c":6.1,"temp_f":43.0,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":177,"wind_dir":"S","pressure_mb":1018.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":50,"cloud":5,"feelslike_c":5.6,"feelslike_f":42.1,"windchill_c":5.6,"windchill_f":42.1,"heatindex_c":6.1,"heatindex_f":43.0,"dewpoint_c":-3.5,"dewpoint_f":25.7,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.9,"vis_miles":12.0},{"time_epoch":1510102800,"time":"2017-11-08 02:00","temp_c":5.9,"temp_f":42.6,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":3.1,"wind_kph":5.0,"wind_degree":187,"wind_dir":"S","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":51,"cloud":6,"feelslike_c":5.2,"feelslike_f":41.4,"windchill_c":5.2,"windchill_f":41.4,"heatindex_c":5.9,"heatindex_f":42.6,"dewpoint_c":-3.5,"dewpoint_f":25.7,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.4,"vis_miles":12.0},{"time_epoch":1510106400,"time":"2017-11-08 03:00","temp_c":5.7,"temp_f":42.3,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":3.4,"wind_kph":5.4,"wind_degree":198,"wind_dir":"SSW","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":51,"cloud":8,"feelslike_c":4.7,"feelslike_f":40.5,"windchill_c":4.7,"windchill_f":40.5,"heatindex_c":5.7,"heatindex_f":42.3,"dewpoint_c":-3.6,"dewpoint_f":25.5,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.8,"vis_miles":11.0},{"time_epoch":1510110000,"time":"2017-11-08 04:00","temp_c":5.5,"temp_f":41.9,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":3.8,"wind_kph":6.1,"wind_degree":209,"wind_dir":"SSW","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":52,"cloud":9,"feelslike_c":4.3,"feelslike_f":39.7,"windchill_c":4.3,"windchill_f":39.7,"heatindex_c":5.5,"heatindex_f":41.9,"dewpoint_c":-3.6,"dewpoint_f":25.5,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.3,"vis_miles":11.0},{"time_epoch":1510113600,"time":"2017-11-08 05:00","temp_c":5.4,"temp_f":41.7,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":3.4,"wind_kph":5.4,"wind_degree":198,"wind_dir":"SSW","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":54,"cloud":20,"feelslike_c":4.5,"feelslike_f":40.1,"windchill_c":4.5,"windchill_f":40.1,"heatindex_c":5.4,"heatindex_f":41.7,"dewpoint_c":-3.1,"dewpoint_f":26.4,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.4,"vis_miles":11.0},{"time_epoch":1510117200,"time":"2017-11-08 06:00","temp_c":5.3,"temp_f":41.5,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":2.9,"wind_kph":4.7,"wind_degree":187,"wind_dir":"S","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":57,"cloud":31,"feelslike_c":4.6,"feelslike_f":40.3,"windchill_c":4.6,"windchill_f":40.3,"heatindex_c":5.3,"heatindex_f":41.5,"dewpoint_c":-2.6,"dewpoint_f":27.3,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.6,"vis_miles":11.0},{"time_epoch":1510120800,"time":"2017-11-08 07:00","temp_c":5.2,"temp_f":41.4,"is_day":0,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/night/116.png","code":1003},"wind_mph":2.5,"wind_kph":4.0,"wind_degree":176,"wind_dir":"S","pressure_mb":1017.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":59,"cloud":42,"feelslike_c":4.8,"feelslike_f":40.6,"windchill_c":4.8,"windchill_f":40.6,"heatindex_c":5.2,"heatindex_f":41.4,"dewpoint_c":-2.1,"dewpoint_f":28.2,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.7,"vis_miles":11.0},{"time_epoch":1510124400,"time":"2017-11-08 08:00","temp_c":5.6,"temp_f":42.1,"is_day":1,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/day/122.png","code":1009},"wind_mph":2.5,"wind_kph":4.0,"wind_degree":168,"wind_dir":"SSE","pressure_mb":1018.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":60,"cloud":57,"feelslike_c":5.3,"feelslike_f":41.5,"windchill_c":5.3,"windchill_f":41.5,"heatindex_c":5.6,"heatindex_f":42.1,"dewpoint_c":-1.5,"dewpoint_f":29.3,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.5,"vis_miles":11.0},{"time_epoch":1510128000,"time":"2017-11-08 09:00","temp_c":5.9,"temp_f":42.6,"is_day":1,"condition":{"text":"Partly cloudy","icon":"//cdn.apixu.com/weather/64x64/day/116.png","code":1003},"wind_mph":2.2,"wind_kph":3.6,"wind_degree":160,"wind_dir":"SSE","pressure_mb":1018.0,"pressure_in":30.5,"precip_mm":0.0,"precip_in":0.0,"humidity":61,"cloud":73,"feelslike_c":5.7,"feelslike_f":42.3,"windchill_c":5.7,"windchill_f":42.3,"heatindex_c":5.9,"heatindex_f":42.6,"dewpoint_c":-0.9,"dewpoint_f":30.4,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.2,"vis_miles":11.0},{"time_epoch":1510131600,"time":"2017-11-08 10:00","temp_c":6.3,"temp_f":43.3,"is_day":1,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/day/122.png","code":1009},"wind_mph":2.2,"wind_kph":3.6,"wind_degree":151,"wind_dir":"SSE","pressure_mb":1019.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":62,"cloud":88,"feelslike_c":6.2,"feelslike_f":43.2,"windchill_c":6.2,"windchill_f":43.2,"heatindex_c":6.3,"heatindex_f":43.3,"dewpoint_c":-0.3,"dewpoint_f":31.5,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.0,"vis_miles":11.0},{"time_epoch":1510135200,"time":"2017-11-08 11:00","temp_c":7.1,"temp_f":44.8,"is_day":1,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"wind_mph":1.8,"wind_kph":2.9,"wind_degree":117,"wind_dir":"ESE","pressure_mb":1019.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":60,"cloud":84,"feelslike_c":7.1,"feelslike_f":44.8,"windchill_c":7.1,"windchill_f":44.8,"heatindex_c":7.1,"heatindex_f":44.8,"dewpoint_c":-0.1,"dewpoint_f":31.8,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.2,"vis_miles":11.0},{"time_epoch":1510138800,"time":"2017-11-08 12:00","temp_c":8.0,"temp_f":46.4,"is_day":1,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/day/122.png","code":1009},"wind_mph":1.3,"wind_kph":2.2,"wind_degree":82,"wind_dir":"E","pressure_mb":1020.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":58,"cloud":80,"feelslike_c":7.9,"feelslike_f":46.2,"windchill_c":7.9,"windchill_f":46.2,"heatindex_c":8.0,"heatindex_f":46.4,"dewpoint_c":0.2,"dewpoint_f":32.4,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.3,"vis_miles":11.0},{"time_epoch":1510142400,"time":"2017-11-08 13:00","temp_c":8.8,"temp_f":47.8,"is_day":1,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"wind_mph":0.9,"wind_kph":1.4,"wind_degree":47,"wind_dir":"NE","pressure_mb":1020.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":55,"cloud":76,"feelslike_c":8.8,"feelslike_f":47.8,"windchill_c":8.8,"windchill_f":47.8,"heatindex_c":8.8,"heatindex_f":47.8,"dewpoint_c":0.4,"dewpoint_f":32.7,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.5,"vis_miles":11.0},{"time_epoch":1510146000,"time":"2017-11-08 14:00","temp_c":9.1,"temp_f":48.4,"is_day":1,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"wind_mph":1.8,"wind_kph":2.9,"wind_degree":143,"wind_dir":"SE","pressure_mb":1020.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":55,"cloud":75,"feelslike_c":8.9,"feelslike_f":48.0,"windchill_c":8.9,"windchill_f":48.0,"heatindex_c":9.1,"heatindex_f":48.4,"dewpoint_c":0.6,"dewpoint_f":33.1,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.0,"vis_miles":11.0},{"time_epoch":1510149600,"time":"2017-11-08 15:00","temp_c":9.4,"temp_f":48.9,"is_day":1,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":238,"wind_dir":"WSW","pressure_mb":1020.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":55,"cloud":74,"feelslike_c":9.1,"feelslike_f":48.4,"windchill_c":9.1,"windchill_f":48.4,"heatindex_c":9.4,"heatindex_f":48.9,"dewpoint_c":0.8,"dewpoint_f":33.4,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.4,"vis_miles":12.0},{"time_epoch":1510153200,"time":"2017-11-08 16:00","temp_c":9.7,"temp_f":49.5,"is_day":1,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/day/119.png","code":1006},"wind_mph":3.6,"wind_kph":5.8,"wind_degree":334,"wind_dir":"NNW","pressure_mb":1020.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":55,"cloud":74,"feelslike_c":9.2,"feelslike_f":48.6,"windchill_c":9.2,"windchill_f":48.6,"heatindex_c":9.7,"heatindex_f":49.5,"dewpoint_c":1.0,"dewpoint_f":33.8,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.9,"vis_miles":12.0},{"time_epoch":1510156800,"time":"2017-11-08 17:00","temp_c":9.3,"temp_f":48.7,"is_day":1,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/day/122.png","code":1009},"wind_mph":3.4,"wind_kph":5.4,"wind_degree":333,"wind_dir":"NNW","pressure_mb":1021.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":59,"cloud":83,"feelslike_c":8.9,"feelslike_f":48.0,"windchill_c":8.9,"windchill_f":48.0,"heatindex_c":9.3,"heatindex_f":48.7,"dewpoint_c":1.5,"dewpoint_f":34.7,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.6,"vis_miles":12.0},{"time_epoch":1510160400,"time":"2017-11-08 18:00","temp_c":8.8,"temp_f":47.8,"is_day":0,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/night/119.png","code":1006},"wind_mph":2.9,"wind_kph":4.7,"wind_degree":333,"wind_dir":"NNW","pressure_mb":1021.0,"pressure_in":30.6,"precip_mm":0.0,"precip_in":0.0,"humidity":63,"cloud":91,"feelslike_c":8.5,"feelslike_f":47.3,"windchill_c":8.5,"windchill_f":47.3,"heatindex_c":8.8,"heatindex_f":47.8,"dewpoint_c":2.1,"dewpoint_f":35.8,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.3,"vis_miles":11.0},{"time_epoch":1510164000,"time":"2017-11-08 19:00","temp_c":8.4,"temp_f":47.1,"is_day":0,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/night/122.png","code":1009},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":332,"wind_dir":"NNW","pressure_mb":1022.0,"pressure_in":30.7,"precip_mm":0.0,"precip_in":0.0,"humidity":67,"cloud":100,"feelslike_c":8.2,"feelslike_f":46.8,"windchill_c":8.2,"windchill_f":46.8,"heatindex_c":8.4,"heatindex_f":47.1,"dewpoint_c":2.6,"dewpoint_f":36.7,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":19.0,"vis_miles":11.0},{"time_epoch":1510167600,"time":"2017-11-08 20:00","temp_c":8.3,"temp_f":46.9,"is_day":0,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/night/119.png","code":1006},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":335,"wind_dir":"NNW","pressure_mb":1022.0,"pressure_in":30.7,"precip_mm":0.0,"precip_in":0.0,"humidity":69,"cloud":92,"feelslike_c":8.1,"feelslike_f":46.6,"windchill_c":8.1,"windchill_f":46.6,"heatindex_c":8.3,"heatindex_f":46.9,"dewpoint_c":2.9,"dewpoint_f":37.2,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.6,"vis_miles":11.0},{"time_epoch":1510171200,"time":"2017-11-08 21:00","temp_c":8.2,"temp_f":46.8,"is_day":0,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/night/122.png","code":1009},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":337,"wind_dir":"NNW","pressure_mb":1022.0,"pressure_in":30.7,"precip_mm":0.0,"precip_in":0.0,"humidity":71,"cloud":83,"feelslike_c":7.9,"feelslike_f":46.2,"windchill_c":7.9,"windchill_f":46.2,"heatindex_c":8.2,"heatindex_f":46.8,"dewpoint_c":3.2,"dewpoint_f":37.8,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":18.3,"vis_miles":11.0},{"time_epoch":1510174800,"time":"2017-11-08 22:00","temp_c":8.1,"temp_f":46.6,"is_day":0,"condition":{"text":"Cloudy","icon":"//cdn.apixu.com/weather/64x64/night/119.png","code":1006},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":340,"wind_dir":"NNW","pressure_mb":1023.0,"pressure_in":30.7,"precip_mm":0.0,"precip_in":0.0,"humidity":73,"cloud":75,"feelslike_c":7.8,"feelslike_f":46.0,"windchill_c":7.8,"windchill_f":46.0,"heatindex_c":8.1,"heatindex_f":46.6,"dewpoint_c":3.5,"dewpoint_f":38.3,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":17.9,"vis_miles":11.0},{"time_epoch":1510178400,"time":"2017-11-08 23:00","temp_c":8.0,"temp_f":46.4,"is_day":0,"condition":{"text":"Overcast","icon":"//cdn.apixu.com/weather/64x64/night/122.png","code":1009},"wind_mph":2.7,"wind_kph":4.3,"wind_degree":229,"wind_dir":"SW","pressure_mb":1023.0,"pressure_in":30.7,"precip_mm":0.0,"precip_in":0.0,"humidity":72,"cloud":81,"feelslike_c":7.7,"feelslike_f":45.9,"windchill_c":7.7,"windchill_f":45.9,"heatindex_c":8.0,"heatindex_f":46.4,"dewpoint_c":3.3,"dewpoint_f":37.9,"will_it_rain":0,"chance_of_rain":"0","will_it_snow":0,"chance_of_snow":"0","vis_km":17.9,"vis_miles":11.0}]}]}} | |
| json = open("weather.json","r").read() | |
| json = json.replace(":", " | |
| ") | |
| print json | |
| Description :json_normalize, open and read json, weather.json, weather, json normalize, pandas, json | |
| -------new entry -------Line No. :77 | |
| ROWID :77 | |
| import sqlite3 | |
| import feedparser | |
| import time | |
| import sqlite3 | |
| Dbase = 'bigfeedfts.db' | |
| conn = sqlite3.connect(Dbase) | |
| c = conn.cursor() | |
| #c.execute(''' | |
| #CREATE TABLE IF NOT EXISTS bbctech | |
| #(head text, feed text) | |
| #'''); | |
| c.execute("" | |
| CREATE VIRTUAL TABLE IF NOT EXISTS bbctech | |
| USING FTS3(head, feed); | |
| "") | |
| count=0 | |
| while count<35: | |
| count=count+1 | |
| if count==1:feed='http://feeds.bbci.co.uk/news/technology/rss.xml' | |
| if count==2:feed='http://www.cbn.com/cbnnews/us/feed/' | |
| if count==3:feed='http://feeds.reuters.com/Reuters/worldNews' | |
| if count==4:feed='http://feeds.bbci.co.uk/news/technology/rss.xml' | |
| if count==5:feed='http://news.sky.com/info/rss' | |
| if count==6:feed='http://www.cbn.com/cbnnews/us/feed/' | |
| if count==7:feed='http://feeds.reuters.com/Reuters/domesticNews' | |
| if count==8:feed='http://news.yahoo.com/rss/' | |
| if count==9:feed='http://www.techradar.com/rss' | |
| if count==10:feed='https://www.wired.com/feed/rss' | |
| if count==11:feed='http://www.zdnet.com/zdnet.opml' | |
| if count==12:feed='http://www.computerweekly.com/rss/All-Computer-Weekly-content.xml' | |
| if count==13:feed='http://gadgets.ndtv.com/rss/feeds' | |
| if count==14:feed='http://feeds.arstechnica.com/arstechnica/index' | |
| if count==15:feed='https://www.techworld.com/news/rss' | |
| if count==16:feed='https://www.infoworld.com/index.rss' | |
| if count==18:feed='https://www.pcworld.com/index.rss' | |
| if count==19:feed='http://tech.economictimes.indiatimes.com/rss/technology' | |
| if count==20:feed='https://www.technologyreview.com/stories.rss' | |
| if count==21:feed='http://tech.economictimes.indiatimes.com/rss/topstories' | |
| if count==22:feed='http://feeds.feedburner.com/digit/latest-from-digit' | |
| if count==23:feed='http://feeds.techsoup.org/TechSoup_Articles' | |
| if count==24:feed='http://rss.sciam.com/ScientificAmerican-News?format=xml' | |
| if count==25:feed='https://www.sciencedaily.com/rss/all.xml' | |
| if count==26:feed='http://feeds.nanowerk.com/nanowerk/agWB' | |
| if count==27:feed='http://feeds.nanowerk.com/NanowerkNanotechnologySpotlight' | |
| if count==28:feed='http://feeds.nanowerk.com/feedburner/NanowerkRoboticsNews' | |
| if count==29:feed='http://feeds.nanowerk.com/NanowerkSpaceExplorationNews' | |
| if count==30:feed='http://www.npr.org/rss/rss.php?id=1019' | |
| if count==31:feed='http://feeds.nature.com/news/rss/news_s16?format=xml' | |
| if count==32:feed='http://feeds.latimes.com/latimes/technology?format=xml' | |
| if count==33:feed='http://feeds.feedburner.com/BadAstronomyBlog?format=xml' | |
| if count==34:feed='http://feeds.newscientist.com/physics-math' | |
| if count==35:feed='http://rss.slashdot.org/Slashdot/slashdotMain' | |
| d = feedparser.parse(feed) | |
| for post in d.entries: | |
| aa = `d['feed']['title'],d['feed']['link'],d.entries[0]['link']` | |
| bb = `post.title + ": " + post.link + ""` | |
| conn = sqlite3.connect(Dbase) | |
| c = conn.cursor() | |
| c.execute("INSERT INTO bbctech VALUES (?,?)", (aa,bb)) | |
| conn.commit() | |
| conn.close() | |
| conn = sqlite3.connect(Dbase) | |
| c = conn.cursor()# Never | |
| count=0 | |
| for row in c.execute('SELECT * FROM bbctech ORDER BY rowid DESC'): | |
| row=str(row) | |
| row=row.replace("(u","");row=row.replace('", u"u'," | |
| ") | |
| row=row.replace("/', u'"," ");row=row.replace('"',"") | |
| row=row.replace("', u'"," ");row=row.replace("')"," ") | |
| row=row.replace("'","");row=row.replace(" , uu"," | |
| ") | |
| count=count+1 | |
| print" | |
| Number :",count," ----- | |
| ",(row) | |
| Description :SQLite inserting files by list, rss feed, rss,feeds, rss to database sqlite - adding files to database by extension | |
| -------new entry -------Line No. :78 | |
| ROWID :78 | |
| %%writefile HashCheck.py | |
| "" USEAGE: | |
| (insert hash) | |
| for example to get hash of a filename: | |
| def md5(Path): | |
| hash_md5 = hashlib.md5() | |
| with open(Path, "rb") as f: | |
| for chunk in iter(lambda: f.read(4096), b""): | |
| hash_md5.update(chunk) | |
| return hash_md5.hexdigest() | |
| Hash = md5(title) | |
| import HashCheck | |
| hasht = "jsudsfhndsfjunjsd" | |
| HashCheck.hashfill(Hash) | |
| --------------- | |
| import HashCheck | |
| HashCheck.hashcheck() | |
| import HashCheck | |
| HashCheck.killmem() | |
| "" | |
| import sqlite3 | |
| import sys | |
| import os | |
| def md5(Path): | |
| hash_md5 = hashlib.md5() | |
| with open(Path, "rb") as f: | |
| for chunk in iter(lambda: f.read(4096), b""): | |
| hash_md5.update(chunk) | |
| return hash_md5.hexdigest() | |
| def hashfill(Hash): | |
| import sqlite3 | |
| database ="tmp-mem" | |
| conn = sqlite3.connect(database) | |
| c = conn.cursor() | |
| c.execute("CREATE TABLE IF NOT EXISTS Hash(hasht TEXT);") | |
| c.execute("INSERT INTO Hash(hasht) VALUES(?)",(Hash,)) | |
| conn.commit() | |
| conn.close() | |
| def hashcheck(): | |
| import sqlite3 | |
| database ="tmp-mem" | |
| conn = sqlite3.connect(database) | |
| c = conn.cursor() | |
| rows = c.execute("SELECT rowid, hasht FROM Hash") | |
| for row in rows: | |
| print row[1] | |
| return | |
| def killmem(): | |
| os.remove(database) | |
| print(database,"Removed!") | |
| Description :verify files, hash , duplicate find duplicate files, hashfill (Hash),HashCheck .py,HashCheck.py | |
| -------new entry -------Line No. :79 | |
| ROWID :79 | |
| #%%writefile GISTstore/post2gist.py | |
| import os | |
| from time import sleep | |
| import sqlite3 | |
| import os, requests, sys, json | |
| import GISTkey | |
| username=GISTkey.gistkey()[0] | |
| password=GISTkey.gistkey()[1] | |
| database ="GISTstore/gist.db" | |
| conn = sqlite3.connect(database) | |
| c = conn.cursor() | |
| c.execute("" | |
| CREATE VIRTUAL TABLE IF NOT EXISTS gist | |
| USING FTS4(content, description, filename); | |
| "") | |
| c.close() | |
| conn.close() | |
| conn = sqlite3.connect(database) | |
| conn.text_factory=str | |
| c = conn.cursor() | |
| if not os.path.exists('GISTstore'): | |
| os.makedirs('GISTstore') | |
| #PATH = "GISTstore/" | |
| #filename = "UnCamel.py" | |
| #content=open(PATH+filename, 'r').read() | |
| filename = "Post_To_Gist _PostToGist.py.ipynb" | |
| content=open(filename, 'r').read() | |
| post = 'https://api.github.com/gists' | |
| r = requests.post(post,json.dumps({'files':{filename:{"content":content}}}), | |
| auth=requests.auth.HTTPBasicAuth(username, password)) | |
| sleep(2) | |
| Url= (r.json()['html_url']) | |
| c.execute("INSERT INTO gist VALUES (?,?,?)", (content, Url, filename)) | |
| conn.commit() | |
| title = "Index" | |
| c.execute("INSERT INTO gist VALUES (?,?,?)", (title, Url, filename)) | |
| conn.commit() | |
| for row in c.execute('SELECT rowid, content, description, filename FROM gist WHERE filename MATCH ?', (filename,)): | |
| print Url," | |
| ",row[0],row[1],row[2],row[3] | |
| c.close() | |
| conn.close() | |
| Description :gist, gist.gisthub , post to gist, save file to sqlite database | |
| -------new entry -------Line No. :80 | |
| ROWID :80 | |
| %%writefile GistIndex.py | |
| #Useage: python GistIndex.py | |
| import sqlite3 | |
| def Index(): | |
| database ="GISTstore/gist.db" | |
| conn = sqlite3.connect(database) | |
| c = conn.cursor() | |
| for row in c.execute('SELECT rowid, content, description, filename FROM gist WHERE content = ?', ("Index",)): | |
| print row[0],row[1],row[2],row[3] | |
| if __name__ == "__main__": | |
| Index() | |
| Description :view gist , see open , gist, gist database gist.gisthub , post to gist, save file to sqlite database | |
| -------new entry -------Line No. :81 | |
| ROWID :81 | |
| simple gist post no database | |
| %%writefile GISTstore/SimpleGistPost.py | |
| #!/usr/bin/python | |
| #USE: python SimpleGistPost.py FileToPost.py | |
| import os, requests, sys, json | |
| import GISTkey | |
| from time import sleep | |
| username=GISTkey.gistkey()[0] | |
| password=GISTkey.gistkey()[1] | |
| filename = os.path.basename(sys.argv[1]) | |
| content=open(filename, 'r').read() | |
| r = requests.post('https://api.github.com/gists',json.dumps({'files':{filename:{"content":content}}}),auth=requests.auth.HTTPBasicAuth(username, password)) | |
| sleep(2) | |
| print(r.json()['html_url']) | |
| Description :simple gist post no database view gist , see open , gist, gist database gist.gisthub , post to gist, save file to sqlite database | |
| -------new entry -------Line No. :82 | |
| ROWID :82 | |
| import os.path | |
| from os import listdir, getcwd | |
| def mp3list(PATH): | |
| abs_path = os.path.join(os.getcwd(),PATH) | |
| dir_files = os.listdir(abs_path) | |
| return dir_files | |
| def mkFile(mfile, PATH): | |
| mFiles = open(mfile, "w");mFiles.close() | |
| lst = mp3list(PATH) | |
| lst = str(lst) | |
| lst = lst.replace(","," | |
| ");lst = lst.replace("'","") | |
| lst = lst.replace("[","");lst = lst.replace("]","") | |
| lst = lst.replace(" ","") | |
| print lst | |
| mFiles = open(mfile, "a") | |
| mFiles.write(lst) | |
| mFiles.close() | |
| PATH = "MP3/" | |
| mfile = "GISTstore/muz.list" | |
| mkFile(mfile, PATH) | |
| --- | |
| list directory, list to file directory, list to file | |
| Description :list directory, list to file directory, list to file | |
| -------new entry -------Line No. :83 | |
| ROWID :83 | |
| def filelen(fname): | |
| with open(fname) as f: | |
| for i, l in enumerate(f): | |
| pass | |
| return i + 1 | |
| get file len file length characters in file | |
| Description :get file len file length characters in file | |
| -------new entry -------Line No. :84 | |
| ROWID :84 | |
| #find table names, unknown sqlite database | |
| import sqlite3 | |
| import sys | |
| from time import sleep | |
| parts = [] | |
| database = "/home/jack/Desktop/text_stuff/database-bak/collection.db" | |
| conn = sqlite3.connect(database) | |
| conn.text_factory = str | |
| c = conn.cursor() | |
| res = c.execute("SELECT name FROM sqlite_master WHERE type='table';") | |
| for name in res: | |
| sleep(2) | |
| print name[0] | |
| #explore a database unknown database find lost table tables sqlite_master sqlite tricks | |
| #recover column names recover field names, unknown database, unknown sqlite | |
| import sqlite3 | |
| database = "/home/jack/Desktop/text_stuff/database-bak/collection.db" | |
| conn = sqlite3.connect(database) | |
| c = conn.cursor() | |
| cur = c.execute('select * from tweets') | |
| #this also works | |
| #names = list(map(lambda x: x[0], cur.description)) | |
| names = [description[0] for description in cur.description] | |
| print names[0:] | |
| Description :explore a database unknown database find lost table tables sqlite_master sqlite tricks | |
| recover column names recover field names, unknown database, unknown sqlite | |
| -------new entry -------Line No. :85 | |
| ROWID :85 | |
| import sqlite3 | |
| from time import sleep | |
| import sys | |
| conn = sqlite3.connect('/home/jack/snippet.db') | |
| conn.text_factory = str | |
| c = conn.cursor() | |
| count=0;req=200 | |
| filein = open("allsnippets.txt","w");filein.close() | |
| filein = open("allsnippets.txt","a") | |
| for row in c.execute('SELECT rowid, * FROM snippet'): | |
| info= (row)[2] | |
| info = str(info) | |
| info = info+" | |
| " | |
| filein.write(info) | |
| filein.close() | |
| Description :snippet to file, snippet.db to file snippet2file | |
| -------new entry -------Line No. :86 | |
| ROWID :86 | |
| # Remove blank ( white spaces ) from filenames recursive recursively remove white spaces | |
| import os | |
| from shutil import copyfile | |
| for dirpath, dirnames, filenames in os.walk("/home/jack/"): | |
| for filename in [f for f in filenames if f.endswith(".ipynb")]: | |
| Path = os.path.join(dirpath, filename) | |
| os.rename(os.path.join(dirpath, filename), os.path.join(dirpath, filename.replace(' ', '-'))) | |
| Description :Remove blank ( white spaces ) from filenames recursive recursively remove white spaces | |
| -------new entry -------Line No. :87 | |
| ROWID :87 | |
| # Remove blank ( white spaces ) from filenames recursive recursively remove white spaces | |
| import os | |
| from shutil import copyfile | |
| for dirpath, dirnames, filenames in os.walk("/home/jack/"): | |
| #The next two lines skip hidden folders and files | |
| filenames = [f for f in filenames if not f[0] == '.'] | |
| dirnames[:] = [d for d in dirnames if not d[0] == '.'] | |
| for filename in [f for f in filenames if f.endswith(".ipynb")]: | |
| Path = os.path.join(dirpath, filename) | |
| os.rename(os.path.join(dirpath, filename), os.path.join(dirpath, filename.replace(' ', '-'))) | |
| --------------------------------- | |
| for root, dirs, files in os.walk(path): | |
| files = [f for f in files if not f[0] == '.'] | |
| dirs[:] = [d for d in dirs if not d[0] == '.'] | |
| Description :Remove blank ( white spaces ) from filenames recursive recursively remove white spaces | |
| lines skip hidden folders and files | |
| -------new entry -------Line No. :88 | |
| ROWID :88 | |
| # Return only the numbers in a string | |
| def only_numerics(seq): | |
| return filter(type(seq).isdigit, seq) | |
| def only_numerics(seq): | |
| seq_type= type(seq) | |
| return seq_type().join(filter(seq_type.isdigit, seq)) | |
| seq = "sjdhdyfjd343555ksishdkdsioj7skdkjshsgjs" | |
| #only_numerics(seq) | |
| only_numerics(seq) | |
| Description : | |
| Return only the numbers in a string | |
| only numerics (seq) seq | |
| only_numerics(seq) | |
| -------new entry -------Line No. :89 | |
| ROWID :89 | |
| Using the RE re module regedit | |
| Metacharacter(s) Description Example | |
| Note that all the if statements return a TRUE value | |
| . Normally matches any character except a newline. Within square brackets the dot is literal. | |
| string1 = "Hello, world." | |
| if re.search(r".....", string1): | |
| print string1 + " has length >= 5" | |
| ( ) Groups a series of pattern elements to a single element. When you match a pattern within parentheses, you can use any of $1, $2, ... later to refer to the previously matched pattern. | |
| string1 = "Hello, world." | |
| m_obj = re.search(r"(H..).(o..)", string1) | |
| if m_obj: | |
| print "We matched '" + m_obj.group(1) + "' and '" + m_obj.group(2) + "'" | |
| Output: We matched 'Hel' and 'o, '; | |
| + Matches the preceding pattern element one or more times. | |
| string1 = "Hello, world." | |
| if re.search(r"l+", string1): | |
| print 'There are one or more consecutive letter "l"' + "'s in " + string1 | |
| Output: There are one or more consecutive letter "l"'s in Hello World | |
| ? Matches the preceding pattern element zero or one times. | |
| string1 = "Hello, world." | |
| if re.search(r"H.?e", string1): | |
| print "There is an 'H' and a 'e' separated by " + "0-1 characters (Ex: He Hoe) | |
| " | |
| ? Modifies the *, +, or {M,N}'d regexp that comes before to match as few times as possible. | |
| string1 = "Hello, world." | |
| if re.search(r"l.+?o", string1): | |
| print "The non-greedy match with 'l' followed by | |
| " + "one or more characters is 'llo' rather than | |
| " + "'llo wo'." | |
| * Matches the preceding pattern element zero or more times. | |
| string1 = "Hello, world." | |
| if re.search(r"el*o", string1): | |
| print "There is an 'e' followed by zero to many | |
| " + "'l' followed by 'o' (eo, elo, ello, elllo)" | |
| {M,N} Denotes the minimum M and the maximum N match count. | |
| string1 = "Hello, world." | |
| if re.search(r"l{1,2}", string1): | |
| print "There exists a substring with at least 1 | |
| " + "and at most 2 l's in " + string1 | |
| [...] Denotes a set of possible character matches. | |
| string1 = "Hello, world." | |
| if re.search(r"[aeiou]+", string1): | |
| print string1 + " contains one or more vowels." | |
| | Separates alternate possibilities. | |
| string1 = "Hello, world." | |
| if re.search(r"(Hello|Hi|Pogo)", string1): | |
| print "At least one of Hello, Hi, or Pogo is " + "contained in " + string1 | |
| Matches a word boundary. | |
| string1 = "Hello World" | |
| if re.search(r"llo", string1): | |
| print "There is a word that ends with 'llo'" | |
| else: | |
| print "There are no words that end with 'llo'" | |
| \w Matches an alphanumeric character, including "_". | |
| string1 = "Hello World" | |
| m_obj = re.search(r"(\w\w)", string1) | |
| if m_obj: | |
| print "The first two adjacent alphanumeric characters" | |
| print "(A-Z, a-z, 0-9, _) in", string1, "were", | |
| print m_obj.group(1) | |
| Output: The first two adjacent alphanumeric characters | |
| (A-Z, a-z, 0-9, _) in Hello World were He | |
| \W Matches a non-alphanumeric character, excluding "_". | |
| string1 = "Hello World" | |
| if re.search(r"\W", string1): | |
| print "The space between Hello and " + "World is not alphanumeric" | |
| \s Matches a whitespace character (space, tab, newline, form feed) | |
| string1 = "Hello World | |
| " | |
| if re.search(r"\s.*\s", string1): | |
| print "There are TWO whitespace characters, which may" | |
| print "be separated by other characters, in", string1 | |
| \S Matches anything BUT a whitespace. | |
| string1 = "Hello World | |
| " | |
| m_obj = re.search(r"(\S*)\s*(\S*)", string1) | |
| if m_obj: | |
| print "The first two groups of NON-whitespace characters" | |
| print "are '%s' and '%s'." % m_obj.groups() | |
| Output: The first two groups of NON-whitespace characters | |
| are 'Hello' and 'World'. | |
| \d Matches a digit, same as [0-9]. | |
| string1 = "99 bottles of beer on the wall." | |
| m_obj = re.search(r"(\d+)", string1) | |
| if m_obj: | |
| print m_obj.group(1), "is the first number in '" + string1 + "'" | |
| Output: 99 is the first number in '99 bottles of beer on the wall.' | |
| \D Matches a non-digit. | |
| string1 = "Hello World" | |
| if re.search(r"\D", string1): | |
| print "There is at least one character in", string1, | |
| print "that is not a digit." | |
| ^ Matches the beginning of a line or string. | |
| string1 = "Hello World" | |
| if re.search(r"^He", string1): | |
| print string1, "starts with the characters 'He'" | |
| $ Matches the end of a line or string. | |
| string1 = "Hello World" | |
| if re.search(r"rld$", string1): | |
| print string1, "is a line or string " + "that ends with 'rld'" | |
| \A Matches the beginning of a string (but not an internal line). | |
| string1 = "Hello | |
| World | |
| " | |
| if re.search(r"\AH", string1): | |
| print string1, "is a string", | |
| print "that starts with 'H'" | |
| Output: Hello | |
| World | |
| is a string that starts with 'H' | |
| \Z Matches the end of a string (but not an internal line). | |
| string1 = "Hello | |
| World | |
| " | |
| if re.search(r"d | |
| \Z", string1): | |
| print string1, "is a string", | |
| print "that ends with 'd\n'" | |
| [^...] Matches every character except the ones inside brackets. | |
| string1 = "Hello World | |
| " | |
| if re.search(r"[^abc]", string1): | |
| print string1 + " contains a character other than " + "a, b, and c" | |
| Description : | |
| string manipulation re module re module examples | |
| -------new entry -------Line No. :90 | |
| ROWID :90 | |
| # Print the first word of a random line from a file | |
| import random | |
| def generate_the_word(infile): | |
| with open(infile) as f: | |
| contents_of_file = f.read() | |
| lines = contents_of_file.splitlines() | |
| line_number = random.randrange(0, len(lines)) | |
| return lines[line_number] | |
| infile = "/home/jack/Desktop/text_stuff/ALL_WIKI_GOOD.txt" | |
| line = generate_the_word(infile) | |
| print line.split(' ', 1)[0] | |
| Description : | |
| Print, first word, random line, file | |
| generate_the_word infile | |
| first word random line, file | |
| -------new entry -------Line No. :91 | |
| ROWID :91 | |
| num = raw_input ("Type Number : ") | |
| search = open("file.txt") | |
| for line in search: | |
| if num in line: | |
| print line | |
| --------------- | |
| search text, locate text, line number, print by line number | |
| search strings, locate strings, line numbers, print by line string | |
| find text in file, search a file | |
| Description : | |
| search text, locate text, line number, print by line number | |
| search strings, locate strings, line numbers, print by line string | |
| find text in file, search a file | |
| -------new entry -------Line No. :92 | |
| ROWID :92 | |
| with open('largeFile', 'r') as inF: | |
| for line in inF: | |
| if 'myString' in line: | |
| # do_something | |
| --------------- | |
| search text, locate text, line number, print by line number | |
| search strings, locate strings, line numbers, print by line string | |
| find text in file, search a file | |
| Description : | |
| search text, locate text, line number, print by line number | |
| search strings, locate strings, line numbers, print by line string | |
| find text in file, search a file | |
| -------new entry -------Line No. :93 | |
| ROWID :93 | |
| import re | |
| def check_string(): | |
| #no need to pass arguments to function if you're not using them | |
| w = raw_input("Input the English word: ") | |
| #open the file using `with` context manager, it'll automatically close the file for you | |
| with open("example.txt") as f: | |
| found = False | |
| for line in f: #iterate over the file one line at a time(memory efficient) | |
| if re.search("{0}".format(w),line): #if string found is in current line then print it | |
| print line | |
| found = True | |
| if not found: | |
| print('The translation cannot be found!') | |
| check_string() #now call the function | |
| --------------- | |
| search function, search with function, | |
| search text, locate text, line number, print by line number | |
| search strings, locate strings, line numbers, print by line string | |
| find text in file, search a file | |
| Description : | |
| search text, locate text, line number, print by line number | |
| search strings, locate strings, line numbers, print by line string | |
| find text in file, search a file | |
| -------new entry -------Line No. :94 | |
| ROWID :94 | |
| The idiomatic way to do this in Python is to use rstrip(' | |
| '): | |
| for line in open('myfile.txt'): # opened in text-mode; all EOLs are converted to ' | |
| ' | |
| line = line.rstrip(' | |
| ') | |
| process(line) | |
| ---------------- | |
| rstrip(' | |
| ') , rstrip(), rmove | |
| , strip | |
| , delete | |
| Description : | |
| rstrip(' | |
| ') , rstrip(), rmove | |
| , strip | |
| , delete | |
| -------new entry -------Line No. :95 | |
| ROWID :95 | |
| #remove characters fron front or rear of a line | |
| STR ="'The idiomatic way to do this in Python is to use rstrip(' | |
| '):'" | |
| STR = STR.strip("'") | |
| #Result: The idiomatic way to do this in Python is to use rstrip(' | |
| # '): | |
| STR = STR.rstrip("(' | |
| '):") | |
| #Result: The idiomatic way to do this in Python is to use rstrip | |
| STR = STR.lstrip("The") | |
| # result: | |
| # idiomatic way to do this in Python is to use rstrip | |
| print STR | |
| ---------------- | |
| rstrip(' | |
| ') , rstrip(), rmove | |
| , strip | |
| , delete | |
| Description : | |
| rstrip(' | |
| ') , rstrip(), rmove | |
| , strip | |
| , delete | |
| -------new entry -------Line No. :96 | |
| ROWID :96 | |
| STR = "Citizen of the Year\u2019 \u2014 \u2018Richly Deserved\u2019." | |
| wordList = re.sub("[^\w]", " ", STR).split() | |
| has_string=False | |
| Find = "u" | |
| count=0 | |
| for item in wordList: | |
| count=count+1 | |
| if Find in item: | |
| has_string=True | |
| print count,"***Has string ",Find," TRUE*****" | |
| print " ".join(wordList), | |
| ------------------ | |
| create list from str string list from string find string in list search list \ | |
| Description : | |
| create list from str string list from string find string in list search list \ | |
| -------new entry -------Line No. :97 | |
| ROWID :97 | |
| from time import sleep | |
| # Method to check a given sentence for given rules | |
| def checkSentence(string): | |
| # Calculate the length of the string. | |
| length = len(string) | |
| # Check that the first character lies in [A-Z]. | |
| # Otherwise return false. | |
| if string[0] < 'A' or string[0] > 'Z': | |
| return False | |
| # If the last character is not a full stop(.) no | |
| # need to check further. | |
| if string[length-1] != '.': | |
| return False | |
| # Maintain 2 states. Previous and current state based | |
| # on which vertex state you are. Initialise both with | |
| # 0 = start state. | |
| prev_state = 0 | |
| curr_state = 0 | |
| # Keep the index to the next character in the string. | |
| index = 1 | |
| # Loop to go over the string. | |
| while (string[index]): | |
| # Set states according to the input characters in the | |
| # string and the rule defined in the description. | |
| # If current character is [A-Z]. Set current state as 0. | |
| if string[index] >= 'A' and string[index] <= 'Z': | |
| curr_state = 0 | |
| # If current character is a space. Set current state as 1. | |
| elif string[index] == ' ': | |
| curr_state = 1 | |
| # If current character is a space. Set current state as 2. | |
| elif string[index] >= 'a' and string[index] <= 'z': | |
| curr_state = 2 | |
| # If current character is a space. Set current state as 3. | |
| elif string[index] == '.': | |
| curr_state = 3 | |
| # Validates all current state with previous state for the | |
| # rules in the description of the problem. | |
| if prev_state == curr_state and curr_state != 2: | |
| return False | |
| # If we have reached last state and previous state is not 1, | |
| # then check next character. If next character is ' ', then | |
| # return true, else false | |
| if prev_state == 2 and curr_state == 0: | |
| return False | |
| # Set previous state as current state before going over | |
| # to the next character. | |
| if curr_state == 3 and prev_state != 1: | |
| return True | |
| index += 1 | |
| prev_state = curr_state | |
| return False | |
| def sent(line): | |
| if line[0] < 'A' or line[0] > 'Z': | |
| return False | |
| def addperiod(line): | |
| line = [ (str.rstrip(' | |
| ') + "." + " | |
| " ) for str in line ] | |
| return line | |
| def period(line): | |
| line = [ (str.rstrip(' | |
| ') + "." + " | |
| " ) for str in line ] | |
| return line | |
| def sent(line): | |
| if line[0] < 'A' or line[0] > 'Z': | |
| return False | |
| infile = open("Alltext_good_no_dup.txt") | |
| lines = infile.readlines() | |
| for line in lines: | |
| line = line.split(".") | |
| line = period(line) | |
| sleep(1) | |
| sentence = line[0] | |
| sentence = sentence.rstrip(" | |
| ").capitalize() | |
| if not sentence[0].isupper() and sentence[-1] != '.': # You can check the last character using sentence[-1] | |
| # both the conditions are not satisfied | |
| print 'Your sentence does not start with a capital letter and has no full stop at the end.' | |
| elif not sentence[0].isupper(): | |
| # sentence does not start with a capital letter | |
| print 'Your sentence does not start with a capital letter.' | |
| elif sentence[-1] != '.': | |
| # sentence does not end with a full stop | |
| print 'Your sentence does not end with a full stop.' | |
| else: | |
| # sentence is perfect | |
| print 'Your sentence is perfect.' | |
| #if "//tw" in sentence:pass | |
| if "//tw" != sentence and len(sentence)> 40: | |
| print sentence | |
| else: print len(sentence),"SKIPPED SKIPPED SKIPPED SKIPPED SKIPPED SKIPPED " | |
| sentence is good verify sentence perfect sentence good sentence | |
| Description : | |
| sentence is good verify sentence perfect sentence good sentence | |
| -------new entry -------Line No. :98 | |
| ROWID :98 | |
| from enchant.checker import SpellChecker | |
| def evaluate(line): | |
| chkr = SpellChecker("en_US") | |
| value = True | |
| chkr.set_text(line) | |
| for err in chkr: | |
| #print "ERROR:", err.word | |
| if err.word > "":value=False;break | |
| #print value | |
| return value | |
| STR = "You have a sequence of items, and you'd like to determine the most frequently occurring." | |
| evaluate(STR) | |
| ------------- | |
| enchant.checker import Spell Checker | |
| enchant spell checker import SpellChecker | |
| Description : | |
| enchant.checker import Spell Checker | |
| enchant spell checker import SpellChecker | |
| -------new entry -------Line No. :99 | |
| ROWID :99 | |
| jack@jack-desktop:~/Desktop/Processing/Processing$ fuser runit.db | |
| /home/jack/Desktop/Processing/Processing/runit.db: 20512 | |
| jack@jack-desktop:~/Desktop/Processing/Processing$ kill 20512 | |
| jack@jack-desktop:~/Desktop/Processing/Processing$ fuser runit.db | |
| jack@jack-desktop:~/Desktop/Processing/Processing$ fuser runit.db | |
| jack@jack-desktop:~/Desktop/Processing/Processing$ kill 20512 | |
| bash: kill: (20512) - No such process | |
| jack@jack-desktop:~/Desktop/Processing/Processing$ fuser nltk.db | |
| /home/jack/Desktop/Processing/Processing/nltk.db: 29997 | |
| jack@jack-desktop:~/Desktop/Processing/Processing$ kill 29997 | |
| jack@jack-desktop:~/Desktop/Processing/Processing$ | |
| ------------- | |
| unlock a locked database unlock db database is locked how to unlock database | |
| Description : | |
| unlock a locked database unlock db database is locked how to unlock database | |
| unlock, locked database, unlock db, database locked, how to unlock database, | |
| -------new entry -------Line No. :100 | |
| ROWID :100 | |
| %%writefile dbINFO.py | |
| "" | |
| Gets tablename and column names from unknown database | |
| just enter database name. | |
| USAGE: | |
| import dbINFO | |
| database= "databases/history.db" | |
| #database= "databases/sat2.db" | |
| #database= "databases/ImageD.db" | |
| dbINFO.dbinfo(database) | |
| "" | |
| import sqlite3 | |
| def dbinfo(database): | |
| conn = sqlite3.connect(database) | |
| conn.text_factory = str | |
| c = conn.cursor() | |
| res = c.execute("SELECT name FROM sqlite_master WHERE type='table';") | |
| row = c.fetchone() | |
| row = str(row) | |
| row = row.replace("(","");row = row.replace(",)","") | |
| row = row.replace("'","") | |
| print row | |
| cur = c.execute("select * from "%s"" % row) | |
| columns = [description[0] for description in cur.description] | |
| return columns | |
| # if run directly | |
| #database = "databases/history.db" | |
| #dbinfo(database) | |
| unknown database get database info get SQLite info lost column names get tablename | |
| retrieve database info get table get rows | |
| Description : | |
| unknown database get database info get SQLite info lost column names get tablename | |
| retrieve database info get table get rows | |
| -------new entry -------Line No. :101 | |
| ROWID :101 | |
| %%writefile UnicodeToAscii.py | |
| def unicodetoascii(text): | |
| uni2ascii = { | |
| ord('’'.decode('utf-8')): ord("'"), | |
| ord('“'.decode('utf-8')): ord('"'), | |
| ord('”'.decode('utf-8')): ord('"'), | |
| ord('„'.decode('utf-8')): ord('"'), | |
| ord('‟'.decode('utf-8')): ord('"'), | |
| ord('é'.decode('utf-8')): ord('e'), | |
| ord('“'.decode('utf-8')): ord('"'), | |
| ord('–'.decode('utf-8')): ord('-'), | |
| ord('‒'.decode('utf-8')): ord('-'), | |
| ord('—'.decode('utf-8')): ord('-'), | |
| ord('—'.decode('utf-8')): ord('-'), | |
| ord('‘'.decode('utf-8')): ord("'"), | |
| ord('‛'.decode('utf-8')): ord("'"), | |
| ord('‐'.decode('utf-8')): ord('-'), | |
| ord('‑'.decode('utf-8')): ord('-'), | |
| ord('′'.decode('utf-8')): ord("'"), | |
| ord('″'.decode('utf-8')): ord("'"), | |
| ord('‴'.decode('utf-8')): ord("'"), | |
| ord('‵'.decode('utf-8')): ord("'"), | |
| ord('‶'.decode('utf-8')): ord("'"), | |
| ord('‷'.decode('utf-8')): ord("'"), | |
| ord('⁺'.decode('utf-8')): ord("+"), | |
| ord('⁻'.decode('utf-8')): ord("-"), | |
| ord('⁼'.decode('utf-8')): ord("="), | |
| ord('⁽'.decode('utf-8')): ord("("), | |
| ord('⁾'.decode('utf-8')): ord(")"), | |
| } | |
| return text.decode('utf-8').translate(uni2ascii).encode('ascii') | |
| #print unicodetoascii("weren’t") | |
| ------------- | |
| text decode utf-8 translate ascii).encode('ascii') | |
| decode('utf-8').translate uni2ascii encode ascii | |
| Description :text decode utf-8 translate ascii).encode('ascii') | |
| decode('utf-8').translate uni2ascii encode ascii | |
| -------new entry -------Line No. :102 | |
| ROWID :102 | |
| import os.path | |
| from os import listdir, getcwd | |
| def mp3list(PATH): | |
| abs_path = os.path.join(os.getcwd(),PATH) | |
| dir_files = os.listdir(abs_path) | |
| return dir_files | |
| def mkFile(mfile, PATH): | |
| mFiles = open(mfile, "w");mFiles.close() | |
| lst = mp3list(PATH) | |
| lst = str(lst) | |
| lst = lst.replace(","," | |
| ");lst = lst.replace("'","") | |
| lst = lst.replace("[","");lst = lst.replace("]","") | |
| lst = lst.replace(" ","") | |
| print lst | |
| mFiles = open(mfile, "a") | |
| mFiles.write(lst) | |
| mFiles.close() | |
| PATH = "MP3/" | |
| mfile = "GISTstore/muz.list" | |
| mkFile(mfile, PATH) | |
| --- | |
| list directory, list to file directory, list to file | |
| Description :list directory, list to file directory, list to file | |
| -------new entry -------Line No. :103 | |
| ROWID :103 | |
| # Find all files with a specific EXT and list in a file | |
| import os | |
| import os.path | |
| title = "MP3/MP3-list.list" | |
| f= open(title,"w") | |
| f.close() | |
| count=0 | |
| for dirpath, dirnames, filenames in os.walk("/home/jack/Desktop/Processing/Processing/MP3/"): | |
| for filename in [f for f in filenames if f.endswith(".mp3")]: | |
| count=count+1 | |
| Path = os.path.join(dirpath, filename) | |
| with open(title, 'a') as outfile: | |
| path = Path+" | |
| " | |
| outfile.write(path) | |
| ------------------------------- | |
| dirpath, dirnames, filenames in os.walk | |
| Description :list directory, specific EXT list to file directory, list to file | |
| dirpath, dirnames, filenames in os.walk | |
| -------new entry --------Line No. :104 | |
| Search : printable | |
| ID : 104 | |
| # filter all bad characters from the a string that are not printable. | |
| # Keywords: bad characters encode error unicode error ascii error UnicodeDecodeError Unicode Decode Error | |
| # 'ascii' codec can't decode byte 0xe2 in position 90: ordinal not in range(128) | |
| # Use string.printable, like this: | |
| s = "somestring. with funnycharacters" | |
| import string | |
| printable = set(string.printable) | |
| filter(lambda x: x in printable, s) | |
| -- KEYWORDS Keywords: bad characters encode error unicode error ascii error UnicodeDecodeError Unicode Decode Error | |
| 'ascii' codec can't decode byte 0xe2 in position 90: ordinal not in range(128) | |
| -------new entry --------Line No. :104 | |
| Search : printable | |
| ID : 104 | |
| # filter all bad characters from the a string that are not printable. | |
| # Keywords: bad characters encode error unicode error ascii error UnicodeDecodeError Unicode Decode Error | |
| # 'ascii' codec can't decode byte 0xe2 in position 90: ordinal not in range(128) | |
| # Use string.printable, like this: | |
| s = "somestring. with funnycharacters" | |
| import string | |
| printable = set(string.printable) | |
| filter(lambda x: x in printable, s) | |
| -- KEYWORDS Keywords: bad characters encode error unicode error ascii error UnicodeDecodeError Unicode Decode Error | |
| 'ascii' codec can't decode byte 0xe2 in position 90: ordinal not in range(128) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment