{k: bigdict[k] for k in (‘l’, ‘m’, ‘n’)}
autoreload 1 aimport mypackage
import scipy.sandbox.netcdf.netcdf as nc outfile=nc.NetCDFFile(outNCFile, ‘w’) outfile.createDimension(‘irLats’,np.size(regIRLat)) outfile.createDimension(‘irLons’,np.size(regIRLon)) outfile.IRgrid=(delIRLat,delIRLon) outfile.gridComment=’latitude,longitude spacings used for histogram (degrees)’ outvar = outfile.createVariable(‘irLats’,regIRLat.dtype.char,(‘irLats’,)) outvar[:]=regIRLat[:] outvar = outfile.createVariable(‘irLons’,regIRLat.dtype.char,(‘irLons’,)) outvar[:]=regIRLon[:] outvar = outfile.createVariable(‘meanwvIR’,meanwvIR.dtype.char,(‘irLats’,’irLons’)) outvar[:,:]=meanwvIR[:,:] outfile.close()
lon.dtype.char
import scipy.sandbox.netcdf.netcdf as nc from dateutil.parser import parse import dateutil.tz as tz import pytz from dateutil.relativedelta import relativedelta
#get the starting time (this is local PDT) base_time=995005103 pacific=pytz.timezone(‘US/Pacific’) localTime=parse(time.ctime(base_time)) #set the timezone (can drop is_dst if you want) localTime = pacific.localize(localTime, is_dst=True) #convert to utc utcTime=localTime.astimezone(tz.tzutc()) #get seconds past start time for seconds in [0,10,4000,10000,20000,150000]: offset=relativedelta(seconds=seconds) print utcTime + offset
import dateutil.tz as tz import pytz central=pytz.timezone(‘US/Central’) centralDate=theDate.astimezone(central)
different behaviour for python datetime and tzinfo <file://///home/phil/NOTES/200702/070209#* python – timezone issues>
import datetime as dt import dateutil.tz as tz theDate=dt.datetime(ymd[0],ymd[1],ymd[2],hms[0],hms[1],hms[2],tzinfo=tz.tzutc()) dateString=theDate.strftime(“%b %d, %Y %H:%M %Z”)
from dateutil.parser import parse
In [116]:test Out[116]:datetime.datetime(2005, 1, 6, 0, 0, tzinfo=tzutc())
In [117]:newtest=str(test)
In [118]:newtest Out[118]:’2005-01-06 00:00:00+00:00’
In [119]:parse(newtest) Out[119]:datetime.datetime(2005, 1, 6, 0, 0, tzinfo=tzutc())
In [127]:test=dt.datetime(2005, 1, 6, 0, 0, tzinfo=tz.tzlocal())
In [128]:str(test) Out[128]:’2005-01-06 00:00:00-08:00’
and add Z onto time to get parser to set gmt as timezone: try: parse(‘2005-01-06 00:00:00Z’)
import datetime import dateutil
import pytz #starting time is 86400 seconds UCT #julian day 169 is June 18 in non leap years utcTime=datetime.datetime(1997, 6, 18,0,0,0,tzinfo=dateutil.tz.tzutc()) theDelta=datetime.timedelta(seconds=86400) starttime=utcTime + theDelta central=pytz.timezone(‘US/Central’) localtime=starttime.astimezone(central) theDelta=datetime.timedelta(minutes=20) for timeStep in range(0,150): newDate=localtime + timeStep*theDelta #see python strftime documentation for format codes theTime=newDate.strftime(‘%a, %b %d, %H:%M CDT, day %j’) print theTime
http://docs.sun.com/app/docs/doc/806-5205/6je7vd60r?a=view locale.setlocale(locale.LC_ALL,’cs_CZ.ISO8859-2’)
can also use StringIO object instead of file
#thefile=StringIO.StringIO(‘sometext’)
from collections import OrderedDict as od import csv filename=’varnames.csv’ keep_dict=od() with open(filename,’r’) as f: out=csv.reader(f,delimiter=’;’) colnames=next(out) colnames=[item.strip() for item in colnames] theDict=csv.DictReader(f,fieldnames=colnames,delimiter=’;’) for item in theDict: newitem={k.strip():v.strip() for k,v in item.items()} keep_dict[newitem[‘varname’]]=newitem
import os,glob,os.path
def getoutput(cmdline): fil = os.popen(cmdline, ‘r’) output = fil.read() fil.close() return output
##print getoutput(command)
out=glob.glob(‘*.nc’) for i in out: name,ext=os.path.splitext(i) newname=name + ‘.v5d’ command= ‘%s %s %s’ % (‘nc2v5d’,i, newname) print command print getoutput(command)
~/repos/pythonlibs/pyutils/check_md5.py
#!/usr/bin/env python3 from __future__ import division import mmap from hashlib import md5 from contextlib import closing
def find_md5(the_map,seek_point,buf_length): the_map.seek(seek_point) data=the_map.read(buf_length) m = md5() m.update(data) return m.hexdigest()
def check_md5(filename,buf_length): with open(filename,”rb”) as infile: with closing(mmap.mmap(infile.fileno(),0,access=mmap.ACCESS_READ)) as the_map: file_size=the_map.size() if file_size > 3*buf_length: start_seek=0 mid_seek=int(file_size/2) end_seek=file_size - buf_length else: start_seek=0 mid_seek=0 end_seek=0 start_hex=find_md5(the_map,start_seek,buf_length) mid_hex=find_md5(the_map,mid_seek,buf_length) end_hex=find_md5(the_map,end_seek,buf_length) return (file_size,start_hex,mid_hex,end_hex)
if __name__ == ‘__main__’: the_file=”/backupspace/newroc_users/nchaparr/nchaparr_owl/thesis/LESRun/Nov302013/data/runs/sam_case6/OUT_3D/NCHAPP1_testing_doscamiopdata_24_0000000780.bin3D” buf_length=int(1.e5) print(check_md5(the_file,buf_length))
with io.open(‘test.json’,’r’,encoding=’utf8’) as f: count=0 for the_line in f: try: the_line.encode(‘ascii’) except UnicodeEncodeError: print(“caught ascii exception”) print(count,the_line) if the_line.find(u’\222’) > 0: new_line=the_line.replace(u’\222’,”’”) print(new_line) count+=1
import io,string with io.open(‘test.json’,’r’,encoding=’utf8’) as f: for the_line in f: try: the_line.encode(‘ascii’) except UnicodeEncodeError: if the_line.find(u’\222’) > 0: print(“caught ascii exception”) print(the_line) new_line=the_line.replace(u’\222’,”’”) print(new_line)
dictionary merge: ~/repos/kcbo/kcbo/utils.py
plus convert generator do list or dict
if table_name in db.tables: print(‘dropping {}’.format(table_name)) table_raw=db.metadata.tables[table_name] table_raw.drop() db.engine.connect().close() db = dataset.connect(dbstring)
import dataset silentremove(dbname) db = dataset.connect(dbstring)
primary_id = ‘ticket’ dbname = ‘tickets’ the_table = db.create_table(dbname, primary_id=primary_id) ticket_list = df_test.to_dict(‘records’)
for ticket in ticket_list: the_table.insert(ticket)
#
# sql = “”” create table {0:s}( ticket integer primary key, foreign key (ticket) references tickets(ticket) ); “”” out = db.engine.execute(sql.format(‘history’))
12pm in vancouver In [111]: item[‘DTSTART’].dt.strftime(‘%Y-%m-%dT%H:%M:%S%z’) Out[111]: ‘2014-09-03T12:00:00-0700’
12pm in vancouver In [113]: test=dateutil.parser.parse(item[‘DTSTART’].dt.strftime(‘%Y-%m-%dT%H:%M:%S%z’)) In [114]: test.strftime(‘%Y-%m-%dT%H:%M:%S%z’) Out[114]: ‘2014-09-03T12:00:00-0700’
7pm in greenwich In [119]: test.astimezone(pytz.utc) Out[119]: datetime.datetime(2014, 9, 3, 19, 0, tzinfo=<UTC>)
12pm in vancouver In [129]: class_utc.astimezone(pytz.timezone(‘US/Pacific’)) Out[129]: datetime.datetime(2014, 9, 3, 12, 0, tzinfo=<DstTzInfo ‘US/Pacific’ PDT-1 day, 17:00:00 DST>)
- get clientsecrets as json file from dashboard
- write out credentials file
#https://github.com/Rentier/ReindeerIkenga/blob/master/authenticate.py
from oauth2client.client import flow_from_clientsecrets
flow = flow_from_clientsecrets(‘client_secrets.json’, scope=’https://www.googleapis.com/auth/calendar’, redirect_uri=’urn:ietf:wg:oauth:2.0:oob’)
auth_uri = flow.step1_get_authorize_url() print(‘Visit this site!’) print(auth_uri) code = raw_input(‘Insert the given code!’) credentials = flow.step2_exchange(code) print(credentials)
with open(‘credentials’, ‘wr’) as f: f.write(credentials.to_json())
http://nbviewer.ipython.org/github/ipython/ipython/blob/1.x/examples/notebooks/Progress%20Bars.ipynb
import sys, time try: from IPython.display import clear_output have_ipython = True except ImportError: have_ipython = False
class ProgressBar: def __init__(self, iterations): self.iterations = iterations self.prog_bar = ‘[]’ self.fill_char = ‘*’ self.width = 40 self.__update_amount(0) if have_ipython: self.animate = self.animate_ipython else: self.animate = self.animate_noipython
def animate_ipython(self, iter): print ‘\r’, self, sys.stdout.flush() self.update_iteration(iter + 1)
def update_iteration(self, elapsed_iter): self.__update_amount((elapsed_iter / float(self.iterations)) * 100.0) self.prog_bar += ’ %d of %s complete’ % (elapsed_iter, self.iterations)
def __update_amount(self, new_amount): percent_done = int(round((new_amount / 100.0) * 100.0)) all_full = self.width - 2 num_hashes = int(round((percent_done / 100.0) * all_full)) self.prog_bar = ‘[’ + self.fill_char * num_hashes + ’ ’ * (all_full - num_hashes) + ‘]’ pct_place = (len(self.prog_bar) // 2) - len(str(percent_done)) pct_string = ‘%d%%’ % percent_done self.prog_bar = self.prog_bar[0:pct_place] + (pct_string + self.prog_bar[pct_place + len(pct_string):])
def __str__(self): return str(self.prog_bar)
nightly crontab:
ls -R -l -Q –time-style=full-iso –time=status tera > ls_tera.txt du -k tera > du_tera.txt
files in /backupspace/stats_newroc
combine old read_du.py with ~/repos/pythonlibs/pyutils/check_md5.py
http://stackoverflow.com/questions/14114729/save-a-file-using-the-python-requests-library
with open(‘output.jpg’, ‘wb’) as handle: response = requests.get(‘http://www.example.com/image.jpg’, stream=True)
if not response.ok:
for block in response.iter_content(1024): if not block: break
handle.write(block)
0INTEGRATED RADIANCE = 1.183E-02 WATTS CM-2 STER-1 integratedFlux=re.compile(r’.*0INTEGRATED RADIANCE =(.*?)WATTS CM-2 STER-1.*’,re.DOTALL)
will match only number
import docx import glob files=glob.glob(“*docx”) for the_file in files: print(“file: “,the_file) doc1=docx.Document(the_file) paras=doc1.paragraphs for the_para in paras: if the_para.style.name == “pha_body”: print(‘-‘*10) print(the_para.text[:70])
def ascii_filt(my_string): return ”.join([i for i in my_string if ord(i) < 128])
transtab={ord(“’”):None,ord(’ ’ ):None,ord(‘-‘):None} row[‘name’] = row[‘name’].translate(transtab)
from random import randint
def random_with_N_digits(n): range_start = 10**(n-1) range_end = (10**n)-1 return randint(range_start, range_end)
except TypeError as ex: ex_type, ex_val, tb = sys.exc_info() print(‘bummer: ‘,ex_val) print(‘\nhere is the traceback:\n’) traceback.print_tb(tb) print(“python3 reads band_names as bytes, won’t split on comma”)
def ascii_filt(my_string): return ”.join([i for i in my_string if ord(i) < 128])
lastname=[ascii_filt(item) for item in presenter_last] transtab={ord(“’”):None,ord(’ ’ ):None,ord(‘-‘):None} lastname=[item.translate(transtab) for item in lastname]
youritem = item.encode(‘ascii’, ‘ignore’).decode(‘ascii’)
plotdir=’{}/{}’.format(os.getcwd(),’plots’) if not os.path.exists(plotdir): os.makedirs(plotdir)
import os import glob, stat import datetime import tzlocal #pip install tzlocal import pytz import subprocess
local_tz = tzlocal.get_localzone() notebooklist=glob.glob(’.ipynb’) pythonlist=glob.glob(‘./python/.py’)
py_dict={} nb_dict={} for the_file in notebooklist: head,ext=os.path.splitext(the_file) the_date=datetime.datetime.fromtimestamp(os.stat(the_file)[stat.ST_MTIME]) the_date=local_tz.localize(the_date) the_date = the_date.astimezone(pytz.utc) nb_dict[head] = the_date
for the_file in pythonlist: head,name=os.path.split(the_file) head,tail=os.path.splitext(name) the_date=datetime.datetime.fromtimestamp(os.stat(the_file)[stat.ST_MTIME]) the_date=local_tz.localize(the_date) the_date = the_date.astimezone(pytz.utc) py_dict[head] = the_date
the_vals=np.arange(12) + 1 dates=[datetime.datetime(2016,month,1,0,0,0,0,tz.utc) for month in the_vals] print([the_date.strftime(‘%b’) for the_date in dates]) [‘Jan’, ‘Feb’, ‘Mar’, ‘Apr’, ‘May’, ‘Jun’, ‘Jul’, ‘Aug’, ‘Sep’, ‘Oct’, ‘Nov’, ‘Dec’]
#notebooks not in pythonlist
py_files=set(py_dict.keys()) nb_files=set(nb_dict.keys())
make_py=nb_files - py_files print(‘rebuilding {}’.format(make_py)) cmdstring=’ipython nbconvert –stdout –to python ../{0:s}.ipynb > python/{0:s}.py’ for the_file in make_py: command=cmdstring.format(the_file) out=subprocess.getstatusoutput(command) print(out)
for the_file in nb_files: if nb_dict[the_file] > py_dict[the_file]: print(‘rebuilding {}’.format(the_file)) command=cmdstring.format(the_file) out=subprocess.getstatusoutput(command) print(out)
import re mailaddr=re.compile(‘.*\<mailto\:(.*)\>.*’) with open(‘addr.txt’,’r’,encoding=’utf-8’) as f: out=f.readlines()
for item in out: out=item.encode(‘ascii’,’ignore’) out=out.decode(‘ascii’) values=out.split(‘;’) for the_value in values: test=mailaddr.match(the_value) if test: print(test.groups(1)[0])
from collections import namedtuple def make_tuple(tupname,in_dict): the_tup = namedtuple(tupname, in_dict.keys()) the_tup = the_tup(**in_dict) return the_tup
the_tup._asdict()
def find_modtime(the_file): “”” remove the .py or .ipynb extenstion from the file name to get the head, and return that name, plus the modification date in UTC. “”” head,ext=os.path.splitext(the_file) print(‘finding modtime for {}’.format(head)) #
# the_date=datetime.datetime.fromtimestamp(os.stat(the_file)[stat.ST_MTIME]) #
# local_tz = tzlocal.get_localzone() the_date=local_tz.localize(the_date) #
# the_date = the_date.astimezone(pytz.utc) #
# head = head.split(‘/’)[-1] return head,the_date
import datetime from datetime import timezone as tz import tzlocal mytz=tzlocal.get_localzone() now=datetime.datetime.now(tz=mytz) now=now.astimezone(tz.utc) now=now.strftime(‘%Y-%m-%d %H:%M:%S UTC’)
from collections import namedtuple def make_tuple(in_dict,tupname=’values’): the_tup = namedtuple(tupname, in_dict.keys()) the_tup = the_tup(**in_dict) return the_tup
def test_scalar(*args): “”” return true if every argument is a scalar “”” isscalar=True for item in args: isscalar = isscalar & np.isscalar(item) return isscalar
import re
testfull=”“” Station name Station number: 89009 Observation time: 130701/0000 Station latitude: -90.00 Station longitude: 0.00 Station elevation: 2835.0 Lifted index: 22.62 LIFT computed using virtual temperature: 22.63 Convective Available Potential Energy: 0.00 CAPE using virtual temperature: 0.00 Convective Inhibition: 0.00 CINS using virtual temperature: 0.00 Bulk Richardson Number: 0.00 Bulk Richardson Number using CAPV: 0.00 Temp [K] of the Lifted Condensation Level: 222.03 Pres [hPa] of the Lifted Condensation Level: 606.36 Mean mixed layer potential temperature: 256.21 Mean mixed layer mixing ratio: 0.08 1000 hPa to 500 hPa thickness: 5107.00 Precipitable water [mm] for entire sounding: 0.33 “”” testfull=testfull.strip()[:400] line = ‘Station Baker 1 2009-11-17 1223.0’ re_text=”“” Station\snumber\:\s(.+?)\n \s+Observation\stime\:\s(.+?)\n \s+Station\slatitude\:\s(.+?)\n \s+Station\slongitude\:\s(.+?)\n \s+Station\selevation\:\s(.+?)\n \s+Lifted “”” the_re=re.compile(re_text,re.DOTALL|re.VERBOSE) print(the_re.findall(testfull))
#http://stackoverflow.com/questions/139180/listing-all-functions-in-a-python-module
~/repos/pythonlibs/pyutils/list_functions.py
import importlib,sys from inspect import getmembers, isfunction, getmodule name=sys.argv[1] importlib.import_module(name, package=None) module=sys.modules[name] the_funcs = getmembers(module, isfunction) for item in the_funcs: try_module=getmodule(item[1]) if try_module == module: print(item[0])
#https://github.com/noamelf/intro-to-python-coroutines/blob/master/intro-to-coroutines.ipynb #http://www.informit.com/articles/article.aspx?p=2320938 effective python recipe 40
def my_coroutine(): while True: received = yield print(‘Received:’, received)
def gen_range(n): i = 0 while i < n: yield i i += 1
def minimize(): current = yield while True: value = yield current current = min(value, current)
if __name__ == “__main__”: it=my_coroutine() next(it) it.send(10) try2 = gen_range(10) for item in try2: print(item) try2.send(20) for item in try2: print(item)
def reader(): “”“A generator that fakes a read from a file, socket, etc.”“” for i in range(4): yield ‘<< %s’ % i
def reader_wrapper(g):
for v in g: yield v
wrap = reader_wrapper(reader()) for i in wrap: print(i)
#equivalent to
def reader_wrapper(g): yield from g
def writer(): “”“A coroutine that writes data sent to it to fd, socket, etc.”“” while True: w = (yield) print(‘>> ‘, w)
def writer_wrapper(coro): coro.send(None) # prime the coro while True: try: x = (yield) # Capture the value that’s sent coro.send(x) # and pass it to the writer except StopIteration: pass
def writer_wrapper(coro): yield from coro
youritem = item.encode(‘ascii’, ‘ignore’).decode(‘ascii’)
import logging logging.basicConfig() log=logging.getLogger(‘log’) log.setLevel(logging.INFO) log.propagate=True oldlevel=log.getEffectiveLevel() log.setLevel(logging.DEBUG) log.debug(“debug bkeys: {}”.format(list(b_lookup.items()))) log.debug(‘before: {}’.format(row)) log.setLevel(oldlevel)
import pprint pp = pprint.PrettyPrinter(indent=4) pp.pprint(gitoutput)
timezone = tzlocal.get_localzone()
def to_utc_timestamp(a_date): “”” convert a naive datetime to a utc timestamp “”” a_date=timezone.localize(a_date) date_utc = a_date.astimezone(pytz.utc) return int(date_utc.strftime(‘%s’))
vec_to_timestamp=np.vectorize(to_utc_timestamp,otypes=[np.int64])
def from_timestamp(a_timestamp): “”” convert a utc timestamp to a datetime in local timezone “”” a_date=datetime.fromtimestamp(a_timestamp,pytz.utc) a_date=a_date.astimezone(timezone) return a_date
vec_to_datetime=np.vectorize(from_timestamp)
with open(caption_file,’w’) as f: ruamel.yaml.dump(yaml_dict,f,Dumper=ruamel.yaml.RoundTripDumper,default_flow_style=False)
with open(caption_file,’r’) as f: input = ruamel.yaml.load(f)
import ruamel.yaml,sys from collections import OrderedDict try: with open(args.dump_info,’r’) as f: input = ruamel.yaml.load(f) except FileNotFoundError as e: yaml_dict=OrderedDict() yaml_dict[‘root’] = ‘/root/path/here’ yaml_dict[‘h5_out’]= ‘/path/to/h5file/here.h5’ with open(e.filename,’w’) as f: ruamel.yaml.dump(yaml_dict,f,Dumper=ruamel.yaml.RoundTripDumper,default_flow_style=False) print(‘created yaml template {}’.format(e.filename)) print(‘edit and rerun’) sys.exit(0)
if f=None then yaml.dump returns a string
tempfile=’tempout’ with open(ls_outfile,’r’) as infile: with open(tempfile,’w’) as outfile: for i in range(10000): outfile.write(next(infile)) import shutil shutil.move(tempfile,ls_outfile)
def bigfig(*args,**kws): ax,fig=plt.subplots(*args,**kws,figsize=10,8) return ax,fig
from functools import partial bigfig = partial(plt.subplots,figsize=(10,8))
for regular packages with a __init__.py
from pathlib import Path util_dir = a405utils.__path__[0] data_dir = Path(util_dir).joinpath(‘../data’) yaml_file = data_dir.joinpath(‘dropgrow.yaml’)
from pathlib import Path home=os.environ[‘HOME’] pythonlibs = Path(home).joinpath(‘repos/pythonlibs’) site.addsitedir(pythonlibs)
#http://stackoverflow.com/questions/7243750/download-file-from-web-in-python-3 import urllib.request import shutil …
with urllib.request.urlopen(url) as response, open(file_name, ‘wb’) as out_file: shutil.copyfileobj(response, out_file)
#http://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py def download_file(url): local_filename = url.split(‘/’)[-1]
r = requests.get(url, stream=True) with open(local_filename, ‘wb’) as f: for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) #f.flush() commented by recommendation from J.F.Sebastian return local_filename
import re from orderedset import OrderedSet notebook = re.compile(‘(\w+.ipynb)’) keepit = OrderedSet() with open(‘index.rst’,’r’) as infile: for line in infile: for item in notebook.finditer(line): keepit.add(item.group(1))
see pyutils.table
no __init__.py then path is
testpath.__path__._path[0]
with __init__.py then path is
testpath.__path__[0]