python reference

python_a_reference.md

Python Reference

CheatSheets

Python Basics: Data Science	importing data	python 3 (pt 1 of 2)	python 3 (pt 2 of 2)
Numpy Basics	Numpy	Matplotlib
Pandas Basics	Pandas (1 of 2)	Pandas (2 of 2)	Pandas

ipynb

Partial list of debugging commands

%pdb on auto-activate debugger on exception
%debug starts the debugger after an errror
%xmode Verbose sets error reporting to verbose %xmode Plain sets error reporting back to default

There are many more available commands for interactive debugging than we've listed here; the following table contains a description of some of the more common and useful ones:

Command	Description
list	Show the current location in the file
h(elp)	Show a list of commands, or find help on a specific command
q(uit)	Quit the debugger and the program
c(ontinue)	Quit the debugger, continue in the program
n(ext)	Go to the next step of the program
<enter>	Repeat the previous command
p(rint)	Print variables
s(tep)	Step into a subroutine
r(eturn)	Return out of a subroutine

For more information, use the help command in the debugger, or take a look at ipdb's online documentation.

String Formatting

from https://www.python.org/dev/peps/pep-3101/ and https://pyformat.info/

templating

'1, 2, 3'        '{0}, {1}, {2}'.format(1, 2, 3)
'1, 2, 3'        '{}, {}, {}'.format(1, 2, 3)
'1, 2, 3'        '{value1}, {value2}, {value2}'.format(value1=1, value2=2, value3=3)
'second'         '{[1]}'.format(['first', 'second', 'third'])
'<stdin>'        '{.name}'.format(sys.stdin)
'7 9'            '{[a]} {[a]}'.format({'a':7},{'a':9})
'7 9'            '{[1][0]} {[1][2]}'.format([[4,5,6],[7,8,9]])
'9'              '{[1][2]}'.format([[4,5,6],[7,8,9]])
'9'              '{0[1][2]}'.format([[4,5,6],[7,8,9]])
'9'              '{[a][1][2]}'.format({'a':[[4,5,6],[7,8,9]]}) property accessing
'9'              '{a[1][2]}'.format(**{'a':[[4,5,6],[7,8,9]]}) property accessing
'9'              '{a[1][2]}'.format(a=[[4,5,6],[7,8,9]])       property accessing

truncation

'ba'             '{:.2}'.format('bar') 

padding

'x    '          '{:5}'.format('x')             implicit left
'x    '          '{:<5}'.format('x')            explicit left
' x  '           '{:^4}'.format('x')            even # chars
'  x  '          '{:^5}'.format('x')            odd  # chars
'    x'          '{:>5}'.format('x')            left padding
'x____'          '{:_<5}'.format('x')           w/chars left
'__x__'          '{:_^5}'.format('x')           w/chars center
'____x'          '{:_>5}'.format('x')           w/chars right
'_b_'            '{:_^3.1}'.format('bar')       w/ truncation

numerical formatting

'33.1'           '{:.3}'.format(33.14159)
'33.141'         '{:.3f}'.format(33.14159)
'      3.14'     '{:10.2f}'.format(3.14159)
'     +3.14'     '{:+10.2f}'.format(3.14159)
'     -3.14'     '{:10.2f}'.format(-3.14159)
'     -3.14'     '{:+10.2f}'.format(-3.14159)
'__-3.14___'     '{:_^10.2f}'.format(-3.14159) implicit sign -
'__-3.14___'     '{:_^+10.2f}'.format(-3.14159) explicit sign -
'__+3.14___'     '{:_^+10.2f}'.format(3.14159)  explicit sign +
'00+3.14000'     '{:0^+10.2f}'.format(3.14159) works with 0's
'+       23'     '{:=+10d}'.format(23)         padding after sign

conversions

'1.53'           '{!s}'.format(1.53)              str()
"'foo'"          '{!r}'.format('foo')             repr
'd9f'            '{:x}'.format(3487)              hex
'D9F'            '{:X}'.format(3487)              hex upper
'0xd9f'          '{:#x}'.format(3487)             hexadecimal w/prefix
'0xD9F'          '{:#X}'.format(3487)             hexadecimal w/prefix upper
'1100100'        '{:b}'.format(100)               binary
'0b1100100'      '{#:b}'.format(100)              binary w/ prefix
'100'            '{:d}'.format(100)               decimal
'1,000'          '{:,}'.format(1000)              decimal with commas
'144'            '{:o}'.format(100)               octal
'0o144'          '{:#o}'.format(100)              octal w/ prefix
'1.0e-2'         '{:e}'.format(0.01)              exponent
'1.0E-2'         '{:E}'.format(0.01)              exponent upper E
'66.000000%'     '{:%}'.format(0.66)              percentage
'd'              '{:c}'.format(100)               character
'1'              '{:g}'.format(1)                 general
'1'              '{:g}'.format(1.)
'1.2345'         '{:g}'.format(1.2345)            
'1'              '{:n}'.format(1)                 general with locale-based separator
'1'              '{:n}'.format(1.)
'1.2345'         '{:n}'.format(1.2345)            
'1'              '{:}'.format(1)                  None, like g, but prints at least 1 digit
'1.0'            '{:}'.format(1.)
'1.2345'         '{:}'.format(1.2345)
'1.000000'       '{:f}'.format(1)                 fixed point
'1.000000'       '{:f}'.format(1.)
'1.234500'       '{:f}'.format(1.2345)

combinations

'__+3.14___'     '{a[1][0]:_^+10.2f}'.format(a=[[],[3.14159]]) combinations
'_+314.16%_'     '{a[1][0]:_^+10.2%}'.format(a=[[],[3.14159]]) combinations
'    x     '     '{:{align}{width}}'.format('x', align='^', width='10') as params
'xy = 2.7'       '{:{prec}} = {:{prec}}'.format('xyz', 2.7182, prec='.2') as params

datetime

'2001-02-03 04:05' '{:%Y-%m-%d %H:%M}'.format(datetime(2001, 2, 3, 4, 5))

special class

class Scare(object):
    def __format__(self, format):
        if (format == 'boo'): return "AHHH!"
        return 'zzzzz'

'AHHH!'          '{:boo}'.format(Scare())
'zzzzz'          '{:notboo}'.format(Scare())

Random

import random
random.randint(0, 5)      # 0|1|2|3|4|5 
random.randrange(0, 5)    # 0|1|2|3|4 
random.randrange(0, 5, 2) # 0|2|4
random.random()           # [0,...,1) num between 0 and 1, excluding 1

ranges

list(range(0,5)) # [0,1,2,3,4]
list(range(0,5,2)) # [0,2,4]

python_functional_snippets.py

# from toolz.functoolz import curry,partial,compose,flip,pipe as datapipe


#logic
def cond(predicateFnMatrix):
  def condInner(*args):
    for (predicate,fn) in predicateFnMatrix:
      if predicate(*args):
        return fn(*args)
  return condInner
all = lambda *fns: lambda *args,**kwargs: reduce((lambda truth,fn:(fn(*args,**kwargs) if truth else truth)),fns,True)
ifElse = lambda predicate,fnTrue,fnFalse: lambda *a,**kw: fnTrue(*a,**kw) if predicate(*a,**kw) else fnFalse(*a,**kw)


# predicates
lt = curry(lambda x,y: y < x)
lte = curry(lambda x,y: y <= x)
gt = curry(lambda x,y: y > x)
gte = curry(lambda x,y: y >= x)
eq = curry(lambda x,y: y == x)
ne = curry(lambda x,y: y != x)
is_ = curry(lambda x,y: x is y)
is_not = curry(lambda x,y: x is not y)

#debugging
plog = lambda *args: print(*args) or args[0]
def logFn(fn,name=""):
    def fn_logger(*args,**kwargs):
        print(name,'args:',args,'kwargs:',kwargs)
        return fn(*args,**kwargs)
    return fn_logger


#math
def len_minus1(collection): return len(collection) - 1
add = curry(lambda x,y:y+x)
sub = curry(lambda x,y:y-x)
mul = curry(lambda x,y:y*x)
truediv = curry(lambda x,y:y/x)
floordiv = curry(lambda x,y:y//x)
pow = curry(lambda x,y:y**x)
mod = curry(lambda x,y: y % x, arity=2))
divisibleBy = lambda x: pipe(mod(x), eq(0))

#function manipulation
from functools import reduce
pipe = lambda fn1,*fns: lambda arg,*args: reduce(lambda a,f: f(a), fns, fn1(arg,*args))
# or, the verbose version
def pipe(firstFn, *fns): 
    def pipeInner(lastArg,*args):
        lastArg = firstFn(lastArg,*args)
        for f in fns:
            lastArg = f(lastArg)
        return lastArg
    return pipeInner
compose = lambda *fns :pipe(*reversed(fns))
fmap = lambda fn: lambda data: (fn(d) for d in data)
mapPipe = compose(fmap,pipe)
spread = lambda fn : lambda iterableArg : fn(*iterableArg)
forkjoin = lambda *fns,mergefn: lambda data:mergefn(*[fn(data) for fn in fns])
nthArg = lambda argNum: lambda *args: args[argNum]
negate = lambda fn: lambda *args,**kwargs:not fn(*args,**kwargs)
over = lambda fns: lambda *args,**kwargs: [f(*args,**kwargs) for f in fns]
def to_transformer(fn,acc):
  def inner(acc,v,k,c):
    fn(acc,v,k,c)
    return acc
  return inner
transformTo = curry(lambda stubFn,fn,coll:reduce(to_transformer(fn),coll,stubFn()),arity=3)
transToDict = transformTo(dict)
listToArgs = lambda fn: lambda *lst,**kwargs: fn(*lst[0],**kwargs)
argsToList = lambda fn: lambda *args,**kwargs: fn(args,**kwargs)

from functools import partial
def curry(fn,arity=2):
  if arity < 1: return fn
  def curryInner(*args,**kwargs):
    if len(args) >= arity : return fn(*args[:arity],**kwargs)
    return curry(partial(fn, *args[:arity],**kwargs), arity=arity-len(args))
  return curryInner

# curry tests
lt = curry(lambda x,y: x > y)
assert lt(2,1) == True
assert lt(2)(1) == True
assert lt(2,1,100) == True
assert lt(2)(1,100) == True
assert lt()(2,1,100) == True

  
# misc utils
identity = lambda x: x
noop = lambda *x: None
constant = lambda x: lambda *ignored: x
stub_0 = constant(0)
stub_True = lambda *x: True
stub_False = lambda *x: False
stub_List = lambda *x: list()

def assign_dicts(dest,*dicts):
  for d in dicts:
    for k in d.keys():
      dest[k]=d[k]
  return dest

def assign_dicts(dest,*dicts):
  for d in dicts:
    for k in d.keys():
      dest[k]=d[k]
  return dest

def analyze_graph(get_adjacent_vertices,collection}):
  graph = dict(
    vertices={id:dict(id=id,data=data) for v in get_adjacent_vertices('start',collection)},
    root_vertices={},
    in_edges={},
    in_paths={},
    out_edges={},
    in_cross={},
    out_cross={},
    in_back={},
    out_back={},
    in_forward={},
    out_forward={}
  )

  node = None
  queue=[v for v in graph['vertices']]
  while(len(queue)>0):
      node=queue.shift()
      for (id,data) in get_adjacent_vertices(node['id'],collection):
        if(id in graph['vertices']):
          pass # visited
          # tests for cross, back, forward edges
        else:
          graph['vertices'][id]=dict(id=id,data=data)
          queue.append()

  for (id,vertex) in graph['vertices'].items():
    if id not in graph['in_edges']:
      root_vertices[id]=vertex
          
  return graph

python_recursion_experiments.py

# Deep Dive into recursion and looping solutions for optimization problems
# Written to help me describe optimization functions' parts, and those parts' permutations,
# to facilitate both making choices between them and effectively articulating those choices

##
# Code Design
##
# - function parts broken down completely
# - only one thing can change per example. Even identical variable names describe identical things in all examples
# - full word variables to save limited working memory for learning (vs storing variable semantics)
#     (1. all functions "jump" from "start_distance" to "goal_distance" in the "fewest_jumps" possible)
#     (2. all functions "i" from "a" to "n" in the "x" possible)
# - only the minimum variables necessary for the permutation to run
# - all the code runs (tested with asserts in the file)




##
# Permutation Definitions
# NOTE: Each function has one permutation of all of these
#       If you write a function I haven't added and/or think of a missing permutation, please add them to the comments.
##

# - PROGRESSION STRATEGY (Recursion|Looping)
# - CACHED (cached) - intermittent results are cached/memoized, or not
# - DIRECTION (bottom-up|top-down)
# - GOAL TEST LOCATION pre|in|post "loop"
#     note that "loop" is used twice, once for the test location, and once for the "looping" strategy
#     Please comment if you know better language that differentiates the two
# ... Permutations with names and concepts I'm still disentangling
# - ______ ? VARIABLE TYPES I don't know if there are standard names for different types of variables.
#             e.g, distance is ____? count is ___? jump_lengths is ___? Maybe Independent vs dependent variables?
# - ______ ? VARIABLE PLACEMENT like dependent in an inner loop and independent in outer (if it matters)
# - _______? ROOT (one|multi) (from a graph standpoint, one or multiple nodes can be a start)
# - _______? TEST STRATEGY___? look behind vs look ahead when testing for the goal... or in recursion, when "wasted" function calls happen
# - _______? ENTRY CONDITIONS (still hashing out what I even mean by this)
# - ____anything-else____?



##
# Test (Just one, since all examples thus far do the same thing in different ways)
##
def should_successfully_count_fewest_jumps_to_goal(fn, tests=[(9,2),(13,2),(19,4)],longer_tests=[(34,3),(43,5)]):
    for pair in tests:assert(fn(pair[0])==pair[1]);
    for pair in longer_tests:assert(fn(pair[0])==pair[1]);





##
# Constants
##

BABY_JUMPS = 10000 # the number of jumps a baby took to go some distance.  Used for defaults and worst case comparisons.
# ^^ Please ignore ethics and practicality of getting a baby to jump. That many times.






##
# Functions
##


## Permutation: recursion|bottom-up|goal-test-pre
def recursion_bottomup_goaltestpre(goal_distance=20, start_distance = 1, jump_lengths = [1,4,5,11]):
    
    def recurse(distance=start_distance, count=0):
        if(distance == goal_distance): return count
        if(distance >  goal_distance): return BABY_JUMPS
        return min(recurse(distance + jump, count + 1) for jump in jump_lengths)
    
    jumps_to_goal = recurse()
    return jumps_to_goal

should_successfully_count_fewest_jumps_to_goal(recurseBottomUpGoalTestPreLoop,longer_tests=[])
## Permutation: recursion|bottom-up|goal-test-in
def recursion_bottomup_goaltestin(goal_distance=20, start_distance = 1, jump_lengths = [1,4,5,11]):
    
    def recurse(distance=start_distance, count=0):
        if(distance == goal_distance): return count
        return min(recurse((distance+jump,count+1) for jump in jump_lengths if distance+jump <= goal_distance),default=BABY_JUMPS)
    
    jumps_to_goal = recurse()
    return jumps_to_goal

should_successfully_count_fewest_jumps_to_goal(recursion_bottomup_goaltestin,longer_tests=[])
## Permutation: recursion|bottom-up|goal-test-post
## Permutation: recursion|top-down|goal-test-pre
## Permutation: recursion|top-down|goal-test-in
## Permutation: recursion|top-down|goal-test-post
## Permutation: (impossible) looping|bottom-up|goal-test-pre
## Permutation: (impossible) looping|bottom-up|goal-test-in
## Permutation: (impossible) looping|bottom-up|goal-test-post
## Permutation: (impossible) looping|top-down|goal-test-pre 
## Permutation: (impossible) looping|top-down|goal-test-in
## Permutation: (impossible) looping|top-down|goal-test-post
# These ^^ looping permutations are impossible to get consistent optimal results from.
# Without a cache, they form greedy functions.  Why do the recursive ones yield optimal results?
# Recursively nested function scopes implicitly cache all results along their recursion path.
# Each min() call compares the results at each cache distance. (like the combine phase of merge sort)

## Permutation: cached|recursion|bottom-up|goal-test-pre
## Permutation: cached|recursion|bottom-up|goal-test-in
def recursion_cached_bottomup_goaltestin(goal_distance=20, start_distance=1, jump_lengths=[1,4,5,11], fewest_jumps={}):
    should_cache = lambda distance,count: distance not in fewest_jumps or count<fewest_jumps[distance]
    
    def recurse (distance=start_distance, count=0):
        fewest_jumps[distance] = count
        if(distance==goal):return count
        return min((recurse(distance+jump, count+1) for jump in jump_lengths if distance<=goal and should_cache(distance+jump,count+1)),default=BABY_JUMPS)
    
    jumps_to_goal = recurse()
    return jumps_to_goal

should_successfully_count_fewest_jumps_to_goal(recursion_cached_bottomup_goaltestin)
## Permutation: cached|recursion|bottom-up|goal-test-post
## Permutation: cached|recursion|top-down|goal-test-pre
## Permutation: cached|recursion|top-down|goal-test-in
## Permutation: cached|recursion|top-down|goal-test-post
## Permutation: cached|looping|bottom-up|goal-test-pre
## Permutation: cached|looping|bottom-up|goal-test-in
def looping_cached_bottomup_goaltestin(goal_distance=20, start_distance=1, jump_lengths=[1,4,5,11], fewest_jumps={}):
    should_cache = lambda distance,count: distance not in fewest_jumps or count<fewest_jumps[distance]
    
    fewest_jumps[start_distance]=0
    for distance in range(start_distance,goal_distance+1):
        count=fewest_jumps[distance]
        for jump in jump_lengths:
            if should_cache(distance+jump, count+1): fewest_jumps[distance+jump]=count+1
    
    jumps_to_goal = fewest_jumps[goal_distance]
    return jumps_to_goal

should_successfully_count_fewest_jumps_to_goal(looping_cached_bottomup_goaltestin)
## Permutation: cached|looping|bottom-up|goal-test-post
## Permutation: cached|looping|top-down|goal-test-pre
## Permutation: cached|looping|top-down|goal-test-in
## Permutation: cached|looping|top-down|goal-test-post




# 
# def ensureValidStart(goal_distance,start_distance=1,jump_lengths=[1,4,5,11]):
#     if(start_distance < 0 or start_distance > goal_distance): raise ValueError("start must be in interval [0,n]")
#     if(len(jump_lengths)==0): raise ValueError("jump_lengths cannot be empty")
#     if not min(goal_distance % jump for jump in jump_lengths) == 0:
#          raise ValueError("goal_distance not reachable from given jump_lengths")
#     jump_lengths = sorted(jump_lengths)
#     if(goal_distance<jump_lengths[0]): raise ValueError("goal_distance must be >= the smallest jump_length")
# 
#     BABY_JUMPS = 10000 # the number of jumps a baby took to get to ___ (used for defaults and worst cases)
#     cache = {}
#     return goal_distance,start_distance,jump_lengths,BABY_JUMPS,cache