The solution

hackyascent.py

# Initiate our vector
auto_next = scipy.misc.imread("starting_file_that_is_32x32x3.png")

# create the thing that pokes in the dx_i direction
def dxi(i,j,k):
  a = np.zeros(shape=(32,32,3))
  a[i][j][k] = 1
  return a

# Create an array of all possible directions to poke in
dxis = np.array([dxi(i,j,k) for i in range(32) for j in range(32) for k in range(3)])

# do a step
def step(auto):
  print model.predict(np.array([auto]))[0][1]
  # Find the (orthogonal) direction that makes the biggest change
  biggest = max([(a[1], i) for (i,a) in enumerate((model.predict(auto+dxis)-model.predict((auto+dxis)-dxis)))])
  print biggest
  # step in that direction
  auto_next = auto + 10*dxis[biggest[1]]
  return auto_next
          
while True:
    auto_next = step(auto_next)
    print model.predict(np.array([auto_next]))
    

This is of course an insane way to do this. I'm almost positive that keras provides some way for me to compute the gradient on the layers, I was just too tired to figure it out at the time.