Generate_pano.py

  HEIGHT=768
  WIDTH=1024 
  OVERLAP = 256
  PROMPT = "360 degree equirectangular panorama photograph, alps, mountains, landscape,\
          trending on artstation, cinematic composition, beautiful lighting, hyper detailed, 8 k, photo, photography"

  INPAINT_FIRST = False # use inpainting to fill in the first frame (force blue, green etc)
  USE_OPEN_JOURNEY = False # open journey is nice, but inpainting works with vanilla SD - sometimes the difference is too big.
  #Also open journey does not work with all resolution sizes
  # First image - no overlap here. 

  if INPAINT_FIRST == False:
      if USE_OPEN_JOURNEY:
          model = replicate.models.get("prompthero/openjourney")
          version = model.versions.get("9936c2001faa2194a261c01381f90e65261879985476014a0a37a334593a05eb")

      else:
          model = replicate.models.get("stability-ai/stable-diffusion")
          version = model.versions.get("6359a0cab3ca6e4d3320c33d79096161208e9024d174b2311e5a21b6c7e1131c")

      output = version.predict(prompt=PROMPT, width=WIDTH, height=HEIGHT)
  else:

      #  use inpainting and force the bottom and top pixels to be blue, green, respectively
      model = replicate.models.get("andreasjansson/stable-diffusion-inpainting")
      version = model.versions.get("8eb2da8345bee796efcd925573f077e36ed5fb4ea3ba240ef70c23cf33f0d848")

      img_initial = Image.new('RGB', (WIDTH, HEIGHT), color = (0, 0, 255))
      # make bottom half black, top half blue
      img_initial = np.array(img_initial)
      img_initial[HEIGHT//2:,:] = (0,255,0)
      img_initial[:HEIGHT//2,:] = (0,0,255)
      img_initial = Image.fromarray(img_initial)
      img_initial.save("initial.png")

      mask_initial = Image.new('L', (WIDTH, HEIGHT), color = 255)
      mask_initial = np.array(mask_initial)
      mask_initial[0:5,:] = 0
      mask_initial[-5:,:] = 0
      mask_initial = Image.fromarray(mask_initial)
      mask_initial.save("mask_initial.png")

      input_object = open("initial.png", "rb")
      mask_object = open("mask_initial.png", "rb")
      output = version.predict(prompt=PROMPT, width=WIDTH, height=HEIGHT, image=input_object, mask=mask_object)
      print(output)



  # download the iamge from the url at output[0]
  image = requests.get(output[0]).content
  image_file = Image.open(io.BytesIO(image))

  #save image as a file
  tempfile = open("output.jpg", "wb")
  tempfile.write(image)

  tempfile.close()
  # create a mask image of size 1024 x 512 that is all black
  mask_img = Image.new('L', (WIDTH, HEIGHT), color = 'white')
  # add white borders of width 128 on either side
  mask_img.paste(0, (0, 0, OVERLAP, HEIGHT))
  mask_img.paste(0, (WIDTH-OVERLAP, 0, WIDTH, HEIGHT))
  mask_img.save('mask.jpg')


  # do second model
  model = replicate.models.get("andreasjansson/stable-diffusion-inpainting")
  version = model.versions.get("8eb2da8345bee796efcd925573f077e36ed5fb4ea3ba240ef70c23cf33f0d848")

  # open output.jpg
  file_object = open("output.jpg", "rb")
  # generate image from file_object
  input_image = Image.open(file_object)
  # save left_most 128 pixels to temp image
  temp_image = input_image.crop((0, 0, OVERLAP, HEIGHT))

  # for input_image, put right-most 128 pixels to the left side
  input_image.paste(input_image.crop((WIDTH-OVERLAP, 0, WIDTH, HEIGHT)), (0, 0, OVERLAP, HEIGHT))

  # put temp_image to the right side
  input_image.paste(temp_image, (WIDTH-OVERLAP, 0, WIDTH, HEIGHT))

  # save the image
  input_image.save('input.jpg')

  input_object = open("input.jpg", "rb")
  mask_object = open("mask.jpg", "rb")


  output = version.predict(prompt=PROMPT, width=WIDTH, height=HEIGHT, image=input_object, mask=mask_object)

  image = requests.get(output[0]).content
  tempfile = open("final_output.jpg", "wb")
  tempfile.write(image)
  tempfile.close()

  # merge output.jpg and final_output.jpg

  output_image = Image.open("output.jpg")
  final_output_image = Image.open("final_output.jpg")

  combined_image = Image.new('RGB', (2*WIDTH-2*OVERLAP, HEIGHT))
  combined_image.paste(output_image, (0, 0))
  combined_image.paste(final_output_image, (WIDTH-OVERLAP, 0))
  combined_image.save('combined.jpg')


  # send combined image to esran model to upsample it 

  img_to_upsample = open("combined.jpg", "rb")

  model = replicate.models.get("nightmareai/real-esrgan")
  version = model.versions.get("42fed1c4974146d4d2414e2be2c5277c7fcf05fcc3a73abf41610695738c1d7b")
  output = version.predict(image=img_to_upsample)


  # save image
  img = requests.get(output).content
  tempfile = open("final_output_upsampled.jpg", "wb")
  tempfile.write(img)
  tempfile.close()

  # compress image
  img = Image.open("final_output_upsampled.jpg")
  img.save("final_output_upsampled.jpg", optimize=True, quality=50)

  # open panellum site
  url_pannellum = "https://cdn.pannellum.org/2.5/pannellum.htm#panorama={}&author=Jasper%20Schoormans&autoLoad=true".format(output)

jrj