maintain_order.py

import time

item = {
  "images": ["img1", "img2", "img1"]
}

while True:
  print("------ new loop")
  
  # original scraper code:
  # https://github.com/revamediadk/pyspiders/blob/e1b6a338e6fc8fe04b9d023c709ea42893b27539/python_spiders/pipelines.py#L154
  # item["images"] = list(set(item["images"]))
  images_out = list(set(item["images"]))
  print(images_out) # deduplicated, order not always preserved

  images_out = list(dict.fromkeys(item["images"]))
  print(images_out) # deduplicated, order preserved

  time.sleep(0.5)