Created
November 1, 2016 20:45
-
-
Save aman-tiwari/2d889c542042b8a37b209d0c998d6de5 to your computer and use it in GitHub Desktop.
read remote image files in tensorflow // url file reader
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import tensorflow as tf | |
from skimage import io | |
import time | |
N_THREADS = 64 | |
BATCH_SIZE = 32 | |
N_IMGS = 1000 | |
random_img = 'url to load' | |
# Urls should be a list of urls. | |
# You could make this function stateless by passing in a placeholder that is the | |
# index of the url you want to load rather than auto incrementing it. | |
# io.imread doesn't engage the GIL so multiple requests can be made in parallel. | |
def url_example_producer(urls): | |
def load_nth_url(n): | |
return io.imread(urls[n]) | |
url_iter = tf.Variable(0, name='url_iter').count_up_to(len(urls)) | |
return tf.py_func(load_nth_url, [url_iter], stateful=True, Tout=[tf.uint8]) | |
one_example = url_example_producer([random_img] * N_IMGS) | |
queue = tf.RandomShuffleQueue(capacity=500, min_after_dequeue=100, dtypes=tf.uint8, shapes=(300, 300, 3)) | |
enq_op = queue.enqueue(one_example) | |
inputs = queue.dequeue_many(BATCH_SIZE) | |
qr = tf.train.QueueRunner(queue, [enq_op] * N_THREADS) | |
with tf.Session() as sess: | |
tf.initialize_all_variables().run() | |
coord = tf.train.Coordinator() | |
enqueue_threads = qr.create_threads(sess, coord=coord, start=True) | |
t = time.time() | |
try: | |
while not coord.should_stop(): | |
# Load images | |
images = sess.run(inputs) | |
print len(images), images[0].shape | |
except tf.errors.OutOfRangeError: | |
print('Done loading -- limit reached', 'Took:', time.time() - t) | |
finally: | |
# When done, ask the threads to stop. | |
coord.request_stop() | |
# Wait for threads to finish. | |
coord.join(enqueue_threads) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
hi , i want to read url image directly ,because we will use this in tensor serving,but ,the serving does not support tf.py_func