Skip to content

Instantly share code, notes, and snippets.

@rohit-gupta
Created September 9, 2017 16:14
Show Gist options
  • Save rohit-gupta/6276695d65fc220408d6b3009906a3ff to your computer and use it in GitHub Desktop.
Save rohit-gupta/6276695d65fc220408d6b3009906a3ff to your computer and use it in GitHub Desktop.
How to write a Keras data generator
def generator(features, labels, batch_size):
# Create empty arrays to contain batch of features and labels#
batch_features = np.zeros((batch_size, 64, 64, 3))
batch_labels = np.zeros((batch_size,1))
while True:
for i in range(batch_size):
# choose random index in features
index= random.choice(len(features),1)
batch_features[i] = some_processing(features[index])
batch_labels[i] = labels[index]
yield batch_features, batch_labels
def generate_arrays_from_file(path):
while 1:
f = open(path)
for line in f:
# create numpy arrays of input data
# and labels, from each line in the file
x1, x2, y = process_line(line)
yield ({'input_1': x1, 'input_2': x2}, {'output': y})
f.close()
model.fit_generator(generate_arrays_from_file('/my_file.txt'),
steps_per_epoch=10000, epochs=10)
from skimage.io import imread
from skimage.transform import resize
import numpy as np
# Here, `x_set` is list of path to the images and `y_set` are the associated classes.
class CIFAR10Sequence(Sequence):
def __init__(self, x_set, y_set, batch_size):
self.X,self.y = x_set,y_set
self.batch_size = batch_size
def __len__(self):
return len(self.X) // self.batch_size
def __getitem__(self,idx):
batch_x = self.X[idx*self.batch_size:(idx+1)*self.batch_size]
batch_y = self.y[idx*self.batch_size:(idx+1)*self.batch_size]
return np.array([
resize(imread(file_name), (200,200))
for file_name in batch_x]), np.array(batch_y)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment