Keras MNIST with Sequence - tracing with multiple processes

keras_mnist_sequence.py

"""
Title: Simple MNIST convnet
Author: [fchollet](https://twitter.com/fchollet)
Date created: 2015/06/19
Last modified: 2020/04/21
Description: A simple convnet that achieves ~99% test accuracy on MNIST.


Modified:
- data wrapped as a Sequence
- addede various prints to trace the processes while multiprocessing

Observations:
- it works for:
  - both fit() and fit_generator() if the sequence is thread-safe
  - both processes/threads
  - all multiprocessing methods
- Keras peeks at the index 0 of the sequence from the main process!
  - GeneratorDataAdapter.__init__() - peek_and_restore()
- on_batch_end() is also called from the main process
- otherwise __getitem__ is called from subprocesses
- the are separate processes for training/validation data
- for fork method the subprocesses are reused across epochs
- for forkserver/spawn methods the subprocesses are renew for each epoch
"""

"""
## Setup
"""

if __name__ == '__main__':
    import multiprocessing as mp
    mp.set_start_method('spawn') # spawn or forkserver (default is fork)

import os

print('new process', os.getpid())

import time

import numpy as np
from tensorflow import keras
from tensorflow.keras import layers
#import tensorflow.compat.v1 as tf
#import tensorflow.compat.v1.keras.backend as K

#K.set_session(tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))))

from tensorflow.keras.utils import Sequence


"""
## Prepare the data
"""

# Model / data parameters
num_classes = 10
input_shape = (28, 28, 1)

# the data, split between train and test sets
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()

# Scale images to the [0, 1] range
x_train = x_train.astype("float32") / 255
x_test = x_test.astype("float32") / 255
# Make sure images have shape (28, 28, 1)
x_train = np.expand_dims(x_train, -1)
x_test = np.expand_dims(x_test, -1)
print("x_train shape:", x_train.shape)
print(x_train.shape[0], "train samples")
print(x_test.shape[0], "test samples")


# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)


# NOTES
# - when multiprocessing=True it runs each data sequence in a separate process
#   but reused across epochs
# - there's on_epoch_end() hook, but not on_epoch_begin()
#   - it gets called from the main process, not the subprocess!

class MnistSequence(Sequence):
    def __init__(self, x, y, batch_size, name):
        self.name = name
        self.x = x
        self.y = y
        self.batch_size = batch_size
        print(self.name, 'MnistSequence.__init__(), PID:', os.getpid(), 'len:', len(self))

    def __len__(self):
        return len(self.x) // self.batch_size

    def __getitem__(self, i):
#         if i == 0:
#             raise ValueError('foo')
#             print(self.name, 'MnistSequence.__getitem__(0), PID:', os.getpid())
        print(self.name, f'MnistSequence.__getitem__({i}), PID:', os.getpid())
        start = i * self.batch_size
        end = start + self.batch_size

#        time.sleep(0.01)

        return self.x[start:end], self.y[start:end]

    def on_epoch_end(self):
        print(self.name, 'MnistSequence.on_epoch_end(), PID:', os.getpid())

batch_size = 1024

if __name__ == '__main__':
    print('main process', os.getpid())
    
    seq_train = MnistSequence(x_train, y_train, batch_size, 'TRAIN')
    seq_test = MnistSequence(x_test, y_test, batch_size, 'VAL')

    """
    ## Build the model
    """

    model = keras.Sequential(
        [
            keras.Input(shape=input_shape),
            layers.Conv2D(32, kernel_size=(3, 3), activation="relu"),
            layers.MaxPooling2D(pool_size=(2, 2)),
            layers.Conv2D(64, kernel_size=(3, 3), activation="relu"),
            layers.MaxPooling2D(pool_size=(2, 2)),
            layers.Flatten(),
            layers.Dropout(0.5),
            layers.Dense(num_classes, activation="softmax"),
        ]
    )

    model.summary()

    """
    ## Train the model
    """

    epochs = 2

    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])

    # model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)

    model.fit(
        #x_train, y_train,
        #validation_data=(x_test, y_test),

        seq_train,
        validation_data=seq_test,
        shuffle=False,

        batch_size=batch_size,
        epochs=epochs,
        #workers=2,
        use_multiprocessing=True,
    )
    
    
#     model.fit_generator(
#         seq_train,
#         validation_data=seq_test,
#         shuffle=False,
#         epochs=epochs,
#         use_multiprocessing=True,
#     )

    # """
    # ## Evaluate the trained model
    # """

    # score = model.evaluate(x_test, y_test, verbose=0)
    # print("Test loss:", score[0])
    # print("Test accuracy:", score[1])

keras_sequence_fork.log

new process 28285
x_train shape: (60000, 28, 28, 1)
60000 train samples
10000 test samples
main process 28285
TRAIN MnistSequence.__init__(), PID: 28285 len: 58
VAL MnistSequence.__init__(), PID: 28285 len: 9
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d (Conv2D)              (None, 26, 26, 32)        320       
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 13, 13, 32)        0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 11, 11, 64)        18496     
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 5, 5, 64)          0         
_________________________________________________________________
flatten (Flatten)            (None, 1600)              0         
_________________________________________________________________
dropout (Dropout)            (None, 1600)              0         
_________________________________________________________________
dense (Dense)                (None, 10)                16010     
=================================================================
Total params: 34,826
Trainable params: 34,826
Non-trainable params: 0
_________________________________________________________________
TRAIN MnistSequence.__getitem__(0), PID: 28285
Epoch 1/2
OrderedEnqueuer: queue.put 0 28285
OrderedEnqueuer: queue.put 1 28285
OrderedEnqueuer: queue.put 2 28285
OrderedEnqueuer: queue.put 3 28285
OrderedEnqueuer: queue.put 4 28285
OrderedEnqueuer: queue.put 5 28285
OrderedEnqueuer: queue.put 6 28285
OrderedEnqueuer: queue.put 7 28285
OrderedEnqueuer: queue.put 8 28285
OrderedEnqueuer: queue.put 9 28285
OrderedEnqueuer: queue.put 10 28285
OrderedEnqueuer: queue.put 11 28285

 1/58 [..............................] - ETA: 0s - loss: 2.3259 - accuracy: 0.0977OrderedEnqueuer: queue.put 12 28285
OrderedEnqueuer: queue.put 13 28285
OrderedEnqueuer: queue.put 14 28285
OrderedEnqueuer: queue.put 15 28285

 5/58 [=>............................] - ETA: 0s - loss: 2.2472 - accuracy: 0.2086OrderedEnqueuer: queue.put 16 28285
OrderedEnqueuer: queue.put 17 28285
OrderedEnqueuer: queue.put 18 28285
OrderedEnqueuer: queue.put 19 28285
OrderedEnqueuer: queue.put 20 28285

10/58 [====>.........................] - ETA: 0s - loss: 2.1480 - accuracy: 0.3295OrderedEnqueuer: queue.put 21 28285
OrderedEnqueuer: queue.put 22 28285
OrderedEnqueuer: queue.put 23 28285
OrderedEnqueuer: queue.put 24 28285
OrderedEnqueuer: queue.put 25 28285

15/58 [======>.......................] - ETA: 0s - loss: 2.0280 - accuracy: 0.4081OrderedEnqueuer: queue.put 26 28285
OrderedEnqueuer: queue.put 27 28285
OrderedEnqueuer: queue.put 28 28285
OrderedEnqueuer: queue.put 29 28285
OrderedEnqueuer: queue.put 30 28285

20/58 [=========>....................] - ETA: 0s - loss: 1.8731 - accuracy: 0.4724OrderedEnqueuer: queue.put 31 28285
OrderedEnqueuer: queue.put 32 28285
OrderedEnqueuer: queue.put 33 28285
OrderedEnqueuer: queue.put 34 28285
OrderedEnqueuer: queue.put 35 28285

25/58 [===========>..................] - ETA: 0s - loss: 1.7099 - accuracy: 0.5192OrderedEnqueuer: queue.put 36 28285
OrderedEnqueuer: queue.put 37 28285
OrderedEnqueuer: queue.put 38 28285
OrderedEnqueuer: queue.put 39 28285
OrderedEnqueuer: queue.put 40 28285

30/58 [==============>...............] - ETA: 0s - loss: 1.5629 - accuracy: 0.5562OrderedEnqueuer: queue.put 41 28285
OrderedEnqueuer: queue.put 42 28285
OrderedEnqueuer: queue.put 43 28285
OrderedEnqueuer: queue.put 44 28285
OrderedEnqueuer: queue.put 45 28285

35/58 [=================>............] - ETA: 0s - loss: 1.4401 - accuracy: 0.5871OrderedEnqueuer: queue.put 46 28285
OrderedEnqueuer: queue.put 47 28285
OrderedEnqueuer: queue.put 48 28285
OrderedEnqueuer: queue.put 49 28285
OrderedEnqueuer: queue.put 50 28285

40/58 [===================>..........] - ETA: 0s - loss: 1.3319 - accuracy: 0.6162OrderedEnqueuer: queue.put 51 28285
OrderedEnqueuer: queue.put 52 28285
OrderedEnqueuer: queue.put 53 28285
OrderedEnqueuer: queue.put 54 28285
OrderedEnqueuer: queue.put 55 28285

45/58 [======================>.......] - ETA: 0s - loss: 1.2451 - accuracy: 0.6390OrderedEnqueuer: queue.put 56 28285
OrderedEnqueuer: queue.put 57 28285

50/58 [========================>.....] - ETA: 0s - loss: 1.1675 - accuracy: 0.6603
55/58 [===========================>..] - ETA: 0s - loss: 1.0964 - accuracy: 0.6806VAL MnistSequence.__getitem__(0), PID: 28285
TRAIN MnistSequence.on_epoch_end(), PID: 28285
get_index 0 28484
TRAIN MnistSequence.__getitem__(0), PID: 28484
get_index 1 28484
TRAIN MnistSequence.__getitem__(1), PID: 28484
get_index 2 28484
TRAIN MnistSequence.__getitem__(2), PID: 28484
get_index 3 28484
TRAIN MnistSequence.__getitem__(3), PID: 28484
get_index 4 28484
TRAIN MnistSequence.__getitem__(4), PID: 28484
get_index 5 28484
TRAIN MnistSequence.__getitem__(5), PID: 28484
get_index 6 28484
TRAIN MnistSequence.__getitem__(6), PID: 28484
get_index 7 28484
TRAIN MnistSequence.__getitem__(7), PID: 28484
get_index 8 28484
TRAIN MnistSequence.__getitem__(8), PID: 28484
get_index 9 28484
TRAIN MnistSequence.__getitem__(9), PID: 28484
get_index 10 28484
TRAIN MnistSequence.__getitem__(10), PID: 28484
get_index 11 28484
TRAIN MnistSequence.__getitem__(11), PID: 28484
get_index 12 28484
TRAIN MnistSequence.__getitem__(12), PID: 28484
get_index 13 28484
TRAIN MnistSequence.__getitem__(13), PID: 28484
get_index 14 28484
TRAIN MnistSequence.__getitem__(14), PID: 28484
get_index 15 28484
TRAIN MnistSequence.__getitem__(15), PID: 28484
get_index 16 28484
TRAIN MnistSequence.__getitem__(16), PID: 28484
get_index 17 28484
TRAIN MnistSequence.__getitem__(17), PID: 28484
get_index 18 28484
TRAIN MnistSequence.__getitem__(18), PID: 28484
get_index 19 28484
TRAIN MnistSequence.__getitem__(19), PID: 28484
get_index 20 28484
TRAIN MnistSequence.__getitem__(20), PID: 28484
get_index 21 28484
TRAIN MnistSequence.__getitem__(21), PID: 28484
get_index 22 28484
TRAIN MnistSequence.__getitem__(22), PID: 28484
get_index 23 28484
TRAIN MnistSequence.__getitem__(23), PID: 28484
get_index 24 28484
TRAIN MnistSequence.__getitem__(24), PID: 28484
get_index 25 28484
TRAIN MnistSequence.__getitem__(25), PID: 28484
get_index 26 28484
TRAIN MnistSequence.__getitem__(26), PID: 28484
get_index 27 28484
TRAIN MnistSequence.__getitem__(27), PID: 28484
get_index 28 28484
TRAIN MnistSequence.__getitem__(28), PID: 28484
get_index 29 28484
TRAIN MnistSequence.__getitem__(29), PID: 28484
get_index 30 28484
TRAIN MnistSequence.__getitem__(30), PID: 28484
get_index 31 28484
TRAIN MnistSequence.__getitem__(31), PID: 28484
get_index 32 28484
TRAIN MnistSequence.__getitem__(32), PID: 28484
get_index 33 28484
TRAIN MnistSequence.__getitem__(33), PID: 28484
get_index 34 28484
TRAIN MnistSequence.__getitem__(34), PID: 28484
get_index 35 28484
TRAIN MnistSequence.__getitem__(35), PID: 28484
get_index 36 28484
TRAIN MnistSequence.__getitem__(36), PID: 28484
get_index 37 28484
TRAIN MnistSequence.__getitem__(37), PID: 28484
get_index 38 28484
TRAIN MnistSequence.__getitem__(38), PID: 28484
get_index 39 28484
TRAIN MnistSequence.__getitem__(39), PID: 28484
get_index 40 28484
TRAIN MnistSequence.__getitem__(40), PID: 28484
get_index 41 28484
TRAIN MnistSequence.__getitem__(41), PID: 28484
get_index 42 28484
TRAIN MnistSequence.__getitem__(42), PID: 28484
get_index 43 28484
TRAIN MnistSequence.__getitem__(43), PID: 28484
get_index 44 28484
TRAIN MnistSequence.__getitem__(44), PID: 28484
get_index 45 28484
TRAIN MnistSequence.__getitem__(45), PID: 28484
get_index 46 28484
TRAIN MnistSequence.__getitem__(46), PID: 28484
get_index 47 28484
TRAIN MnistSequence.__getitem__(47), PID: 28484
get_index 48 28484
TRAIN MnistSequence.__getitem__(48), PID: 28484
get_index 49 28484
TRAIN MnistSequence.__getitem__(49), PID: 28484
get_index 50 28484
TRAIN MnistSequence.__getitem__(50), PID: 28484
get_index 51 28484
TRAIN MnistSequence.__getitem__(51), PID: 28484
get_index 52 28484
TRAIN MnistSequence.__getitem__(52), PID: 28484
get_index 53 28484
TRAIN MnistSequence.__getitem__(53), PID: 28484
get_index 54 28484
TRAIN MnistSequence.__getitem__(54), PID: 28484
get_index 55 28484
TRAIN MnistSequence.__getitem__(55), PID: 28484
get_index 56 28484
TRAIN MnistSequence.__getitem__(56), PID: 28484
get_index 57 28484
TRAIN MnistSequence.__getitem__(57), PID: 28484
OrderedEnqueuer: queue.put 0 28285
OrderedEnqueuer: queue.put 1 28285
OrderedEnqueuer: queue.put 2 28285
OrderedEnqueuer: queue.put 3 28285
OrderedEnqueuer: queue.put 4 28285
OrderedEnqueuer: queue.put 5 28285
OrderedEnqueuer: queue.put 6 28285
OrderedEnqueuer: queue.put 7 28285
OrderedEnqueuer: queue.put 8 28285
OrderedEnqueuer: queue.put 9 28285
OrderedEnqueuer: queue.put 10 28285
OrderedEnqueuer: queue.put 0 28285
OrderedEnqueuer: queue.put 1 28285
OrderedEnqueuer: queue.put 2 28285
OrderedEnqueuer: queue.put 3 28285
OrderedEnqueuer: queue.put 4 28285
OrderedEnqueuer: queue.put 5 28285
OrderedEnqueuer: queue.put 6 28285
OrderedEnqueuer: queue.put 7 28285
OrderedEnqueuer: queue.put 8 28285
VAL MnistSequence.on_epoch_end(), PID: 28285

58/58 [==============================] - 1s 18ms/step - loss: 1.0545 - accuracy: 0.6930 - val_loss: 0.2796 - val_accuracy: 0.9256
TRAIN MnistSequence.on_epoch_end(), PID: 28285
Epoch 2/2
get_index 0 28509
TRAIN MnistSequence.__getitem__(0), PID: 28509
get_index 1 28509
TRAIN MnistSequence.__getitem__(1), PID: 28509
get_index 2 28509
TRAIN MnistSequence.__getitem__(2), PID: 28509
get_index 3 28509
TRAIN MnistSequence.__getitem__(3), PID: 28509
get_index 4 28509
TRAIN MnistSequence.__getitem__(4), PID: 28509
get_index 5 28509
TRAIN MnistSequence.__getitem__(5), PID: 28509
get_index 6 28509
TRAIN MnistSequence.__getitem__(6), PID: 28509
get_index 7 28509
TRAIN MnistSequence.__getitem__(7), PID: 28509
get_index 8 28509
TRAIN MnistSequence.__getitem__(8), PID: 28509
get_index 9 28509
TRAIN MnistSequence.__getitem__(9), PID: 28509
get_index 10 28509
TRAIN MnistSequence.__getitem__(10), PID: 28509
OrderedEnqueuer: queue.put 0 28285
OrderedEnqueuer: queue.put 1 28285
OrderedEnqueuer: queue.put 2 28285
OrderedEnqueuer: queue.put 3 28285
OrderedEnqueuer: queue.put 4 28285
OrderedEnqueuer: queue.put 5 28285
OrderedEnqueuer: queue.put 6 28285
OrderedEnqueuer: queue.put 7 28285
OrderedEnqueuer: queue.put 8 28285
OrderedEnqueuer: queue.put 9 28285
OrderedEnqueuer: queue.put 10 28285
OrderedEnqueuer: queue.put 11 28285

 1/58 [..............................] - ETA: 0s - loss: 0.3856 - accuracy: 0.8887get_index 0 28517
VAL MnistSequence.__getitem__(0), PID: 28517
get_index 1 28517
VAL MnistSequence.__getitem__(1), PID: 28517
get_index 2 28517
VAL MnistSequence.__getitem__(2), PID: 28517
get_index 3 28517
VAL MnistSequence.__getitem__(3), PID: 28517
get_index 4 28517
VAL MnistSequence.__getitem__(4), PID: 28517
get_index 5 28517
VAL MnistSequence.__getitem__(5), PID: 28517
get_index 6 28517
VAL MnistSequence.__getitem__(6), PID: 28517
get_index 7 28517
VAL MnistSequence.__getitem__(7), PID: 28517
get_index 8 28517
VAL MnistSequence.__getitem__(8), PID: 28517
OrderedEnqueuer: queue.put 12 28285
OrderedEnqueuer: queue.put 13 28285
OrderedEnqueuer: queue.put 14 28285
OrderedEnqueuer: queue.put 15 28285

 5/58 [=>............................] - ETA: 0s - loss: 0.3392 - accuracy: 0.9037OrderedEnqueuer: queue.put 16 28285
OrderedEnqueuer: queue.put 17 28285
OrderedEnqueuer: queue.put 18 28285
OrderedEnqueuer: queue.put 19 28285

 9/58 [===>..........................] - ETA: 0s - loss: 0.3415 - accuracy: 0.9002OrderedEnqueuer: queue.put 20 28285
OrderedEnqueuer: queue.put 21 28285
OrderedEnqueuer: queue.put 22 28285
OrderedEnqueuer: queue.put 23 28285
OrderedEnqueuer: queue.put 24 28285

14/58 [======>.......................] - ETA: 0s - loss: 0.3366 - accuracy: 0.9003OrderedEnqueuer: queue.put 25 28285
OrderedEnqueuer: queue.put 26 28285
OrderedEnqueuer: queue.put 27 28285
OrderedEnqueuer: queue.put 28 28285
OrderedEnqueuer: queue.put 29 28285

19/58 [========>.....................] - ETA: 0s - loss: 0.3274 - accuracy: 0.9020OrderedEnqueuer: queue.put 30 28285
OrderedEnqueuer: queue.put 31 28285
OrderedEnqueuer: queue.put 32 28285
OrderedEnqueuer: queue.put 33 28285

23/58 [==========>...................] - ETA: 0s - loss: 0.3149 - accuracy: 0.9052OrderedEnqueuer: queue.put 34 28285
OrderedEnqueuer: queue.put 35 28285
OrderedEnqueuer: queue.put 36 28285
OrderedEnqueuer: queue.put 37 28285
OrderedEnqueuer: queue.put 38 28285

28/58 [=============>................] - ETA: 0s - loss: 0.3086 - accuracy: 0.9069OrderedEnqueuer: queue.put 39 28285
OrderedEnqueuer: queue.put 40 28285
OrderedEnqueuer: queue.put 41 28285
OrderedEnqueuer: queue.put 42 28285
OrderedEnqueuer: queue.put 43 28285

33/58 [================>.............] - ETA: 0s - loss: 0.3060 - accuracy: 0.9079OrderedEnqueuer: queue.put 44 28285
OrderedEnqueuer: queue.put 45 28285
OrderedEnqueuer: queue.put 46 28285
OrderedEnqueuer: queue.put 47 28285
OrderedEnqueuer: queue.put 48 28285

38/58 [==================>...........] - ETA: 0s - loss: 0.2965 - accuracy: 0.9106OrderedEnqueuer: queue.put 49 28285
OrderedEnqueuer: queue.put 50 28285
OrderedEnqueuer: queue.put 51 28285
OrderedEnqueuer: queue.put 52 28285
OrderedEnqueuer: queue.put 53 28285

43/58 [=====================>........] - ETA: 0s - loss: 0.2901 - accuracy: 0.9126OrderedEnqueuer: queue.put 54 28285
OrderedEnqueuer: queue.put 55 28285
OrderedEnqueuer: queue.put 56 28285
OrderedEnqueuer: queue.put 57 28285

48/58 [=======================>......] - ETA: 0s - loss: 0.2846 - accuracy: 0.9141
53/58 [==========================>...] - ETA: 0s - loss: 0.2799 - accuracy: 0.9159
58/58 [==============================] - ETA: 0s - loss: 0.2704 - accuracy: 0.9190OrderedEnqueuer: queue.put 0 28285
OrderedEnqueuer: queue.put 1 28285
OrderedEnqueuer: queue.put 2 28285
OrderedEnqueuer: queue.put 3 28285
OrderedEnqueuer: queue.put 4 28285
OrderedEnqueuer: queue.put 5 28285
OrderedEnqueuer: queue.put 6 28285
OrderedEnqueuer: queue.put 7 28285
OrderedEnqueuer: queue.put 8 28285
TRAIN MnistSequence.on_epoch_end(), PID: 28285
get_index 0 28524
TRAIN MnistSequence.__getitem__(0), PID: 28524
get_index 1 28524
TRAIN MnistSequence.__getitem__(1), PID: 28524
get_index 2 28524
TRAIN MnistSequence.__getitem__(2), PID: 28524
get_index 3 28524
TRAIN MnistSequence.__getitem__(3), PID: 28524
get_index 4 28524
TRAIN MnistSequence.__getitem__(4), PID: 28524
get_index 5 28524
TRAIN MnistSequence.__getitem__(5), PID: 28524
get_index 6 28524
TRAIN MnistSequence.__getitem__(6), PID: 28524
get_index 7 28524
TRAIN MnistSequence.__getitem__(7), PID: 28524
get_index 8 28524
TRAIN MnistSequence.__getitem__(8), PID: 28524
get_index 9 28524
TRAIN MnistSequence.__getitem__(9), PID: 28524
get_index 10 28524
TRAIN MnistSequence.__getitem__(10), PID: 28524
get_index 11 28524
TRAIN MnistSequence.__getitem__(11), PID: 28524
get_index 12 28524
TRAIN MnistSequence.__getitem__(12), PID: 28524
get_index 13 28524
TRAIN MnistSequence.__getitem__(13), PID: 28524
get_index 14 28524
TRAIN MnistSequence.__getitem__(14), PID: 28524
get_index 15 28524
TRAIN MnistSequence.__getitem__(15), PID: 28524
get_index 16 28524
TRAIN MnistSequence.__getitem__(16), PID: 28524
get_index 17 28524
TRAIN MnistSequence.__getitem__(17), PID: 28524
get_index 18 28524
TRAIN MnistSequence.__getitem__(18), PID: 28524
get_index 19 28524
TRAIN MnistSequence.__getitem__(19), PID: 28524
get_index 20 28524
TRAIN MnistSequence.__getitem__(20), PID: 28524
get_index 21 28524
TRAIN MnistSequence.__getitem__(21), PID: 28524
get_index 22 28524
TRAIN MnistSequence.__getitem__(22), PID: 28524
get_index 23 28524
TRAIN MnistSequence.__getitem__(23), PID: 28524
get_index 24 28524
TRAIN MnistSequence.__getitem__(24), PID: 28524
get_index 25 28524
TRAIN MnistSequence.__getitem__(25), PID: 28524
get_index 26 28524
TRAIN MnistSequence.__getitem__(26), PID: 28524
get_index 27 28524
TRAIN MnistSequence.__getitem__(27), PID: 28524
get_index 28 28524
TRAIN MnistSequence.__getitem__(28), PID: 28524
get_index 29 28524
TRAIN MnistSequence.__getitem__(29), PID: 28524
get_index 30 28524
TRAIN MnistSequence.__getitem__(30), PID: 28524
get_index 31 28524
TRAIN MnistSequence.__getitem__(31), PID: 28524
get_index 32 28524
TRAIN MnistSequence.__getitem__(32), PID: 28524
get_index 33 28524
TRAIN MnistSequence.__getitem__(33), PID: 28524
get_index 34 28524
TRAIN MnistSequence.__getitem__(34), PID: 28524
get_index 35 28524
TRAIN MnistSequence.__getitem__(35), PID: 28524
get_index 36 28524
TRAIN MnistSequence.__getitem__(36), PID: 28524
get_index 37 28524
TRAIN MnistSequence.__getitem__(37), PID: 28524
get_index 38 28524
TRAIN MnistSequence.__getitem__(38), PID: 28524
get_index 39 28524
TRAIN MnistSequence.__getitem__(39), PID: 28524
get_index 40 28524
TRAIN MnistSequence.__getitem__(40), PID: 28524
get_index 41 28524
TRAIN MnistSequence.__getitem__(41), PID: 28524
get_index 42 28524
TRAIN MnistSequence.__getitem__(42), PID: 28524
get_index 43 28524
TRAIN MnistSequence.__getitem__(43), PID: 28524
get_index 44 28524
TRAIN MnistSequence.__getitem__(44), PID: 28524
get_index 45 28524
TRAIN MnistSequence.__getitem__(45), PID: 28524
get_index 46 28524
TRAIN MnistSequence.__getitem__(46), PID: 28524
get_index 47 28524
TRAIN MnistSequence.__getitem__(47), PID: 28524
get_index 48 28524
TRAIN MnistSequence.__getitem__(48), PID: 28524
get_index 49 28524
TRAIN MnistSequence.__getitem__(49), PID: 28524
get_index 50 28524
TRAIN MnistSequence.__getitem__(50), PID: 28524
get_index 51 28524
TRAIN MnistSequence.__getitem__(51), PID: 28524
get_index 52 28524
TRAIN MnistSequence.__getitem__(52), PID: 28524
get_index 53 28524
TRAIN MnistSequence.__getitem__(53), PID: 28524
get_index 54 28524
TRAIN MnistSequence.__getitem__(54), PID: 28524
get_index 55 28524
TRAIN MnistSequence.__getitem__(55), PID: 28524
get_index 56 28524
TRAIN MnistSequence.__getitem__(56), PID: 28524
get_index 57 28524
TRAIN MnistSequence.__getitem__(57), PID: 28524
get_index 0 28531
VAL MnistSequence.__getitem__(0), PID: 28531
get_index 1 28531
VAL MnistSequence.__getitem__(1), PID: 28531
get_index 2 28531
VAL MnistSequence.__getitem__(2), PID: 28531
get_index 3 28531
VAL MnistSequence.__getitem__(3), PID: 28531
get_index 4 28531
VAL MnistSequence.__getitem__(4), PID: 28531
get_index 5 28531
VAL MnistSequence.__getitem__(5), PID: 28531
get_index 6 28531
VAL MnistSequence.__getitem__(6), PID: 28531
get_index 7 28531
VAL MnistSequence.__getitem__(7), PID: 28531
get_index 8 28531
VAL MnistSequence.__getitem__(8), PID: 28531
OrderedEnqueuer: queue.put 0 28285
OrderedEnqueuer: queue.put 1 28285
OrderedEnqueuer: queue.put 2 28285
OrderedEnqueuer: queue.put 3 28285
OrderedEnqueuer: queue.put 4 28285
OrderedEnqueuer: queue.put 5 28285
OrderedEnqueuer: queue.put 6 28285
OrderedEnqueuer: queue.put 7 28285
OrderedEnqueuer: queue.put 8 28285
OrderedEnqueuer: queue.put 9 28285
OrderedEnqueuer: queue.put 10 28285
VAL MnistSequence.on_epoch_end(), PID: 28285

58/58 [==============================] - 1s 16ms/step - loss: 0.2704 - accuracy: 0.9190 - val_loss: 0.1454 - val_accuracy: 0.9588
TRAIN MnistSequence.on_epoch_end(), PID: 28285
get_index 0 28536
TRAIN MnistSequence.__getitem__(0), PID: 28536
get_index 1 28536
TRAIN MnistSequence.__getitem__(1), PID: 28536
get_index 2 28536
TRAIN MnistSequence.__getitem__(2), PID: 28536
get_index 3 28536
TRAIN MnistSequence.__getitem__(3), PID: 28536
get_index 4 28536
TRAIN MnistSequence.__getitem__(4), PID: 28536
get_index 5 28536
TRAIN MnistSequence.__getitem__(5), PID: 28536
get_index 6 28536
TRAIN MnistSequence.__getitem__(6), PID: 28536
get_index 7 28536
TRAIN MnistSequence.__getitem__(7), PID: 28536
get_index 8 28536
TRAIN MnistSequence.__getitem__(8), PID: 28536
get_index 9 28536
TRAIN MnistSequence.__getitem__(9), PID: 28536
get_index 10 28536
TRAIN MnistSequence.__getitem__(10), PID: 28536

keras_sequence_spawn.log

new process 27656
x_train shape: (60000, 28, 28, 1)
60000 train samples
10000 test samples
main process 27656
TRAIN MnistSequence.__init__(), PID: 27656 len: 58
VAL MnistSequence.__init__(), PID: 27656 len: 9
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d (Conv2D)              (None, 26, 26, 32)        320       
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 13, 13, 32)        0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 11, 11, 64)        18496     
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 5, 5, 64)          0         
_________________________________________________________________
flatten (Flatten)            (None, 1600)              0         
_________________________________________________________________
dropout (Dropout)            (None, 1600)              0         
_________________________________________________________________
dense (Dense)                (None, 10)                16010     
=================================================================
Total params: 34,826
Trainable params: 34,826
Non-trainable params: 0
_________________________________________________________________
TRAIN MnistSequence.__getitem__(0), PID: 27656
Epoch 1/2
OrderedEnqueuer: queue.put 0 27656
OrderedEnqueuer: queue.put 1 27656
OrderedEnqueuer: queue.put 2 27656
OrderedEnqueuer: queue.put 3 27656
OrderedEnqueuer: queue.put 4 27656
OrderedEnqueuer: queue.put 5 27656
OrderedEnqueuer: queue.put 6 27656
OrderedEnqueuer: queue.put 7 27656
OrderedEnqueuer: queue.put 8 27656
OrderedEnqueuer: queue.put 9 27656
OrderedEnqueuer: queue.put 10 27656
OrderedEnqueuer: queue.put 11 27656

 1/58 [..............................] - ETA: 0s - loss: 2.3068 - accuracy: 0.1240OrderedEnqueuer: queue.put 12 27656
OrderedEnqueuer: queue.put 13 27656
OrderedEnqueuer: queue.put 14 27656
OrderedEnqueuer: queue.put 15 27656

 5/58 [=>............................] - ETA: 0s - loss: 2.2439 - accuracy: 0.2193OrderedEnqueuer: queue.put 16 27656
OrderedEnqueuer: queue.put 17 27656
OrderedEnqueuer: queue.put 18 27656
OrderedEnqueuer: queue.put 19 27656

 9/58 [===>..........................] - ETA: 0s - loss: 2.1708 - accuracy: 0.3194OrderedEnqueuer: queue.put 20 27656
OrderedEnqueuer: queue.put 21 27656
OrderedEnqueuer: queue.put 22 27656
OrderedEnqueuer: queue.put 23 27656
OrderedEnqueuer: queue.put 24 27656

14/58 [======>.......................] - ETA: 0s - loss: 2.0505 - accuracy: 0.4100OrderedEnqueuer: queue.put 25 27656
OrderedEnqueuer: queue.put 26 27656
OrderedEnqueuer: queue.put 27 27656
OrderedEnqueuer: queue.put 28 27656

18/58 [========>.....................] - ETA: 0s - loss: 1.9343 - accuracy: 0.4603OrderedEnqueuer: queue.put 29 27656
OrderedEnqueuer: queue.put 30 27656
OrderedEnqueuer: queue.put 31 27656
OrderedEnqueuer: queue.put 32 27656
OrderedEnqueuer: queue.put 33 27656

23/58 [==========>...................] - ETA: 0s - loss: 1.7664 - accuracy: 0.5124OrderedEnqueuer: queue.put 34 27656
OrderedEnqueuer: queue.put 35 27656
OrderedEnqueuer: queue.put 36 27656
OrderedEnqueuer: queue.put 37 27656
OrderedEnqueuer: queue.put 38 27656

28/58 [=============>................] - ETA: 0s - loss: 1.6086 - accuracy: 0.5524OrderedEnqueuer: queue.put 39 27656
OrderedEnqueuer: queue.put 40 27656
OrderedEnqueuer: queue.put 41 27656
OrderedEnqueuer: queue.put 42 27656

32/58 [===============>..............] - ETA: 0s - loss: 1.5078 - accuracy: 0.5759OrderedEnqueuer: queue.put 43 27656
OrderedEnqueuer: queue.put 44 27656
OrderedEnqueuer: queue.put 45 27656
OrderedEnqueuer: queue.put 46 27656
OrderedEnqueuer: queue.put 47 27656

37/58 [==================>...........] - ETA: 0s - loss: 1.3879 - accuracy: 0.6060OrderedEnqueuer: queue.put 48 27656
OrderedEnqueuer: queue.put 49 27656
OrderedEnqueuer: queue.put 50 27656
OrderedEnqueuer: queue.put 51 27656
OrderedEnqueuer: queue.put 52 27656

42/58 [====================>.........] - ETA: 0s - loss: 1.2888 - accuracy: 0.6315OrderedEnqueuer: queue.put 53 27656
OrderedEnqueuer: queue.put 54 27656
OrderedEnqueuer: queue.put 55 27656
OrderedEnqueuer: queue.put 56 27656
OrderedEnqueuer: queue.put 57 27656

47/58 [=======================>......] - ETA: 0s - loss: 1.2043 - accuracy: 0.6537
52/58 [=========================>....] - ETA: 0s - loss: 1.1320 - accuracy: 0.6739
57/58 [============================>.] - ETA: 0s - loss: 1.0618 - accuracy: 0.6938VAL MnistSequence.__getitem__(0), PID: 27656
TRAIN MnistSequence.on_epoch_end(), PID: 27656
new process 27873
x_train shape: (60000, 28, 28, 1)
60000 train samples
10000 test samples
get_index 0 27873
TRAIN MnistSequence.__getitem__(0), PID: 27873
get_index 1 27873
TRAIN MnistSequence.__getitem__(1), PID: 27873
get_index 2 27873
TRAIN MnistSequence.__getitem__(2), PID: 27873
get_index 3 27873
TRAIN MnistSequence.__getitem__(3), PID: 27873
get_index 4 27873
TRAIN MnistSequence.__getitem__(4), PID: 27873
get_index 5 27873
TRAIN MnistSequence.__getitem__(5), PID: 27873
get_index 6 27873
TRAIN MnistSequence.__getitem__(6), PID: 27873
get_index 7 27873
TRAIN MnistSequence.__getitem__(7), PID: 27873
get_index 8 27873
TRAIN MnistSequence.__getitem__(8), PID: 27873
get_index 9 27873
TRAIN MnistSequence.__getitem__(9), PID: 27873
get_index 10 27873
TRAIN MnistSequence.__getitem__(10), PID: 27873
get_index 11 27873
TRAIN MnistSequence.__getitem__(11), PID: 27873
get_index 12 27873
TRAIN MnistSequence.__getitem__(12), PID: 27873
get_index 13 27873
TRAIN MnistSequence.__getitem__(13), PID: 27873
get_index 14 27873
TRAIN MnistSequence.__getitem__(14), PID: 27873
get_index 15 27873
TRAIN MnistSequence.__getitem__(15), PID: 27873
get_index 16 27873
TRAIN MnistSequence.__getitem__(16), PID: 27873
get_index 17 27873
TRAIN MnistSequence.__getitem__(17), PID: 27873
get_index 18 27873
TRAIN MnistSequence.__getitem__(18), PID: 27873
get_index 19 27873
TRAIN MnistSequence.__getitem__(19), PID: 27873
get_index 20 27873
TRAIN MnistSequence.__getitem__(20), PID: 27873
get_index 21 27873
TRAIN MnistSequence.__getitem__(21), PID: 27873
get_index 22 27873
TRAIN MnistSequence.__getitem__(22), PID: 27873
get_index 23 27873
TRAIN MnistSequence.__getitem__(23), PID: 27873
get_index 24 27873
TRAIN MnistSequence.__getitem__(24), PID: 27873
get_index 25 27873
TRAIN MnistSequence.__getitem__(25), PID: 27873
get_index 26 27873
TRAIN MnistSequence.__getitem__(26), PID: 27873
get_index 27 27873
TRAIN MnistSequence.__getitem__(27), PID: 27873
get_index 28 27873
TRAIN MnistSequence.__getitem__(28), PID: 27873
get_index 29 27873
TRAIN MnistSequence.__getitem__(29), PID: 27873
get_index 30 27873
TRAIN MnistSequence.__getitem__(30), PID: 27873
get_index 31 27873
TRAIN MnistSequence.__getitem__(31), PID: 27873
get_index 32 27873
TRAIN MnistSequence.__getitem__(32), PID: 27873
get_index 33 27873
TRAIN MnistSequence.__getitem__(33), PID: 27873
get_index 34 27873
TRAIN MnistSequence.__getitem__(34), PID: 27873
get_index 35 27873
TRAIN MnistSequence.__getitem__(35), PID: 27873
get_index 36 27873
TRAIN MnistSequence.__getitem__(36), PID: 27873
get_index 37 27873
TRAIN MnistSequence.__getitem__(37), PID: 27873
get_index 38 27873
TRAIN MnistSequence.__getitem__(38), PID: 27873
get_index 39 27873
TRAIN MnistSequence.__getitem__(39), PID: 27873
get_index 40 27873
TRAIN MnistSequence.__getitem__(40), PID: 27873
get_index 41 27873
TRAIN MnistSequence.__getitem__(41), PID: 27873
get_index 42 27873
TRAIN MnistSequence.__getitem__(42), PID: 27873
get_index 43 27873
TRAIN MnistSequence.__getitem__(43), PID: 27873
get_index 44 27873
TRAIN MnistSequence.__getitem__(44), PID: 27873
get_index 45 27873
TRAIN MnistSequence.__getitem__(45), PID: 27873
get_index 46 27873
TRAIN MnistSequence.__getitem__(46), PID: 27873
get_index 47 27873
TRAIN MnistSequence.__getitem__(47), PID: 27873
get_index 48 27873
TRAIN MnistSequence.__getitem__(48), PID: 27873
get_index 49 27873
TRAIN MnistSequence.__getitem__(49), PID: 27873
get_index 50 27873
TRAIN MnistSequence.__getitem__(50), PID: 27873
get_index 51 27873
TRAIN MnistSequence.__getitem__(51), PID: 27873
get_index 52 27873
TRAIN MnistSequence.__getitem__(52), PID: 27873
get_index 53 27873
TRAIN MnistSequence.__getitem__(53), PID: 27873
get_index 54 27873
TRAIN MnistSequence.__getitem__(54), PID: 27873
get_index 55 27873
TRAIN MnistSequence.__getitem__(55), PID: 27873
get_index 56 27873
TRAIN MnistSequence.__getitem__(56), PID: 27873
get_index 57 27873
TRAIN MnistSequence.__getitem__(57), PID: 27873
OrderedEnqueuer: queue.put 0 27656
OrderedEnqueuer: queue.put 1 27656
OrderedEnqueuer: queue.put 2 27656
OrderedEnqueuer: queue.put 3 27656
OrderedEnqueuer: queue.put 4 27656
OrderedEnqueuer: queue.put 5 27656
OrderedEnqueuer: queue.put 6 27656
OrderedEnqueuer: queue.put 7 27656
OrderedEnqueuer: queue.put 8 27656
OrderedEnqueuer: queue.put 9 27656
OrderedEnqueuer: queue.put 10 27656
OrderedEnqueuer: queue.put 0 27656
OrderedEnqueuer: queue.put 1 27656
OrderedEnqueuer: queue.put 2 27656
OrderedEnqueuer: queue.put 3 27656
OrderedEnqueuer: queue.put 4 27656
OrderedEnqueuer: queue.put 5 27656
OrderedEnqueuer: queue.put 6 27656
OrderedEnqueuer: queue.put 7 27656
OrderedEnqueuer: queue.put 8 27656
VAL MnistSequence.on_epoch_end(), PID: 27656

58/58 [==============================] - 4s 67ms/step - loss: 1.0477 - accuracy: 0.6979 - val_loss: 0.2774 - val_accuracy: 0.9240
new process 27914
x_train shape: (60000, 28, 28, 1)
60000 train samples
10000 test samples
get_index 0 27914
TRAIN MnistSequence.__getitem__(0), PID: 27914
get_index 1 27914
TRAIN MnistSequence.__getitem__(1), PID: 27914
get_index 2 27914
TRAIN MnistSequence.__getitem__(2), PID: 27914
get_index 3 27914
TRAIN MnistSequence.__getitem__(3), PID: 27914
get_index 4 27914
TRAIN MnistSequence.__getitem__(4), PID: 27914
get_index 5 27914
TRAIN MnistSequence.__getitem__(5), PID: 27914
get_index 6 27914
TRAIN MnistSequence.__getitem__(6), PID: 27914
get_index 7 27914
TRAIN MnistSequence.__getitem__(7), PID: 27914
get_index 8 27914
TRAIN MnistSequence.__getitem__(8), PID: 27914
get_index 9 27914
TRAIN MnistSequence.__getitem__(9), PID: 27914
get_index 10 27914
TRAIN MnistSequence.__getitem__(10), PID: 27914
TRAIN MnistSequence.on_epoch_end(), PID: 27656
Epoch 2/2
new process 27933
x_train shape: (60000, 28, 28, 1)
60000 train samples
10000 test samples
get_index 0 27933
VAL MnistSequence.__getitem__(0), PID: 27933
get_index 1 27933
VAL MnistSequence.__getitem__(1), PID: 27933
get_index 2 27933
VAL MnistSequence.__getitem__(2), PID: 27933
get_index 3 27933
VAL MnistSequence.__getitem__(3), PID: 27933
get_index 4 27933
VAL MnistSequence.__getitem__(4), PID: 27933
get_index 5 27933
VAL MnistSequence.__getitem__(5), PID: 27933
get_index 6 27933
VAL MnistSequence.__getitem__(6), PID: 27933
get_index 7 27933
VAL MnistSequence.__getitem__(7), PID: 27933
get_index 8 27933
VAL MnistSequence.__getitem__(8), PID: 27933
OrderedEnqueuer: queue.put 0 27656
OrderedEnqueuer: queue.put 1 27656
OrderedEnqueuer: queue.put 2 27656
OrderedEnqueuer: queue.put 3 27656
OrderedEnqueuer: queue.put 4 27656
OrderedEnqueuer: queue.put 5 27656
OrderedEnqueuer: queue.put 6 27656
OrderedEnqueuer: queue.put 7 27656
OrderedEnqueuer: queue.put 8 27656
OrderedEnqueuer: queue.put 9 27656
OrderedEnqueuer: queue.put 10 27656
OrderedEnqueuer: queue.put 11 27656

 1/58 [..............................] - ETA: 0s - loss: 0.3669 - accuracy: 0.8906OrderedEnqueuer: queue.put 12 27656
OrderedEnqueuer: queue.put 13 27656
OrderedEnqueuer: queue.put 14 27656
OrderedEnqueuer: queue.put 15 27656

 5/58 [=>............................] - ETA: 0s - loss: 0.3378 - accuracy: 0.9016OrderedEnqueuer: queue.put 16 27656
OrderedEnqueuer: queue.put 17 27656
OrderedEnqueuer: queue.put 18 27656
OrderedEnqueuer: queue.put 19 27656

 9/58 [===>..........................] - ETA: 0s - loss: 0.3377 - accuracy: 0.9019OrderedEnqueuer: queue.put 20 27656
OrderedEnqueuer: queue.put 21 27656
OrderedEnqueuer: queue.put 22 27656
OrderedEnqueuer: queue.put 23 27656

13/58 [=====>........................] - ETA: 0s - loss: 0.3346 - accuracy: 0.9001OrderedEnqueuer: queue.put 24 27656
OrderedEnqueuer: queue.put 25 27656
OrderedEnqueuer: queue.put 26 27656
OrderedEnqueuer: queue.put 27 27656
OrderedEnqueuer: queue.put 28 27656

18/58 [========>.....................] - ETA: 0s - loss: 0.3283 - accuracy: 0.9006OrderedEnqueuer: queue.put 29 27656
OrderedEnqueuer: queue.put 30 27656
OrderedEnqueuer: queue.put 31 27656
OrderedEnqueuer: queue.put 32 27656
OrderedEnqueuer: queue.put 33 27656

23/58 [==========>...................] - ETA: 0s - loss: 0.3163 - accuracy: 0.9039OrderedEnqueuer: queue.put 34 27656
OrderedEnqueuer: queue.put 35 27656
OrderedEnqueuer: queue.put 36 27656
OrderedEnqueuer: queue.put 37 27656

27/58 [============>.................] - ETA: 0s - loss: 0.3113 - accuracy: 0.9052OrderedEnqueuer: queue.put 38 27656
OrderedEnqueuer: queue.put 39 27656
OrderedEnqueuer: queue.put 40 27656
OrderedEnqueuer: queue.put 41 27656
OrderedEnqueuer: queue.put 42 27656

32/58 [===============>..............] - ETA: 0s - loss: 0.3068 - accuracy: 0.9070OrderedEnqueuer: queue.put 43 27656
OrderedEnqueuer: queue.put 44 27656
OrderedEnqueuer: queue.put 45 27656
OrderedEnqueuer: queue.put 46 27656
OrderedEnqueuer: queue.put 47 27656

37/58 [==================>...........] - ETA: 0s - loss: 0.2977 - accuracy: 0.9097OrderedEnqueuer: queue.put 48 27656
OrderedEnqueuer: queue.put 49 27656
OrderedEnqueuer: queue.put 50 27656
OrderedEnqueuer: queue.put 51 27656
OrderedEnqueuer: queue.put 52 27656

42/58 [====================>.........] - ETA: 0s - loss: 0.2907 - accuracy: 0.9121OrderedEnqueuer: queue.put 53 27656
OrderedEnqueuer: queue.put 54 27656
OrderedEnqueuer: queue.put 55 27656
OrderedEnqueuer: queue.put 56 27656
OrderedEnqueuer: queue.put 57 27656

47/58 [=======================>......] - ETA: 0s - loss: 0.2859 - accuracy: 0.9131
52/58 [=========================>....] - ETA: 0s - loss: 0.2803 - accuracy: 0.9151
57/58 [============================>.] - ETA: 0s - loss: 0.2710 - accuracy: 0.9181new process 27995
x_train shape: (60000, 28, 28, 1)
60000 train samples
10000 test samples
get_index 0 27995
TRAIN MnistSequence.__getitem__(0), PID: 27995
get_index 1 27995
TRAIN MnistSequence.__getitem__(1), PID: 27995
get_index 2 27995
TRAIN MnistSequence.__getitem__(2), PID: 27995
get_index 3 27995
TRAIN MnistSequence.__getitem__(3), PID: 27995
get_index 4 27995
TRAIN MnistSequence.__getitem__(4), PID: 27995
get_index 5 27995
TRAIN MnistSequence.__getitem__(5), PID: 27995
get_index 6 27995
TRAIN MnistSequence.__getitem__(6), PID: 27995
get_index 7 27995
TRAIN MnistSequence.__getitem__(7), PID: 27995
get_index 8 27995
TRAIN MnistSequence.__getitem__(8), PID: 27995
get_index 9 27995
TRAIN MnistSequence.__getitem__(9), PID: 27995
get_index 10 27995
TRAIN MnistSequence.__getitem__(10), PID: 27995
get_index 11 27995
TRAIN MnistSequence.__getitem__(11), PID: 27995
get_index 12 27995
TRAIN MnistSequence.__getitem__(12), PID: 27995
get_index 13 27995
TRAIN MnistSequence.__getitem__(13), PID: 27995
get_index 14 27995
TRAIN MnistSequence.__getitem__(14), PID: 27995
get_index 15 27995
TRAIN MnistSequence.__getitem__(15), PID: 27995
get_index 16 27995
TRAIN MnistSequence.__getitem__(16), PID: 27995
get_index 17 27995
TRAIN MnistSequence.__getitem__(17), PID: 27995
get_index 18 27995
TRAIN MnistSequence.__getitem__(18), PID: 27995
get_index 19 27995
TRAIN MnistSequence.__getitem__(19), PID: 27995
get_index 20 27995
TRAIN MnistSequence.__getitem__(20), PID: 27995
get_index 21 27995
TRAIN MnistSequence.__getitem__(21), PID: 27995
get_index 22 27995
TRAIN MnistSequence.__getitem__(22), PID: 27995
get_index 23 27995
TRAIN MnistSequence.__getitem__(23), PID: 27995
get_index 24 27995
TRAIN MnistSequence.__getitem__(24), PID: 27995
get_index 25 27995
TRAIN MnistSequence.__getitem__(25), PID: 27995
get_index 26 27995
TRAIN MnistSequence.__getitem__(26), PID: 27995
get_index 27 27995
TRAIN MnistSequence.__getitem__(27), PID: 27995
get_index 28 27995
TRAIN MnistSequence.__getitem__(28), PID: 27995
get_index 29 27995
TRAIN MnistSequence.__getitem__(29), PID: 27995
get_index 30 27995
TRAIN MnistSequence.__getitem__(30), PID: 27995
get_index 31 27995
TRAIN MnistSequence.__getitem__(31), PID: 27995
get_index 32 27995
TRAIN MnistSequence.__getitem__(32), PID: 27995
get_index 33 27995
TRAIN MnistSequence.__getitem__(33), PID: 27995
get_index 34 27995
TRAIN MnistSequence.__getitem__(34), PID: 27995
get_index 35 27995
TRAIN MnistSequence.__getitem__(35), PID: 27995
get_index 36 27995
TRAIN MnistSequence.__getitem__(36), PID: 27995
get_index 37 27995
TRAIN MnistSequence.__getitem__(37), PID: 27995
get_index 38 27995
TRAIN MnistSequence.__getitem__(38), PID: 27995
get_index 39 27995
TRAIN MnistSequence.__getitem__(39), PID: 27995
get_index 40 27995
TRAIN MnistSequence.__getitem__(40), PID: 27995
get_index 41 27995
TRAIN MnistSequence.__getitem__(41), PID: 27995
get_index 42 27995
TRAIN MnistSequence.__getitem__(42), PID: 27995
get_index 43 27995
TRAIN MnistSequence.__getitem__(43), PID: 27995
get_index 44 27995
TRAIN MnistSequence.__getitem__(44), PID: 27995
get_index 45 27995
TRAIN MnistSequence.__getitem__(45), PID: 27995
get_index 46 27995
TRAIN MnistSequence.__getitem__(46), PID: 27995
get_index 47 27995
TRAIN MnistSequence.__getitem__(47), PID: 27995
get_index 48 27995
TRAIN MnistSequence.__getitem__(48), PID: 27995
get_index 49 27995
TRAIN MnistSequence.__getitem__(49), PID: 27995
get_index 50 27995
TRAIN MnistSequence.__getitem__(50), PID: 27995
get_index 51 27995
TRAIN MnistSequence.__getitem__(51), PID: 27995
get_index 52 27995
TRAIN MnistSequence.__getitem__(52), PID: 27995
get_index 53 27995
TRAIN MnistSequence.__getitem__(53), PID: 27995
get_index 54 27995
TRAIN MnistSequence.__getitem__(54), PID: 27995
get_index 55 27995
TRAIN MnistSequence.__getitem__(55), PID: 27995
get_index 56 27995
TRAIN MnistSequence.__getitem__(56), PID: 27995
get_index 57 27995
TRAIN MnistSequence.__getitem__(57), PID: 27995
TRAIN MnistSequence.on_epoch_end(), PID: 27656
OrderedEnqueuer: queue.put 0 27656
OrderedEnqueuer: queue.put 1 27656
OrderedEnqueuer: queue.put 2 27656
OrderedEnqueuer: queue.put 3 27656
OrderedEnqueuer: queue.put 4 27656
OrderedEnqueuer: queue.put 5 27656
OrderedEnqueuer: queue.put 6 27656
OrderedEnqueuer: queue.put 7 27656
OrderedEnqueuer: queue.put 8 27656
VAL MnistSequence.on_epoch_end(), PID: 27656

58/58 [==============================] - 4s 61ms/step - loss: 0.2684 - accuracy: 0.9189 - val_loss: 0.1444 - val_accuracy: 0.9583
new process 28084
x_train shape: (60000, 28, 28, 1)
60000 train samples
10000 test samples
get_index 0 28084
VAL MnistSequence.__getitem__(0), PID: 28084
get_index 1 28084
VAL MnistSequence.__getitem__(1), PID: 28084
get_index 2 28084
VAL MnistSequence.__getitem__(2), PID: 28084
get_index 3 28084
VAL MnistSequence.__getitem__(3), PID: 28084
get_index 4 28084
VAL MnistSequence.__getitem__(4), PID: 28084
get_index 5 28084
VAL MnistSequence.__getitem__(5), PID: 28084
get_index 6 28084
VAL MnistSequence.__getitem__(6), PID: 28084
get_index 7 28084
VAL MnistSequence.__getitem__(7), PID: 28084
get_index 8 28084
VAL MnistSequence.__getitem__(8), PID: 28084
TRAIN MnistSequence.on_epoch_end(), PID: 27656
new process 28100
x_train shape: (60000, 28, 28, 1)
60000 train samples
10000 test samples

keras_sequence_thread.log

new process 28850
x_train shape: (60000, 28, 28, 1)
60000 train samples
10000 test samples
main process 28850
TRAIN MnistSequence.__init__(), PID: 28850 len: 58
VAL MnistSequence.__init__(), PID: 28850 len: 9
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d (Conv2D)              (None, 26, 26, 32)        320       
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 13, 13, 32)        0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 11, 11, 64)        18496     
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 5, 5, 64)          0         
_________________________________________________________________
flatten (Flatten)            (None, 1600)              0         
_________________________________________________________________
dropout (Dropout)            (None, 1600)              0         
_________________________________________________________________
dense (Dense)                (None, 10)                16010     
=================================================================
Total params: 34,826
Trainable params: 34,826
Non-trainable params: 0
_________________________________________________________________
TRAIN MnistSequence.__getitem__(0), PID: 28850
Epoch 1/2
TRAIN MnistSequence.__getitem__(0), PID: 28850
TRAIN MnistSequence.__getitem__(1), PID: 28850

 1/58 [..............................] - ETA: 0s - loss: 2.3016 - accuracy: 0.0850TRAIN MnistSequence.__getitem__(2), PID: 28850
TRAIN MnistSequence.__getitem__(3), PID: 28850
TRAIN MnistSequence.__getitem__(4), PID: 28850
TRAIN MnistSequence.__getitem__(5), PID: 28850
TRAIN MnistSequence.__getitem__(6), PID: 28850

 6/58 [==>...........................] - ETA: 0s - loss: 2.2262 - accuracy: 0.2528TRAIN MnistSequence.__getitem__(7), PID: 28850
TRAIN MnistSequence.__getitem__(8), PID: 28850
TRAIN MnistSequence.__getitem__(9), PID: 28850
TRAIN MnistSequence.__getitem__(10), PID: 28850
TRAIN MnistSequence.__getitem__(11), PID: 28850

11/58 [====>.........................] - ETA: 0s - loss: 2.1398 - accuracy: 0.3550TRAIN MnistSequence.__getitem__(12), PID: 28850
TRAIN MnistSequence.__getitem__(13), PID: 28850
TRAIN MnistSequence.__getitem__(14), PID: 28850
TRAIN MnistSequence.__getitem__(15), PID: 28850
TRAIN MnistSequence.__getitem__(16), PID: 28850

16/58 [=======>......................] - ETA: 0s - loss: 2.0294 - accuracy: 0.4228TRAIN MnistSequence.__getitem__(17), PID: 28850
TRAIN MnistSequence.__getitem__(18), PID: 28850
TRAIN MnistSequence.__getitem__(19), PID: 28850
TRAIN MnistSequence.__getitem__(20), PID: 28850
TRAIN MnistSequence.__getitem__(21), PID: 28850

21/58 [=========>....................] - ETA: 0s - loss: 1.8838 - accuracy: 0.4851TRAIN MnistSequence.__getitem__(22), PID: 28850
TRAIN MnistSequence.__getitem__(23), PID: 28850
TRAIN MnistSequence.__getitem__(24), PID: 28850
TRAIN MnistSequence.__getitem__(25), PID: 28850
TRAIN MnistSequence.__getitem__(26), PID: 28850

26/58 [============>.................] - ETA: 0s - loss: 1.7252 - accuracy: 0.5284TRAIN MnistSequence.__getitem__(27), PID: 28850
TRAIN MnistSequence.__getitem__(28), PID: 28850
TRAIN MnistSequence.__getitem__(29), PID: 28850
TRAIN MnistSequence.__getitem__(30), PID: 28850
TRAIN MnistSequence.__getitem__(31), PID: 28850

31/58 [===============>..............] - ETA: 0s - loss: 1.5850 - accuracy: 0.5624TRAIN MnistSequence.__getitem__(32), PID: 28850
TRAIN MnistSequence.__getitem__(33), PID: 28850
TRAIN MnistSequence.__getitem__(34), PID: 28850
TRAIN MnistSequence.__getitem__(35), PID: 28850
TRAIN MnistSequence.__getitem__(36), PID: 28850

36/58 [=================>............] - ETA: 0s - loss: 1.4564 - accuracy: 0.5941TRAIN MnistSequence.__getitem__(37), PID: 28850
TRAIN MnistSequence.__getitem__(38), PID: 28850
TRAIN MnistSequence.__getitem__(39), PID: 28850
TRAIN MnistSequence.__getitem__(40), PID: 28850
TRAIN MnistSequence.__getitem__(41), PID: 28850

41/58 [====================>.........] - ETA: 0s - loss: 1.3490 - accuracy: 0.6215TRAIN MnistSequence.__getitem__(42), PID: 28850
TRAIN MnistSequence.__getitem__(43), PID: 28850
TRAIN MnistSequence.__getitem__(44), PID: 28850
TRAIN MnistSequence.__getitem__(45), PID: 28850
TRAIN MnistSequence.__getitem__(46), PID: 28850

46/58 [======================>.......] - ETA: 0s - loss: 1.2573 - accuracy: 0.6446TRAIN MnistSequence.__getitem__(47), PID: 28850
TRAIN MnistSequence.__getitem__(48), PID: 28850
TRAIN MnistSequence.__getitem__(49), PID: 28850
TRAIN MnistSequence.__getitem__(50), PID: 28850
TRAIN MnistSequence.__getitem__(51), PID: 28850

51/58 [=========================>....] - ETA: 0s - loss: 1.1793 - accuracy: 0.6656TRAIN MnistSequence.__getitem__(52), PID: 28850
TRAIN MnistSequence.__getitem__(53), PID: 28850
TRAIN MnistSequence.__getitem__(54), PID: 28850
TRAIN MnistSequence.__getitem__(55), PID: 28850
TRAIN MnistSequence.__getitem__(56), PID: 28850

56/58 [===========================>..] - ETA: 0s - loss: 1.1076 - accuracy: 0.6852TRAIN MnistSequence.__getitem__(57), PID: 28850
VAL MnistSequence.__getitem__(0), PID: 28850
VAL MnistSequence.__getitem__(0), PID: 28850
VAL MnistSequence.__getitem__(1), PID: 28850
VAL MnistSequence.__getitem__(2), PID: 28850
VAL MnistSequence.__getitem__(3), PID: 28850
VAL MnistSequence.__getitem__(4), PID: 28850
VAL MnistSequence.__getitem__(5), PID: 28850
VAL MnistSequence.__getitem__(6), PID: 28850
VAL MnistSequence.__getitem__(7), PID: 28850
VAL MnistSequence.__getitem__(8), PID: 28850
VAL MnistSequence.on_epoch_end(), PID: 28850

58/58 [==============================] - 1s 14ms/step - loss: 1.0789 - accuracy: 0.6934 - val_loss: 0.2847 - val_accuracy: 0.9243
TRAIN MnistSequence.on_epoch_end(), PID: 28850
Epoch 2/2
TRAIN MnistSequence.__getitem__(0), PID: 28850
TRAIN MnistSequence.__getitem__(1), PID: 28850

 1/58 [..............................] - ETA: 0s - loss: 0.3943 - accuracy: 0.8818TRAIN MnistSequence.__getitem__(2), PID: 28850
TRAIN MnistSequence.__getitem__(3), PID: 28850
TRAIN MnistSequence.__getitem__(4), PID: 28850
TRAIN MnistSequence.__getitem__(5), PID: 28850
TRAIN MnistSequence.__getitem__(6), PID: 28850

 6/58 [==>...........................] - ETA: 0s - loss: 0.3405 - accuracy: 0.8979TRAIN MnistSequence.__getitem__(7), PID: 28850
TRAIN MnistSequence.__getitem__(8), PID: 28850
TRAIN MnistSequence.__getitem__(9), PID: 28850
TRAIN MnistSequence.__getitem__(10), PID: 28850
TRAIN MnistSequence.__getitem__(11), PID: 28850

11/58 [====>.........................] - ETA: 0s - loss: 0.3339 - accuracy: 0.8999TRAIN MnistSequence.__getitem__(12), PID: 28850
TRAIN MnistSequence.__getitem__(13), PID: 28850
TRAIN MnistSequence.__getitem__(14), PID: 28850
TRAIN MnistSequence.__getitem__(15), PID: 28850
TRAIN MnistSequence.__getitem__(16), PID: 28850

16/58 [=======>......................] - ETA: 0s - loss: 0.3366 - accuracy: 0.8976TRAIN MnistSequence.__getitem__(17), PID: 28850
TRAIN MnistSequence.__getitem__(18), PID: 28850
TRAIN MnistSequence.__getitem__(19), PID: 28850
TRAIN MnistSequence.__getitem__(20), PID: 28850
TRAIN MnistSequence.__getitem__(21), PID: 28850

21/58 [=========>....................] - ETA: 0s - loss: 0.3233 - accuracy: 0.9021TRAIN MnistSequence.__getitem__(22), PID: 28850
TRAIN MnistSequence.__getitem__(23), PID: 28850
TRAIN MnistSequence.__getitem__(24), PID: 28850
TRAIN MnistSequence.__getitem__(25), PID: 28850
TRAIN MnistSequence.__getitem__(26), PID: 28850

26/58 [============>.................] - ETA: 0s - loss: 0.3125 - accuracy: 0.9060TRAIN MnistSequence.__getitem__(27), PID: 28850
TRAIN MnistSequence.__getitem__(28), PID: 28850
TRAIN MnistSequence.__getitem__(29), PID: 28850
TRAIN MnistSequence.__getitem__(30), PID: 28850
TRAIN MnistSequence.__getitem__(31), PID: 28850

31/58 [===============>..............] - ETA: 0s - loss: 0.3087 - accuracy: 0.9076TRAIN MnistSequence.__getitem__(32), PID: 28850
TRAIN MnistSequence.__getitem__(33), PID: 28850
TRAIN MnistSequence.__getitem__(34), PID: 28850
TRAIN MnistSequence.__getitem__(35), PID: 28850
TRAIN MnistSequence.__getitem__(36), PID: 28850

36/58 [=================>............] - ETA: 0s - loss: 0.2993 - accuracy: 0.9103TRAIN MnistSequence.__getitem__(37), PID: 28850
TRAIN MnistSequence.__getitem__(38), PID: 28850
TRAIN MnistSequence.__getitem__(39), PID: 28850
TRAIN MnistSequence.__getitem__(40), PID: 28850
TRAIN MnistSequence.__getitem__(41), PID: 28850

41/58 [====================>.........] - ETA: 0s - loss: 0.2926 - accuracy: 0.9121TRAIN MnistSequence.__getitem__(42), PID: 28850
TRAIN MnistSequence.__getitem__(43), PID: 28850
TRAIN MnistSequence.__getitem__(44), PID: 28850
TRAIN MnistSequence.__getitem__(45), PID: 28850
TRAIN MnistSequence.__getitem__(46), PID: 28850

46/58 [======================>.......] - ETA: 0s - loss: 0.2877 - accuracy: 0.9134TRAIN MnistSequence.__getitem__(47), PID: 28850
TRAIN MnistSequence.__getitem__(48), PID: 28850
TRAIN MnistSequence.__getitem__(49), PID: 28850
TRAIN MnistSequence.__getitem__(50), PID: 28850
TRAIN MnistSequence.__getitem__(51), PID: 28850

51/58 [=========================>....] - ETA: 0s - loss: 0.2823 - accuracy: 0.9153TRAIN MnistSequence.__getitem__(52), PID: 28850
TRAIN MnistSequence.__getitem__(53), PID: 28850
TRAIN MnistSequence.__getitem__(54), PID: 28850
TRAIN MnistSequence.__getitem__(55), PID: 28850
TRAIN MnistSequence.__getitem__(56), PID: 28850

56/58 [===========================>..] - ETA: 0s - loss: 0.2742 - accuracy: 0.9181TRAIN MnistSequence.__getitem__(57), PID: 28850
VAL MnistSequence.__getitem__(0), PID: 28850
VAL MnistSequence.__getitem__(1), PID: 28850
VAL MnistSequence.__getitem__(2), PID: 28850
VAL MnistSequence.__getitem__(3), PID: 28850
VAL MnistSequence.__getitem__(4), PID: 28850
VAL MnistSequence.__getitem__(5), PID: 28850
VAL MnistSequence.__getitem__(6), PID: 28850
VAL MnistSequence.__getitem__(7), PID: 28850
VAL MnistSequence.__getitem__(8), PID: 28850
VAL MnistSequence.on_epoch_end(), PID: 28850

58/58 [==============================] - 1s 12ms/step - loss: 0.2695 - accuracy: 0.9195 - val_loss: 0.1456 - val_accuracy: 0.9588
TRAIN MnistSequence.on_epoch_end(), PID: 28850