MNISTGRNN_multi.py

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pickle as pk
import multiprocessing
import time

MemoryValue = 0.55

start_time = time.time()
reservoir_weightsDF = pd.read_csv('WeightMatrix.csv', header=0, index_col=0)
transcriptomicsDF = pd.read_csv('InterpolateData.csv', header=0, index_col=0)
imageData = pd.read_csv('ImageClassification/ResizedTrainImage.csv', header=None, index_col=0)
print(time.time() - start_time)
print(imageData.head())

print(transcriptomicsDF.head())
# exit()
GeneLegend = list(reservoir_weightsDF.columns)

WeightMatrix = reservoir_weightsDF.values
# print(GeneLegend)

TranscriptomicData = transcriptomicsDF[GeneLegend]

InitialMemory = list(TranscriptomicData.iloc[0])
print("InitialMemory", InitialMemory)



def getMaxExpression(TranscriptomicDF, GeneID):
    return TranscriptomicDF[GeneID].max()

def worker(a, b, result, row):
    n = len(b[0])
    for j in range(n):
        result[row][j] = sum(a[row][k] * b[k][j] for k in range(len(b)))

def worker_multi(a, b, result, start_index, end_index):
    n = len(b[0])
    for row in range(start_index, end_index):
        for j in range(n):
            result[row][j] = sum(a[row][k] * b[k][j] for k in range(len(b)))

def rc_output(rc_weights, input_array):
    memory = np.maximum(rc_weights.dot(input_array), 0)
    return memory

NUM_PROCESSES=4

def split(a, n):
    k, m = divmod(len(a), n)
    return (a[i*k+min(i, m):(i+1)*k+min(i+1, m)] for i in range(n))

def parallel_rc_output(rc_weights, input_array):
    a = rc_weights
    b = input_array
    #reservoir_weights[:len(expanded_inputs), :]
    rows = len(a)
    print(b[0])
    result = multiprocessing.Array('d', rows * len(b[0]))
    result_np = np.frombuffer(result.get_obj()).reshape((rows, len(b[0])))    
    processes = []

    # num_rows_per_proc = ceil(rows / NUM_PROCESSES)
    # index = 0
    # for i in range(NUM_PROCESSES):
    #     start_index = index
    #     end_index = max(index + num_rows_per_proc, rows)
    #     index = end_index + 1

    #     p = multiprocessing.Process(target=worker, args=(a, b, result_np, start_index, end_index))
    #     processes.append(p)
    #     p.start()   
    for i in range(rows):
    # only run NUM_PROCESSES, tell worker array of rows=[1,2,3,...]
    # merge reults
        p = multiprocessing.Process(target=worker, args=(a, b, result_np, i))
        processes.append(p)
        p.start()   
    for p in processes:
        p.join()    
    return np.maximum(result_np, 0)

def input_padding(input_geneID, input_value, GeneLegend, initMem):
    input_matrix = np.array(initMem, dtype=float)

    for ig in range(0, len(input_geneID)):
        input_matrix[GeneLegend.index(input_geneID[ig])] = input_value[ig]*getMaxExpression(TranscriptomicData, input_geneID[ig])


    return input_matrix


#pick 15 random genes from GeneLegend
InputGenes = np.random.choice(GeneLegend, 15, replace=False)
print("InputGene", InputGenes)
# exit()

output_matrix = []
for i in range(0, 1):
# for i in range(0, len(imageData)):

    imgTemp = imageData.iloc[i].values
    # normalize the image
    # imgTemp = imgTemp / 255
    imgDims = np.sqrt(len(imgTemp)).astype(int)
    img = imgTemp.reshape(imgDims, imgDims)
    print(img)

    for j in range(0, imgDims):
        valueArray = img[:,j]
        print(valueArray)

        if j == 0:
            input_array = input_padding(InputGenes, valueArray, GeneLegend, InitialMemory)
            Output = parallel_rc_output(WeightMatrix, input_array)
        else:
            input_array = input_padding(InputGenes, valueArray, GeneLegend, Output*MemoryValue)
            Output = parallel_rc_output(WeightMatrix, input_array)
    output_matrix.append(Output)

with open('ImageClassification/output_matrix.pkl', 'wb') as f:
    pk.dump(output_matrix, f)

#to csv
#output_matrixDF = pd.DataFrame(output_matrix)

#add column names
#output_matrixDF.columns = GeneLegend
#print(output_matrixDF.head())
#output_matrixDF.to_csv('MNISTData/test' + '1' + '.csv')