NEURAL NETWORK IN CUDA

gistfile1.c

#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <math.h>


typedef struct {
    int n_inputs;
    int n_hidden;
    int n_outputs;
    
    float *out_input;
    float *out_hidden;
    float *out_output;

    float **changes_input_hidden;
    float **changes_hidden_output;
    
    float **w_input_hidden;
    float **w_hidden_output;
} NeuralNet;

typedef struct {
    int *result;
    int *data;
} Pattern;


// Calculate output for one layer
__device__ void calculateOutputForOneLayer(float *output, float *input_layer, float *weight_layer, int n_inputs_in_layer, int n_outputs_in_layer) {
   
}


void buildLayer(float *arr, int n, float initial) {
    int i=0;
    while(i < n){
        *arr = initial;
        arr++;
        i++;
    }
}

float** buildWeightsLayer(int outer_n, int inner_n, float seed) {
    int i;
    int total = outer_n * inner_n;
    float *data = (float*) malloc(sizeof(float) * total);
    for(i=0; i < total; i++) {
	if (seed == -1) {
	  data[i] = (float)rand()/(float)RAND_MAX;
	} else {
	  data[i] = seed;
	}
        
    }
    
    float **ptr_arr = (float **)malloc(sizeof(float*) * outer_n);
    for(i=0; i < outer_n; i++) {
        ptr_arr[i] = data + (i* inner_n);
    }
    return ptr_arr;
}

NeuralNet buildNeuralNet(int n_inputs, int n_outputs, int n_hidden) {

    float *out_input = (float *)malloc(sizeof(float) * (n_inputs + 1));
    float *out_hidden = (float *)malloc(sizeof(float) * n_hidden);
    float *out_output = (float *)malloc(sizeof(float) * n_outputs);

    buildLayer(out_input, n_inputs + 1, 1.0f);
    buildLayer(out_hidden, n_hidden, 1.0f);
    buildLayer(out_output, n_outputs, 1.0f);
    
    // Build changes layer
    float **changes_input_hidden = buildWeightsLayer(n_inputs + 1, n_hidden, 0.0f);
    float **changes_hidden_output = buildWeightsLayer(n_hidden, n_outputs, 0.0f);
    
    // Build weight matrix
    float **w_input_hidden = buildWeightsLayer(n_inputs + 1, n_hidden, -1.0f);
    float **w_hidden_output = buildWeightsLayer(n_hidden, n_outputs, -1.0f);

    NeuralNet nn;

    nn.n_inputs = n_inputs + 1;
    nn.n_outputs = n_outputs;
    nn.n_hidden = n_hidden;

    nn.out_input = out_input;
    nn.out_hidden = out_hidden;
    nn.out_output = out_output;

    nn.changes_input_hidden = changes_input_hidden;
    nn.changes_hidden_output = changes_hidden_output;

    nn.w_input_hidden = w_input_hidden;
    nn.w_hidden_output = w_hidden_output;

    return nn;
}

float *update_pattern(Pattern pattern, NeuralNet nn) {
    // Write inputs
    int i, j;
    float weighted_sum;
    for(i=0; i < nn.n_inputs -1; i++) {
        nn.out_input[i] = pattern.data[i];
    }
    
    // Write hidden
    for(i=0; i < nn.n_hidden; i++) {
      weighted_sum = 0.0f;
      for(j=0; j < nn.n_inputs; j++) {
          weighted_sum += nn.out_input[j] * nn.w_input_hidden[j][i];
      }
      nn.out_hidden[i] = tanh(weighted_sum);
      //printf("Hidden is : %f\n", nn.out_hidden[i]);
    }

    // Write output
    for(i=0; i < nn.n_outputs; i++) {
      weighted_sum = 0.0f;
      for(j=0; j < nn.n_hidden; j++) {
          weighted_sum += nn.out_hidden[j] * nn.w_hidden_output[j][i];
      }
      nn.out_output[i] = tanh(weighted_sum);
      //printf("Output is : %f\n", nn.out_output[i]);
    }

    return nn.out_output;
}

float dsigmoid(float y) {
    return 1.0 - pow(y,2.0f);
}

float back_propagate_network(Pattern p, NeuralNet n) {
    // Calculate deltas
    int i, j;
    float *output_delta =(float *) malloc(sizeof(float) * n.n_outputs);
	float *hidden_delta = (float *) malloc(sizeof(float) * n.n_hidden);
    
    // Calculate output delta
    for (i=0; i < n.n_outputs; i++) {
        float error = p.result[i] - n.out_output[i];
        output_delta[i] = dsigmoid(n.out_output[i]) * error;
    }
    
    
    // Calculate hidden delta
    for(i=0; i < n.n_hidden; i++) {
        float error = 0.0f;
        for (j=0; j < n.n_outputs; j++) {
            error += output_delta[j] * n.w_hidden_output[i][j];
        }
        hidden_delta[i] = dsigmoid(n.out_hidden[i]) * error;
    }
    
    // Set hidden-output weights
    for(i=0; i < n.n_hidden; i++) {
      for (j=0; j < n.n_outputs; j++) {
        float change = output_delta[j] * n.out_hidden[i];
        n.w_hidden_output[i][j] += 0.5 * change + 0.5 * n.changes_hidden_output[i][j];
        n.changes_hidden_output[i][j] = change;
      }
    }
    
    // Set input-hidden weights
    for(i=0; i < n.n_inputs; i++) {
      for(j=0; j < n.n_hidden; j++) {
        float change = hidden_delta[j] * n.out_input[i];
        n.w_input_hidden[i][j] += 0.5 * change + 0.5 * n.changes_input_hidden[i][j];
        n.changes_input_hidden[i][j] = change;
      }
    }
    
    // Calculate error
    float error = 0.0f;
    for (i=0; i < n.n_outputs; i++) {
    	error = error + 0.5f * pow(p.result[i] - n.out_output[i], 2);
    }
    return error;
}

void train_network(Pattern *patterns, int n_patterns, int n_iterations, NeuralNet nn) {
  int i, j;
  for (i=0; i < n_iterations; i++) {
    float error = 0;
    for (j=0; j < n_patterns; j++) {
       update_pattern(patterns[j], nn);
       error += back_propagate_network(patterns[j], nn);
    }
    if (i % 100 == 0) {
       printf("Error is: %-.5f\n", error);
    }
  }
}

Pattern makePatternSingleOutput(int *data, int result) {
    Pattern p;
    p.data = data;

    p.result = (int *)malloc(sizeof(int));
    p.result[0] = result;

    return p;
}


int main() {
    srand((unsigned)time(NULL));
    
    int n_inputs = 2;
    int n_hidden = 4;
    int n_outputs = 1;
    
    // Build output layer
    NeuralNet nn = buildNeuralNet(n_inputs, n_outputs, n_hidden);
    
    // Build training samples
    int _p1[] = {0,0};
    Pattern p1 = makePatternSingleOutput(_p1, 0);
    int _p2[] = {0,1};
    Pattern p2 = makePatternSingleOutput(_p2, 1);
    int _p3[] = {1,1};
    Pattern p3 = makePatternSingleOutput(_p3, 1);
    int _p4[] = {1,0};
    Pattern p4 = makePatternSingleOutput(_p4, 1);
    
    Pattern patterns[] = {p1, p2, p3, p4};
    
    // Train the network
    train_network(patterns, 4, 10000, nn);

    printf("\n\nTesting the network\n");
    update_pattern(p1, nn);
    for (int i=0; i < nn.n_outputs; i++) {
        printf("Output: %f, expected: %i\n", nn.out_output[i], p1.result[i]);
    }
    
}