Autoencoder test with MNIST 10k dataset traning digits

gistfile1.txt

//Add visualize hidden nodes
//Add so you can save weight by press <S>
//Autoencoder learning test with raspicam
//press <Y> at start It's tested with dataset from
/// t10k-images-idx3-ubyte
/// http://yann.lecun.com/exdb/mnist/

//If you comment out USE_MNIST_DATABASE switch then simpler pattern used from a picture https://blog.webkid.io/datasets-for-machine-learning/
//0..9 28x28 pixel digits pattern mnist.png filname 289x289 orginal
//orginal image taken from
//       https://blog.webkid.io/datasets-for-machine-learning/

//#define USE_RASPICAM_INPUT //If you want to use raspicam input data

#define USE_MNIST_DATABASE// Here read the t10k-images-idx3-ubyte file
/// Input data from
/// t10k-images-idx3-ubyte
/// http://yann.lecun.com/exdb/mnist/
const int MNIST_pix_size = 28*28;
char data_10k_MNIST[10000][MNIST_pix_size];
const int MNIST_header_offset = 16;
 /*
TRAINING SET IMAGE FILE (train-images-idx3-ubyte):

[offset] [type]          [value]          [description]
0000     32 bit integer  0x00000803(2051) magic number
0004     32 bit integer  60000            number of images
0008     32 bit integer  28               number of rows
0012     32 bit integer  28               number of columns
0016     unsigned byte   ??               pixel
0017     unsigned byte   ??               pixel
........
xxxx     unsigned byte   ??               pixel
*/

#define USE_RELU //other wise use sigmoid

const float Relu_neg_gain = 0.1f;//0.0f pure rectify 1.0 full linear

#include <opencv2/highgui/highgui.hpp>  // OpenCV window I/O
#include <opencv2/imgproc/imgproc.hpp> // Gaussian Blur
#include <stdio.h>
#include <raspicam/raspicam_cv.h>
#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp>        // Basic OpenCV structures (cv::Mat, Scalar)
#include <cstdlib>
#include <ctime>
#include <math.h>  // exp
#include <stdlib.h>// exit(0);
//#include <iostream>
    char filename[100];
    char filename2[100];
    char filename_dst[100];

using namespace std;
using namespace cv;
//const float LearningRate =0.05;
//const float Momentum = 0.025;
#ifdef USE_RELU
const float LearningRate =0.0015;
const float Momentum = 0.001;
float noise_percent =50.0f;//40
const int Const_hidd_node = 40;
//float noise_amplitude = 0.0f;//0..1,0
float noise_amplitude = 1.0f;
//float noise_offset = -0.5f;
float noise_offset = 0.0f;//-0.5..+0.5
#else
const float LearningRate =0.02;
const float Momentum = 0.00;
float noise_percent =40.0f;
const int Const_hidd_node = 80;
   float noise_amplitude = 1.0f;
   //float noise_offset = -0.5f;
    float noise_offset = 0.0f;

#endif // USE_RELU

//  float noise_percent =50.0f;
//#define USE_IND_NOISE
    float start_weight_noise_range = 0.025f;//+/- Weight startnoise range
    float noise = 0.0f;//Noise added on input stimulu

#ifdef USE_MNIST_DATABASE
    const int Const_nr_pic = 10000;
#else
    const int Const_nr_pic = 100;//Use only the 289x289 input pattern or raspicam input
#endif // USE_MNIST_DATABASE


void sigmoid_mat(Mat image)
{
	float* ptr_src_index;
	ptr_src_index = image.ptr<float>(0);
    int nRows = image.rows;
    int nCols = image.cols;
    for(int i=0;i<nRows;i++)
    {
		for(int j=0;j<nCols;j++)
		{
			*ptr_src_index = 1.0/(1.0 + exp(-(*ptr_src_index)));//Sigmoid function
			ptr_src_index++;
		}
	}
}

void relu_mat(Mat image)
{
	float* ptr_src_index;
	ptr_src_index = image.ptr<float>(0);
    int nRows = image.rows;
    int nCols = image.cols;
    for(int i=0;i<nRows;i++)
    {
		for(int j=0;j<nCols;j++)
		{
		    if(*ptr_src_index < 0.0)
            {
                *ptr_src_index = *ptr_src_index * Relu_neg_gain;
            }
			ptr_src_index++;
		}
	}
}



#include <termios.h>
#include <unistd.h>
#include <fcntl.h>

int kbhit(void)
{
  struct termios oldt, newt;
  int ch;
  int oldf;

  tcgetattr(STDIN_FILENO, &oldt);
  newt = oldt;
  newt.c_lflag &= ~(ICANON | ECHO);
  tcsetattr(STDIN_FILENO, TCSANOW, &newt);
  oldf = fcntl(STDIN_FILENO, F_GETFL, 0);
  fcntl(STDIN_FILENO, F_SETFL, oldf | O_NONBLOCK);

  ch = getchar();

  tcsetattr(STDIN_FILENO, TCSANOW, &oldt);
  fcntl(STDIN_FILENO, F_SETFL, oldf);

  if(ch != EOF)
  {
    ungetc(ch, stdin);
    return 1;
  }

  return 0;
}

void local_normalizing(Mat gray)
{

	#define BLUR_FLT_NUMERATOR 2
	#define BLUR_FLT_DENOMINATOR 20

    Mat float_gray, blur, num, den, store_gray;
	store_gray = gray;//Initialize size
    // convert to floating-point image
    gray.convertTo(float_gray, CV_32F, 1.0/255.0);

    // numerator = img - gauss_blur(img)
    cv::GaussianBlur(float_gray, blur, Size(0,0), BLUR_FLT_NUMERATOR, BLUR_FLT_NUMERATOR);
    num = float_gray - blur;

    // denominator = sqrt(gauss_blur(img^2))
    cv::GaussianBlur(num.mul(num), blur, Size(0,0), BLUR_FLT_DENOMINATOR, BLUR_FLT_DENOMINATOR);
    cv::pow(blur, 0.5, den);

    // output = numerator / denominator
    gray = num / den;

    // normalize output into [0,1]
    cv::normalize(gray, gray, 0.0, 1.0, NORM_MINMAX, -1);

    // Display
    //namedWindow("demo", CV_WINDOW_AUTOSIZE );
    gray.convertTo(store_gray, CV_8U, 255);
    //imshow("demo", gray);

}
int get_MNIST_file_size(void)
{
    int file_size=0;
    FILE *fp2;
    fp2 = fopen("t10k-images-idx3-ubyte", "r");
    if (fp2 == NULL)
    {
        puts("Error while opening file t10k-images-idx3-ubyte");
        exit(0);
    }

    fseek(fp2, 0L, SEEK_END);
    file_size = ftell(fp2);
    printf("file_size %d\n", file_size);
    rewind(fp2);
    fclose(fp2);
    return file_size;
}

//attach_weight_2_mat(ptr_M_matrix, i, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);
void attach_weight_2_mat(float* ptr_M_matrix, int i, Mat src, int sqr_of_H_nod_plus1, int  Hidden_nodes, int Height, int Width)
{
    float *start_corner_offset = src.ptr<float>(0);
    int start_offset=0;
    float *src_zero_ptr = src.ptr<float>(0);
    float *src_ptr = src.ptr<float>(0);
    for(int j=0; j<Hidden_nodes ; j++)
    {

        start_offset = (j/sqr_of_H_nod_plus1)*Height*src.cols + (j%sqr_of_H_nod_plus1)*Width;
        start_corner_offset = start_offset + src_zero_ptr;
        src_ptr = start_corner_offset + (i/Width)*src.cols + (i%Width);
        *src_ptr = *ptr_M_matrix;
        ptr_M_matrix++;
    }
}
int main()
{
    FILE *fp2;
    int MNIST_file_size=0;
#ifdef USE_MNIST_DATABASE
    MNIST_file_size = get_MNIST_file_size();
//read_10k_MNIST();
    char *MNIST_data;
    MNIST_data = new char[MNIST_file_size];
   FILE *fp;
   char c_data=0;
   fp = fopen("t10k-images-idx3-ubyte","r");

   if(fp == NULL)
   {
      perror("Error in opening t10k-images-idx3-ubyte file");
      return(-1);
   }
   int MN_index=0;
   for(int i=0;i<MNIST_file_size;i++)
   {

      c_data = fgetc(fp);
      if( feof(fp) )
      {
         break;
      }
      //printf("c_data %d\n", c_data);
      MNIST_data[MN_index] = c_data;
      if((MNIST_header_offset-1)<i)
      {
          MN_index++;
      }
   }
fclose(fp);

#endif // USE_MNIST_DATABASE
    float Rando=0.0f;
 //   float start_weight_noise_range = 0.25f;//+/- start weight noise range

    printf("Auto encoder test program\n");
    //int get_data =0;
    int Height=0;
    int Width=0;
    int nr_of_pixels=0;
    int Hidden_nodes=0;
    int nr_of_pictures=0;
    int training_image=0;
    char answer_character;
    printf("Do you want default settings <Y>/<N> ? \n");
	answer_character = getchar();
	if(answer_character == 'Y' || answer_character == 'y')
	{
      //  Height = 48;
      //  Width = 64;
        Height = 28;
        Width = 28;
        Hidden_nodes = Const_hidd_node;
        nr_of_pictures = Const_nr_pic;
	}
	else
    {
    printf("Enter number of pixel Height of input pictures\n");
    scanf("%d", &Height);
    printf("Enter number of pixel Width of input pictures\n");
    scanf("%d", &Width);
    printf("Enter number of hidden nodes of input pictures\n");
    scanf("%d", &Hidden_nodes);
    printf("Enter number of hidden training pictures posXXX.JPG \n");
    scanf("%d", &nr_of_pictures);

    }
    nr_of_pixels = Height * Width;
    printf("Number of pixels = %d\n", nr_of_pixels);

///*************************************************************************************************
///************* Make a visual Mat to show the hidden nodes as a image representation **************
///*************************************************************************************************

    int sqr_r_hidd_nodes=0;
    sqr_r_hidd_nodes = sqrt(Hidden_nodes);
    int test_sqr_r_n=0;
    test_sqr_r_n = sqr_r_hidd_nodes * sqr_r_hidd_nodes;
    while(test_sqr_r_n < Hidden_nodes)
    {
        sqr_r_hidd_nodes++;
        test_sqr_r_n = sqr_r_hidd_nodes * sqr_r_hidd_nodes;
    }
    printf("sqr_r_hidd_nodes %d\n", sqr_r_hidd_nodes);
    Mat vis_hidd_node;
    vis_hidd_node.create(sqr_r_hidd_nodes, sqr_r_hidd_nodes, CV_32F);
    float *zero_ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);
    float *ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);

    Mat colour_vis_h;
    colour_vis_h.create(sqr_r_hidd_nodes, sqr_r_hidd_nodes, CV_8UC3);
    CvPoint P1;
///*************************************************************************************************
///*************************************************************************************************

	Mat src, image, mnist,m_gray;
///************** Raspicam **************
#ifdef USE_RASPICAM_INPUT
	raspicam::RaspiCam_Cv Camera;
	Camera.set( CV_CAP_PROP_FRAME_WIDTH, Width);
	Camera.set( CV_CAP_PROP_FRAME_HEIGHT, Height);
	Camera.set( CV_CAP_PROP_FORMAT, CV_8UC1 );
//Camera.set( CV_CAP_PROP_FORMAT, CV_8U );
	//Open camera
	cout<<"Opening Camera..."<<endl;
	if (!Camera.open()) {cerr<<"Error opening the camera"<<endl;return -1;}

	Camera.grab();
	Camera.retrieve (src);
	waitKey(1);
	src.convertTo(image, CV_32F, 1.0/255.0);//Convert pixels from 0..255 char to float 0..1

#endif // USE_RASPICAM_INPUT
///***************************************

#ifndef USE_RASPICAM_INPUT
///************Open mnist image ********
#ifndef USE_MNIST_DATABASE
			sprintf(filename2, "mnist.png");//Assigne a filename "pos" with index number added

			mnist = imread( filename2, 1 );
			if ( !mnist.data )
			{
				printf("\n");
				printf("==================================================\n");
				printf("No image data Error! Probably not find mnist.png \n");
				printf("==================================================\n");
				printf("\n");
				//return -1;
			}


      cvtColor(mnist,m_gray,CV_BGR2GRAY);
      imshow("m_gray", m_gray);
        waitKey(1);

        int mnist_h1 = 28;
        int mnist_w1 = 28;
        int mnh=0;
        int mnw=0;
        int pos = 0;
        int ret;
   //     Mat m1;
//#ifndef USE_MNIST_DATABASE
for(int j=0;j<10;j++)
{
    for(int i=0;i<10;i++)
    {
//    Mat m1(image, Rect(0, 0, T0_FRAME_WIDTH, T0_FRAME_HEIGHT));// Rect(<start_x>, <start_y>, <width>, <hight>)
//48 x 45
        mnh = j*mnist_h1+j*1;
        mnw = i*mnist_w1+i*1;
        Mat m1(m_gray, Rect(mnw, mnh, mnist_w1, mnist_h1));// Rect(<start_x>, <start_y>, <width>, <hight>)
        cv::imwrite("temporary_file.JPG",m1);

            sprintf(filename_dst, "pos%d.JPG", pos);
            ret = rename("temporary_file.JPG", filename_dst);
            if(ret == 0)
            {
                printf("File renamed successfully");
            }
            else
            {
                printf("Error: unable to rename the file");
            }
        pos++;

    }
}
#endif

///*************************************

#endif // USE_RASPICAM_INPUT

   // mat_H =
    float *input_node;
    input_node = new float[nr_of_pixels];
    float *hidden_node;
    hidden_node = new float[Hidden_nodes];
    float *output_node;
    output_node = new float[nr_of_pixels];
    float **weight_matrix_M;//Pointer fo a dynamic array. This weight_matrix_M will then have a size of rows = nr_of_pixels, colums = Hidden_nodes.
    float **change_weight_M;//Pointer fo a dynamic array. This weight_matrix_M will then have a size of rows = nr_of_pixels, colums = Hidden_nodes.
    weight_matrix_M = new float *[nr_of_pixels];
    change_weight_M = new float *[nr_of_pixels];
    for(int i=0; i < nr_of_pixels; i++)
    {
        weight_matrix_M[i] = new float[Hidden_nodes];
        change_weight_M[i] = new float[Hidden_nodes];
    }
       float *f_data;
       f_data = new float[nr_of_pixels*Hidden_nodes];
        int ix=0;
        printf("size of weight_matrix_M[2][0] =%d\n", sizeof weight_matrix_M[2][0]);

    int sqr_of_H_nod_plus1=0;
    sqr_of_H_nod_plus1 = sqrt(Hidden_nodes);
    sqr_of_H_nod_plus1 += 1;
    printf("sqr_of_H_nod_plus1 %d\n", sqr_of_H_nod_plus1);
    float* ptr_M_matrix;
    Mat visual_all_feature, outimg;
    visual_all_feature.create(Height * sqr_of_H_nod_plus1, Width * sqr_of_H_nod_plus1,CV_32F);
    outimg.create(Height, Width, CV_32F);

    srand (static_cast <unsigned> (time(0)));//Seed the randomizer
for(int i=0;i<nr_of_pixels;i++)
{
    ptr_M_matrix = &weight_matrix_M[i][0];
    for(int j=0;j<(Hidden_nodes);j++)
    {
        Rando = (float) (rand() % 65535) / 65536;//0..1.0 range
        Rando -= 0.5f;
        Rando *= start_weight_noise_range;
        *ptr_M_matrix = Rando;
        ptr_M_matrix++;
        change_weight_M[i][j] = 0.0f;
    }
}
		printf("Would you like to load stored weight_matrix_M.dat <Y>/<N> \n");
		getchar();
        answer_character = getchar();
		if(answer_character == 'Y' || answer_character == 'y')
		{
			sprintf(filename, "weight_matrix_M.dat");
			fp2 = fopen(filename, "r");
			if (fp2 == NULL)
			{
				printf("Error while opening file weight_matrix_M.dat");
				exit(0);
			}
        fread(f_data, sizeof f_data[0], (nr_of_pixels*Hidden_nodes), fp2);
        ix=0;
 		for(int n=0;n<Hidden_nodes;n++)
        {
            for(int p=0;p<nr_of_pixels;p++)
            {
                weight_matrix_M[p][n] = f_data[ix];
                ix++;
            }
        }

			fclose(fp2);
		}
    for(int i=0;i<nr_of_pixels;i++)
    {
        ptr_M_matrix = &weight_matrix_M[i][0];
        attach_weight_2_mat(ptr_M_matrix, i, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);
    }
 //   local_normalizing(visual_all_feature);

//test on


        visual_all_feature += 0.5f;
    Mat color_img, gray, noised_input;
     srand (static_cast <unsigned> (time(0)));//Seed the randomizer


    noised_input.create(Height, Width, CV_32F);
   float error=0.0;
   float *zero_ptr_gray = gray.ptr<float>(0);
   float *ptr_gray = gray.ptr<float>(0);
   float *ptr_noised_inp = noised_input.ptr<float>(0);
   float *zero_ptr_noised_inp = noised_input.ptr<float>(0);
   ptr_gray = zero_ptr_gray + (gray.cols * (gray.rows/2)) + gray.cols/2;
   float test_pixel_value = *ptr_gray;
   int *noise_pixels;
   noise_pixels = new int[nr_of_pixels];
   int noise_p_counter=0;

   float noise_ratio=0.0f;
   int nr_of_noise_rand_ittr=0;
   int rand_pix_pos=0;
    char keyboard;
    int started = 0;
#ifdef USE_MNIST_DATABASE
 gray.create(28,28,CV_32F);
#endif // USE_MNIST_DATABASE
    while(1)
    {
#ifdef USE_RASPICAM_INPUT
   	Camera.grab();
	Camera.retrieve (src);
	waitKey(1);
	src.convertTo(image, CV_32F, 1.0/255.0);//Convert pixels from 0..255 char to float 0..1
    imshow("image", image);
    gray = image;//Connect raspberry camera instead of posXXX.jpg image

#else

#ifndef USE_MNIST_DATABASE
        training_image = rand() % nr_of_pictures;
     //   printf("training_image %d\n", training_image);
			sprintf(filename, "pos%d.JPG", training_image);//Assigne a filename "pos" with index number added
			color_img = imread( filename, 1 );
			if ( !color_img.data )
			{
				printf("\n");
				printf("==================================================\n");
				printf("No image data Error! Probably not find pos%d.JPG \n", training_image);
				printf("==================================================\n");
				printf("\n");
				//return -1;
			}
      cvtColor(color_img,gray,CV_BGR2GRAY);

#else
///*********************************************************************
///read data from ************* t10k-images-idx3-ubyte ************ file
///*********************************************************************
    training_image = rand() % nr_of_pictures;
/// read from data
      zero_ptr_gray = gray.ptr<float>(0);
      for(int n=0;n<nr_of_pixels;n++)
      {
          ptr_gray = zero_ptr_gray + n;
          *ptr_gray = MNIST_data[(nr_of_pixels*training_image) +n];
      }

    color_img.create(28,28,CV_32F);

#endif // USE_MNIST_DATABASE

      imshow("color_img", color_img);


      gray.convertTo(gray, CV_32F, 1.0/255.0);//Convert pixels from 0..255 char to float 0..1

//  local_normalizing(gray);

#endif // USE_RASPICAM_INPUT
#ifdef USE_RELU

     relu_mat(gray);
 //    gray *= 0.5f;
 //    gray += 0.5f;

#else
      sigmoid_mat(gray);
#endif // USE_RELU
      zero_ptr_gray = gray.ptr<float>(0);
      ptr_gray = gray.ptr<float>(0);

///******************************************************
///************ Select noise pixels positions ***********
///******************************************************
    noise_p_counter=0;
    noise_ratio=0.0f;
    nr_of_noise_rand_ittr=0;
    rand_pix_pos=0;

    for(int n=0;n<nr_of_pixels;n++)
    {
        noise_pixels[n] = 0;
    }

    while(noise_ratio < (noise_percent*0.01f))
    {
        rand_pix_pos = (int) (rand() % nr_of_pixels);
            if(noise_pixels[rand_pix_pos] == 0)
            {
                noise_p_counter++;
            }
            noise_pixels[rand_pix_pos] = 1;
        noise_ratio = ((float)noise_p_counter) / ((float)nr_of_pixels);
        nr_of_noise_rand_ittr++;
        if(nr_of_noise_rand_ittr > 2*nr_of_pixels)
        {
            printf("give up fill random up noise this turn\n");
            printf("noise_ratio %f\n", noise_ratio);
            break;
        }
    }
#ifndef USE_IND_NOISE
///****************************************************
///********** Select noise level ******************
///****************************************************
noise = (float) (rand() % 65535) / 65536;//0..1.0 range
noise -= 0.5f;
noise = noise * noise_amplitude;
noise += 0.5f;
noise += noise_offset;
#endif // USE_IND_NOISE


///****************************************************
///**************** Feed forward **********************
///****************************************************
    ///Connect image pixel and also insert noise to input_node[]
    for(int n=0;n<nr_of_pixels;n++)
    {
        if(noise_pixels[n] == 1 && started == 1)
        {
#ifdef USE_IND_NOISE
///****************************************************
///********** Select noise level ******************
///****************************************************
noise = (float) (rand() % 65535) / 65536;//0..1.0 range
noise -= 0.5f;
noise = noise * noise_amplitude;
noise += 0.5f;
noise += noise_offset;
#endif // USE_IND_NOISE
            input_node[n] = noise;///Insert noise 0.5f instead of real pixel value
        }
        else
        {
            ptr_gray = zero_ptr_gray + n;
            input_node[n] = *ptr_gray;///Insert pixel value
        }
        ptr_noised_inp = zero_ptr_noised_inp + n;
        *ptr_noised_inp = input_node[n];//To show
    }
    imshow("noised_input ", noised_input);
    ///*********** Forward to hidden nodes ****************
    for(int j=0;j<Hidden_nodes;j++)
    {
        hidden_node[j] = 0;
        for(int i=0;i<nr_of_pixels;i++)
        {
            hidden_node[j] += input_node[i] * weight_matrix_M[i][j];///Sum up values from input gained with weight
        }
#ifdef USE_RELU
        ///*** Relu this node ***
        if(hidden_node[j] < 0.0f)
        {
            hidden_node[j] = hidden_node[j] * Relu_neg_gain;///Relu function
        }
#else
        ///*** Sigmoid this node ***
        hidden_node[j] = 1.0/(1.0 + exp(-(hidden_node[j])));///Sigmoid function
#endif // USE_RELU


    }
    ///********** Forward to output nodes *******************
    for(int i=0;i<nr_of_pixels;i++)
    {
        output_node[i] = 0.0f;
        for(int j=0;j<Hidden_nodes;j++)
        {
            output_node[i] += hidden_node[j] * weight_matrix_M[i][j];///Sum up values from hidden gained with weight
        }
#ifdef USE_RELU
        ///*** Relu this node ***
        if(output_node[i] < 0.0f)
        {
            output_node[i] = output_node[i] * Relu_neg_gain;///Relu function
        }
#else
        ///*** Sigmoid this node ***
        output_node[i] = 1.0/(1.0 + exp(-(output_node[i])));///Sigmoid function
#endif // USE_RELU


    }

    float input_pixel =0.0f;
///****************************************************
///************* Calculate total loss *****************
///****************************************************
    /// k = nr_of_pixel
    /// loss = -SUM k (output[k] * log(input[k]) + (1.0 - output[k]) * log(1.0 - input[k]))
    /// or
    /// loss = 1/2 SUM k (input[k] - ouput[k])²
    float loss=0.0f;
    loss=0.0f;
    for(int i=0;i<nr_of_pixels;i++)
    {
        ptr_gray = zero_ptr_gray + i;///pick real input pixel
        input_pixel = *ptr_gray;///Insert pixel value
        loss += (input_pixel - output_node[i]) * (input_pixel - output_node[i]);/// loss = 1/2 SUM k (input[k] - ouput[k])²
    }
    static int print_loss=0;
    if(print_loss<10)
    {
        print_loss++;
    }
    else
    {
        printf("loss error = %f\n", loss);
       print_loss=0;
    }

///****************************************************
///**************** Backpropagation *******************
///****************************************************

    /// **** make Delta value for backpropagation and loss calculation ****
    float delta_pixel=0.0f;
    for(int i=0;i<nr_of_pixels;i++)
    {
        ptr_gray = zero_ptr_gray + i;///pick real input pixel
        input_pixel = *ptr_gray;///Insert pixel value
        delta_pixel = input_pixel - output_node[i] ;/// detal calculus

        /// **** update tied weight regarding delta
        for(int j=0;j<Hidden_nodes;j++)
        {
            if(started == 1)
            {
                change_weight_M[i][j] = LearningRate * hidden_node[j] * delta_pixel + Momentum * change_weight_M[i][j];
                weight_matrix_M[i][j] += change_weight_M[i][j];

            }
        }
    }

      float *zero_ptr_outimg = outimg.ptr<float>(0);
      float *ptr_outimg = outimg.ptr<float>(0);

      for(int i=0;i<nr_of_pixels;i++)
      {
          ptr_outimg = zero_ptr_outimg + i;
          *ptr_outimg = output_node[i];
      }

 //     gray -= 0.5;
      outimg -= 0.5;
 //     visual_all_feature -= 0.5;
 //     gray *= 2;
      outimg *= 2;
 //     visual_all_feature *= 2;
 //     gray += 0.5;
      outimg += 0.5;
 //     visual_all_feature += 0.5;


    for(int i=0;i<nr_of_pixels;i++)
    {
        ptr_M_matrix = &weight_matrix_M[i][0];
        attach_weight_2_mat(ptr_M_matrix, i, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);
    }


        visual_all_feature += 0.5f;

      imshow("visual_all_feature", visual_all_feature);

      imshow("gray", gray);

        imshow("outimg", outimg);

///******** start stop
		if(kbhit()){
            keyboard = getchar();
            if(keyboard== ' ')
            {
				if(started == 1)
				{
					started = 0;
					printf("Stop training\n");
				}
				else
				{
					started = 1;
					printf("Start training\n");
				}

			}
            if(keyboard== 'S' || keyboard== 's')
            {
                //Save weights
		sprintf(filename, "weight_matrix_M.dat");//Assigne a filename with index number added
		fp2 = fopen(filename, "w+");
		//size_t fread(void *ptr, size_t size_of_elements, size_t number_of_elements, FILE *a_file);

        ix=0;
 		for(int n=0;n<Hidden_nodes;n++)
        {
            for(int p=0;p<nr_of_pixels;p++)
            {
                f_data[ix] = weight_matrix_M[p][n];
                ix++;
            }
        }
        fwrite(f_data, sizeof f_data[0], (nr_of_pixels*Hidden_nodes), fp2);



		fclose(fp2);
		printf("weights are saved at weight_matrix_M.dat file\n");

            }

        }

///***************
    if(started == 0)
    {
        waitKey(2000);
        printf("Training stop now only feed forward\n");
        printf("Hit <Space> to start / stop traning \n");
        printf("Hit <S> to save all weights to weight_matrix_M.dat file\n");

    }
///*********** Visualize hidden nodes on a image **********
    cvtColor(visual_all_feature,colour_vis_h,CV_GRAY2RGB);

    zero_ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);
    ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);
    int index_hid_nodes=0;
    int pix_n=0;
    index_hid_nodes=0;
    pix_n=0;
    for(int ro=0;ro<vis_hidd_node.rows;ro++)
    {
        for(int co=0;co<vis_hidd_node.rows;co++)
        {
            if(index_hid_nodes<Hidden_nodes)
            {

                ptr_vis_hidd_node = zero_ptr_vis_hidd_node + pix_n;
                *ptr_vis_hidd_node = hidden_node[index_hid_nodes];

                ///************* visualize on vis_hidd_node pattern **********
                ///************* with green light ****************************
                int circle_zize = 4;
                int x,y;
                float light_level=0.0f;
                float light_red=0.0f;
                light_level = *ptr_vis_hidd_node;
                if(light_level > 1.0f)
                {
                    light_level=1.0f;
                }
                if(light_level < 0.0f)
                {
                    light_level=0.0f;
                }
                light_red = 1.0f - light_level;
                x = (index_hid_nodes%sqr_of_H_nod_plus1)*Width + (Width/2);
                y = (index_hid_nodes/sqr_of_H_nod_plus1)*Height + (Height/2);
                P1.x = x;
                P1.y = y;
                circle(colour_vis_h, P1, circle_zize, Scalar(0,light_level,light_red), 2, -1);
                index_hid_nodes++;
                ///***********************************************************

            }
            else
            {
                *ptr_vis_hidd_node = 0.0f;
            }
            pix_n++;
        }
    }
    imshow("vis_hidd_node", vis_hidd_node);
    imshow("colour_vis_h", colour_vis_h);

///********************************************************
    waitKey(1);
    }
    return 0;
}