Autoencoder train 10x10 patches from realtime video raspicam on Raspberry pi

gistfile1.txt

///Now USE_IND_NOISE switch ON make more realistoc gabor filter like feature 

           /// Fix Bugg replace hidden_node[j] and output_node[i] with Bias_level
           //   change_weight_in2hid[j] = (LearningRate/2) * hidden_node[j] * hid_node_delta[j] + Momentum * change_weight_in2hid[j];
           //   change_weight_hid2out[i] = (LearningRate/2) * output_node[i] * delta_pixel + Momentum * change_weight_hid2out[i];
///#define USE_LIM_BIAS//

//Here use real image from raspicam take random part 10x10 pixel and put in to the autoencoder

const int cam_h = 240;
const int cam_w = 320;

//const int cam_h = 480;
//const int cam_w = 640;
//const int cam_h = 960;
//const int cam_w = 1280;
//const int cam_h = 120;
//const int cam_w = 160;


///Now also Add bias nodes showed in the last 2 patches
const float Bias_level = 1.0f;
const float Bias_w_n_range = -0.5f;
const float Bias_w_p_range = 0.5f;
const float change_bias_weight_range = 0.2f;


//Add visualize hidden nodes
//Add so you can save weight by press <S>
//Autoencoder learning test with raspicam
//press <Y> at start It's tested with dataset from
/// t10k-images-idx3-ubyte
/// http://yann.lecun.com/exdb/mnist/

//If you comment out USE_MNIST_DATABASE switch then simpler pattern used from a picture https://blog.webkid.io/datasets-for-machine-learning/
//0..9 28x28 pixel digits pattern mnist.png filname 289x289 orginal
//orginal image taken from
//       https://blog.webkid.io/datasets-for-machine-learning/

#define USE_RASPICAM_INPUT //If you want to use raspicam input data

//#define USE_MNIST_DATABASE// Here read the t10k-images-idx3-ubyte file
/// Input data from
/// t10k-images-idx3-ubyte
/// http://yann.lecun.com/exdb/mnist/
const int MNIST_pix_size = 28*28;
char data_10k_MNIST[10000][MNIST_pix_size];
const int MNIST_header_offset = 16;
/*
TRAINING SET IMAGE FILE (train-images-idx3-ubyte):

[offset] [type]          [value]          [description]
0000     32 bit integer  0x00000803(2051) magic number
0004     32 bit integer  60000            number of images
0008     32 bit integer  28               number of rows
0012     32 bit integer  28               number of columns
0016     unsigned byte   ??               pixel
0017     unsigned byte   ??               pixel
........
xxxx     unsigned byte   ??               pixel
*/

#define USE_RELU //other wise use sigmoid
#define USE_DELTA_RELU_REDUCED_NEG

const float Relu_neg_gain = 0.1f;//0.0f pure rectify 1.0 full linear

#include <opencv2/highgui/highgui.hpp>  // OpenCV window I/O
#include <opencv2/imgproc/imgproc.hpp> // Gaussian Blur
#include <stdio.h>
#include <raspicam/raspicam_cv.h>
#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp>        // Basic OpenCV structures (cv::Mat, Scalar)
#include <cstdlib>
#include <ctime>
#include <math.h>  // exp
#include <stdlib.h>// exit(0);
//#include <iostream>
char filename[100];
char filename2[100];
char filename_dst[100];

using namespace std;
using namespace cv;
//const float LearningRate =0.05;
//const float Momentum = 0.025;
#ifdef USE_RELU
const float LearningRate =0.0015;
const float Momentum = 0.001;

#ifdef USE_RASPICAM_INPUT
float noise_percent =30.0f;//25
const int Const_hidd_node = 20;
#else
float noise_percent =50.0f;//50
const int Const_hidd_node = 40;
#endif // USE_RASPICAM_INPUT
//float noise_amplitude = 0.0f;//0..1,0
float noise_amplitude = 1.0f;
//float noise_offset = -0.5f;
float noise_offset = 0.0f;//-0.5..+0.5
#else
const float LearningRate =0.02;
const float Momentum = 0.00;
float noise_percent =30.0f;
const int Const_hidd_node = 20;
float noise_amplitude = 1.0f;
//float noise_offset = -0.5f;
float noise_offset = 0.0f;

#endif // USE_RELU

//  float noise_percent =50.0f;
#define USE_IND_NOISE///Now USE_IND_NOISE switch ON make more realistoc gabor filter like feature on first layer  
//float start_weight_noise_range = 0.025f;//+/- Weight startnoise range
    float start_weight_noise_range = 0.15f;//+/- Weight startnoise range

float noise = 0.0f;//Noise added on input stimulu

#ifdef USE_MNIST_DATABASE
const int Const_nr_pic = 10000;
#else
const int Const_nr_pic = 100;//Use only the 289x289 input pattern or raspicam input
#endif // USE_MNIST_DATABASE


void sigmoid_mat(Mat image)
{
    float* ptr_src_index;
    ptr_src_index = image.ptr<float>(0);
    int nRows = image.rows;
    int nCols = image.cols;
    for(int i=0; i<nRows; i++)
    {
        for(int j=0; j<nCols; j++)
        {
            *ptr_src_index = 1.0/(1.0 + exp(-(*ptr_src_index)));//Sigmoid function
            ptr_src_index++;
        }
    }
}

void relu_mat(Mat image)
{
    float* ptr_src_index;
    ptr_src_index = image.ptr<float>(0);
    int nRows = image.rows;
    int nCols = image.cols;
    for(int i=0; i<nRows; i++)
    {
        for(int j=0; j<nCols; j++)
        {
            if(*ptr_src_index < 0.0)
            {
                *ptr_src_index = *ptr_src_index * Relu_neg_gain;
            }
            ptr_src_index++;
        }
    }
}



#include <termios.h>
#include <unistd.h>
#include <fcntl.h>

int kbhit(void)
{
    struct termios oldt, newt;
    int ch;
    int oldf;

    tcgetattr(STDIN_FILENO, &oldt);
    newt = oldt;
    newt.c_lflag &= ~(ICANON | ECHO);
    tcsetattr(STDIN_FILENO, TCSANOW, &newt);
    oldf = fcntl(STDIN_FILENO, F_GETFL, 0);
    fcntl(STDIN_FILENO, F_SETFL, oldf | O_NONBLOCK);

    ch = getchar();

    tcsetattr(STDIN_FILENO, TCSANOW, &oldt);
    fcntl(STDIN_FILENO, F_SETFL, oldf);

    if(ch != EOF)
    {
        ungetc(ch, stdin);
        return 1;
    }

    return 0;
}

void local_normalizing(Mat gray)
{

//#define BLUR_FLT_NUMERATOR 2
//#define BLUR_FLT_DENOMINATOR 20
#define BLUR_FLT_NUMERATOR 10
#define BLUR_FLT_DENOMINATOR 20

    Mat float_gray, blur, num, den, store_gray;
    store_gray = gray;//Initialize size
    // convert to floating-point image
    gray.convertTo(float_gray, CV_32F, 1.0/255.0);

    // numerator = img - gauss_blur(img)
    cv::GaussianBlur(float_gray, blur, Size(0,0), BLUR_FLT_NUMERATOR, BLUR_FLT_NUMERATOR);
    num = float_gray - blur;

    // denominator = sqrt(gauss_blur(img^2))
    cv::GaussianBlur(num.mul(num), blur, Size(0,0), BLUR_FLT_DENOMINATOR, BLUR_FLT_DENOMINATOR);
    cv::pow(blur, 0.5, den);

    // output = numerator / denominator
    gray = num / den;

    // normalize output into [0,1]
    cv::normalize(gray, gray, 0.0, 1.0, NORM_MINMAX, -1);

    // Display
    //namedWindow("demo", CV_WINDOW_AUTOSIZE );
    gray.convertTo(store_gray, CV_8U, 255);
    //imshow("demo", gray);

}
int get_MNIST_file_size(void)
{
    int file_size=0;
    FILE *fp2;
    fp2 = fopen("t10k-images-idx3-ubyte", "r");
    if (fp2 == NULL)
    {
        puts("Error while opening file t10k-images-idx3-ubyte");
        exit(0);
    }

    fseek(fp2, 0L, SEEK_END);
    file_size = ftell(fp2);
    printf("file_size %d\n", file_size);
    rewind(fp2);
    fclose(fp2);
    return file_size;
}

//attach_weight_2_mat(ptr_M_matrix, i, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);
void attach_weight_2_mat(float* ptr_M_matrix, int i, Mat src, int sqr_of_H_nod_plus1, int  Hidden_nodes, int Height, int Width)
{
    float *start_corner_offset = src.ptr<float>(0);
    int start_offset=0;
    float *src_zero_ptr = src.ptr<float>(0);
    float *src_ptr = src.ptr<float>(0);
    for(int j=0; j<Hidden_nodes ; j++)
    {

        start_offset = (j/sqr_of_H_nod_plus1)*Height*src.cols + (j%sqr_of_H_nod_plus1)*Width;
        start_corner_offset = start_offset + src_zero_ptr;
        src_ptr = start_corner_offset + (i/Width)*src.cols + (i%Width);
        *src_ptr = *ptr_M_matrix;
        ptr_M_matrix++;
    }
}

void attach_in2hid_w_2_mat(float* ptr_bias_weights, Mat src, int sqr_of_H_nod_plus1, int  Hidden_nodes, int Height, int Width)
{
    float *start_corner_offset = src.ptr<float>(0);
    int start_offset=0;
    float *src_zero_ptr = src.ptr<float>(0);
    float *src_ptr = src.ptr<float>(0);
    int j=Hidden_nodes;///Hidden_nodes+0 is the patch position where in2hid weight should be visualized
    start_offset = (j/sqr_of_H_nod_plus1)*Height*src.cols + (j%sqr_of_H_nod_plus1)*Width;
    start_corner_offset = start_offset + src_zero_ptr;
    for(int i=0; i<Hidden_nodes; i++)
    {
        if(i>(Height*Width-1))
        {
            break;///The hidden nodes may be larger then one visualize patches then break so it not point out in neverland
        }
        src_ptr = start_corner_offset + (i/Width)*src.cols + (i%Width);
        *src_ptr = *ptr_bias_weights;
        ptr_bias_weights++;
    }
}

void attach_hid2out_w_2_mat(float* ptr_bias_weights, Mat src, int sqr_of_H_nod_plus1, int  Hidden_nodes, int Height, int Width)
{
    float *start_corner_offset = src.ptr<float>(0);
    int start_offset=0;
    float *src_zero_ptr = src.ptr<float>(0);
    float *src_ptr = src.ptr<float>(0);
    int j=Hidden_nodes+1;///Hidden_nodes+1 is the patch position where hid2out weight should be visualized
    start_offset = (j/sqr_of_H_nod_plus1)*Height*src.cols + (j%sqr_of_H_nod_plus1)*Width;
    start_corner_offset = start_offset + src_zero_ptr;
    for(int i=0; i<(Height*Width); i++)
    {
        src_ptr = start_corner_offset + (i/Width)*src.cols + (i%Width);
        *src_ptr = *ptr_bias_weights;
        ptr_bias_weights++;
    }
}


int main()
{
    FILE *fp2;
    FILE *fp3;
    FILE *fp4;
    int MNIST_file_size=0;
#ifdef USE_MNIST_DATABASE
    MNIST_file_size = get_MNIST_file_size();
//read_10k_MNIST();
    char *MNIST_data;
    MNIST_data = new char[MNIST_file_size];
    FILE *fp;
    char c_data=0;
    fp = fopen("t10k-images-idx3-ubyte","r");

    if(fp == NULL)
    {
        perror("Error in opening t10k-images-idx3-ubyte file");
        return(-1);
    }
    int MN_index=0;
    for(int i=0; i<MNIST_file_size; i++)
    {

        c_data = fgetc(fp);
        if( feof(fp) )
        {
            break;
        }
        //printf("c_data %d\n", c_data);
        MNIST_data[MN_index] = c_data;
        if((MNIST_header_offset-1)<i)
        {
            MN_index++;
        }
    }
    fclose(fp);

#endif // USE_MNIST_DATABASE
    float Rando=0.0f;
//   float start_weight_noise_range = 0.25f;//+/- start weight noise range

    printf("Auto encoder test program\n");
    //int get_data =0;
    int Height=0;
    int Width=0;
    int nr_of_pixels=0;
    int Hidden_nodes=0;
    int nr_of_pictures=0;
    int training_image=0;
    int Use_local_normaliz=0;
    char answer_character;

    printf("Do you want to use local normalize on the input image <Y>/<N> ? \n");
    answer_character = getchar();
    if(answer_character == 'Y' || answer_character == 'y')
    {
        Use_local_normaliz=1;
    }
    else
    {
        Use_local_normaliz=0;
    }
    getchar();
    printf("Do you want default settings <Y>/<N> ? \n");
    answer_character = getchar();
    if(answer_character == 'Y' || answer_character == 'y')
    {
        //  Height = 48;
        //  Width = 64;
   //     Height = 28;
   //     Width = 28;
#ifdef USE_RASPICAM_INPUT
        Height = 10;
        Width = 10;
#else
        Height = 28;
        Width = 28;
#endif // USE_RASPICAM_INPUT


        Hidden_nodes = Const_hidd_node;
        nr_of_pictures = Const_nr_pic;
    }
    else
    {
        printf("Enter number of pixel Height of input pictures\n");
        scanf("%d", &Height);
        printf("Enter number of pixel Width of input pictures\n");
        scanf("%d", &Width);
        printf("Enter number of hidden nodes of input pictures\n");
        scanf("%d", &Hidden_nodes);
        printf("Enter number of hidden training pictures posXXX.JPG \n");
        scanf("%d", &nr_of_pictures);

    }
    nr_of_pixels = Height * Width;
    printf("Number of pixels = %d\n", nr_of_pixels);

///*************************************************************************************************
///************* Make a visual Mat to show the hidden nodes as a image representation **************
///*************************************************************************************************

    int sqr_r_hidd_nodes=0;
    sqr_r_hidd_nodes = sqrt(Hidden_nodes+2);//+1 because make space for input bias weight and hidden bias weight
    int test_sqr_r_n=0;
    test_sqr_r_n = sqr_r_hidd_nodes * sqr_r_hidd_nodes;
    while(test_sqr_r_n < Hidden_nodes)
    {
        sqr_r_hidd_nodes++;
        test_sqr_r_n = sqr_r_hidd_nodes * sqr_r_hidd_nodes;
    }
    printf("sqr_r_hidd_nodes %d\n", sqr_r_hidd_nodes);
    Mat vis_hidd_node;
    vis_hidd_node.create(sqr_r_hidd_nodes, sqr_r_hidd_nodes, CV_32F);
    float *zero_ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);
    float *ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);

    Mat colour_vis_h;
    colour_vis_h.create(sqr_r_hidd_nodes, sqr_r_hidd_nodes, CV_8UC3);
    CvPoint P1;
///*************************************************************************************************
///*************************************************************************************************

    Mat src, image, cam_part, mnist,m_gray;
///************** Raspicam **************
#ifdef USE_RASPICAM_INPUT
    raspicam::RaspiCam_Cv Camera;


    Camera.set( CV_CAP_PROP_FRAME_WIDTH, cam_w);
    Camera.set( CV_CAP_PROP_FRAME_HEIGHT, cam_h);
    Camera.set( CV_CAP_PROP_FORMAT, CV_8UC1 );
//Camera.set( CV_CAP_PROP_FORMAT, CV_8U );
    //Open camera
    cout<<"Opening Camera..."<<endl;
    if (!Camera.open())
    {
        cerr<<"Error opening the camera"<<endl;
        return -1;
    }

    Camera.grab();
    Camera.retrieve (src);
    waitKey(1);
    src.convertTo(image, CV_32F, 1.0/255.0);//Convert pixels from 0..255 char to float 0..1
  //  cam_part.create(Height, Width, CV_32F);//take only small random parts of camera image

#endif // USE_RASPICAM_INPUT
///***************************************

#ifndef USE_RASPICAM_INPUT
///************Open mnist image ********
#ifndef USE_MNIST_DATABASE
    sprintf(filename2, "mnist.png");//Assigne a filename "pos" with index number added

    mnist = imread( filename2, 1 );
    if ( !mnist.data )
    {
        printf("\n");
        printf("==================================================\n");
        printf("No image data Error! Probably not find mnist.png \n");
        printf("==================================================\n");
        printf("\n");
        //return -1;
    }


    cvtColor(mnist,m_gray,CV_BGR2GRAY);
    imshow("m_gray", m_gray);
    waitKey(1);

    int mnist_h1 = 28;
    int mnist_w1 = 28;
    int mnh=0;
    int mnw=0;
    int pos = 0;
    int ret;
    //     Mat m1;
//#ifndef USE_MNIST_DATABASE
    for(int j=0; j<10; j++)
    {
        for(int i=0; i<10; i++)
        {
//    Mat m1(image, Rect(0, 0, T0_FRAME_WIDTH, T0_FRAME_HEIGHT));// Rect(<start_x>, <start_y>, <width>, <hight>)
//48 x 45
            mnh = j*mnist_h1+j*1;
            mnw = i*mnist_w1+i*1;
            Mat m1(m_gray, Rect(mnw, mnh, mnist_w1, mnist_h1));// Rect(<start_x>, <start_y>, <width>, <hight>)
            cv::imwrite("temporary_file.JPG",m1);

            sprintf(filename_dst, "pos%d.JPG", pos);
            ret = rename("temporary_file.JPG", filename_dst);
            if(ret == 0)
            {
                printf("File renamed successfully");
            }
            else
            {
                printf("Error: unable to rename the file");
            }
            pos++;

        }
    }
#endif

///*************************************

#endif // Not USE_RASPICAM_INPUT

///****************************************************************************
///******* Start of dynamic declaration of variables **************************
///****************************************************************************

    float *input_node;
    input_node = new float[nr_of_pixels];
    float *hidden_node;
    hidden_node = new float[Hidden_nodes];
    float *output_node;
    output_node = new float[nr_of_pixels];
    float **weight_matrix_M;//Pointer fo a dynamic array. This weight_matrix_M will then have a size of rows = nr_of_pixels, colums = Hidden_nodes.
    float **change_weight_M;//Pointer fo a dynamic array. This weight_matrix_M will then have a size of rows = nr_of_pixels, colums = Hidden_nodes.
    weight_matrix_M = new float *[nr_of_pixels];
    change_weight_M = new float *[nr_of_pixels];
    for(int i=0; i < nr_of_pixels; i++)
    {
        weight_matrix_M[i] = new float[Hidden_nodes];
        change_weight_M[i] = new float[Hidden_nodes];
    }
    float *f_data;///File data read/write connect to tied weights
    f_data = new float[nr_of_pixels*Hidden_nodes];///File data is same size as all tied weights
    int ix=0;///index to f_data[ix]
///*****************************************************************************
///****** Bias weights this is not tied weight *********************************
/// Bias weights is separate in 2 groups:
/// 1. input layer  +1 bias to all hidden nodes
/// 2. hidden layer +1 bias to all output nodes
/// Therefor this weight must be treated diffrent compared with the tied weights
///*****************************************************************************

    float *bias_weight_in2hid;
    float *bias_weight_hid2out;
    float *change_weight_in2hid;
    float *change_weight_hid2out;
    float *hid_node_delta;///this is used only for backpropagation to me in2hid wehigt updates (not needed for tied weight)
    bias_weight_in2hid    = new float[Hidden_nodes];///This weight go from the input layer bias node (how always have a fix value +1) to all the hidden nodes (but not to the hidden bias node how is fix +1.0)
    bias_weight_hid2out   = new float[nr_of_pixels];///This weight go from the hidden layer bias node (how always have a fix value +1) to all the output nodes how are nr_of_pixels wide vector
    change_weight_in2hid  = new float [Hidden_nodes];
    change_weight_hid2out = new float [nr_of_pixels];
    hid_node_delta        = new float [Hidden_nodes];///this is used only for backpropagation to me in2hid wehigt updates (not needed for tied weight)

///*****************************************************************************
///************ End of dynamic declaration *************************************
///*****************************************************************************

    int sqr_of_H_nod_plus1=0;
    sqr_of_H_nod_plus1 = sqrt(Hidden_nodes+2);//+2 for the visualize bias weights also
    sqr_of_H_nod_plus1 += 1;//
    printf("sqr_of_H_nod_plus1 %d\n", sqr_of_H_nod_plus1);
    float* ptr_M_matrix;
    float* ptr_bias_w;
    Mat visual_all_feature, outimg;
    visual_all_feature.create(Height * sqr_of_H_nod_plus1, Width * sqr_of_H_nod_plus1,CV_32F);
    outimg.create(Height, Width, CV_32F);

    srand (static_cast <unsigned> (time(0)));//Seed the randomizer
    for(int i=0; i<nr_of_pixels; i++)
    {
        ptr_M_matrix = &weight_matrix_M[i][0];
        for(int j=0; j<(Hidden_nodes); j++)
        {
            Rando = (float) (rand() % 65535) / 65536;//0..1.0 range
            Rando -= 0.5f;
            Rando *= start_weight_noise_range;
            *ptr_M_matrix = Rando;
            ptr_M_matrix++;
            change_weight_M[i][j] = 0.0f;
        }
    }

    for(int i=0; i<Hidden_nodes; i++)
    {
        ///Add start randomize noise to bias_weight_in2hid weights
        Rando = (float) (rand() % 65535) / 65536;//0..1.0 range
        Rando -= 0.5f;
        Rando *= start_weight_noise_range;
        bias_weight_in2hid[i] = Rando;
        change_weight_in2hid[i] = 0.0f;///Initialize with zero
    }

    for(int i=0; i<nr_of_pixels; i++)
    {
        ///Add start randomize noise to bias_weight_hid2out weights
        Rando = (float) (rand() % 65535) / 65536;//0..1.0 range
        Rando -= 0.5f;
        Rando *= start_weight_noise_range;
        bias_weight_hid2out[i] = Rando;
        change_weight_hid2out[i] = 0.0f;///Initialize with zero
    }

    printf("Would you like to load stored weight_matrix_M.dat <Y>/<N> \n");
    getchar();
    answer_character = getchar();
    if(answer_character == 'Y' || answer_character == 'y')
    {
        sprintf(filename, "weight_matrix_M.dat");
        fp2 = fopen(filename, "r");
        if (fp2 == NULL)
        {
            printf("Error while opening file weight_matrix_M.dat");
            exit(0);
        }
        fread(f_data, sizeof f_data[0], (nr_of_pixels*Hidden_nodes), fp2);
        ix=0;
        for(int n=0; n<Hidden_nodes; n++)
        {
            for(int p=0; p<nr_of_pixels; p++)
            {
                weight_matrix_M[p][n] = f_data[ix];///File data put in to tied weights
                ix++;
            }
        }
        fclose(fp2);

///*** Load bias weight from in2hid layer *******
        sprintf(filename, "bias_in2hid_weight.dat");//Assigne a filename with index number added
        fp2 = fopen(filename, "r");
        if (fp2 == NULL)
        {
            printf("Error while opening file bias_in2hid_weight.dat");
            exit(0);
        }
        fread(bias_weight_in2hid, sizeof bias_weight_in2hid[0], Hidden_nodes, fp2);
        fclose(fp2);
        printf("weights are loaded to bias_in2hid_weight.dat file\n");
///************ Saved ***************************

///*** Load bias weight from hid2out layer *******
        sprintf(filename, "bias_hid2out_weight.dat");//Assigne a filename with index number added
        fp2 = fopen(filename, "r");
        if (fp2 == NULL)
        {
            printf("Error while opening file bias_hid2out_weight.dat");
            exit(0);
        }
        fread(bias_weight_hid2out, sizeof bias_weight_hid2out[0], nr_of_pixels, fp2);
        fclose(fp2);
        printf("weights are loaded to bias_hid2out_weight.dat file\n");
///************ Saved ***************************


    }

    /*
        for(int i=0;i<nr_of_pixels;i++)
        {
            ptr_M_matrix = &weight_matrix_M[i][0];///Set the start pos of pointer so function below could read out the weight array
            attach_weight_2_mat(ptr_M_matrix, i, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);
        }
    */

//   local_normalizing(visual_all_feature);

//test on


    visual_all_feature += 0.5f;
    Mat color_img, gray, noised_input, normalized_local;
    srand (static_cast <unsigned> (time(0)));//Seed the randomizer


    noised_input.create(Height, Width, CV_32F);
    float error=0.0;
    float *zero_ptr_gray = gray.ptr<float>(0);
    float *ptr_gray = gray.ptr<float>(0);
    float *ptr_noised_inp = noised_input.ptr<float>(0);
    float *zero_ptr_noised_inp = noised_input.ptr<float>(0);
    ptr_gray = zero_ptr_gray + (gray.cols * (gray.rows/2)) + gray.cols/2;
    float test_pixel_value = *ptr_gray;
    int *noise_pixels;
    noise_pixels = new int[nr_of_pixels];
    int noise_p_counter=0;

    float noise_ratio=0.0f;
    int nr_of_noise_rand_ittr=0;
    int rand_pix_pos=0;
    char keyboard;
    int started = 0;
#ifdef USE_MNIST_DATABASE
    gray.create(28,28,CV_32F);
#endif // USE_MNIST_DATABASE
#ifdef USE_RASPICAM_INPUT
    int rand_x_start=0;//Used to pick up a small part of camera image and feed in to the auto encoder
    int rand_y_start=0;//Used to pick up a small part of camera image and feed in to the auto encoder
    int max_point_x_or_y=0;
#endif // USE_RASPICAM_INPUT
    while(1)
    {
#ifdef USE_RASPICAM_INPUT
        Camera.grab();
        Camera.retrieve (src);
       // waitKey(1);
        src.convertTo(image, CV_32F, 1.0/255.0);//Convert pixels from 0..255 char to float 0..1
        imshow("image", image);

        max_point_x_or_y = image.cols - Width;//Not allowed to set start x point so the small clip out rectangel can go outside the camera source image
        rand_x_start = (int) (rand() % max_point_x_or_y);// range
        max_point_x_or_y = image.rows - Height;//Not allowed to set start y point so the small clip out rectangel can go outside the camera source image
        rand_y_start = (int) (rand() % max_point_x_or_y);// range
   //     printf("rand_x_start %d\n", rand_x_start);
   //     printf("rand_y_start %d\n", rand_y_start);
        Mat cam_part2(image, Rect(rand_x_start, rand_y_start, Width, Height));//Pick a small part of image and feed autoencoder with this
        imshow("cam_part2", cam_part2);
        gray = cam_part2.clone();//Connect raspberry camera instead of posXXX.jpg image
        normalized_local = image.clone();

if(Use_local_normaliz==1)
{
        local_normalizing(gray);
        local_normalizing(normalized_local);
}
        imshow("normalized", normalized_local);

#else

#ifndef USE_MNIST_DATABASE
        training_image = rand() % nr_of_pictures;
        //   printf("training_image %d\n", training_image);
        sprintf(filename, "pos%d.JPG", training_image);//Assigne a filename "pos" with index number added
        color_img = imread( filename, 1 );
        if ( !color_img.data )
        {
            printf("\n");
            printf("==================================================\n");
            printf("No image data Error! Probably not find pos%d.JPG \n", training_image);
            printf("==================================================\n");
            printf("\n");
            //return -1;
        }
        cvtColor(color_img,gray,CV_BGR2GRAY);

#else
///*********************************************************************
///read data from ************* t10k-images-idx3-ubyte ************ file
///*********************************************************************
        training_image = rand() % nr_of_pictures;
/// read from data
        zero_ptr_gray = gray.ptr<float>(0);
        for(int n=0; n<nr_of_pixels; n++)
        {
            ptr_gray = zero_ptr_gray + n;
            *ptr_gray = MNIST_data[(nr_of_pixels*training_image) +n];
        }

        color_img.create(28,28,CV_32F);

#endif // USE_MNIST_DATABASE

        imshow("color_img", color_img);

        gray.convertTo(gray, CV_32F, 1.0/255.0);//Convert pixels from 0..255 char to float 0..1

//  local_normalizing(gray);

#endif // USE_RASPICAM_INPUT
#ifdef USE_RELU

        relu_mat(gray);
//    gray *= 0.5f;
//    gray += 0.5f;

#else
        sigmoid_mat(gray);
#endif // USE_RELU
        zero_ptr_gray = gray.ptr<float>(0);
        ptr_gray = gray.ptr<float>(0);
   //     zero_ptr_gray = cam_part2.ptr<float>(0);
   //     ptr_gray = cam_part2.ptr<float>(0);


///******************************************************
///************ Select noise pixels positions ***********
///******************************************************
        noise_p_counter=0;
        noise_ratio=0.0f;
        nr_of_noise_rand_ittr=0;
        rand_pix_pos=0;

        for(int n=0; n<nr_of_pixels; n++)
        {
            noise_pixels[n] = 0;
        }

        while(noise_ratio < (noise_percent*0.01f))
        {
            rand_pix_pos = (int) (rand() % nr_of_pixels);
            if(noise_pixels[rand_pix_pos] == 0)
            {
                noise_p_counter++;
            }
            noise_pixels[rand_pix_pos] = 1;
            noise_ratio = ((float)noise_p_counter) / ((float)nr_of_pixels);
            nr_of_noise_rand_ittr++;
            if(nr_of_noise_rand_ittr > 2*nr_of_pixels)
            {
                printf("give up fill random up noise this turn\n");
                printf("noise_ratio %f\n", noise_ratio);
                break;
            }
        }
#ifndef USE_IND_NOISE
///****************************************************
///********** Select noise level ******************
///****************************************************
        noise = (float) (rand() % 65535) / 65536;//0..1.0 range
        noise -= 0.5f;
        noise = noise * noise_amplitude;
        noise += 0.5f;
        noise += noise_offset;
#endif // USE_IND_NOISE


///****************************************************
///**************** Feed forward **********************
///****************************************************
        ///Connect image pixel and also insert noise to input_node[]
        for(int n=0; n<nr_of_pixels; n++)
        {
            if(noise_pixels[n] == 1 && started == 1)
            {
#ifdef USE_IND_NOISE
///****************************************************
///********** Select noise level ******************
///****************************************************
                noise = (float) (rand() % 65535) / 65536;//0..1.0 range
                noise -= 0.5f;
                noise = noise * noise_amplitude;
                noise += 0.5f;
                noise += noise_offset;
#endif // USE_IND_NOISE
                input_node[n] = noise;///Insert noise 0.5f instead of real pixel value
            }
            else
            {
                ptr_gray = zero_ptr_gray + n;
                input_node[n] = *ptr_gray;///Insert pixel value
            }
            ptr_noised_inp = zero_ptr_noised_inp + n;
            *ptr_noised_inp = input_node[n];//To show
        }
        imshow("noised_input ", noised_input);
        ///*********** Forward to hidden nodes ****************
        for(int j=0; j<Hidden_nodes; j++)
        {
            hidden_node[j] = 0;
            for(int i=0; i<nr_of_pixels; i++)
            {
                hidden_node[j] += input_node[i] * weight_matrix_M[i][j];///Sum up values from input gained with weight
            }
            hidden_node[j] += Bias_level * bias_weight_in2hid[j];///Add weighted bias signal also

#ifdef USE_RELU
            ///*** Relu this node ***
            if(hidden_node[j] < 0.0f)
            {
                hidden_node[j] = hidden_node[j] * Relu_neg_gain;///Relu function
            }
#else
            ///*** Sigmoid this node ***
            hidden_node[j] = 1.0/(1.0 + exp(-(hidden_node[j])));///Sigmoid function
#endif // USE_RELU


        }
        ///********** Forward to output nodes *******************
        for(int i=0; i<nr_of_pixels; i++)
        {
            output_node[i] = 0.0f;
            for(int j=0; j<Hidden_nodes; j++)
            {
                output_node[i] += hidden_node[j] * weight_matrix_M[i][j];///Sum up values from hidden gained with weight
            }
            output_node[i] += Bias_level * bias_weight_hid2out[i];///Add weighted bias signal also

#ifdef USE_RELU
            ///*** Relu this node ***
            if(output_node[i] < 0.0f)
            {
                output_node[i] = output_node[i] * Relu_neg_gain;///Relu function
            }
#else
            ///*** Sigmoid this node ***
            output_node[i] = 1.0/(1.0 + exp(-(output_node[i])));///Sigmoid function
#endif // USE_RELU


        }

        float input_pixel =0.0f;
///****************************************************
///************* Calculate total loss *****************
///****************************************************
        /// k = nr_of_pixel
        /// loss = -SUM k (output[k] * log(input[k]) + (1.0 - output[k]) * log(1.0 - input[k]))
        /// or
        /// loss = 1/2 SUM k (input[k] - ouput[k])²
        float loss=0.0f;
        loss=0.0f;
        for(int i=0; i<nr_of_pixels; i++)
        {
            ptr_gray = zero_ptr_gray + i;///pick real input pixel
            input_pixel = *ptr_gray;///Insert pixel value
            loss += (input_pixel - output_node[i]) * (input_pixel - output_node[i]);/// loss = 1/2 SUM k (input[k] - ouput[k])²
        }
        static int print_loss=0;
        if(print_loss<10)
        {
            print_loss++;
        }
        else
        {
            printf("loss error = %f\n", loss);
            print_loss=0;
        }

#ifdef USE_LIM_BIAS
///*********** Limit bias ******************
            for(int j=0; j<Hidden_nodes; j++)
            {
                if(bias_weight_in2hid[j] > Bias_w_p_range)
                {
                    bias_weight_in2hid[j] = Bias_w_p_range;
                }
                if(bias_weight_in2hid[j] < Bias_w_n_range)
                {
                    bias_weight_in2hid[j] = Bias_w_n_range;
                }
                if(change_weight_in2hid[j] > change_bias_weight_range)
                {
                    change_weight_in2hid[j] = change_bias_weight_range;
                }
                if(change_weight_in2hid[j] < -change_bias_weight_range)
                {
                    change_weight_in2hid[j] = change_bias_weight_range;
                }
            }
            for(int i=0; i<nr_of_pixels; i++)
            {
                if(bias_weight_hid2out[i] > Bias_w_p_range)
                {
                    bias_weight_hid2out[i] = Bias_w_p_range;
                }
                if(bias_weight_hid2out[i] < Bias_w_n_range)
                {
                    bias_weight_hid2out[i] = Bias_w_n_range;
                }

                if(change_weight_hid2out[i] > change_bias_weight_range)
                {
                    change_weight_hid2out[i] = change_bias_weight_range;
                }
                if(change_weight_hid2out[i] < -change_bias_weight_range)
                {
                    change_weight_hid2out[i] = change_bias_weight_range;
                }
            }
///*********** End Limit bias ******************
#endif // USE_LIM_BIAS

///****************************************************
///**************** Backpropagation *******************
///****************************************************

        /// **** make Delta value for backpropagation and loss calculation ****

        for(int j=0; j<Hidden_nodes; j++)
        {
            hid_node_delta[j] = 0.0f;///clear delta regarding Bias backpropagation
        }

        float delta_pixel=0.0f;
        for(int i=0; i<nr_of_pixels; i++)
        {
            ptr_gray = zero_ptr_gray + i;///pick real input pixel
            input_pixel = *ptr_gray;///Insert pixel value
            delta_pixel = input_pixel - output_node[i] ;/// detal calculus
#ifdef USE_DELTA_RELU_REDUCED_NEG
            if(output_node[i] < 0.0f)
            {
                delta_pixel *= Relu_neg_gain;
            }
#endif // USE_DELTA_RELU_REDUCED_NEG
            for(int j=0; j<Hidden_nodes; j++)
            {
                ///*** Bias backpropagation ***
                ///Back propagation from output nodes to hid2out weight
                hid_node_delta[j] += (delta_pixel/2) * weight_matrix_M[i][j];///delta_pixel/2 because this delta is normaly used for tied weight

                ///*** End bias backprop ***********
                if(started == 1)
                {
                    /// **** update tied weight regarding delta
                    change_weight_M[i][j] = LearningRate * hidden_node[j] * delta_pixel + Momentum * change_weight_M[i][j];
                    weight_matrix_M[i][j] += change_weight_M[i][j];
                }
            }

            if(started == 1)
            {
                ///Update hidden bias to output weights bias_weight_hid2out[]
           /// Bugg
           //   change_weight_hid2out[i] = (LearningRate/2) * output_node[i] * delta_pixel + Momentum * change_weight_hid2out[i];
              change_weight_hid2out[i] = (LearningRate/2) * Bias_level * delta_pixel + Momentum * change_weight_hid2out[i];
              bias_weight_hid2out[i] += change_weight_hid2out[i];
            }

        }///End for(int i=0;i<nr_of_pixels;i++)

///***************************************************
///************* End backprop ************************
///***************************************************

///==========================================================================
///*** Update input bias to hidden weights bias_weight_in2hid
        for(int j=0; j<Hidden_nodes; j++)
        {
            if(started == 1)
            {
                ///Now use the delta from hid_node_delta[j]
           /// Bugg
           //   change_weight_in2hid[j] = (LearningRate/2) * hidden_node[j] * hid_node_delta[j] + Momentum * change_weight_in2hid[j];
                change_weight_in2hid[j] = (LearningRate/2) * Bias_level * hid_node_delta[j] + Momentum * change_weight_in2hid[j];
                bias_weight_in2hid[j] += change_weight_in2hid[j];
            }
        }
///*** End of update bias to hidden weights bias_weight_in2hid **************
///==========================================================================
///***************************************************
///************* End update weights ******************
///***************************************************



///********** Visualization **************************
        float *zero_ptr_outimg = outimg.ptr<float>(0);
        float *ptr_outimg = outimg.ptr<float>(0);
        for(int i=0; i<nr_of_pixels; i++)
        {
            ptr_outimg = zero_ptr_outimg + i;
            *ptr_outimg = output_node[i];
        }

//     gray -= 0.5;
        outimg -= 0.5;
//     visual_all_feature -= 0.5;
//     gray *= 2;
        outimg *= 2;
//     visual_all_feature *= 2;
//     gray += 0.5;
        outimg += 0.5;
//     visual_all_feature += 0.5;


        for(int i=0; i<nr_of_pixels; i++)
        {
            ptr_M_matrix = &weight_matrix_M[i][0];
            attach_weight_2_mat(ptr_M_matrix, i, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);
        }
        /// *******************************************************************
        /// ********* Also show bias weight with 2 patches ********************
        /// attach_in2hid_w_2_mat() will add 1 patches to show input to hidden bias weights
        ptr_bias_w = &bias_weight_in2hid[0];///Set the start pos of pointer so function below could read out the weight array
        attach_in2hid_w_2_mat(ptr_bias_w, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);///Hidden_nodes is because the function needs to know were to end visualization of patches

        /// attach_hid2out_w_2_mat() will add 1 patches to show hidden to output bias weights
        ptr_bias_w = &bias_weight_hid2out[0];///Set the start pos of pointer so function below could read out the weight array
        attach_hid2out_w_2_mat(ptr_bias_w, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);///Hidden_nodes is because the function needs to know were to end visualization of patches
        /// *******************************************************************
        /// *******************************************************************


        visual_all_feature += 0.5f;

        imshow("visual_all_feature", visual_all_feature);

        imshow("gray", gray);

        imshow("outimg", outimg);

///******** start stop
        if(kbhit())
        {
            keyboard = getchar();
            if(keyboard== ' ')
            {
                if(started == 1)
                {
                    started = 0;
                    printf("Stop training\n");
                }
                else
                {
                    started = 1;
                    printf("Start training\n");
                }

            }
            if(keyboard== 'S' || keyboard== 's')
            {
                //Save weights
                sprintf(filename, "weight_matrix_M.dat");//Assigne a filename with index number added
                fp2 = fopen(filename, "w+");
                if (fp2 == NULL)
                {
                    printf("Error while opening file weight_matrix_M.dat");
                    exit(0);
                }

                //size_t fread(void *ptr, size_t size_of_elements, size_t number_of_elements, FILE *a_file);
                ix=0;
                for(int n=0; n<Hidden_nodes; n++)
                {
                    for(int p=0; p<nr_of_pixels; p++)
                    {
                        f_data[ix] = weight_matrix_M[p][n];
                        ix++;
                    }
                }
                fwrite(f_data, sizeof f_data[0], (nr_of_pixels*Hidden_nodes), fp2);
                fclose(fp2);
                printf("weights are saved at weight_matrix_M.dat file\n");

///*** Save bias weight from in2hid layer *******
                sprintf(filename, "bias_in2hid_weight.dat");//Assigne a filename with index number added
                fp2 = fopen(filename, "w+");
                if (fp2 == NULL)
                {
                    printf("Error while opening file bias_in2hid_weight.dat");
                    exit(0);
                }
                fwrite(bias_weight_in2hid, sizeof bias_weight_in2hid[0], Hidden_nodes, fp2);
                fclose(fp2);
                printf("weights are saved at bias_in2hid_weight.dat file\n");
///************ Saved ***************************

///*** Save bias weight from hid2out layer *******
                sprintf(filename, "bias_hid2out_weight.dat");//Assigne a filename with index number added
                fp2 = fopen(filename, "w+");
                if (fp2 == NULL)
                {
                    printf("Error while opening file bias_hid2out_weight.dat");
                    exit(0);
                }
                fwrite(bias_weight_hid2out, sizeof bias_weight_hid2out[0], nr_of_pixels, fp2);
                fclose(fp2);
                printf("weights are saved at bias_hid2out_weight.dat file\n");
///************ Saved ***************************


            }

        }

///***************
        if(started == 0)
        {
            waitKey(2000);
            printf("Training stop now only feed forward\n");
            printf("Hit <Space> to start / stop traning \n");
            printf("Hit <S> to save all weights to weight_matrix_M.dat file\n");

        }
///*********** Visualize hidden nodes on a image **********
        cvtColor(visual_all_feature,colour_vis_h,CV_GRAY2RGB);

        zero_ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);
        ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);
        int index_hid_nodes=0;
        int pix_n=0;
        index_hid_nodes=0;
        pix_n=0;
        for(int ro=0; ro<vis_hidd_node.rows; ro++)
        {
            for(int co=0; co<vis_hidd_node.rows; co++)
            {
                if(index_hid_nodes<Hidden_nodes)
                {

                    ptr_vis_hidd_node = zero_ptr_vis_hidd_node + pix_n;
                    *ptr_vis_hidd_node = hidden_node[index_hid_nodes];

                    ///************* visualize on vis_hidd_node pattern **********
                    ///************* with green light ****************************
                    int circle_zize = 2;
                    int x,y;
                    float light_level=0.0f;
                    float light_red=0.0f;
                    light_level = *ptr_vis_hidd_node;
                    if(light_level > 1.0f)
                    {
                        light_level=1.0f;
                    }
                    if(light_level < 0.0f)
                    {
                        light_level=0.0f;
                    }
                    light_red = 1.0f - light_level;
                    x = (index_hid_nodes%sqr_of_H_nod_plus1)*Width + (Width/2);
                    y = (index_hid_nodes/sqr_of_H_nod_plus1)*Height + (Height/2);
                    P1.x = x;
                    P1.y = y;
                    circle(colour_vis_h, P1, circle_zize, Scalar(0,light_level,light_red), 2, -1);
                    index_hid_nodes++;
                    ///***********************************************************

                }
                else
                {
                    *ptr_vis_hidd_node = 0.0f;
                }
                pix_n++;
            }
        }
        imshow("vis_hidd_node", vis_hidd_node);
        imshow("colour_vis_h", colour_vis_h);

///********************************************************
        waitKey(1);
    }
    return 0;
}