Skip to content

Instantly share code, notes, and snippets.

@ollewelin

ollewelin/main.cpp

Last active July 7, 2017 17:16
Show Gist options
  • Save ollewelin/5a0eaeb12b2506eaa00732e97c835239 to your computer and use it in GitHub Desktop.
Save ollewelin/5a0eaeb12b2506eaa00732e97c835239 to your computer and use it in GitHub Desktop.
Download ZIP
Autoencoder unsupervised learning Relu tied weight and Supervised Learning Logistic Regression. MNIST test
Raw
main.cpp
///Now with fully connected Logistic Regression network Supervised Learning
///************* Parameters and things regarding fully connected network **************
int fully_conn_backprop =0;
const int C_fully_hidd_nodes = 200;
const int C_fully_out_nodes = 10;
int fully_hidd_nodes = C_fully_hidd_nodes;
int fully_out_nodes = C_fully_out_nodes;
int drop_out_percent = 50;/// 50% dropout percent hidden nodes during training
int verification = 0;
float Error_level=0.0f;
const float fc_start_weight_noise_range = 0.15f;//+/- Weight startnoise range
///float fc_LearningRate = 0.025f;
///float fc_Momentum = 0.9;
float fc_LearningRate = 0.025f;///0.025f
float fc_Momentum = 0.96;///0.9
const float High_Target_value = 1.0f;
const float Low_Target_value = 0.0f;
int Learning_fc = 1;
///*************************************************************************************
int save_first_images=0;
///L2 removed
int Pause_cam =0;
int norm_typ_gain_ON=0;
//#define USE_RASPICAM_INPUT //If you want to use raspicam input data
//Here use real image from raspicam take random part 10x10 pixel and put in to the autoencoder
//test stacking autoencoder 2 layer 4x input same Layer 1 (L1) filter down to the Layer 2 (L2)
//L2 input nodes = 4x L1 nodes
///#define USE_LIM_BIAS//
#define USE_DELTA_RELU_REDUCED_NEG
//const float C_delta_damping_full_backprop = 0.1;
//const float drop_probability = 40.0f;///don't work
//int Enable_drop_out=0;///don't work always 0
int Lock_L1=0;///Lock feature training L1
//const int cam_h = 240;
//const int cam_w = 320;
//const int cam_h = 480;
//const int cam_w = 640;
const int cam_h = 960;
const int cam_w = 1280;
///Now also Add bias nodes showed in the last 2 patches
const float Bias_level = 1.0f;
const float Bias_w_n_range = -0.5f;
const float Bias_w_p_range = 0.5f;
const float change_bias_weight_range = 0.2f;
/// Input data from
/// t10k-images-idx3-ubyte
/// t10k-labels-idx1-ubyte
/// train-images-idx3-ubyte
/// train-labels-idx1-ubyte
/// http://yann.lecun.com/exdb/mnist/
const int MNIST_pix_size = 28*28;
///char data_10k_MNIST[10000][MNIST_pix_size];
///char data_60k_MNIST[60000][MNIST_pix_size];
const int MNIST_header_offset = 16;
const int MNIST_lable_offset = 8;
/*
TEST SET LABEL FILE (t10k-labels-idx1-ubyte):
[offset] [type] [value] [description]
0000 32 bit integer 0x00000801(2049) magic number (MSB first)
0004 32 bit integer 10000 number of items
0008 unsigned byte ?? label
0009 unsigned byte ?? label
........
xxxx unsigned byte ?? label
The labels values are 0 to 9.
TEST SET IMAGE FILE (t10k-images-idx3-ubyte):
[offset] [type] [value] [description]
0000 32 bit integer 0x00000803(2051) magic number
0004 32 bit integer 10000 number of images
0008 32 bit integer 28 number of rows
0012 32 bit integer 28 number of columns
0016 unsigned byte ?? pixel
0017 unsigned byte ?? pixel
........
xxxx unsigned byte ?? pixel
Pixels are organized row-wise. Pixel values are 0 to 255. 0 means background (white), 255 means foreground (black).
*/
const float Relu_neg_gain = 0.1f;//0.0f pure rectify 1.0 full linear
#include <opencv2/highgui/highgui.hpp> // OpenCV window I/O
#include <opencv2/imgproc/imgproc.hpp> // Gaussian Blur
#include <stdio.h>
#include <raspicam/raspicam_cv.h>
#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp> // Basic OpenCV structures (cv::Mat, Scalar)
#include <cstdlib>
#include <ctime>
#include <math.h> // exp
#include <stdlib.h>// exit(0);
//#include <iostream>
char filename[100];
char filename2[100];
char filename_dst[100];
using namespace std;
using namespace cv;
//const float LearningRate =0.05;
//const float Momentum = 0.025;
int H_MIN = 0;
int H_MAX = 100;
int H_MAX_1000 = 1000;
int L1_noise_int = 50;
const float C_noise_percent =50.0f;//25
float noise_percent = C_noise_percent;//25
const int Const_hidd_node = 50;//
const float feature_type_gain = 0.05f;
const string trackbarWindowName = "Trackbars";
float abs_value(float signed_value)
{
float abs_value;
abs_value = signed_value;
if(abs_value < 0)
{
abs_value = -abs_value;
}
return abs_value;
}
void sigmoid_mat(Mat image)
{
float* ptr_src_index;
ptr_src_index = image.ptr<float>(0);
int nRows = image.rows;
int nCols = image.cols;
for(int i=0;i<nRows;i++)
{
for(int j=0;j<nCols;j++)
{
*ptr_src_index = 1.0/(1.0 + exp(-(*ptr_src_index)));//Sigmoid function
ptr_src_index++;
}
}
}
void randomize_dropoutHid(int *zero_ptr_dropoutHidden, int HiddenNodes, int verification)
{
int drop_out_part = HiddenNodes * drop_out_percent/100;//
// int dropoutHidden[HiddenNodes];
// srand (static_cast <unsigned> (time(0)));//Seed the randomizer
// q = rand() % PatternCount;
int*ptr_dropoutHidden;
for(int i=0; i<HiddenNodes; i++)
{
ptr_dropoutHidden = zero_ptr_dropoutHidden + i;
*ptr_dropoutHidden = 0;//reset
}
int check_how_many_dropout = 0;
if(verification == 0)
{
for(int k=0; k<HiddenNodes*2; k++) ///Itterate max HiddenNodes*2 number of times then give up to reach drop_out_part
{
for(int i=0; i<(drop_out_part-check_how_many_dropout); i++)
{
int r=0;
r = rand() % (HiddenNodes-1);
ptr_dropoutHidden = zero_ptr_dropoutHidden + r;
///dropoutHidden[r] = 1;//add a dropout node
*ptr_dropoutHidden = 1;///
}
check_how_many_dropout = 0;
for(int j=0; j<HiddenNodes; j++)
{
ptr_dropoutHidden = zero_ptr_dropoutHidden + j;
check_how_many_dropout += *ptr_dropoutHidden;
}
if(check_how_many_dropout >= drop_out_part)
{
break;
}
}
// printf("check_how_many_dropout =%d\n", check_how_many_dropout);
}
}
void on_trackbar( int, void* )
{//This function gets called whenever a
// trackbar position is changed
noise_percent = (float) L1_noise_int;
}
int fine_tune_L1_int = 0;
int fine_tune_L1_moment_int =0;
void createTrackbars(){
//create window for trackbars
namedWindow(trackbarWindowName,0);
//create memory to store trackbar name on window
char TrackbarName[50];
sprintf( TrackbarName, "Control values");
createTrackbar( "L1 noise [%] ", trackbarWindowName, &L1_noise_int, H_MAX, on_trackbar );
createTrackbar( "L1 finetune learning gain [*0.001] ", trackbarWindowName, &fine_tune_L1_int, 1000, on_trackbar );
createTrackbar( "L1 finetune momentum [*0.001] ", trackbarWindowName, &fine_tune_L1_int, 1000, on_trackbar );
}
const float C_LearningRate =0.00020;///0.0015
const float C_Momentum = 0.0;///0.0f
float LearningRate = C_LearningRate;///depend on the state of Lock_L1
float Momentum = C_Momentum;///depend on the state of Lock_L1
//const float L2_LearningRate =0.0015;
//const float L2_Momentum = 0.001;
float supervised_L1_LearningRate = 0.001*((float)fine_tune_L1_int);
float supervised_L1_Momentum = 0.001*((float)fine_tune_L1_moment_int);
//float noise_amplitude = 0.0f;//0..1,0
float noise_amplitude = 1.0f;
//float noise_offset = -0.5f;
float noise_offset = 0.0f;//-0.5..+0.5
// float noise_percent =50.0f;
#define USE_IND_NOISE ///Good to use this for netural image
//#define USE_WHITE_NOISE_ONLY
//float start_weight_noise_range = 0.025f;//+/- Weight startnoise range
float start_weight_noise_range = 0.15f;//+/- Weight startnoise range
float noise = 0.0f;//Noise added on input stimulu
const int Const_nr_pic = 60000;
void relu_mat(Mat image)
{
float* ptr_src_index;
ptr_src_index = image.ptr<float>(0);
int nRows = image.rows;
int nCols = image.cols;
for(int i=0; i<nRows; i++)
{
for(int j=0; j<nCols; j++)
{
if(*ptr_src_index < 0.0)
{
*ptr_src_index = *ptr_src_index * Relu_neg_gain;
}
ptr_src_index++;
}
}
}
void print_help(void)
{
printf("Hit <?> or <Space> show HELP menu\n");
printf("Training stop now only feed forward\n");
printf("Hit <Space> to start / stop traning \n");
printf("Hit <S> to save all weights to weight_matrix_M.dat file\n");
printf("Hit <B> Turn OFF noise to layer L2\n");
printf("Hit <C> clear fully connected network weights\n");
printf("Hit <N> Turn ON noise to layer L2\n");
printf("Hit <U> Unlock L1 features training \n");
printf("Hit <L> Lock L1 features training \n");
printf("Hit <P> Pause Raspicam and store image to paused_camera.JPG file\n");
printf("Hit <R> Run Raspicam \n");
printf("Hit <I> read paused image from paused_camera.JPG how was stored when <P>\n");
printf("Hit <F> Start Logistic Regression fully connected network training. Stop Autoencoder learning\n");
printf("Hit <G> Start Autoencoder learning. Stop Logistic Regression fully connected network training\n");
printf("Hit <T> Stop fully connected learning only <P>\n");
printf("Hit <H> Start fully connected learning only <P>\n");
printf("Hit <Z> Normalize feature gain \n");
}
#include <termios.h>
#include <unistd.h>
#include <fcntl.h>
int kbhit(void)
{
struct termios oldt, newt;
int ch;
int oldf;
tcgetattr(STDIN_FILENO, &oldt);
newt = oldt;
newt.c_lflag &= ~(ICANON | ECHO);
tcsetattr(STDIN_FILENO, TCSANOW, &newt);
oldf = fcntl(STDIN_FILENO, F_GETFL, 0);
fcntl(STDIN_FILENO, F_SETFL, oldf | O_NONBLOCK);
ch = getchar();
tcsetattr(STDIN_FILENO, TCSANOW, &oldt);
fcntl(STDIN_FILENO, F_SETFL, oldf);
if(ch != EOF)
{
ungetc(ch, stdin);
return 1;
}
return 0;
}
void local_normalizing(Mat gray)
{
//#define BLUR_FLT_NUMERATOR 2
//#define BLUR_FLT_DENOMINATOR 20
#define BLUR_FLT_NUMERATOR 10
#define BLUR_FLT_DENOMINATOR 20
Mat float_gray, blur, num, den, store_gray;
store_gray = gray;//Initialize size
// convert to floating-point image
gray.convertTo(float_gray, CV_32F, 1.0/255.0);
// numerator = img - gauss_blur(img)
cv::GaussianBlur(float_gray, blur, Size(0,0), BLUR_FLT_NUMERATOR, BLUR_FLT_NUMERATOR);
num = float_gray - blur;
// denominator = sqrt(gauss_blur(img^2))
cv::GaussianBlur(num.mul(num), blur, Size(0,0), BLUR_FLT_DENOMINATOR, BLUR_FLT_DENOMINATOR);
cv::pow(blur, 0.5, den);
// output = numerator / denominator
gray = num / den;
// normalize output into [0,1]
cv::normalize(gray, gray, 0.0, 1.0, NORM_MINMAX, -1);
// Display
//namedWindow("demo", CV_WINDOW_AUTOSIZE );
gray.convertTo(store_gray, CV_8U, 255);
//imshow("demo", gray);
}
int get_MNIST_file_size(void)
{
int file_size=0;
FILE *fp2;
fp2 = fopen("train-images-idx3-ubyte", "r");
if (fp2 == NULL)
{
puts("Error while opening file train-images-idx3-ubyte");
exit(0);
}
fseek(fp2, 0L, SEEK_END);
file_size = ftell(fp2);
printf("file_size %d\n", file_size);
rewind(fp2);
fclose(fp2);
return file_size;
}
int get_MNIST_lable_file_size(void)
{
int file_size=0;
FILE *fp2;
fp2 = fopen("train-labels-idx1-ubyte", "r");
if (fp2 == NULL)
{
puts("Error while opening file train-labels-idx1-ubyte");
exit(0);
}
fseek(fp2, 0L, SEEK_END);
file_size = ftell(fp2);
printf("file_size %d\n", file_size);
rewind(fp2);
fclose(fp2);
return file_size;
}
//attach_weight_2_mat(ptr_M_matrix, i, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);
void attach_weight_2_mat(float* ptr_M_matrix, int i, Mat src, int sqr_of_H_nod_plus1, int Hidden_nodes, int Height, int Width)
{
float *start_corner_offset = src.ptr<float>(0);
int start_offset=0;
float *src_zero_ptr = src.ptr<float>(0);
float *src_ptr = src.ptr<float>(0);
for(int j=0; j<Hidden_nodes ; j++)
{
start_offset = (j/sqr_of_H_nod_plus1)*Height*src.cols + (j%sqr_of_H_nod_plus1)*Width;
start_corner_offset = start_offset + src_zero_ptr;
src_ptr = start_corner_offset + (i/Width)*src.cols + (i%Width);
*src_ptr = *ptr_M_matrix;
ptr_M_matrix++;
}
}
void attach_in2hid_w_2_mat(float* ptr_bias_weights, Mat src, int sqr_of_H_nod_plus1, int Hidden_nodes, int Height, int Width)
{
float *start_corner_offset = src.ptr<float>(0);
int start_offset=0;
float *src_zero_ptr = src.ptr<float>(0);
float *src_ptr = src.ptr<float>(0);
int j=Hidden_nodes;///Hidden_nodes+0 is the patch position where in2hid weight should be visualized
start_offset = (j/sqr_of_H_nod_plus1)*Height*src.cols + (j%sqr_of_H_nod_plus1)*Width;
start_corner_offset = start_offset + src_zero_ptr;
for(int i=0; i<Hidden_nodes; i++)
{
if(i>(Height*Width-1))
{
break;///The hidden nodes may be larger then one visualize patches then break so it not point out in neverland
}
src_ptr = start_corner_offset + (i/Width)*src.cols + (i%Width);
*src_ptr = *ptr_bias_weights;
ptr_bias_weights++;
}
}
void attach_hid2out_w_2_mat(float* ptr_bias_weights, Mat src, int sqr_of_H_nod_plus1, int Hidden_nodes, int Height, int Width)
{
float *start_corner_offset = src.ptr<float>(0);
int start_offset=0;
float *src_zero_ptr = src.ptr<float>(0);
float *src_ptr = src.ptr<float>(0);
int j=Hidden_nodes+1;///Hidden_nodes+1 is the patch position where hid2out weight should be visualized
start_offset = (j/sqr_of_H_nod_plus1)*Height*src.cols + (j%sqr_of_H_nod_plus1)*Width;
start_corner_offset = start_offset + src_zero_ptr;
for(int i=0; i<(Height*Width); i++)
{
src_ptr = start_corner_offset + (i/Width)*src.cols + (i%Width);
*src_ptr = *ptr_bias_weights;
ptr_bias_weights++;
}
}
int main()
{
createTrackbars();
int save_vis_all_feature_jpg=0;
FILE *fp2;
FILE *fp3;
FILE *fp4;
int MNIST_file_size=0;
///Read database train-images-idx3-ubyte
MNIST_file_size = get_MNIST_file_size();
//read_10k_MNIST();
char *MNIST_data;
MNIST_data = new char[MNIST_file_size];
FILE *fp;
char c_data=0;
fp = fopen("train-images-idx3-ubyte","r");
if(fp == NULL)
{
perror("Error in opening train-images-idx3-ubyte file");
return(-1);
}
int MN_index=0;
for(int i=0; i<MNIST_file_size; i++)
{
c_data = fgetc(fp);
if( feof(fp) )
{
break;
}
//printf("c_data %d\n", c_data);
MNIST_data[MN_index] = c_data;
if((MNIST_header_offset-1)<i)
{
MN_index++;
}
}
fclose(fp);
printf("train-images-idx3-ubyte file is successfully loaded in to MNIST_data[MN_index] memory\n");
///Read lable
///Read train-labels-idx1-ubyte
MNIST_file_size = get_MNIST_lable_file_size();
//read_10k_MNIST();
char *MNIST_lable;
MNIST_lable = new char[MNIST_file_size];
// FILE *fp;
c_data=0;
fp = fopen("train-labels-idx1-ubyte","r");
if(fp == NULL)
{
perror("Error in opening train-labels-idx1-ubyte file");
return(-1);
}
MN_index=0;
for(int i=0; i<MNIST_file_size; i++)
{
c_data = fgetc(fp);
if( feof(fp) )
{
break;
}
//printf("c_data %d\n", c_data);
MNIST_lable[MN_index] = c_data;
if((MNIST_lable_offset-1)<i)
{
MN_index++;
}
}
fclose(fp);
printf("train-labels-idx1-ubyte file is successfully loaded in to MNIST_lable[MN_index] memory\n");
float Rando=0.0f;
// float start_weight_noise_range = 0.25f;//+/- start weight noise range
printf("Neural Network test program with 2 stacked Autoencoder then fully connected sigmoid Logistic Regression Network\n");
//int get_data =0;
int Height=0;
int Width=0;
int nr_of_pixels=0;
int Hidden_nodes=0;
int nr_of_pictures=0;
int training_image=0;
int Use_local_normaliz=0;
char answer_character;
printf("Do you want to use local normalize on the input image <Y>/<N> ? \n");
answer_character = getchar();
if(answer_character == 'Y' || answer_character == 'y')
{
Use_local_normaliz=1;
}
else
{
Use_local_normaliz=0;
}
getchar();
/*
printf("Do you want to full backpropagation to L2 <Y>/<N> ? \n");
answer_character = getchar();
if(answer_character == 'Y' || answer_character == 'y')
{
full_backprop_L2=1;
}
else
{
full_backprop_L2=0;
}
getchar();
printf("full_backprop_L2 = %d\n", full_backprop_L2);
*/
printf("Do you want default settings <Y>/<N> ? \n");
answer_character = getchar();
if(answer_character == 'Y' || answer_character == 'y')
{
// Height = 48;
// Width = 64;
// Height = 28;
// Width = 28;
///#ifdef USE_RASPICAM_INPUT
/// Height = 15;
/// Width = 15;
///#else
Height = 28;
Width = 28;
///#endif // USE_RASPICAM_INPUT
Hidden_nodes = Const_hidd_node;
nr_of_pictures = Const_nr_pic;
fully_hidd_nodes = C_fully_hidd_nodes;
fully_out_nodes = C_fully_out_nodes;
}
else
{
printf("Enter number of pixel Height of input pictures L1 \n");
scanf("%d", &Height);
printf("Enter number of pixel Width of input pictures L1 \n");
scanf("%d", &Width);
printf("Enter number of hidden nodes of input pictures L1\n");
scanf("%d", &Hidden_nodes);
printf("Enter number of hidden training pictures posXXX.JPG \n");
scanf("%d", &nr_of_pictures);
printf("Enter number of fully connected hidden nodes \n");
scanf("%d", &fully_hidd_nodes);
printf("Enter number supervised categorys = nr of fully connected output nodes \n");
scanf("%d", &fully_out_nodes);
}
nr_of_pixels = Height * Width;
printf("Number of pixels = %d\n", nr_of_pixels);
///*************************************************************************************************
///************* Make a visual Mat to show the hidden nodes L1 as a image representation **************
///*************************************************************************************************
int sqr_r_hidd_nodes=0;
sqr_r_hidd_nodes = sqrt(Hidden_nodes+2);//+1 because make space for input bias weight and hidden bias weight
int test_sqr_r_n=0;
test_sqr_r_n = sqr_r_hidd_nodes * sqr_r_hidd_nodes;
while(test_sqr_r_n < Hidden_nodes+2)
{
sqr_r_hidd_nodes++;
test_sqr_r_n = sqr_r_hidd_nodes * sqr_r_hidd_nodes;
}
printf("sqr_r_hidd_nodes %d\n", sqr_r_hidd_nodes);
Mat vis_hidd_node;
vis_hidd_node.create(sqr_r_hidd_nodes, sqr_r_hidd_nodes, CV_32F);
float *zero_ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);
float *ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);
Mat colour_vis_h;
colour_vis_h.create(sqr_r_hidd_nodes, sqr_r_hidd_nodes, CV_8UC3);
CvPoint P1;
///*************************************************************************************************
///*************************************************************************************************
Mat src, image, cam_part, mnist,m_gray;
///************** Raspicam **************
#ifdef USE_RASPICAM_INPUT
raspicam::RaspiCam_Cv Camera;
Camera.set( CV_CAP_PROP_FRAME_WIDTH, cam_w);
Camera.set( CV_CAP_PROP_FRAME_HEIGHT, cam_h);
Camera.set( CV_CAP_PROP_FORMAT, CV_8UC1 );
//Camera.set( CV_CAP_PROP_FORMAT, CV_8U );
//Open camera
cout<<"Opening Camera..."<<endl;
if (!Camera.open())
{
cerr<<"Error opening the camera"<<endl;
return -1;
}
Camera.grab();
Camera.retrieve (src);
waitKey(1);
src.convertTo(image, CV_32F, 1.0/255.0);//Convert pixels from 0..255 char to float 0..1
// cam_part.create(Height, Width, CV_32F);//take only small random parts of camera image
#endif // USE_RASPICAM_INPUT
///***************************************
///****************************************************************************
///******* Start of dynamic declaration of variables **************************
///****************************************************************************
///======================================
///L1 layer1.
///======================================
float *input_node;
input_node = new float[nr_of_pixels];
float *hidden_node;
hidden_node = new float[Hidden_nodes];
float *output_node;
output_node = new float[nr_of_pixels];
float **weight_matrix_M;//Pointer fo a dynamic array. This weight_matrix_M will then have a size of rows = nr_of_pixels, colums = Hidden_nodes.
float **change_weight_M;//Pointer fo a dynamic array. This weight_matrix_M will then have a size of rows = nr_of_pixels, colums = Hidden_nodes.
weight_matrix_M = new float *[nr_of_pixels];
change_weight_M = new float *[nr_of_pixels];
for(int i=0; i < nr_of_pixels; i++)
{
weight_matrix_M[i] = new float[Hidden_nodes];
change_weight_M[i] = new float[Hidden_nodes];
}
float *f_data;///File data read/write connect to tied weights
f_data = new float[nr_of_pixels*Hidden_nodes];///File data is same size as all tied weights
int ix=0;///index to f_data[ix]
///*****************************************************************************
///****** Bias weights this is not tied weight *********************************
/// Bias weights is separate in 2 groups:
/// 1. input layer +1 bias to all hidden nodes
/// 2. hidden layer +1 bias to all output nodes
/// Therefor this weight must be treated diffrent compared with the tied weights
///*****************************************************************************
///======================================
///L1 layer1.
///======================================
float *bias_weight_in2hid;
float *bias_weight_hid2out;
float *change_bias_weight_in2hid;
float *change_bias_weight_hid2out;
float *hid_node_delta;///this is used only for backpropagation to me in2hid wehigt updates (not needed for tied weight)
bias_weight_in2hid = new float[Hidden_nodes];///This weight go from the input layer bias node (how always have a fix value +1) to all the hidden nodes (but not to the hidden bias node how is fix +1.0)
bias_weight_hid2out = new float[nr_of_pixels];///This weight go from the hidden layer bias node (how always have a fix value +1) to all the output nodes how are nr_of_pixels wide vector
change_bias_weight_in2hid = new float [Hidden_nodes];
change_bias_weight_hid2out = new float [nr_of_pixels];
hid_node_delta = new float [Hidden_nodes];///this is used only for backpropagation to me in2hid wehigt updates (not needed for tied weight)
///**********************************************************************************
///*** Regarding the Fully connected sigmoid Logistic regression neural network *****
///**********************************************************************************
float *fc_input_node;
float *fc_hidden_node;
float *fc_output_node;
float *fc_target_node;
float *fc_output_delta;
float *fc_hidden_delta;
float *fc_input_delta;
int *dropoutHidden;///dropout table
dropoutHidden = new int[fully_hidd_nodes];///data 0 normal fc_hidden_node. 1= dropout this fc_hidden_node this training turn.
fc_input_node = new float[Hidden_nodes];///fc means fully connected. sigmoid of this L2_hidden_node = new float[L2_Hidden_nodes];
fc_hidden_node = new float[fully_hidd_nodes];///
fc_output_node = new float[fully_out_nodes];
fc_target_node = new float[fully_out_nodes];///The target value from lable file should be put in this
fc_output_delta = new float[fully_out_nodes];
fc_hidden_delta = new float[fully_hidd_nodes];///
fc_input_delta = new float[Hidden_nodes];
float **fc_hidden_weight;
float **fc_output_weight;
float **fc_change_hidden_weight;
float **fc_change_output_weight;
fc_hidden_weight = new float*[Hidden_nodes+1];///+1 for bias node connection
fc_change_hidden_weight = new float*[Hidden_nodes+1];
for(int i=0;i<Hidden_nodes+1;i++)///+1 for bias node connection
{
fc_hidden_weight[i] = new float[fully_hidd_nodes];
fc_change_hidden_weight[i] = new float[fully_hidd_nodes];
}
fc_output_weight = new float*[fully_hidd_nodes+1];///+1 for bias node connection
fc_change_output_weight = new float*[fully_hidd_nodes+1];
for(int i=0;i<fully_hidd_nodes+1;i++)///+1 for bias node connection
{
fc_output_weight[i] = new float[fully_out_nodes];
fc_change_output_weight[i] = new float[fully_out_nodes];
}
float *f_data_fc_h_w;///File data read/write connect to fc_hidden_weight weights
f_data_fc_h_w = new float[fully_hidd_nodes * (Hidden_nodes+1)];///File data is same size as fc_hidden_weight
float *f_data_fc_o_w;///File data read/write connect to fc_output_weight weights
f_data_fc_o_w = new float[(fully_hidd_nodes+1) * fully_out_nodes];///File data is same size as fc_output_weight
///*****************************************************************************
///************ End of dynamic declaration *************************************
///*****************************************************************************
///======================================
///L1 layer1. setup L1 tied weight's
///======================================
int sqr_of_H_nod_plus1=0;
sqr_of_H_nod_plus1 = sqrt(Hidden_nodes+2);//+2 for the visualize bias weights also
sqr_of_H_nod_plus1 += 1;//
printf("sqr_of_H_nod_plus1 %d\n", sqr_of_H_nod_plus1);
float* ptr_M_matrix;
float* ptr_bias_w;
Mat visual_all_feature, outimg;
visual_all_feature.create(Height * sqr_of_H_nod_plus1, Width * sqr_of_H_nod_plus1,CV_32F);
outimg.create(Height, Width, CV_32F);
srand (static_cast <unsigned> (time(0)));//Seed the randomizer
for(int i=0; i<nr_of_pixels; i++)
{
ptr_M_matrix = &weight_matrix_M[i][0];
for(int j=0; j<(Hidden_nodes); j++)
{
Rando = (float) (rand() % 65535) / 65536;//0..1.0 range
Rando -= 0.5f;
Rando *= start_weight_noise_range;
*ptr_M_matrix = Rando;
ptr_M_matrix++;
change_weight_M[i][j] = 0.0f;
}
}
for(int i=0; i<Hidden_nodes; i++)
{
///Add start randomize noise to bias_weight_in2hid weights
Rando = (float) (rand() % 65535) / 65536;//0..1.0 range
Rando -= 0.5f;
Rando *= start_weight_noise_range;
bias_weight_in2hid[i] = Rando;
change_bias_weight_in2hid[i] = 0.0f;///Initialize with zero
}
for(int i=0; i<nr_of_pixels; i++)
{
///Add start randomize noise to bias_weight_hid2out weights
Rando = (float) (rand() % 65535) / 65536;//0..1.0 range
Rando -= 0.5f;
Rando *= start_weight_noise_range;
bias_weight_hid2out[i] = Rando;
change_bias_weight_hid2out[i] = 0.0f;///Initialize with zero
}
///============================================
///************* Initialize randomized noise on fc_weight *********
for(int i=0; i<Hidden_nodes+1; i++)
{
for(int j=0;j<fully_hidd_nodes;j++)
{
Rando = (float) (rand() % 65535) / 65536;//0..1.0 range
Rando -= 0.5f;
Rando *= fc_start_weight_noise_range;
fc_hidden_weight[i][j] = Rando;///Noise around 0.5f
fc_change_hidden_weight[i][j] = 0.0f;///Initialize with zero
}
}
for(int i=0; i<fully_hidd_nodes+1; i++)
{
for(int j=0;j<fully_out_nodes;j++)
{
Rando = (float) (rand() % 65535) / 65536;//0..1.0 range
Rando -= 0.5f;
Rando *= fc_start_weight_noise_range;
fc_output_weight[i][j] = Rando;///Noise around 0.5f
fc_change_output_weight[i][j] = 0.0f;///Initialize with zero
}
}
///***********************************
printf("Would you like to load stored weight_matrix_M.dat <Y>/<N> \n");
getchar();
answer_character = getchar();
if(answer_character == 'Y' || answer_character == 'y')
{
///L1
sprintf(filename, "weight_matrix_M.dat");
fp2 = fopen(filename, "r");
if (fp2 == NULL)
{
printf("Error while opening file weight_matrix_M.dat");
exit(0);
}
fread(f_data, sizeof f_data[0], (nr_of_pixels*Hidden_nodes), fp2);
ix=0;
for(int n=0; n<Hidden_nodes; n++)
{
for(int p=0; p<nr_of_pixels; p++)
{
weight_matrix_M[p][n] = f_data[ix];///File data put in to tied weights
ix++;
}
}
fclose(fp2);
printf("weights are loaded from weight_matrix_M.dat file\n");
///L1
///*** Load bias weight from in2hid layer *******
sprintf(filename, "bias_in2hid_weight.dat");//Assigne a filename with index number added
fp2 = fopen(filename, "r");
if (fp2 == NULL)
{
printf("Error while opening file bias_in2hid_weight.dat");
exit(0);
}
fread(bias_weight_in2hid, sizeof bias_weight_in2hid[0], Hidden_nodes, fp2);
fclose(fp2);
printf("weights are loaded from bias_in2hid_weight.dat file\n");
///************ Loaded ***************************
///*** Load bias weight from hid2out layer *******
sprintf(filename, "bias_hid2out_weight.dat");//Assigne a filename with index number added
fp2 = fopen(filename, "r");
if (fp2 == NULL)
{
printf("Error while opening file bias_hid2out_weight.dat");
exit(0);
}
fread(bias_weight_hid2out, sizeof bias_weight_hid2out[0], nr_of_pixels, fp2);
fclose(fp2);
printf("weights are loaded from bias_hid2out_weight.dat file\n");
///************ Loaded ***************************
///**************** Load fully connected network weights. ********************
///********** Load fc_hidden_weight ********************
/// fc_hidden_weight[Hidden_nodes+1][fully_hidd_nodes]
sprintf(filename, "fc_hidden_weight.dat");//Assigne a filename with index number added
fp2 = fopen(filename, "r");
if (fp2 == NULL)
{
printf("Error while opening file fc_hidden_weight.dat");
exit(0);
}
fread(f_data_fc_h_w, sizeof f_data_fc_h_w[0], ((Hidden_nodes+1)*fully_hidd_nodes), fp2);
ix=0;
for(int n=0; n<fully_hidd_nodes; n++)
{
for(int p=0; p<Hidden_nodes+1; p++)
{
fc_hidden_weight[p][n] = f_data_fc_h_w[ix];
ix++;
}
}
fclose(fp2);
printf("weights are loaded at fc_hidden_weight.dat file\n");
///********** End Load fc_hidden_weight ********************
///********** Load fc_output_weight ********************
/// fc_output_weight[fully_hidd_nodes+1][fully_out_nodes]
sprintf(filename, "fc_output_weight.dat");//Assigne a filename with index number added
fp2 = fopen(filename, "r");
if (fp2 == NULL)
{
printf("Error while opening file fc_output_weight.dat");
exit(0);
}
fread(f_data_fc_o_w, sizeof f_data_fc_o_w[0], ((fully_hidd_nodes+1)*fully_out_nodes), fp2);
ix=0;
for(int n=0; n<fully_out_nodes; n++)
{
for(int p=0; p<fully_hidd_nodes+1; p++)
{
fc_output_weight[p][n] = f_data_fc_o_w[ix];
ix++;
}
}
fclose(fp2);
printf("weights are loaded at fc_output_weight.dat file\n");
///********** End Load fc_output_weight ********************
///************ Loaded ***************************
}
visual_all_feature += 0.5f;
Mat color_img, gray, noised_input, normalized_local;///
// outimg.create(Height, Width, CV_32F);
// srand (static_cast <unsigned> (time(0)));//Seed the randomizer
// noised_input.create(Height, Width, CV_32F);
noised_input.create(Height, Width, CV_32F);
float error=0.0;
float *zero_ptr_gray = gray.ptr<float>(0);
float *ptr_gray = gray.ptr<float>(0);
float *ptr_noised_inp = noised_input.ptr<float>(0);
float *zero_ptr_noised_inp = noised_input.ptr<float>(0);
ptr_gray = zero_ptr_gray + (gray.cols * (gray.rows/2)) + gray.cols/2;
float test_pixel_value = *ptr_gray;
int *noise_pixels;
noise_pixels = new int[nr_of_pixels];
int noise_p_counter=0;
float noise_ratio=0.0f;
int nr_of_noise_rand_ittr=0;
int rand_pix_pos=0;
char keyboard;
int started = 0;
gray.create(28,28,CV_32F);
print_help();
int print_only_100=0;
Mat labeling(80,80, CV_8UC3, Scalar(0,0,40));
while(1)
{
#ifdef USE_RASPICAM_INPUT
if(Pause_cam==0)
{
Camera.grab();
Camera.retrieve (src);
}
int rand_x_start=0;//Used to pick up a small part of camera image and feed in to the auto encoder
int rand_y_start=0;//Used to pick up a small part of camera image and feed in to the auto encoder
int max_point_x_or_y=0;
// waitKey(1);
src.convertTo(image, CV_32F, 1.0/255.0);//Convert pixels from 0..255 char to float 0..1
imshow("image", image);
max_point_x_or_y = image.cols - (Width*2);//Not allowed to set start x point so the small clip out rectangel can go outside the camera source image
rand_x_start = (int) (rand() % max_point_x_or_y);// range
max_point_x_or_y = image.rows - (Height*2);//Not allowed to set start y point so the small clip out rectangel can go outside the camera source image
rand_y_start = (int) (rand() % max_point_x_or_y);// range
// printf("rand_x_start %d\n", rand_x_start);
// printf("rand_y_start %d\n", rand_y_start);
Mat cam_part2(image, Rect(rand_x_start, rand_y_start, Width*2, Height*2));//Pick a small part of image and feed autoencoder with this
// imshow("cam_part2", cam_part2);
x4_window = cam_part2.clone();//Connect raspberry camera instead of posXXX.jpg image
normalized_local = image.clone();
if(Use_local_normaliz==1)
{
local_normalizing(x4_window);
// local_normalizing(normalized_local);//Only for view
}
// imshow("normalized", normalized_local);
imshow("x4_window", x4_window);
#endif // USE_RASPICAM_INPUT
///*********************************************************************
///read data from ************* train-images-idx3-ubyte ************ file
///*********************************************************************
training_image = (int) (rand() % nr_of_pictures);
/// read from data
zero_ptr_gray = gray.ptr<float>(0);
for(int n=0; n<nr_of_pixels; n++)
{
ptr_gray = zero_ptr_gray + n;
*ptr_gray = MNIST_data[(nr_of_pixels*training_image) +n];
}
color_img.create(28,28,CV_32F);
/// imshow("color_img", color_img);
gray.convertTo(gray, CV_32F, 1.0/255.0);//Convert pixels from 0..255 char to float 0..1
// local_normalizing(x4_window);
relu_mat(gray);
zero_ptr_gray = gray.ptr<float>(0);
ptr_gray = gray.ptr<float>(0);
// zero_ptr_gray = cam_part2.ptr<float>(0);
// ptr_gray = cam_part2.ptr<float>(0);
///******************************************************
///************ Select noise pixels positions ***********
///******************************************************
if(fully_conn_backprop==0)
{
///L1 noise
noise_p_counter=0;
noise_ratio=0.0f;
nr_of_noise_rand_ittr=0;
rand_pix_pos=0;
for(int n=0; n<nr_of_pixels; n++)
{
noise_pixels[n] = 0;
}
while(noise_ratio < (noise_percent*0.01f))
{
rand_pix_pos = (int) (rand() % nr_of_pixels);
if(noise_pixels[rand_pix_pos] == 0)
{
noise_p_counter++;
}
noise_pixels[rand_pix_pos] = 1;
noise_ratio = ((float)noise_p_counter) / ((float)nr_of_pixels);
nr_of_noise_rand_ittr++;
if(nr_of_noise_rand_ittr > 2*nr_of_pixels)
{
printf("give up fill random up noise this turn\n");
printf("noise_ratio %f\n", noise_ratio);
break;
}
}
#ifndef USE_IND_NOISE
///****************************************************
///********** Select noise level ******************
///****************************************************
noise = (float) (rand() % 65535) / 65536;//0..1.0 range
noise -= 0.5f;
noise = noise * noise_amplitude;
noise += 0.5f;
noise += noise_offset;
#endif // USE_IND_NOISE
}
///****************************************************
///**************** Feed forward **********************
///****************************************************
///Connect image pixel and also insert noise to input_node[]
for(int n=0; n<nr_of_pixels; n++)
{
if(noise_pixels[n] == 1 && started == 1 && fully_conn_backprop==0)
{
#ifdef USE_IND_NOISE
///****************************************************
///********** Select noise level ******************
///****************************************************
noise = (float) (rand() % 65535) / 65536;//0..1.0 range
noise -= 0.5f;
noise = noise * noise_amplitude;
noise += 0.5f;
noise += noise_offset;
#endif // USE_IND_NOISE
#ifdef USE_WHITE_NOISE_ONLY
input_node[n] = 1.0f;///Insert noise 0.5f instead of real pixel value
#else
input_node[n] = noise;///Insert noise 0.5f instead of real pixel value
#endif // USE_WHITE_NOISE_ONLY
}
else
{
ptr_gray = zero_ptr_gray + n;
input_node[n] = *ptr_gray;///Insert pixel value
}
ptr_noised_inp = zero_ptr_noised_inp + n;
*ptr_noised_inp = input_node[n];//To show
}
imshow("noised_input ", noised_input);
///*********** Forward to hidden nodes ****************
for(int j=0; j<Hidden_nodes; j++)
{
hidden_node[j] = 0;
for(int i=0; i<nr_of_pixels; i++)
{
hidden_node[j] += input_node[i] * weight_matrix_M[i][j];///Sum up values from input gained with weight
}
hidden_node[j] += Bias_level * bias_weight_in2hid[j];///Add weighted bias signal also
///*** Relu this node ***
if(hidden_node[j] < 0.0f)
{
hidden_node[j] = hidden_node[j] * Relu_neg_gain;///Relu function
}
}
///********** Forward to L1 output nodes *******************
for(int i=0; i<nr_of_pixels; i++)
{
output_node[i] = 0.0f;
for(int j=0; j<Hidden_nodes; j++)
{
output_node[i] += hidden_node[j] * weight_matrix_M[i][j];///Sum up values from hidden gained with weight
}
output_node[i] += Bias_level * bias_weight_hid2out[i];///Add weighted bias signal also
///*** Relu this node ***
if(output_node[i] < 0.0f)
{
output_node[i] = output_node[i] * Relu_neg_gain;///Relu function
}
}
float input_pixel =0.0f;
///****************************************************
///************* Calculate total loss *****************
///****************************************************
if(fully_conn_backprop==0)
{
/// k = nr_of_pixel
/// loss = -SUM k (output[k] * log(input[k]) + (1.0 - output[k]) * log(1.0 - input[k]))
/// or
/// loss = 1/2 SUM k (input[k] - ouput[k])²
///L1 loss
float loss=0.0f;
loss=0.0f;
for(int i=0; i<nr_of_pixels; i++)
{
ptr_gray = zero_ptr_gray + i;///pick real input pixel
input_pixel = *ptr_gray;///Insert pixel value
loss += (input_pixel - output_node[i]) * (input_pixel - output_node[i]);/// loss = 1/2 SUM k (input[k] - ouput[k])²
}
static int print_loss=0;
if(print_loss<10)
{
print_loss++;
}
else
{
if(started == 1)
{
if(Lock_L1==0)
{
printf("L1 loss error = %f\n", loss);
}
}
print_loss=0;
}
}
if(fully_conn_backprop==0)
{
///****************************************************
///**************** Backpropagation Autoencoder *******
///****************************************************
///L1 Backprop
/// **** make Delta value for backpropagation and loss calculation ****
for(int j=0; j<Hidden_nodes; j++)
{
hid_node_delta[j] = 0.0f;///clear delta regarding Bias backpropagation
}
float delta_pixel=0.0f;
for(int i=0; i<nr_of_pixels; i++)
{
ptr_gray = zero_ptr_gray + i;///pick real input pixel
input_pixel = *ptr_gray;///Insert pixel value
delta_pixel = input_pixel - output_node[i] ;/// detal calculus
#ifdef USE_DELTA_RELU_REDUCED_NEG
if(output_node[i] < 0.0f)
{
delta_pixel *= Relu_neg_gain;
}
#endif // USE_DELTA_RELU_REDUCED_NEG
for(int j=0; j<Hidden_nodes; j++)
{
///*** Bias backpropagation ***
///Back propagation from output nodes to hid2out weight
hid_node_delta[j] += (delta_pixel/2) * weight_matrix_M[i][j];///delta_pixel/2 because this delta is normaly used for tied weight
///*** End bias backprop ***********
if(started == 1)
{
/// **** update tied weight regarding delta
change_weight_M[i][j] = LearningRate * hidden_node[j] * delta_pixel + Momentum * change_weight_M[i][j];
weight_matrix_M[i][j] += change_weight_M[i][j];
}
}
if(started == 1)
{
///Update hidden bias to output weights bias_weight_hid2out[]
/// Buggfix Bias_level instead of ...node[i]
change_bias_weight_hid2out[i] = (LearningRate/2) * Bias_level * delta_pixel + Momentum * change_bias_weight_hid2out[i];
bias_weight_hid2out[i] += change_bias_weight_hid2out[i];
}
}///End for(int i=0;i<nr_of_pixels;i++)
///End L1 backprop
///***************************************************
///************* End backprop ************************
///***************************************************
///==========================================================================
///*** Update input bias to hidden weights L1 bias_weight_in2hid
for(int j=0; j<Hidden_nodes; j++)
{
if(started == 1)
{
///Now use the delta from hid_node_delta[j]
/// Bugg
// change_bias_weight_in2hid[j] = (LearningRate/2) * hidden_node[j] * hid_node_delta[j] + Momentum * change_bias_weight_in2hid[j];
/// Buggfix Bias_level instead of hidden_node[j]
change_bias_weight_in2hid[j] = (LearningRate/2) * Bias_level * hid_node_delta[j] + Momentum * change_bias_weight_in2hid[j];
bias_weight_in2hid[j] += change_bias_weight_in2hid[j];
}
}
///*** End of update bias to hidden weights L1 bias_weight_in2hid **************
///==========================================================================
///***************************************************
///************* End update weights ******************
///***************************************************
}/// End Autoencoder backpropagate unsupervised learning backprop
else /// fully_conn_backprop == 1
{
///***********************************************************************************************************************************
///***********************************************************************************************************************************
///********************************** Supervised Learning Logistic regression ********************************************************
///***********************************************************************************************************************************
///***********************************************************************************************************************************
///***************************************************************************
///************* Feed forward L1 hidden node to fc_input_node ****************
///***************************************************************************
float Accum=0.0f;
///void randomize_dropoutHid(int *zero_ptr_dropoutHidden, int HiddenNodes, int verification)
if(started==1 && Learning_fc == 1)
{
verification=0;
}
else
{
verification=1;
}
randomize_dropoutHid(&dropoutHidden[0], fully_hidd_nodes, verification);///select dropout node to the hidden node
for(int i=0; i<Hidden_nodes; i++)///Connect and sigmoid the Autencoder L2 nodes to fully connected neural network
{
fc_input_node[i] = 1.0/(1.0 + exp(-(hidden_node[i])));///Sigmoid function. x = 1.0/(1.0 + exp(-(x)))
/*
if(started==0)
{
printf("hidden_node[%d] = %f\n", i , hidden_node[i]);
printf("fc_input_node[%d] = %f\n", i, fc_input_node[i]);
}
*/
}
for(int j=0; j<fully_hidd_nodes; j++)
{
Accum = fc_hidden_weight[Hidden_nodes][j] * Bias_level;///Begin with the Bias node as the start value
for(int i=0; i<Hidden_nodes; i++)
{
Accum += fc_hidden_weight[i][j] * fc_input_node[i];///Weight in each input data modified by sigmoid from autoencoder feature map into the fully connected neural network hidden nodes
}
if(dropoutHidden[j] == 0)
{
///Normal forward not drop out this node
fc_hidden_node[j] = 1.0/(1.0 + exp(-(Accum)));///Sigmoid function. x = 1.0/(1.0 + exp(-(x)))
}
else
{
fc_hidden_node[j] = 0.0f;
}
/// printf("fc_hidden_node[%d] =%f\n", j, fc_hidden_node[j]);
}
///***************************************************************************
///***************************************************************************
///***************************************************************************
///Compare the actual output value from feed forward to target
///Insert target value into the fc_target_node from lable database
for(int n=0; n<fully_out_nodes; n++)
{
if(n == ((int) MNIST_lable[training_image]))
{
///fc_target_node[n] = 1.0f;///This is the node how is correspond to the lable digits
fc_target_node[n] = High_Target_value;///This is the node how is correspond to the lable digits
}
else
{
///fc_target_node[n] = 0.0f;///This is NOT the node how is correspond to the lable digits
fc_target_node[n] = Low_Target_value;///This is the node how is correspond to the lable digits
}
}
Error_level = 0.0f;
///***************************************************************************
///*************** Feed forward fc_hidden_node[] to fc_output_node[] *********
///***************************************************************************
for(int j=0; j<fully_out_nodes; j++)
{
Accum = fc_output_weight[fully_hidd_nodes][j] * Bias_level;///Begin with the Bias node as the start value
for(int i=0; i<fully_hidd_nodes; i++)
{
Accum += fc_output_weight[i][j] * fc_hidden_node[i];///
}
fc_output_node[j] = 1.0/(1.0 + exp(-(Accum)));///Sigmoid function. x = 1.0/(1.0 + exp(-(x)))
fc_output_delta[j] = (fc_target_node[j] - fc_output_node[j]) * fc_output_node[j] * (1.0f - fc_output_node[j]);
/*
if(started == 0)
{
// printf("debug_counter %d training target %d j = %d\n", debug_counter, training_image, j);
printf("fc_output_node[%d] = %f\n",j, fc_output_node[j]);
printf("fc_target_node[%d] = %f\n", j, fc_target_node[j]);
printf("fc_output_delta[j] = %f\n", fc_output_delta[j]);
}
*/
Error_level += 0.5 * (fc_target_node[j] - fc_output_node[j]) * (fc_target_node[j] - fc_output_node[j]);///
}
if(started == 0 || print_only_100==0)
{
printf("MNIST_lable[training_image] %d\n", MNIST_lable[training_image]);
printf("Error_level =%f\n", Error_level);
}
if(print_only_100<100)
{
print_only_100++;
}
else
{
print_only_100=0;
}
///***************************************************************************
///**************** End feed forward *****************************************
///***************************************************************************
///***************************************************************************
///**************** Backpropagate fully connected network ********************
///***************************************************************************
/******************************************************************
* Backpropagate errors from output layer to hidden layer
******************************************************************/
for(int i = 0 ; i < fully_hidd_nodes ; i++ )
{
Accum = 0.0 ;
for(int j = 0 ; j < fully_out_nodes ; j++ ) {
Accum += fc_output_weight[i][j] * fc_output_delta[j] ;
}
if(dropoutHidden[i] == 0)
{
fc_hidden_delta[i] = Accum * fc_hidden_node[i] * (1.0 - fc_hidden_node[i]);///Backpropagate gradiant decent
}
else
{
fc_hidden_delta[i] = 0.0f;/// Hidden node delta zero when drop out no change of the weight regarding this backprop
}
}
/******************************************************************
* Backpropagate errors from hidden layer to fc input layer
******************************************************************/
for(int i = 0 ; i < Hidden_nodes ; i++ )
{
Accum = 0.0 ;
for(int j = 0 ; j < fully_hidd_nodes ; j++ ) {
Accum += fc_hidden_weight[i][j] * fc_hidden_delta[j] ;
}
fc_input_delta[i] = Accum * fc_input_node[i] * (1.0 - fc_input_node[i]) ;///Backpropagate gradiant decent
}
/******************************************************************
* Backpropagate errors from fc_input_delta[] layer to hid_node_delta[j]
******************************************************************/
///hid_node_delta[j]
for(int j = 0 ; j < Hidden_nodes ; j++ )
{
hid_node_delta[j] = fc_input_delta[j];
}
///**************** End backprop **************
if(started == 1)
{
/******************************************************************
* Update Inner-->Hidden Weights
******************************************************************/
if(Learning_fc == 1)
{
for(int i = 0 ; i < fully_hidd_nodes ; i++ )
{
fc_change_hidden_weight[Hidden_nodes][i] = fc_LearningRate * Bias_level * fc_hidden_delta[i] + fc_Momentum * fc_change_hidden_weight[Hidden_nodes][i] ;///Begin with update Bias change weights
if(dropoutHidden[i] == 0)
{
fc_hidden_weight[Hidden_nodes][i] += fc_change_hidden_weight[Hidden_nodes][i];///Begin with update Bias weights
for(int j = 0 ; j < Hidden_nodes ; j++ )
{
fc_change_hidden_weight[j][i] = fc_LearningRate * fc_input_node[j] * fc_hidden_delta[i] + fc_Momentum * fc_change_hidden_weight[j][i];
fc_hidden_weight[j][i] += fc_change_hidden_weight[j][i] ;
}
}
}
}
/******************************************************************
* Update Hidden-->Output Weights
******************************************************************/
if(Learning_fc == 1)
{
for(int i = 0 ; i < fully_out_nodes ; i++ )
{
fc_change_output_weight[fully_hidd_nodes][i] = fc_LearningRate * Bias_level * fc_output_delta[i] + fc_Momentum * fc_change_output_weight[fully_hidd_nodes][i] ;///Begin with update Bias change weights
fc_output_weight[fully_hidd_nodes][i] += fc_change_output_weight[fully_hidd_nodes][i];///Begin with update Bias weights
for(int j = 0 ; j < fully_hidd_nodes ; j++ )
{
if(dropoutHidden[j] == 0)
{
fc_change_output_weight[j][i] = fc_LearningRate * fc_hidden_node[j] * fc_output_delta[i] + fc_Momentum * fc_change_output_weight[j][i];
fc_output_weight[j][i] += fc_change_output_weight[j][i] ;
}
}
}
}
///***********
}///End if (started == 1)
///***************************************************************************
///**************** END Backpropagate fully connected network ****************
///***************************************************************************
///*******************************************************************************************************
///**************** Backpropagate and fine tune L2 and L1 autoencoder feature layers ********************
///*******************************************************************************************************
/******************************************************************
* Update L1 weights fine tune Supervised learning
******************************************************************/
for(int i=0; i<(nr_of_pixels); i++)
{
for(int j=0; j<(Hidden_nodes); j++)
{
if(started == 1)
{
/// **** update tied weight regarding supervised backprop delta fc_input_delta
change_weight_M[i][j] = supervised_L1_LearningRate * input_node[i] * hid_node_delta[j] + supervised_L1_Momentum * change_weight_M[i][j];
weight_matrix_M[i][j] += change_weight_M[i][j];
}
}
}
///Update bias node
for(int j=0; j<(Hidden_nodes); j++)
{
if(started == 1)
{
///Update bias node
change_bias_weight_in2hid[j] = supervised_L1_LearningRate * Bias_level * hid_node_delta[j] + supervised_L1_Momentum * change_bias_weight_in2hid[j];
bias_weight_in2hid[j] += change_bias_weight_in2hid[j];
}
}
///xxxxxxxxxxxx
///xxxxxxxxxxxx
///xxxxxxxxxxxx
///xxxxxxxxxxxx
///xxxxxxxxxxxx
}
///********** Visualization **************************
float *zero_ptr_outimg = outimg.ptr<float>(0);
float *ptr_outimg = outimg.ptr<float>(0);
for(int i=0; i<nr_of_pixels; i++)
{
ptr_outimg = zero_ptr_outimg + i;
*ptr_outimg = output_node[i];
}
imshow("gray", gray);
imshow("outimg", outimg);
if(started == 0)
{
waitKey(500);
}
else
{
waitKey(1);
}
// gray -= 0.5;
// outimg -= 0.5;
// visual_all_feature -= 0.5;
// gray *= 2;
// outimg *= 2;
// visual_all_feature *= 2;
// gray += 0.5;
// outimg += 0.5;
// visual_all_feature += 0.5;
///L1 visual
for(int i=0; i<nr_of_pixels; i++)
{
ptr_M_matrix = &weight_matrix_M[i][0];
attach_weight_2_mat(ptr_M_matrix, i, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);
}
/// *******************************************************************
/// ********* Also show bias weight with 2 patches ********************
/// attach_in2hid_w_2_mat() will add 1 patches to show input to hidden bias weights
ptr_bias_w = &bias_weight_in2hid[0];///Set the start pos of pointer so function below could read out the weight array
attach_in2hid_w_2_mat(ptr_bias_w, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);///Hidden_nodes is because the function needs to know were to end visualization of patches
/// attach_hid2out_w_2_mat() will add 1 patches to show hidden to output bias weights
ptr_bias_w = &bias_weight_hid2out[0];///Set the start pos of pointer so function below could read out the weight array
attach_hid2out_w_2_mat(ptr_bias_w, visual_all_feature, sqr_of_H_nod_plus1, Hidden_nodes, Height, Width);///Hidden_nodes is because the function needs to know were to end visualization of patches
/// *******************************************************************
/// *******************************************************************
visual_all_feature += 0.5f;
imshow("visual_all_feature", visual_all_feature);
///******** start stop
if(kbhit())
{
keyboard = getchar();
if(keyboard== ' ')
{
if(started == 1)
{
started = 0;
printf("Stop training\n");
printf("Training stop now only feed forward\n");
}
else
{
started = 1;
printf("Start training\n");
}
}
if(keyboard== 'L' || keyboard== 'l')
{
Lock_L1 = 1;
printf("Lock_L1 = %d\n", Lock_L1);
}
save_vis_all_feature_jpg=0;
if(keyboard== 'P' || keyboard== 'p')
{
Pause_cam = 1;///Pause camera
printf("Pause_cam = %d\n", Pause_cam);
cv::imwrite("paused_camera.JPG",src);
save_vis_all_feature_jpg=1;
}
if(keyboard== 'I' || keyboard== 'i')
{
printf("Pause and Read image from paused_camera.JPG file now\n");
Pause_cam = 1;///Pause camera
printf("Pause_cam = %d\n", Pause_cam);
Mat jpg_image_src,jpg_image_dst;
jpg_image_src = cv::imread("paused_camera.JPG", 1);
Size size(cam_w,cam_h);//the dst image size,e.g.100x100
resize(jpg_image_src,jpg_image_dst,size);//resize image
cvtColor(jpg_image_dst,src,CV_BGR2GRAY);
}
if(keyboard== 'C' || keyboard== 'c')
{
///************* Initialize randomized noise on fc_weight *********
for(int i=0; i<Hidden_nodes+1; i++)
{
for(int j=0; j<fully_hidd_nodes; j++)
{
Rando = (float) (rand() % 65535) / 65536;//0..1.0 range
Rando -= 0.5f;
Rando *= fc_start_weight_noise_range;
fc_hidden_weight[i][j] = Rando;///Noise around 0.5f
fc_change_hidden_weight[i][j] = 0.0f;///Initialize with zero
}
}
for(int i=0; i<fully_hidd_nodes+1; i++)
{
for(int j=0; j<fully_out_nodes; j++)
{
Rando = (float) (rand() % 65535) / 65536;//0..1.0 range
Rando -= 0.5f;
Rando *= fc_start_weight_noise_range;
fc_output_weight[i][j] = Rando;///Noise around 0.5f
fc_change_output_weight[i][j] = 0.0f;///Initialize with zero
}
}
printf("Cleared with noise the fully connected network weights\n");
waitKey(5000);
///***********************************
}
if(keyboard== 'R' || keyboard== 'r')
{
Pause_cam = 0;///run camera
printf("Pause_cam = %d\n", Pause_cam);
}
/*
if(keyboard== 'Q' || keyboard== 'q')
{
fully_conn_backprop =1;
printf("fully_conn_backprop = %d\n", fully_conn_backprop);
}
if(keyboard== 'W' || keyboard== 'w')
{
fully_conn_backprop =0;
printf("fully_conn_backprop = %d\n", fully_conn_backprop);
}
*/
if(keyboard== 'T' || keyboard== 't')
{
Learning_fc =0;
printf("Learning_fc = %d\n", Learning_fc);
}
if(keyboard== 'H' || keyboard== 'h')
{
Learning_fc =1;
printf("Learning_fc = %d\n", Learning_fc);
}
if(keyboard== 'U' || keyboard== 'u')
{
Lock_L1 = 0;
printf("Lock_L1 = %d\n", Lock_L1);
}
if(keyboard== 'F' || keyboard== 'f')
{
fully_conn_backprop = 1;
printf(" fully_conn_backprop = %d\n", fully_conn_backprop);
}
if(keyboard== 'G' || keyboard== 'g')
{
fully_conn_backprop = 0;
printf(" fully_conn_backprop = %d\n", fully_conn_backprop);
}
if(keyboard== 'Z' || keyboard== 'z')
{
if(norm_typ_gain_ON>0)
{
norm_typ_gain_ON = 0;
}
else
{
norm_typ_gain_ON = 1;
}
printf("norm_typ_gain_ON = %d\n", norm_typ_gain_ON);
}
if(keyboard== '?')
{
started = 0;
print_help();
}
if(Lock_L1==0)
{
LearningRate =C_LearningRate;///depend on the state of Lock_L1
Momentum = C_Momentum;///depend on the state of Lock_L1
noise_percent = C_noise_percent;
// L2_noise_percent = 0.0f;
}
else
{
LearningRate =0.0f;///depend on the state of Lock_L1
Momentum = 0.0f;///depend on the state of Lock_L1
noise_percent = 0.0f;
// L2_noise_percent = C_L2_noise_percent;
}
if(keyboard== 'S' || keyboard== 's')
{
///L1 save weights
//Save weights
sprintf(filename, "weight_matrix_M.dat");//Assigne a filename with index number added
fp2 = fopen(filename, "w+");
if (fp2 == NULL)
{
printf("Error while opening file weight_matrix_M.dat");
exit(0);
}
//size_t fread(void *ptr, size_t size_of_elements, size_t number_of_elements, FILE *a_file);
ix=0;
for(int n=0; n<Hidden_nodes; n++)
{
for(int p=0; p<nr_of_pixels; p++)
{
f_data[ix] = weight_matrix_M[p][n];
ix++;
}
}
fwrite(f_data, sizeof f_data[0], (nr_of_pixels*Hidden_nodes), fp2);
fclose(fp2);
printf("weights are saved at weight_matrix_M.dat file\n");
///*** Save bias weight from in2hid layer *******
sprintf(filename, "bias_in2hid_weight.dat");//Assigne a filename with index number added
fp2 = fopen(filename, "w+");
if (fp2 == NULL)
{
printf("Error while opening file bias_in2hid_weight.dat");
exit(0);
}
fwrite(bias_weight_in2hid, sizeof bias_weight_in2hid[0], Hidden_nodes, fp2);
fclose(fp2);
printf("weights are saved at bias_in2hid_weight.dat file\n");
///************ Saved ***************************
///*** Save bias weight from hid2out layer *******
sprintf(filename, "bias_hid2out_weight.dat");//Assigne a filename with index number added
fp2 = fopen(filename, "w+");
if (fp2 == NULL)
{
printf("Error while opening file bias_hid2out_weight.dat");
exit(0);
}
fwrite(bias_weight_hid2out, sizeof bias_weight_hid2out[0], nr_of_pixels, fp2);
fclose(fp2);
printf("weights are saved at bias_hid2out_weight.dat file\n");
///************ L1 Saved ***************************
///********** Save fc_hidden_weight ********************
/// fc_hidden_weight[Hidden_nodes+1][fully_hidd_nodes]
sprintf(filename, "fc_hidden_weight.dat");//Assigne a filename with index number added
fp2 = fopen(filename, "w+");
if (fp2 == NULL)
{
printf("Error while opening file fc_hidden_weight.dat");
exit(0);
}
ix=0;
for(int n=0; n<fully_hidd_nodes; n++)
{
for(int p=0; p<Hidden_nodes+1; p++)
{
f_data_fc_h_w[ix] = fc_hidden_weight[p][n];
ix++;
}
}
fwrite(f_data_fc_h_w, sizeof f_data_fc_h_w[0], ((Hidden_nodes+1)*fully_hidd_nodes), fp2);
fclose(fp2);
printf("weights are saved at fc_hidden_weight.dat file\n");
///********** End Save fc_hidden_weight ********************
///********** Save fc_output_weight ********************
/// fc_output_weight[fully_hidd_nodes+1][fully_out_nodes]
sprintf(filename, "fc_output_weight.dat");//Assigne a filename with index number added
fp2 = fopen(filename, "w+");
if (fp2 == NULL)
{
printf("Error while opening file fc_output_weight.dat");
exit(0);
}
ix=0;
for(int n=0; n<fully_out_nodes; n++)
{
for(int p=0; p<fully_hidd_nodes+1; p++)
{
f_data_fc_o_w[ix] = fc_output_weight[p][n];
ix++;
}
}
fwrite(f_data_fc_o_w, sizeof f_data_fc_o_w[0], ((fully_hidd_nodes+1)*fully_out_nodes), fp2);
fclose(fp2);
printf("weights are saved at fc_output_weight.dat file\n");
///********** End Save fc_output_weight ********************
}
}
///***************
///*********** Visualize hidden nodes on a image **********
cvtColor(visual_all_feature,colour_vis_h,CV_GRAY2RGB);
if(save_vis_all_feature_jpg == 1)
{
Mat cloned_vis_h;
cloned_vis_h = colour_vis_h.clone();
cloned_vis_h *= 255;
cv::imwrite("cloned_vis_h.JPG",cloned_vis_h);
}
zero_ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);
ptr_vis_hidd_node = vis_hidd_node.ptr<float>(0);
int index_hid_nodes=0;
int pix_n=0;
index_hid_nodes=0;
pix_n=0;
for(int ro=0; ro<vis_hidd_node.rows; ro++)
{
for(int co=0; co<vis_hidd_node.rows; co++)
{
if(index_hid_nodes<Hidden_nodes)
{
ptr_vis_hidd_node = zero_ptr_vis_hidd_node + pix_n;
*ptr_vis_hidd_node = hidden_node[index_hid_nodes];
///************* visualize on vis_hidd_node pattern **********
///************* with green light ****************************
int circle_zize = 2;
int x,y;
float light_level=0.0f;
float light_red=0.0f;
light_level = *ptr_vis_hidd_node;
if(light_level > 1.0f)
{
light_level=1.0f;
}
if(light_level < 0.0f)
{
light_level=0.0f;
}
light_red = 1.0f - light_level;
x = (index_hid_nodes%sqr_of_H_nod_plus1)*Width + (Width/2);
y = (index_hid_nodes/sqr_of_H_nod_plus1)*Height + (Height/2);
P1.x = x;
P1.y = y;
circle(colour_vis_h, P1, circle_zize, Scalar(0,light_level,light_red), 2, -1);
index_hid_nodes++;
///***********************************************************
}
else
{
*ptr_vis_hidd_node = 0.0f;
}
pix_n++;
}
}
imshow("vis_hidd_node", vis_hidd_node);
imshow("colour_vis_h", colour_vis_h);
///******************** Visualize the highest category **********************************************
char Highest_rate =0;
string highest = "x";
std::string::iterator It = highest.begin();
It = highest.begin();
float compare_outpnodes = 0.0f;
Highest_rate =0;
compare_outpnodes = 0.0f;
for(int i = 0 ; i < fully_out_nodes ; i++ )
{
if(compare_outpnodes < fc_output_node[i])
{
compare_outpnodes = fc_output_node[i];
Highest_rate = i;
}
}
waitKey(1);
//CvPoint num_pos = (15,100);
*It = Highest_rate+48;
labeling.setTo(cv::Scalar(50,0,0));
cv::putText(labeling, highest, cvPoint(15,60), CV_FONT_HERSHEY_PLAIN, 4, cvScalar(0,255,0),3);
//void putText(Mat& img, const string& text, Point org, int fontFace, double fontScale, Scalar color, int thickness=1, int lineType=8, bool bottomLeftOrigin=false )¶
imshow("lable", labeling);
///********************************************************
if(started == 0)
{
waitKey(3000);
}
supervised_L1_LearningRate = 0.001*((float)fine_tune_L1_int);
supervised_L1_Momentum = 0.001*((float)fine_tune_L1_moment_int);
if(save_first_images < 6)
{
Mat color_image,save_image,gray2;
sprintf(filename, "first_image%d.JPG", save_first_images);//Assigne a filename with index number added
gray2= gray.clone();
gray2=gray2*-1.0;
gray2+=1.0;
cvtColor(gray2,color_image,CV_GRAY2RGB);
save_image = color_image.clone();
save_image *= 255;
cv::imwrite(filename,save_image);
save_first_images++;
}
if(norm_typ_gain_ON==1)
{
/// feature_type_gain
float sum_up_weights= 0.0f;
float feature_auto_gain= 0.0f;
for(int j=0; j<Hidden_nodes; j++)
{
sum_up_weights= 0.0f;
for(int i=0; i<nr_of_pixels; i++)
{
sum_up_weights += abs_value(weight_matrix_M[i][j]);
}
if(sum_up_weights == 0.0f)
{
sum_up_weights = 0.00000001f;///Protect agains zero divitions
}
sum_up_weights = sum_up_weights / nr_of_pixels;
///kernel_gain_level[0][go_t_f_ker] = FEAT0_TYP_GAIN / kernel_gain_level[0][go_t_f_ker];
feature_auto_gain = feature_type_gain / sum_up_weights;
for(int i=0; i<nr_of_pixels; i++)
{
weight_matrix_M[i][j] *= feature_auto_gain;
}
}
/// absolut_value = abs_value(Feature0Kernel[go_t_f_ker][kernel_index]);
}
waitKey(1);
}
return 0;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment