multi-layer.cpp

#include <iostream>
#include <vector>
#include <cmath>
#include <ctime>
#include <cstdlib>
#include <iomanip>
#include <thread>
#include <mutex>

using namespace std;


inline unsigned long long rdtsc()
{
	#ifdef _MSC_VER
	return __rdtsc();
	#else
		unsigned long long ret;
		asm volatile("rdtsc": "=A"(ret));
		return ret;
	#endif
}

inline double sigmoid(double value)
{
	return 	1/(1+exp(-value));
}

inline double sigmoid_derivative(double value)
{
	return sigmoid(value)*(1-sigmoid(value));
}

inline double rational_sigmoid(double value)
{
	return value/(abs(value)+1);
}

inline double rational_sigmoid_derivative(double value)
{
	return ((abs(value)+1)-value)/((abs(value)+1)*(abs(value)+1));
}

inline double hyp_tan(double value)
{
	return tanh(value);
}

inline double hyp_tan_derivative(double value)
{
	return 1-tanh(value)*tanh(value);
}

template<class FnType>
void do_parallel(unsigned int threads_count, FnType fn)
{
	std::vector<std::thread> threads;
	for(unsigned int it=0; it<threads_count; it++)
	{
		threads.push_back(std::thread(fn));
	}
	for(auto& it: threads)
	{
		it.join();
	}
}


class neuron
{
public:

	neuron(): result_value(0), error_signal(0){}

	neuron(unsigned int inputs_count): result_value(0), error_signal(0)
	{
		setup_weights(inputs_count);
	}

	void setup_weights(unsigned int inputs_count)
	{
		srand((unsigned int)rdtsc());
		weights.assign(inputs_count, 0);
		for(auto& it: weights)
		{
			it=((double)rand()/(double)RAND_MAX);
		}
	}

	void calculate(std::vector<double>& inputs)
	{
		double temp_result=0;
		for(unsigned int it=0; it<inputs.size(); it++)
		{
			temp_result+=inputs[it]*weights[it];
		}
		result_value=activation(temp_result);
	}

	inline double activation(double value)
	{
		return hyp_tan(value);
	}

	void correct_weights(std::vector<double>& inputs, double learning_speed, double error)
	{
		for(unsigned int it=0; it<weights.size(); it++)
		{
			double current_weight_error=error*weights[it];
			weights[it]+=(-learning_speed)*current_weight_error*hyp_tan_derivative(result_value)*inputs[it];
		}
	}

	double result_value;
	double error_signal;
	std::vector<double> weights;
};

class neuron_layer
{
public:

	neuron_layer(){}

	neuron_layer(unsigned int inputs_count, unsigned int neurons_count)
	{
		neurons.assign(neurons_count, neuron(inputs_count));
	}

	void calculate(std::vector<double> inputs)
	{
		for(unsigned int it=0; it<inputs.size(); it++)
		{
			neurons[it].calculate(inputs);
		}
	}

	void calculate(neuron_layer& prev_layer)
	{
		std::vector<double> prev_layer_results(prev_layer.neurons.size(), 0);
		for(unsigned int it=0; it<prev_layer_results.size(); it++)
		{
			prev_layer_results[it]=prev_layer.neurons[it].result_value;
		}
		for(unsigned int it=0; it<neurons.size(); it++)
		{
			neurons[it].calculate(prev_layer_results);
		}
	}

	//update error for non output layer
	void update_error_signal(neuron_layer& prev_layer)
	{
		for(unsigned int it=0; it<neurons.size(); it++)
		{
			double temp_error=0;
			for(unsigned int st=0; st<prev_layer.neurons.size(); st++)
			{
				temp_error+=prev_layer.neurons[st].weights[it]*prev_layer.neurons[st].error_signal;
			}
			neurons[it].error_signal=temp_error;
		}
	}

	//update error for output layer
	void update_error_signal(std::vector<double>& expected_outputs)
	{
		for(unsigned int it=0; it<neurons.size(); it++)
		{
			neurons[it].error_signal=expected_outputs[it]-neurons[it].result_value;
		}
	}

	//correct weights for non input layer
	void correct_weights(neuron_layer& prev_layer, double learning_speed)
	{
		for(unsigned int it=0; it<neurons.size(); it++)
		{
			for(unsigned int st=0; st<neurons[it].weights.size(); st++)
			{
				neurons[it].weights[st]+=learning_speed*neurons[it].error_signal*
					sigmoid_derivative(neurons[it].result_value)*prev_layer.neurons[st].result_value;
			}
		}
	}

	//correct weights for input layer
	void correct_weights(std::vector<double>& inputs, double learning_speed)
	{
		for(unsigned int it=0; it<neurons.size(); it++)
		{
			for(unsigned int st=0; st<neurons[it].weights.size(); st++)
			{
				neurons[it].weights[st]+=learning_speed*neurons[it].error_signal*
					sigmoid_derivative(neurons[it].result_value)*inputs[st];
			}
		}
	}

	const std::vector<neuron>* const get_neurons() const
	{
		return &neurons;
	}

private:

	std::vector<neuron> neurons;
};

struct layer_config
{
	unsigned int inputs_count;
	unsigned int neurons_count;
};

class neural_net
{
public:

	neural_net(unsigned int inputs_count, std::vector<layer_config> layers_config, 
		unsigned int outputs_count, double learning): learning_speed(learning)
	{
		layers.push_back(neuron_layer(layers_config[0].inputs_count, layers_config[0].neurons_count));
		for(unsigned int it=1; it<layers_config.size(); it++)
		{
			layers.push_back(neuron_layer(layers_config[it].inputs_count, layers_config[it].neurons_count));
		}
		layers.push_back(neuron_layer((layers_config.end()-1)->neurons_count, outputs_count));
		outputs.assign(outputs_count, 0);
	}

	void calculate(std::vector<double> inputs)
	{
		(layers.begin())->calculate(inputs);
		for(unsigned int it=1; it<layers.size(); it++)
		{
			layers[it].calculate(layers[it-1]);
		}
		for(unsigned int it=0; it<outputs.size(); it++)
		{
			outputs[it]=(layers.end()-1)->get_neurons()->at(it).result_value;
		}
	}

	void update_error_signal(std::vector<double>& expected_outputs)
	{
		(layers.end()-1)->update_error_signal(expected_outputs);
		for(int it=layers.size()-2; it>-1; it--)
		{
			layers[it].update_error_signal(layers[it+1]);
		}
	}

	void correct_weights(std::vector<double>& inputs)
	{
		(layers.begin())->correct_weights(inputs, learning_speed);
		for(unsigned int it=1; it<layers.size(); it++)
		{
			layers[it].correct_weights(layers[it-1], learning_speed);
		}
	}

	void teach(std::vector<double> inputs, std::vector<double> expected_outputs)
	{
		calculate(inputs);
		update_error_signal(expected_outputs);
		correct_weights(inputs);
	}

	double calc_average_output_error()
	{
		double temp_err=0;
		for(auto& it: *(layers.end()-1)->get_neurons())
		{
			temp_err+=it.error_signal;
		}
		return temp_err/(layers.end()-1)->get_neurons()->size();
	}

	double learning_speed;
	std::vector<neuron_layer> layers;
	std::vector<double> outputs;
};

std::mutex cout_guard;

void train_network(neural_net& net, double diff_error, unsigned int iterations, unsigned int stat_show)
{
	//cout<<"Starting network training with "<<iterations<<" iterations"<<endl;
	time_t timer=clock();
	double best_avg_err=1;
	for(unsigned int it=0; it<iterations; it++)
	{
		if(it>iterations/10 && abs(net.calc_average_output_error())<=diff_error)
		{
			cout<<abs(net.calc_average_output_error())<<endl;
			break;
		}
		if(it>iterations/10 && abs(net.calc_average_output_error())<best_avg_err)
		{
			best_avg_err=abs(net.calc_average_output_error());
		}
		if(it%stat_show==0)
		{
			cout_guard.lock();
			cout<<this_thread::get_id()<<" Iteration: "<<it<<" Avg err: "<<net.calc_average_output_error()<<endl;
			cout_guard.unlock();
		}
		net.teach(std::vector<double>(
		{
			1, 1, 1,
			1, 0, 1,
			1, 0, 1,
			1, 0, 1,
			1, 1, 1
		}), std::vector<double>({1, 0, 0, 0, 0, 0, 0, 0, 0, 0}));

		net.teach(std::vector<double>(
		{
			0, 0, 1,
			0, 0, 1,
			0, 0, 1,
			0, 0, 1,
			0, 0, 1
		}), std::vector<double>({0, 1, 0, 0, 0, 0, 0, 0, 0, 0}));

		net.teach(std::vector<double>(
		{
			1, 1, 1,
			0, 0, 1,
			1, 1, 1,
			1, 0, 0,
			1, 1, 1
		}), std::vector<double>({0, 0, 1, 0, 0, 0, 0, 0, 0, 0}));

		net.teach(std::vector<double>(
		{
			1, 1, 1,
			0, 0, 1,
			1, 1, 1,
			0, 0, 1,
			1, 1, 1
		}), std::vector<double>({0, 0, 0, 1, 0, 0, 0, 0, 0, 0}));

		net.teach(std::vector<double>(
		{
			1, 0, 1,
			1, 0, 1,
			1, 1, 1,
			0, 0, 1,
			0, 0, 1
		}), std::vector<double>({0, 0, 0, 0, 1, 0, 0, 0, 0, 0}));

		net.teach(std::vector<double>(
		{
			1, 1, 1,
			1, 0, 0,
			1, 1, 1,
			0, 0, 1,
			1, 1, 1
		}), std::vector<double>({0, 0, 0, 0, 0, 1, 0, 0, 0, 0}));

		net.teach(std::vector<double>(
		{
			1, 1, 1,
			1, 0, 0,
			1, 1, 1,
			1, 0, 1,
			1, 1, 1
		}), std::vector<double>({0, 0, 0, 0, 0, 0, 1, 0, 0, 0}));

		net.teach(std::vector<double>(
		{
			1, 1, 1,
			0, 0, 1,
			0, 0, 1,
			0, 0, 1,
			0, 0, 1
		}), std::vector<double>({0, 0, 0, 0, 0, 0, 0, 1, 0, 0}));

		net.teach(std::vector<double>(
		{
			1, 1, 1,
			1, 0, 1,
			1, 1, 1,
			1, 0, 1,
			1, 1, 1
		}), std::vector<double>({0, 0, 0, 0, 0, 0, 0, 0, 1, 0}));

		net.teach(std::vector<double>(
		{
			1, 1, 1,
			1, 0, 1,
			1, 1, 1,
			0, 0, 1,
			1, 1, 1
		}), std::vector<double>({0, 0, 0, 0, 0, 0, 0, 0, 0, 1}));
	}

	//cout<<"Network training ended"<<endl;
	cout_guard.lock();
	cout<<"Thread: "<<this_thread::get_id()<<endl;
	cout<<"Best avg err: "<<best_avg_err<<endl;
	cout<<"Time: "<<(float)(clock()-timer)/CLOCKS_PER_SEC<<endl;
	cout_guard.unlock();
}

int main(int argc, char** argv)
{
	cout<<fixed<<setprecision(15);
	std::vector<std::thread> threads;
	auto thread_fn=[]()
	{
		neural_net net(15, std::vector<layer_config>(
		{
			layer_config{15, 30}
		}), 10, 0.01);

		train_network(net, 0.000000001, 100000, 1000);

		net.calculate(std::vector<double>(
		{
			1, 0, 1,
			1, 0, 1,
			1, 1, 1,
			0, 0, 1,
			0, 0, 1
		}));
		
		cout_guard.lock();
		double max_value=net.outputs[0];
		unsigned int max_pos=0;
		for(unsigned int it=0; it<net.outputs.size(); it++)
		{
			cout<<it<<": "<<net.outputs[it]<<endl;
			if(net.outputs[it]>max_value)
			{
				max_value=net.outputs[it];
				max_pos=it;
			}
		}
		cout<<endl;
		cout<<"I guess it is "<<max_pos<<" ("<<max_value<<")"<<endl;
		cout_guard.unlock();
	};
	for(unsigned int it=0; it<4; it++)
	{
		threads.push_back(thread(thread_fn));
	}

	for(auto& it: threads)
	{
		it.join();
	}

	system("pause");
	return 0;
}