vwsPlugin.mu.cronjobs.js

var cronJob = require('cron').CronJob,
	bubSubConverse = require('./lib').bubSubConverse,
	talkToMammut = require('./lib').talkToMammut,
	thisCronjobs = require('./thisCronjobs');

var cronjobs = function () {
	this.cronjobs = {};
	
	return this;
};

cronjobs.prototype.add = function (job, time) {
	
	var thisInstance = this;
	if (typeof thisInstance['_' + job] !== 'function') {
		log.add('no job:' + job, 'error', 'cronjob.add', 3);
		return;
	}
	
	thisInstance.cronjobs[job] = [];
	
	function cb(){
		var ts = thisInstance.cronjobs[job].shift().timestamp;

		log.add('done: '+job+' in ' + thisInstance.getHumanReadableTimediff(ts), 'job', 'cronjob', 3);

		if (thisInstance.cronjobs[job].length > 0){
			thisInstance['_' + job](cb);
		}
	}

	var cjob = new cronJob(time, function(){
		thisInstance.cronjobs[job].push({timestamp : Date.now()});
		if (thisInstance.cronjobs[job].length > 1) {
			log.add('cronjob ' + job + ' pushed in query (length: ' + thisInstance.cronjobs[job].length + ')', 'job', 'cronjob', 3);
			return;
		}
		thisInstance['_' + job](cb);
	}, null, true);

	
	log.add('add: ' + job, 'plain', 'cronjob', 3);
	return this;
};

cronjobs.prototype.getHumanReadableTimediff = function(timestamp) {
	var runningTime = Date.now() - timestamp;
	var timearray = [];
	
	timearray.unshift(runningTime%1000 + 'ms');
	runningTime = Math.round(runningTime/1000);
	
	if (runningTime === 0) {
		return timearray.join(' ');
	}
	
	timearray.unshift(runningTime%60 + 's');
	runningTime = Math.round(runningTime/60);
	
	if (runningTime === 0) {
		return timearray.join(' ');
	}
	
	timearray.unshift(runningTime%60 + 'min');
	runningTime = Math.round(runningTime/60);
	
	if (runningTime === 0) {
		return timearray.join(' ');
	}
	
	timearray.unshift(runningTime%24 + 'h');
	runningTime = Math.round(runningTime/24);
	
	if (runningTime === 0) {
		return timearray.join(' ');
	}
	
	timearray.unshift(runningTime + 'd');
	
	return runningTime;
}

cronjobs.prototype._usersync = function (done) {
	var thisInstance = this,
		userData = [],
		completeLength,
		comm = new thisCronjobs.Usersync('mu');
	
	function onlyPaul(_id){
		return _id === '50098823ccafbf5d55000618';
	}
	
	function nextSync() {
		if (userData.length%10 === 0) {
			log.add('unsyncUser: ' + userData.length + '/' + completeLength, 'job', 'cronjob.usersync');
			if (userData.length === 0) return done();
		};
		comm.getUser(userData.pop(), billings);
	}
	
	function billings(err, user){
		if (err) return setImmediate(nextSync);
		comm.getKbumSyncUser(user, mails);
	}
	
	function mails(err, user){
		if (err) return setImmediate(nextSync);
		comm.updateOptivo(user, nextSync);
	}
	
	comm.getUserIds(function(err, users){
		userData = users
		// .filter(onlyPaul);
		
		completeLength = userData.length;

		setImmediate(nextSync);
	});
};

cronjobs.prototype._sendInvoices = function (done) {
	// don't send invoices, if kbum should offer
	return done();
	
	var invoices = [],
		comm = new thisCronjobs.SendInvoices('mu');
	
	function nextSync(){
		if (invoices.length%100 === 0){
			if (invoices.length === 0) return done();
			log.add('invoices unsend: ' + invoices.length, 'job', 'cronjob', 3);
		}
		var invoice = invoices.pop();
		comm.getInvoice(invoice.fk_customer_id, invoice.invoice_id, function(err, invoiceBuffer){
			var file = {
				buffer : invoiceBuffer,
				filename : invoice.invoice_number + '.pdf'
			}
			if (err) return done(error);
			comm.getCustomer(invoice.fk_customer_id, function(err, user){
				if (err) {
					comm.sendErrorMail(err, invoice.fk_customer_id, file, function(){
						comm.setMailDate(invoice.invoice_id, nextSync)
					});
					return;
				}
				send(invoice, user, file);
			});		
		});
	}
	
	function send(invoice, user, file){
		var isInvoice = invoice.invoice_paymentterms === 'payment_invoice';				
		comm.sendInvoice(user.email, isInvoice, file, function(err, optivoData){
			if (err) return done(error);
			
			optivoData.isInvoice = isInvoice;
			optivoData.invoiceNumber = invoice.invoice_number;
			optivoData._id = user._id;
			comm.setMailDate(invoice.invoice_id, function(){
				comm.trackInvoice(optivoData, nextSync)
			});
		})
		
	}
	
	comm.getNotMailedInvoices(function(err, data){
		if (err) return done(err);
		invoices = data;
		log.add('invoices to send: ' + invoices.length, 'job','cronjob', 3);
		setImmediate(nextSync);
	});
};

cronjobs.prototype._sendDunnings = function (done) {
	
	var dunnings = [],
		comm = new thisCronjobs.SendDunnings('mu');
	
	function mapDunnings(dunning) {
		return {
			id : parseInt(dunning.dunning_id, 10),
			customerId : parseInt(dunning.fk_customer_id, 10),
			step : parseInt(dunning.max_dunning_step, 10),
			filename : dunning.invoice_number + '_' + dunning.max_dunning_step + '.pdf',
			invoiceNumber : parseInt(dunning.invoice_number, 10),
		};
	}

	// esclator Steps

	function getDunning (next, esc) {
		console.log(esc.dunning.filename);
		comm.getDunning(esc.dunning.customerId, esc.dunning.id, function(err, dunningBuffer) {
			if (err) return esc.onError(err, esc);
			esc.file.filename = esc.dunning.filename;
			esc.file.buffer = dunningBuffer;
			setImmediate(next);
		});
	}
	
	function getCustomer(next, esc) {
		comm.getCustomer(esc.dunning.customerId, function(err, user){
			if (err) return esc.onError(err, esc);
			esc.user = user;
			setImmediate(next);
		})
	}

	function sendDunning(next, esc) {
		var dunningMailData = {
			maxDunningStep : esc.dunning.step,
			Anrede 	: 
				(esc.user.invoiceAddress.gender === 'm') ? 'Herr' :
				(esc.user.invoiceAddress.gender === 'f') ? 'Frau' : '',
			Vorname : esc.user.invoiceAddress.firstName,
			Nachname: esc.user.invoiceAddress.lastName,
			email : esc.user.email,
			buffer : esc.file.buffer,
			filename : esc.file.filename
		};
		comm.sendDunning(dunningMailData, function(err, data){
			if (err) return esc.onError(err);
			esc.enqueued = data.enqueued
			setImmediate(next);
		});
	}
	
	function setMailDate(next, esc) {
		comm.setMailDate(esc.dunning.id, next);
	}

	function track(next, esc) {
		var data = {
			enqueued : esc.enqueued,
			invoiceNumber :  esc.dunning.invoiceNumber,
			maxDunningStep : esc.dunning.step,
			_id : esc.user._id
		}
		comm.trackOnUser(data, next);
	}

	function onError(err, esc){
		var mailingsErrors = [
			'cronjob_e_003',
			'cronjob_e_004'
		];

		if (mailingsErrors.indexOf(err.code) === -1) return setImmediate(nextSync);

		comm.sendError(err, esc.dunning.customerId, esc.dunning.id, function(err, data){
			comm.setMailDate(esc.dunning.id, nextSync);
		});
	}

	// recursive Function

	function nextSync(){
		if (dunnings.length%100 === 0) log.add('dunnings unsend: ' + dunnings.length, 'job', 'cronjob');
		if (dunnings.length === 0) return setImmediate(done);

		new escalator('sendDunnings')
		.set('dunning', dunnings.pop())
		.set('user', {})
		.set('file', {filename : "", buffer : new Buffer(0)})
		.add(getDunning)
		.add(getCustomer)
		.add(sendDunning)
		.add(setMailDate)
		.add(track)
		.set('onError', onError)
		.set('onFinish', nextSync)
		.start();
	}

	comm.getNotMailedDunnings(function(err, data){
		if (err) return done(err);
		dunnings = data.map(mapDunnings);
		log.add('dunnings to send: ' + dunnings.length,'job','cronjob', 3);
		setImmediate(nextSync);
	});

};

cronjobs.prototype._optinResend = function (done) {
	
	var unconfirmedUser = [],
		comm = new thisCronjobs.OptinResend('mu');

	function next(){
		if (unconfirmedUser.length%100 === 0){
			if (unconfirmedUser.length === 0) return done();
			log.add('undone resend: ' + unconfirmedUser.length, 'job', 'cronjob');
		}
		
		var id = unconfirmedUser.pop();
		comm.getUser(id, function(err, user){
			if (err) return setImmediate(next);
			comm.sendMail(user, function(err, data){
				if (err) return setImmediate(next);
				comm.updateUser(id, function(err, data){
					setImmediate(next);
				});
			})
		});
	}
	
	comm.getUnconfirmedUserIds(function(err, users){
		if (err) return setImmediate(done);
		unconfirmedUser = users;
		// unconfirmedUser = ['50098823ccafbf5d55000618']
		setImmediate(next);
	})
};

cronjobs.prototype._checkInvoiceStat = function (done) {

	var users = [],
		comm = new thisCronjobs.CheckInvoiceStat('mu');

	function next(){
	
		if (users.length%100 === 0) log.add('uncheckInvoiceStat: ' + users.length, 'job', 'cronjob');
		if (users.length === 0) return done();
		
		var user = users.pop();

		comm.getInvoices(user.customerId, function(err, invoices){
			checkInvoices(user, invoices);
		});
	}
	
	function checkInvoices(user, kbumInvoices){
		var updateInvoices = {},
			userInvoives = user.invoices,
			invoiceNumbers = kbumInvoices.map(function(invoice){
			return invoice.invoice_number;
		});
		
		// kbum daten auf mongo abbilden
		kbumInvoices.forEach(function(kbumInvoice){
			var iNr = kbumInvoice.invoice_number;
			var thisUserInvoice = typeof userInvoives[iNr] !== 'undefined' ? userInvoives[iNr] : {}; 
			var payed = kbumInvoice.invoice_payed === "1";
			var type = parseInt(kbumInvoice.invoice_isCreditnote, 10) > 0 && parseInt(kbumInvoice.fk_invoice_id_forCreditnote) > 0
				? 'Gutschrift'
				: 'Rechnung';
				
			if (thisUserInvoice.type !== type){
				updateInvoices[iNr + ".type"] = type;
			}
			if (thisUserInvoice.exists !== true) {
				updateInvoices[iNr + ".exists"] = true;
			}
			if (thisUserInvoice.payed !== payed) {
				updateInvoices[iNr + ".payed"] = payed;
			}
		});
		
		// mongo daten auf kbum abbilden
		Object.keys(userInvoives)
		.filter(function(iNr){
			return userInvoives[iNr].exists !== (invoiceNumbers.indexOf(iNr) > -1);
		})
		.forEach(function(iNr){
			updateInvoices[iNr + ".exists"] = invoiceNumbers.indexOf(iNr) > -1;
		})

		if (Object.keys(updateInvoices).length === 0) return setImmediate(next);

		comm.putInvoiceStat(user._id, updateInvoices, next);
	}

	comm.getInvoiceChecks(function(err, data){
		users = data;
		setImmediate(next);
	});
};

cronjobs.prototype._documentPipeline = function (done) {
	var thisInstance = this;
	var start = Date.now();
	log.add('start pipeline for rae', 'job', 'cronjob.documents', 3);
	thisInstance.__documentPipeline('rae', function(){
		log.add('start pipeline for rbd', 'job', 'cronjob.documents', 3);
		thisInstance.__documentPipeline('rbd', function(){
			log.add('done in ' + ((Date.now() - start) / 1000)  + ' seconds', 'job', 'cronjob.documents', 3);
			done();
		})
	})
}

cronjobs.prototype.__documentPipeline = function (realm, done) {
	var thisInstance = this;
	thisInstance.__documents(realm, function(){
		thisInstance.__mindItem(realm, function(){
			thisInstance.__lucene(realm, done);
		});
	});
};

cronjobs.prototype._documents = function(done) {
	this.__documents('rae', done)
}

cronjobs.prototype.__documents = function(realm, done) {
	
	var documents = new thisCronjobs.Documents(realm);
	
	var hexxlerData = [],
		self = this;

	function nextDocument () {
		log.add('todo: ' + hexxlerData.length, 'job', 'cronjob.documents', 3);
		if (hexxlerData.length === 0) return setImmediate(done);
		self.document(hexxlerData.pop(), {realm : realm}, nextDocument);
	}

	documents.getHexxlerDocument(function(err, data){
		hexxlerData = data
		// .filter(function(d){
			// return d.document_id === 6477;
		// })
		setImmediate(nextDocument);
	});
};

cronjobs.prototype.document = function(data, options, callback) {

	var documents = new thisCronjobs.Documents(options.realm),
		withMammutCheck = false,
		override = false;
	
	function errCb (err, escalator) {
		log.add(err, 'error', 'cronjob.documents', 3);
		callback(err);
	}
	
	function onFinish (esc) {
		log.add('document done in ' + (Date.now() - esc.starttime), 'jobs', 'cronjob', 3);
		callback(null);
	}
	
	function checkExists (next, thisEscalator) {
		documents.getMongoDocumentByHexxlerId(thisEscalator.hexxlerData.document_id, function(err, mongoData){
			thisEscalator.mongoData = mongoData;
			setImmediate(next);
		})
	}
	
	function postDocument (next, thisEscalator) {
		if (typeof thisEscalator.mongoData !== 'undefined') return setImmediate(next);
		documents.postOriginalDocumentFromHexxlerData(thisEscalator.hexxlerData, function(err, mongoData){
			thisEscalator.mongoData = mongoData;
			setImmediate(next);
		});
	}
	
	function toMammut (next, thisEscalator) {
		if (thisEscalator.mongoData.files.registed.images.length > 0 && !withMammutCheck){
			return setImmediate(next);
		}
		var options = {
			override : override
		}
		documents.postMammutDocument(thisEscalator.mongoData, options, function(err, mammutData){
			if (err) return errCb(err, thisEscalator);
			thisEscalator.mammutData = mammutData;
			setImmediate(next);
		});
	}
	
	function putDoc (next, thisEscalator) {
		if (typeof thisEscalator.mammutData === 'undefined') return setImmediate(next);
		var censored = thisEscalator.mammutData.censored,
			uncensored = thisEscalator.mammutData.uncensored,
			update = {
			_id : thisEscalator.mongoData._id,
			files : {},
			status : {}
		};
		
		if (thisEscalator.mongoData.files.registed.images.length < uncensored.images.length) {
			update.files.registed = uncensored;
			update.files.unregisted = censored;
			update.files.unregisted.download = update.files.registed.download = thisEscalator.mongoData.hexxler.path;
		}
		// das ist bugfix
		if (typeof thisEscalator.mammutData.pages === 'number' && thisEscalator.mongoData.pages !== thisEscalator.mammutData.pages) {
			update.pages = thisEscalator.mammutData.pages;
		}
		if (typeof thisEscalator.mammutData.status.ngfs === 'boolean' && thisEscalator.mongoData.status.ngfs !== thisEscalator.mammutData.status.ngfs){
			update.status.ngfs = thisEscalator.mammutData.status.ngfs;
		}
		if (typeof thisEscalator.mammutData.status.exists === 'boolean' && thisEscalator.mongoData.status.exists !== thisEscalator.mammutData.status.exists) {
			update.status.exists = thisEscalator.mammutData.status.exists;
		}
		
		if (Object.keys(update.files).length === 0 && Object.keys(update.status).length === 0) return setImmediate(next);
		
		documents.putDocumentFilesById(update, function(err, mongoData) {
			if (err) return errCb(err, thisEscalator);
			setImmediate(next);
		});
	}

	new escalator('document')
	.add(checkExists)
	.add(postDocument)
	.add(toMammut)
	.add(putDoc)
	.set('onError', errCb)
	.set('onFinish', onFinish)
	.set('hexxlerData', data)
	.set('starttime', Date.now())
	.start();
}

cronjobs.prototype._mindItem = function(done){
	this.__mindItem('rae', done)
}

cronjobs.prototype.__mindItem = function (realm, done) {
	
	var mindItems = [],
		updatedMindItems = [],
		thisInstance = this,
		tags = {},
		communication = new thisCronjobs.MindItem(realm);
	
	// helper
	
	function uniqueTag() {
		var topicIds = [];
		return function(tag){
			var unique = topicIds.indexOf(tag.topicId) === -1;
			if (unique) topicIds.push(tag.topicId);
			return unique;
		}
	}

	function withTopicId(tag){
		return typeof tag.topicId === 'string' && tag.topicId.length > 0;
	}

	function unique(value, i, arr){
		return arr.indexOf(value) === i
	}
	
	function enumerationLNMedia (enumeration) {
		var arr = [];
		enumeration
		.replace(/^\|/g, "")
		.replace(/\|$/g, "")
		.split('||')
		.forEach(function(document){
			if (document === '') return;
			arr.push((isNaN(document)) ? document : parseInt(document,10));
		});
		return arr.sort(function(a,b){return a-b;});
	}
	
	function removeControlCharacter (str) {
		if (typeof str !== 'string') return '';
		var charCodes = [];
		// space is controlchar 32 (\u0020)
		for (var code,i = 0; i < 32; i++) {
			code = i.toString(16);
			switch(code.length) {
				case 1 : code = "0" + code;
				case 2 : code = "0" + code;
				case 3 : code = "0" + code;
				default : code = "\\u" + code;
			}
			charCodes.push(code);
		}
		var regex = RegExp('[' + charCodes.join("") + ']','g');
		return str.replace(regex, "");
	}
	
	function mappingKernTags (tag, set) {
		var ignoreKernTags = [
			'Zeitschriftenname (nur falls Zeitschrift)',
			'Fremdrechte enthalten',
			'CD/DVD Verweis enthalten',
			'Dokumententyp',
			''
		];
		
		if (ignoreKernTags.indexOf(tag.kerntag_name) > -1) return;
		
		tag.kerntags = removeControlCharacter(tag.kerntags);
		
		if (tag.kerntags === "") return;
		
		switch	(tag.kerntag_name) {
			case 'AuflageNr.':
			case 'AusgabenNr (nur Zeitschrift)':
				set.edition = tag.kerntags;
				break;
			case  'Autor':
				tag.kerntags = tag.kerntags.replace(/\;/g,", ").replace(/\,/g,", ").replace(/\ \ /g," ").replace(/^\ /g,'').replace(/\ $/g,'').split(", ");
				set.publisher = [];
				set.author = [];
				for (var i in tag.kerntags) {
					if (tag.kerntags[i].indexOf('(Hg.)') > 0) set.publisher.push(tag.kerntags[i]);
					else set.author.push(tag.kerntags[i]);
				}
				break;
			case 'Beschreibung'		: set['description'] = tag.kerntags; break;
			// case 'Bestell-Nr.' 		: set = { 'ordering.orderingNumber' : tag.kerntags}; // verlag wird ignoriert, da es von ausgangsdokument geholt wird
			case 'ISBN' 			: set['ordering.isbn'] = tag.kerntags; break;
			case 'ISSN' 			: set['ordering.issn'] = tag.kerntags; break;
			case 'Erscheinungsdatum': set['releaseDate'] = tag.kerntags; break;
			case 'Fach'				: set['subject'] = enumerationLNMedia(tag.kerntags).filter(unique);break;
			case 'Herausgeber'		: set['publisher'] = tag.kerntags.replace(/\,/,", ").replace(/\;/g,", ").replace(/\,/g,", ").replace(/\ \ /g," ").replace(/^\ /g,'').replace(/\ $/g,'').split(", ");break;
			case 'Illustrator' 		:
			case 'Illustratoren'	: set['illustrator'] = tag.kerntags.replace(/\,/,", ").replace(/\;/g,", ").replace(/\,/g,", ").replace(/\ \ /g," ").replace(/^\ /g,'').replace(/\ $/g,'').split(", ");break;
			case 'Klassenstufe'		: set['classYears'] = enumerationLNMedia(tag.kerntags).filter(unique);break;
			case 'Schulart'	: 
				var schoolType = enumerationLNMedia(tag.kerntags);
				if (schoolType.indexOf('Förderschule / Inklusion') > -1){
					schoolType[schoolType.indexOf('Förderschule / Inklusion')] = 'Förderschule';
				}
				if (schoolType.indexOf('Förderschule GB') > -1){
					schoolType[schoolType.indexOf('Förderschule GB')] = 'Förderschule';
				}
				set.schoolType = schoolType.filter(unique);break;
			case 'Thema'	: set.issue = tag.kerntags.split("\n");break;
			case 'Titel 1'	: set.title = tag.kerntags;break;
			case 'Titel 2'	: set.subtitle = tag.kerntags;break;
			// case 'Verlag'	: set.publishingHouse = tag.kerntags;break; // verlag wird ignoriert, da es von ausgangsdokument geholt wird
			default : return set;
		}

		return set;

	}
	
	function buildPagesArray(images){
		return function(page, i) {
			var obj = {};
			Object.keys(images[page-1]).forEach(function(key){
				obj[key] = key === 'pos' ? (i+1) : images[page-1][key];
			})
			return obj;
		}
	}
	
	function isDauer(tag) {
		return tag.strukturtag_name === 'Dauer' || tag.kerntag_name === 'Dauer';
	}
	
	function kernTag2structurTag(kernTag) {
		return {
			strukturtag_name : kernTag.kerntag_name,
			strukturtag : kernTag.kerntags
		}
	}
	//esclatorsteps
	
	function nextMindItem () {
		if (mindItems.length === 0) return process.nextTick(done);
		if (mindItems.length%100 === 0) log.add('unchecked mindItems: ' + mindItems.length, 'job', 'cronjob', 3);
		
		thisInstance.mindItemEsc = new escalator('singleMindItem')
		.set('mindItem', mindItems.shift())
		.set('upsertData', {})
		.set('onError', onError)
		.set('onFinish', nextMindItem)
		.add(getData)
		.add(getDocument)
		.add(compareMindItem)
		.add(compareTags)
		.add(compareKernTags)
		.add(compareStructurtags)
		.add(putToFileserver)
		.add(updateMindItem)
		.start();
	}
	
	function getData (next, thisEscalator) {

		thisEscalator.data = {};
		var id = thisEscalator.mindItem;
		var counter = 5;
		var doNext = function(){
			counter -= 1;
			if (counter > 0) return
			process.nextTick(next);
		}
		
		communication.getTags(id, function(err, tags){
			thisEscalator.data.tags = tags;
			process.nextTick(doNext);
		})
		communication.getHexxlerMindItem(id, function(err, mindItem){
			thisEscalator.data.mindItem = mindItem;
			process.nextTick(doNext);
		});
		communication.getKerntags(id, function(err, kerntags){
			thisEscalator.data.kerntags = kerntags;
			process.nextTick(doNext);
		});
		communication.getStructurtags(id, function(err, structurtags){
			thisEscalator.data.structurtags = structurtags;
			process.nextTick(doNext);
		});
		communication.getMongoMinditem(id, function(err, mongoMindItem){
			thisEscalator.data.mongoMindItem = mongoMindItem;
			process.nextTick(doNext);
		});
	}
	
	function getDocument (next, thisEscalator) {
		communication.getOriginalDocument(thisEscalator.data.mindItem.fk_document_id, function(err, doc){
			if (typeof doc === 'undefined') {
				var error = new aError('cronjob_e_005','no document found for mindItem', thisEscalator.mindItem, 2);
				return thisEscalator.onError(error);	
			}
			thisEscalator.data.document = doc;
			process.nextTick(next);
		})
	}
	
	function compareMindItem (next, thisEscalator) {
	
		var hexxlerData = thisEscalator.data.mindItem,
			mongo = thisEscalator.data.mongoMindItem;
		var pages = enumerationLNMedia(hexxlerData.minditem_pages).filter(function(page){ 
			return !isNaN(page) && thisEscalator.data.document.files.registed.images.length >= page;
		}), 
			path = hexxlerData.minditem_path.replace(/data\/output\//,""),
			registed = pages.map(buildPagesArray(thisEscalator.data.document.files.registed.images)),
			unregisted = pages.map(buildPagesArray(thisEscalator.data.document.files.unregisted.images));
		thisEscalator.updates = {};
		
		if (typeof mongo === 'undefined'){
			var mindItem = {
				status : {
					active : true,
					hexxlerRelease : true
				},
				files : {
					registed : {
						images : registed,
						imagePath : thisEscalator.data.document.files.registed.imagePath,
						download : thisEscalator.data.document.files.registed.download.replace(/.pdf$/, "_partId" + hexxlerData.minditem_id+".pdf")
					},
					unregisted : {
						images : unregisted,
						imagePath : thisEscalator.data.document.files.unregisted.imagePath,
						download : ''
					}
				},
				hexxler : {
					id : hexxlerData.minditem_id,
					path : path
				},
				parent : {
					_id : thisEscalator.data.document._id,
					hexxlerId : hexxlerData.fk_document_id,
					pages : pages
				},
				title : hexxlerData.minditem_name,
				qualifications : {
					publishingHouse : thisEscalator.data.document.qualifications.publishingHouse,
					ordering : {
						orderingNumber : thisEscalator.data.document.qualifications.ordering.orderingNumber
					}
				}
			};
			communication.postMindItem(mindItem, function(err, mongoMindItem){
				thisEscalator.data.mongoMindItem = mongoMindItem;
				communication.putPart(thisEscalator.data.document._id, mongoMindItem._id, next);
			})
			return;
		}
		hexxlerData.minditem_name = hexxlerData.minditem_name.replace(/^[\s]+|[\s]+$/g,"");
		if (mongo.title !== hexxlerData.minditem_name) {
			thisEscalator.updates['title'] = hexxlerData.minditem_name;
		}
		if (typeof mongo.hexxler !== 'object' || mongo.hexxler.id !== hexxlerData.minditem_id) {
			thisEscalator.updates["hexxler.id"] = hexxlerData.minditem_id;
		}
		if (typeof mongo.hexxler !== 'object' || mongo.hexxler.path !== path) {
			thisEscalator.updates["hexxler.path"] = path;
		}
		if (mongo.qualifications.publishingHouse !== thisEscalator.data.document.qualifications.publishingHouse) {
			thisEscalator.updates['qualifications.publishingHouse'] = thisEscalator.data.document.qualifications.publishingHouse;
		}
		if (mongo.qualifications.ordering.orderingNumber !== thisEscalator.data.document.qualifications.ordering.orderingNumber) {
			thisEscalator.updates['qualifications.ordering.orderingNumber'] = thisEscalator.data.document.qualifications.ordering.orderingNumber;
		}
		if (typeof mongo.parent !== 'object' || mongo.parent._id.toString() !== thisEscalator.data.document._id.toString()) {
			thisEscalator.updates["parent._id"] = thisEscalator.data.document._id;
		}
		if (typeof mongo.parent !== 'object' || mongo.parent.hexxlerId !== hexxlerData.fk_document_id) {
			thisEscalator.updates["parent.hexxlerId"] = hexxlerData.fk_document_id;
		}
		if (typeof mongo.parent !== 'object' || mongo.parent.pages.join() !== pages.join()) {
			thisEscalator.updates["parent.pages"] = pages;
		}
		if (typeof mongo.status !== 'object' || mongo.status.active !== true) {
			thisEscalator.updates["status.active"] = true;
		}
		if (typeof mongo.status !== 'object' || mongo.status.hexxlerRelease !== true) {
			thisEscalator.updates["status.hexxlerRelease"] = true;
		}
		//files checken!
		process.nextTick(next);
	}
	
	function compareTags (next, thisEscalator) {
		var	hexxlerArray = thisEscalator.data.tags
		.filter(withTopicId)
		.filter(uniqueTag())
		.map(function(tag){
			if (typeof tags[tag.topicId] !== 'undefined') {
				tag.tag = tags[tag.topicId];
			}
			return {
				topicId : tag.topicId,
				tag : tag.tag.split('(')[0].trim()
			}
		});
		
		if (JSON.stringify(hexxlerArray) !== JSON.stringify(thisEscalator.data.mongoMindItem.qualifications.tags)) {
			thisEscalator.updates['qualifications.tags'] = hexxlerArray;
		}
		
		process.nextTick(next);
	}
	
	function compareKernTags (next, thisEscalator) {
		
		var updates = {}, quali = {},
			mongo = thisEscalator.data.mongoMindItem.qualifications,
			kerntags = {
			edition : "",
			description : "",
			publisher : [],
			author : [],
			'ordering.isbn' : "",
			"ordering.issn" : "",
			releaseDate : "",
			subject : [],
			illustrator : [],
			classYears : [],
			schoolType : [],
			issue : [],
			title : "",
			subtitle : "",
		};
		
		// dirty mapping. better: mapping class by realm
		if (Array.isArray(thisEscalator.data.structurtags) && !thisEscalator.data.structurtags.some(isDauer)) {
			var structurTagMapping = thisEscalator.data.kerntags
			.filter(isDauer)
			.map(kernTag2structurTag);
			
			thisEscalator.data.structurtags = thisEscalator.data.structurtags.concat(structurTagMapping)
		}
			
		for (var i in thisEscalator.data.kerntags){
			mappingKernTags(thisEscalator.data.kerntags[i], kerntags);
		}
		
		// checkes if somethings changed
		if (mongo.edition !== kerntags.edition) {
			quali.edition = kerntags.edition;
		}
		if (mongo.description !== kerntags.description) {
			quali.description = kerntags.description;
		}
		if (mongo.publisher.join() !== kerntags.publisher.join()) {
			quali.publisher = kerntags.publisher;
		}
		if (mongo.author.join() !== kerntags.author.join()) {
			quali.author = kerntags.author;
		}
		if (mongo.ordering.isbn !== kerntags["ordering.isbn"]) {
			quali["ordering.isbn"] = kerntags["ordering.isbn"];
		}	
		if (mongo.ordering.issn !== kerntags["ordering.issn"]) {
			quali["ordering.issn"] = kerntags["ordering.issn"];
		}	
		if (mongo.releaseDate !== kerntags.releaseDate) {
			quali.releaseDate = kerntags.releaseDate;
		}		
		if (mongo.subject.join() !== kerntags.subject.join()) {
			quali.subject = kerntags.subject;
		}
		if (mongo.illustrator.join() !== kerntags.illustrator.join()) {
			quali.illustrator = kerntags.illustrator;
		}
		if (mongo.classYears instanceof Array && kerntags.classYears instanceof Array && mongo.classYears.join() !== kerntags.classYears.join()) {
			quali.classYears = kerntags.classYears;
		}
		if (mongo.schoolType.join() !== kerntags.schoolType.join()) {
			quali.schoolType = kerntags.schoolType;
		}
		if (mongo.issue instanceof Array === false || mongo.issue.join() !== kerntags.issue.join()) {
			quali.issue = kerntags.issue;
		}
		if (mongo.title !== kerntags.title) {
			quali.title = kerntags.title;
		}	
		if (mongo.subtitle !== kerntags.subtitle) {
			quali.subtitle = kerntags.subtitle;
		}
		
		//'i' is already defined.
		for (var i in quali){
			thisEscalator.updates['qualifications.' + i] = quali[i];
		}
		
		process.nextTick(next);
	}
	
	function compareStructurtags (next, thisEscalator) {
		
		// Hexxler-Mongo Mapping
		var mapping = {
			'Aufgabenform' : 'form',
			'Dauer' : 'duration',
			'Lehr-Lernformate' : 'learningFormat',
			'Lernstand ermitteln' : 'learningLevel',
			'Materialart' : 'kind',
			'Materialumfang' : 'amount',
			'Unterrichtsformat' : 'format',
			// jrcontent,
			"Kompetenz" : 'competence',
			"Methode" : 'method',
			"Sozialform" : 'socialForm',
			"Unterrichtsform" : 'teachingForm'
		};
		mapping[unescape('Didaktisches Material f%FCr Lehrer')] = 'didaktikMaterial';
		
		// init Structurtags
		var structurTags = Object.keys(mapping)
		.map(function(hexxlerTag){
			return mapping[hexxlerTag];
		})
		.reduce(function(init, mongoTag){
			init[mongoTag] = [];
			return init;
		}, {});
		
		// fill Strukturtags
		(Array.isArray(thisEscalator.data.structurtags) ? thisEscalator.data.structurtags : [])
		.filter(function(sTag){
			return typeof mapping[sTag.strukturtag_name] !== 'undefined'
		})
		.forEach(function(sTag){
			structurTags[mapping[sTag.strukturtag_name]].push(sTag.strukturtag);
		});
		
		// extend updates
		thisEscalator.updates = Object.keys(structurTags)
		.filter(function(mongoTag){
			var mongoStrukturTags = thisEscalator.data.mongoMindItem.qualifications.structurTags[mongoTag];
			var structurTagExists = Array.isArray(mongoStrukturTags);
			var differentStructurTag = structurTagExists && (mongoStrukturTags.join() !== structurTags[mongoTag].join());
			
			return !structurTagExists || differentStructurTag;
		})
		.reduce(function(updates, mongoTag){
			updates["qualifications.structurTags." + mongoTag] = structurTags[mongoTag];
			return updates;
		}, thisEscalator.updates)
		
		process.nextTick(next);
	}
	
	function putToFileserver (next, thisEscalator) {
		communication.putToFileServer(thisEscalator.data.mongoMindItem, function(err, data){
			thisEscalator.mergeLinksToPdf = err ? false : true; 
			process.nextTick(next);
		});
	}

	function updateMindItem (next, thisEscalator) {
		if (typeof thisEscalator.mergeLinksToPdf !== 'undefined' && 
			thisEscalator.data.mongoMindItem.status.exists !== thisEscalator.mergeLinksToPdf) {
			thisEscalator.updates['status.exists'] = thisEscalator.mergeLinksToPdf;
		}
		communication.updateMindItem(thisEscalator.data.mongoMindItem._id, thisEscalator.updates, next);
	}
	
	// steps
	function onError (err) {
		var error = new aError('cronjob_e_007', 'escalator onError - mindItem',{err: err.stack, mindItem : thisInstance.mindItemEsc.mindItem}, 2);
		process.nextTick(nextMindItem);
	}

	function getAndSetTopics (next) {
		communication.getAllTopics(function(err, data){
			tags = data;
			log.add('found Tags: ' + Object.keys(tags).length, 'job', 'cronjob', 3);
			process.nextTick(next);
		})
	}
	
	function getMindActiveMindItems (next, thisEsc) {
		communication.getReleaseableMindItems(function(err, data){
			mindItems = data;
			log.add('releaseableMindItems ' + mindItems.length, 'job', 'cronjob', 3);
			process.nextTick(next);
		});
	}
	
	function getExludeMindItemsFromMongo (next, thisEsc) {
		communication.getAllMindItemsExcludeByHexxlerIds(mindItems, function(err, active, release){
			thisEsc.excludeHexxlerIds.active = active;
			thisEsc.excludeHexxlerIds.release = release;
			process.nextTick(next);
		})
	}
	
	function getExludeMindItemsFromHexxler (next, thisEsc) {
		var active = thisEsc.excludeHexxlerIds.active,
			release = thisEsc.excludeHexxlerIds.release;
		
		communication.getExludeMindItemsFromHexxler(active, release, function(err, deleteIds, unreleaseIds){
			thisEsc.deleteIds = deleteIds;
			thisEsc.unreleaseIds = unreleaseIds;		
			process.nextTick(next);
		});
	}
	
	function updateExludeMindItems (next, thisEsc) {		
		var counter = 2;
		log.add('noRelease: ' + thisEsc.unreleaseIds.length + ' / toDelete: ' + thisEsc.deleteIds.length, 'job', 'cronjob', 3);
		communication.deleteMindItems(thisEsc.deleteIds, function(err){
			counter -= 1;
			if (counter === 0) process.nextTick(next);
		});
		communication.unreleaseMindItems(thisEsc.unreleaseIds, function(err){
			counter -= 1;
			if (counter === 0) process.nextTick(next);
		});
	}
	
	new escalator('mindItemGetData')
	.set('onFinish', nextMindItem)
	.set('onError', done)
	.add(getAndSetTopics)
	.add(getMindActiveMindItems)
	.set('excludeHexxlerIds',  {active : {}, release : {}})
	.add(getExludeMindItemsFromMongo)
	.set('deleteIds', [])
	.set('unreleaseIds',[])
	.add(getExludeMindItemsFromHexxler)
	.add(updateExludeMindItems)
	.start();
		
};

cronjobs.prototype._lucene = function(done){
	this.__lucene('rae', done);
}

cronjobs.prototype.__lucene = function(realm, done) {
	
	var communication = new thisCronjobs.Lucene(realm);
	
	function setParentId(obj, mindItem) {
		obj[mindItem.parent._id] = true;
		return obj;
	}
	
	function setHexxlerPath(obj, doc) {
		obj[doc._id] = doc.hexxler.path;
		return obj;
	}
	
	//steps
	function getData(next, thisEscalator){
		communication.getSearchengineFiles(function(err, mindItems){
			var documents = mindItems.reduce(setParentId, {});
			log.add('searchengineFiles: ' + mindItems.length, 'job', 'cronjob', 3);
			thisEscalator.mindItems = mindItems;
		
			communication.getDocumentsById(Object.keys(documents), function(err, data){
				thisEscalator.documents = data.reduce(setHexxlerPath, {});
				process.nextTick(next);
			})
		});
	}
	
	function getNavigation(next, thisEscalator){
		communication.getNavigation(function(err, navigation){
			if (err) return thisEscalator.onError(err);
			thisEscalator.navigation = navigation;
			process.nextTick(next);
		});
	}
	
	function sendData(next, thisEscalator){
		var data = {
			mindItems : thisEscalator.mindItems,
			documents : thisEscalator.documents,
			navigation : thisEscalator.navigation,
		};
		communication.buildLucenceIndex(data, function(err, data){
			if (err) {
				var error = new aError('cronjob_e_008', 'buildLucenceIndex', err, 2);
				thisEscalator.onError(error);
				return;
			}
			thisEscalator.updateDocuments = data;
			process.nextTick(next);
		});
	}
	
	function updateFiles(next, thisEscalator) {
		thisEscalator.counter = 2;
		communication.putStatusOn(thisEscalator.updateDocuments, next);
		communication.putOtherStatusOff(thisEscalator.updateDocuments, next);
	}
	
	function startCrawler4Index(next, thisEscalator) {
		if (thisEscalator.counter > 0) return;
		process.nextTick(next);
	}
		
	function rsyncLogs(next, thisEscalator) {
		process.nextTick(next);
	}
	
	function startCrawler4Autocomplete(next, thisEscalator) {
		process.nextTick(next);
	}
	
	new escalator('lucene')
	.add(getData)
	.add(getNavigation)
	.add(sendData)
	.add(updateFiles)
	.add(startCrawler4Index)
	.add(rsyncLogs)
	.add(startCrawler4Autocomplete)
	.set('onError', function(err){
		console.log('error');
		console.log(err);
	})
	.set('onFinish', done)
	.start();
};

cronjobs.prototype._test = function(done){
	
	talkToMammut('/test', {}, function(err, data){
		console.log('test1 done', err);
	});
	talkToMammut('/test', {}, function(err, data){
		console.log('test2 done', err);
		talkToMammut('/test', {}, function(err, data){
			console.log('test3 done', err);
			done();
		})
	});
}

module.exports = cronjobs;