#AEM #CQ #LINUX #CURL #NODEJS Bulk Content Importer

README.md

AEM CQ Bulk TSV Importer

While there is some use case specific logic this script takes a tab separated txt file which contains the AEM path, Node Property, and Node Value and updates accordingly.

##This more or less works but was abandoned.

aem_bulk_tsv_exporter.js

var httpsync = require('httpsync');
var Qs = require('qs');
var Q = require('q')
var csv = require('ya-csv');

require('events')

var credentials = "admin:admin"
var host_port = "localhost:4502"
var parentNode = '/content/property-master/en';

var server = "http://"+credentials+"@"+host_port+"/";
var body = '';
var httpcode = '';

var dataArr = [];

var getObj = doGet(server,parentNode+'.infinity.json',dataArr)
httpcode = getObj.statusCode
body = getObj.data.toString()
json = JSON.parse(body)


var filterArr = ['jcr:created','jcr:primaryType','jcr:mixinTypes','jcr:createdBy','cq:lastReplicationAction','jcr:versionHistory','cq:template','cq:lastReplicatedBy','be_co_uuid','jcr:predecessors','jcr:created','cq:lastReplicated','co_optimization','cq:lastModified','jcr:baseVersion','jcr:isCheckedOut','jcr:uuid','sling:resourceType','cq:lastModifiedBy','jcr:lastModifiedBy','jcr:lastModified','alignment','textIsRich','bullet','maxLevel','imageCrop','selection','height','width','imageRotate','cq:keywords','hideInNav','jcr:mimeType'];
//var filterArr = [''];

//console.log('path\titem\tvalue')
nodeLooper(parentNode,json)
function nodeLooper(node,json) {
	var writer = new csv.CsvWriter(process.stdout, {
		'separator': '\t',
		'quote' : ''
	})
	var children = [];
	var c = []
	for (var attributename in json) {
		if(filterArr.indexOf(attributename) > -1) {
			//console.log('Skipping: '+attributename);
		} else {
			if(json[attributename] == '') {
				//writer.writeRecord([ node,attributename,""])
				//console.log('empty:'+attributename)
			} else if (json[attributename].length) { 
				writer.writeRecord([ node,attributename,json[attributename]])
				//console.log('value:'+attributename)
			} else { 
				c = doGet(server,node+'/'+attributename+'.infinity.json')
				try {
					nodeLooper(node+'/'+attributename,JSON.parse(c.data.toString()))
				} catch(err) {
					console.log('ERROR: '+node+': '+c.statusCode)
				}
			}
		}
	}
}
function doPost(server,path,parms) {
	var req = httpsync.request({
	  url: server+path,
	  method: "POST",
	  useragent: "NodeJS",
	  headers: {
	    Tag: "EIS",
	    String: "Custom"
	  }
	});
	req.write(Qs.stringify(parms))
	return res = req.end();
}
function doGet(server,path,parms) {
	(parms) ? url = server+path+'?'+Qs.stringify(parms) : url = server+path;
	//console.log(url)
	var req = httpsync.request({
	  url: server+path+'?'+Qs.stringify(parms),
	  method: "GET",
	  useragent: "NodeJS",
	  headers: {
	    Tag: "EIS",
	    String: "Custom"
	  }
	});
	return res = req.end();
}
function doDelete(server,path) {
	var req = httpsync.request({
	  url: server+path,
	  method: "DELETE",
	  useragent: "NodeJS",
	  headers: {
	    Tag: "TGB3123",
	    String: "Custom"
	  }
	});
	return res = req.end();
}

aem_bulk_tsv_importer.sh

HOST_PORT="${1}"
CREDENTIALS="${2}"
NL=$'\n'
dos2unix input.tsv.txt
sed -i input.tsv.txt -e 's/"//g;s/^M//g;/^\s*$/d'
cat input.tsv.txt |while read line; do
	_PATH="`echo "${line}" |awk -F $'\t' '{print $1}' |sed -e 's/ //g;'`";
	_PARM_NAME="`echo "${line}" |awk -F $'\t' '{print $2}'`";
	_PARM_VALUE="`echo "${line}" |awk -F $'\t' '{print $3}'`";
	_COMPONENT_NODE_PATH=`echo ${_PATH} |sed -e 's/\(.*leftlist\).*/\1/g;s/\/leftlist//g;s/\(.*rightlist\).*/\1/g;s/\/rightlist//g;'`
	_SNIFF="`curl --silent -u "${CREDENTIALS}" "http://${HOST_PORT}${_COMPONENT_NODE_PATH}.json" -w "%{http_code}\n" -o /dev/null`"
	if [ "${_SNIFF}" = "200" ]; then
		curl --silent -u "${CREDENTIALS}" "http://${HOST_PORT}${_PATH}" -F "${_PARM_NAME}=${_PARM_VALUE}" -o /dev/null
		if [ "$?" = "0" ]; then
			#echo "${_PATH},${_PARM_NAME},${_PARM_VALUE} SUCCESS,"
			echo -e "${_PATH}\t${_PARM_NAME}\t${_PARM_VALUE}\tSUCCESS" |tee -a output.tsv.txt
		else
			echo -e "${_PATH}\t${_PARM_NAME}\t${_PARM_VALUE}\tFAILURE" |tee -a output.tsv.txt
		fi
	else
		echo -e "${_PATH}\t${_PARM_NAME}\t${_PARM_VALUE}\tFAILURE" |tee -a output.tsv.txt
	fi
done;

getcqcontent.js

var httpsync = require('httpsync');
var Qs = require('qs');
var Q = require('q')
var csv = require('ya-csv');

require('events')

var credentials = "admin:admin"
var host_port = "localhost:4502"
var parentNode = '/content/property-master/en';

var server = "http://"+credentials+"@"+host_port+"/";
var body = '';
var httpcode = '';

var dataArr = [];

var getObj = doGet(server,parentNode+'.infinity.json',dataArr)
httpcode = getObj.statusCode
body = getObj.data.toString()
json = JSON.parse(body)


var filterArr = ['jcr:created','jcr:primaryType','jcr:mixinTypes','jcr:createdBy','cq:lastReplicationAction','jcr:versionHistory','cq:template','cq:lastReplicatedBy','be_co_uuid','jcr:predecessors','jcr:created','cq:lastReplicated','co_optimization','cq:lastModified','jcr:baseVersion','jcr:isCheckedOut','jcr:uuid','sling:resourceType','cq:lastModifiedBy','jcr:lastModifiedBy','jcr:lastModified','alignment','textIsRich','bullet','maxLevel','imageCrop','selection','height','width','imageRotate','cq:keywords','hideInNav','jcr:mimeType'];
var filterArr = [''];

console.log('path,item,value')
nodeLooper(parentNode,json)
function nodeLooper(node,json) {
	var writer = new csv.CsvWriter(process.stdout, {
		'separator': ','		
	})
	var children = [];
	var c = []
	for (var attributename in json) {
		if(filterArr.indexOf(attributename) > -1) {
			//console.log('Skipping: '+attributename);
		} else {
			if(json[attributename] == '') {
				//writer.writeRecord([ node,attributename,""])
				//console.log('empty:'+attributename)
			} else if (json[attributename].length) { 
				writer.writeRecord([ node,attributename,json[attributename]])
				//console.log('value:'+attributename)
			} else { 
				c = doGet(server,node+'/'+attributename+'.infinity.json')
				try {
					nodeLooper(node+'/'+attributename,JSON.parse(c.data.toString()))
				} catch(err) {
					console.log('ERROR: '+node+': '+c.statusCode)
				}
			}
		}
	}
}
function doPost(server,path,parms) {
	var req = httpsync.request({
	  url: server+path,
	  method: "POST",
	  useragent: "NodeJS",
	  headers: {
	    Tag: "EIS",
	    String: "Custom"
	  }
	});
	req.write(Qs.stringify(parms))
	return res = req.end();
}
function doGet(server,path,parms) {
	(parms) ? url = server+path+'?'+Qs.stringify(parms) : url = server+path;
	//console.log(url)
	var req = httpsync.request({
	  url: server+path+'?'+Qs.stringify(parms),
	  method: "GET",
	  useragent: "NodeJS",
	  headers: {
	    Tag: "EIS",
	    String: "Custom"
	  }
	});
	return res = req.end();
}
function doDelete(server,path) {
	var req = httpsync.request({
	  url: server+path,
	  method: "DELETE",
	  useragent: "NodeJS",
	  headers: {
	    Tag: "TGB3123",
	    String: "Custom"
	  }
	});
	return res = req.end();
}

loadcqcontent.js

var httpsync = require('httpsync');
var Qs = require('qs');
var Q = require('q')
var csv = require('ya-csv');

//Stops part way through use shell instead for now

var credentials = "admin:admin"
var host_port = "localhost:4502"

var server = "http://"+credentials+"@"+host_port+"/";

var reader = csv.createCsvFileReader('input.tsv.txt', {
	columnsFromHeader: true,
	'separator': '\t',
	'quote' : ''
});
var parseCSVreturnArray = function () {
	csvPromise = Q.defer();
	arr = [];

	reader.addListener('data', function (data) {
		arr.push([ data['path'],data['item'],data['value'] ])
	})
	reader.addListener("end", function () {
		csvPromise.resolve(arr)
        });
	return csvPromise.promise
}
parseCSVreturnArray()
.then(function(csvArr) {
	csvArr.forEach(function(entry) {
		dataArr = []
		if(entry[2].split('\t')) {
			split = entry[2].split('\t')
			dataArr[entry[1]] = [ split[0],split[1] ]
			console.log(dataArr)
			doPost(server,entry[0],dataArr)
		} else { 
			dataArr[entry[1]] = entry[2];
			console.log(dataArr)
			doPost(server,entry[0],dataArr)
		}
	})	
})

function nodeLooper(node,json,writer) {
	var children = [];
	var c = []
	for (var attributename in json) {
		if(json[attributename] == '') {
			writer.writeRecord([ node,attributename,""])
		} else if (json[attributename].length) { 
			writer.writeRecord([ node,attributename,json[attributename]])
		} else { 
			c = doGet(server,node+'/'+attributename+'.json')
			try {
				nodeLooper(node+'/'+attributename,JSON.parse(c.data.toString()),writer)
			} catch(err) {
				console.log('ERROR: '+node+': '+c.statusCode)
			}
		}
	}
}
function doPost(server,path,parms) {
	var req = httpsync.request({
	  url: server+path,
	  method: "POST",
	  useragent: "NodeJS",
	  headers: {
	    Tag: "EIS",
	    String: "Custom"
	  }
	});
	x = Qs.stringify(parms).replace('%5B0%5D','').replace('%5B1%5D','')
	console.log(x)
	req.write(x)
	return res = req.end();
}
function doGet(server,path,parms) {
	(parms) ? url = server+path+'?'+Qs.stringify(parms) : url = server+path;
	var req = httpsync.request({
	  url: server+path+'?'+Qs.stringify(parms),
	  method: "GET",
	  useragent: "NodeJS",
	  headers: {
	    Tag: "EIS",
	    String: "Custom"
	  }
	});
	return res = req.end();
}
function doDelete(server,path) {
	var req = httpsync.request({
	  url: server+path,
	  method: "DELETE",
	  useragent: "NodeJS",
	  headers: {
	    Tag: "TGB3123",
	    String: "Custom"
	  }
	});
	return res = req.end();
}