ImportJob.java

package com.aquent.upgrade;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.fileupload.ParameterParser;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.json.JSONArray;
import org.json.JSONObject;

import com.dotmarketing.beans.Host;
import com.dotmarketing.beans.Identifier;
import com.dotmarketing.beans.Permission;
import com.dotmarketing.business.APILocator;
import com.dotmarketing.business.IdentifierAPI;
import com.dotmarketing.business.UserAPI;
import com.dotmarketing.cache.FieldsCache;
import com.dotmarketing.cache.StructureCache;
import com.dotmarketing.portlets.categories.business.CategoryAPI;
import com.dotmarketing.portlets.categories.model.Category;
import com.dotmarketing.portlets.contentlet.business.ContentletAPI;
import com.dotmarketing.portlets.contentlet.business.HostAPI;
import com.dotmarketing.portlets.contentlet.model.Contentlet;
import com.dotmarketing.portlets.fileassets.business.FileAssetAPI;
import com.dotmarketing.portlets.folders.business.FolderAPI;
import com.dotmarketing.portlets.folders.model.Folder;
import com.dotmarketing.portlets.structure.business.StructureAPI;
import com.dotmarketing.portlets.structure.factories.FieldFactory;
import com.dotmarketing.portlets.structure.factories.RelationshipFactory;
import com.dotmarketing.portlets.structure.factories.StructureFactory;
import com.dotmarketing.portlets.structure.model.Field;
import com.dotmarketing.portlets.structure.model.Field.FieldType;
import com.dotmarketing.portlets.structure.model.Relationship;
import com.dotmarketing.portlets.structure.model.Structure;
import com.dotmarketing.util.Config;
import com.dotmarketing.util.InodeUtils;
import com.dotmarketing.util.Logger;
import com.dotmarketing.util.UtilMethods;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
import com.liferay.portal.model.User;

public class ImportJob {

	private volatile boolean flag = true;
	private volatile boolean finished = false;
	
	private Map<String, String> succeeded =  new HashMap<String, String>();
	private Map<String, String> failed =  new HashMap<String, String> ();
	private Map<String, String> identifierMap = new HashMap<String, String> ();
	
	private ContentletAPI conAPI = APILocator.getContentletAPI();
	private IdentifierAPI idAPI = APILocator.getIdentifierAPI();
	private FileAssetAPI fileAPI = APILocator.getFileAssetAPI();
	private StructureAPI stAPI = APILocator.getStructureAPI();
	private CategoryAPI catAPI = APILocator.getCategoryAPI();
	private FolderAPI folderAPI = APILocator.getFolderAPI();
	private UserAPI userAPI = APILocator.getUserAPI();
	private HostAPI hostAPI = APILocator.getHostAPI();
	
	private static final String JSONFILENAME = "stData.json"; // The name of the Structures Json File 
	private static final String OLDHOST = "http://aquent.com"; // The url of the host you are importing from
	private static final String CATPAGE = "/extra/upgrade/get-cats.htm"; // The uri of the page that gives categories for a given id
	
	private JSONObject stJSON = null;
	
	private String assetsDir;
	private static final String CSVSPLITTER = ",";
	
	public Map<String, String> getSucceeded() {
		return Collections.unmodifiableMap(succeeded);
	}
	
	public Map<String, String> getFailed() {
		return Collections.unmodifiableMap(failed);
	}
	
	public boolean isFinished() {
		return finished;
	}
	
	public synchronized void start() {
		Runnable importerRunnable = new Runnable() {
			public void run() {	
				try {
					while(flag) {
						runImport();
					}
				} catch(Exception e) {
					finished = true;
					flag = false;
				} finally {
					finished = true;
					flag = false;
				}
			}
		};
		Thread importerThread = new Thread(importerRunnable);
		importerThread.start();
	}
	
	private String getAssetsDir() {
		if(!UtilMethods.isSet(assetsDir)) {
			if (UtilMethods.isSet(Config.getStringProperty("ASSET_REAL_PATH"))) {
				assetsDir = Config.getStringProperty("ASSET_REAL_PATH");
			} else {
				assetsDir = Config.CONTEXT.getRealPath(File.separator + Config.getStringProperty("ASSET_PATH"));
			}	
		}
		
		return assetsDir;
	}
	
	private synchronized void runImport() throws Exception {
		Logger.info(this, "***** Starting the Import Job");
		
		buildData();
		importCategories();
		importStructures();
		
		finished=true;
		flag=false;
		
		Logger.info(this, "***** Finished the Import Job");
	}
	
	private void buildData() throws Exception {
		Logger.info(this, "*** Building Data");
		
		// Get the json data
		String jsonPath = getAssetsDir() + File.separator + JSONFILENAME;
		
		Logger.debug(this, "Data Path = "+jsonPath);
		File jsonFile = new File(jsonPath);
		String jsonString = Files.toString(jsonFile, Charsets.UTF_8);
		
		try {
			stJSON = new JSONObject(jsonString);
		} catch (Exception e) {
			Logger.error(this, "Unable to build the structure data.", e);
			failed.put("init", "Could not build the structure data: "+e.getMessage()+"\n"+jsonString);
			throw new Exception(e);
		}		
	}
	
	private void importCategories() throws Exception {
		Logger.info(this, "*** Importing Categories");
		
		// Get the categories from the json
		JSONArray cats;
		try {
			cats = stJSON.getJSONArray("categories");
		} catch (Exception e) {
			failed.put("init", "Could not get categories from json: "+e.getMessage());
			throw new Exception(e);
		}
		
		// Loop through and build the categories
		for(int i = 0; i<cats.length(); i++) {
			JSONObject cat;
			try {
				cat = cats.getJSONObject(i);
			} catch (Exception e) {
				failed.put("cat-"+i, "Could get the json object for this category: "+e.getMessage());
				continue;
			}
			
			// Get the varName
			String key = "";
			try {
				key = cat.getString("key");
			} catch (Exception e) {
				failed.put("cat-"+i, "Could get the var for this category: "+e.getMessage());
				continue;
			}
			
			// First delete the existing category
			try {
				Category c = catAPI.findByKey(key, userAPI.getSystemUser(), false);
				if(c != null ) {
					deleteChildrenCats(c);
					catAPI.delete(c, userAPI.getSystemUser(), false);
				}
			} catch(Exception e) {
				failed.put("cat-"+key, "Could not find or delete the existing category: "+e.getMessage());
				continue;
			}
			
			try {
				importCategory(key, cat, null);
			} catch(Exception e) {
				continue;
			}
		}
	}
	
	private void deleteChildrenCats(Category parent) throws Exception {
		List<Category> children = catAPI.getChildren(parent, userAPI.getSystemUser(), false);
		for(Category child : children) {
			deleteChildrenCats(child);
			catAPI.delete(child, userAPI.getSystemUser(), false);
		}
	}
	
	private void importCategory(String key, JSONObject cat, Category parent) throws Exception {
		String varName;
		String name;
		int order;
		//JSONArray children = null; // Not used yet
		String childrencsv = null;
		boolean hasChildren = false;
		try {
			varName = cat.getString("varname");
			name = cat.getString("name");
			order = cat.getInt("order");
			if(cat.has("children")) {
				hasChildren = true;
				//children = cat.getJSONArray("children");
			} else if(cat.has("children-csv")) {
				childrencsv = cat.getString("children-csv");
			}
		} catch (Exception e) {
			failed.put("cat-"+key, "Could get the properties for this category: "+e.getMessage());
			throw new Exception(e);
		}
		
		Category child = new Category();
		child.setActive(true);
		child.setCategoryName(name);
		child.setCategoryVelocityVarName(varName);
		child.setKey(key);
		child.setSortOrder(order);
		
		try {
			catAPI.save(parent, child, userAPI.getSystemUser(), false);
		} catch (Exception e) {
			Logger.error(this, "Could not create the category", e);
			failed.put("cat-"+key, "Could not create the category: "+e.getMessage());
			throw new Exception(e);
		}
		
		if(hasChildren) {
			//TODO:  Not sure we need this yet 
		} else if(UtilMethods.isSet(childrencsv)) {
			String childrenFile = getAssetsDir() + File.separator + "categories" + File.separator + childrencsv;
			BufferedReader br = null;
			String line = "";
			
			try {
				br = new BufferedReader(new FileReader(childrenFile));
				while((line = br.readLine()) != null) {
					Logger.debug(this, "line="+line);
										
					String[] values = line.split(CSVSPLITTER);
										
					// remove quotes 
					values[0]=values[0].replaceAll("^\"", "").replaceAll("\"$", "").replaceAll("\\\\\"", "\"");
					values[1]=values[1].replaceAll("^\"", "").replaceAll("\"$", "").replaceAll("\\\\\"", "\"");
					values[2]=values[2].replaceAll("^\"", "").replaceAll("\"$", "").replaceAll("\\\\\"", "\"");
					values[3]=values[3].replaceAll("^\"", "").replaceAll("\"$", "").replaceAll("\\\\\"", "\"");
										
					// Avoid blank lines and the headers row
					if(!UtilMethods.isSet(values[0])) continue;
					if(values[0].equalsIgnoreCase("name")) continue;
					
					// Build the json for this child
					StringBuilder jsonString = new StringBuilder();
					jsonString.append("{\"name\":");
					jsonString.append('"' + values[0].trim() + '"');
					jsonString.append(",\"key\":");
					jsonString.append('"' + values[1].trim() + '"');
					jsonString.append(",\"varname\":");
					jsonString.append('"' + values[2].trim() + '"');
					jsonString.append(",\"order\":");
					jsonString.append('"' + values[3].trim() + "\"}");
										
					JSONObject c = new JSONObject(jsonString.toString());
					
					Logger.debug(this, "Importing Child Category: "+c.toString(2));
					
					importCategory(values[1], c, child);
				}
			} catch (Exception e) {
				failed.put("cat-"+key, "Exception reading from children csv file or creating a child category: "+e.getMessage());
				throw new Exception(e);
			} finally {
				if(br != null) {
					try {
						br.close();
					} catch (Exception e) {
						failed.put("cat-"+key, "Exception closing the children csv file: "+e.getMessage());
						throw new Exception(e);
					}
				}
			}
		}
	}
	
	private void importStructures() throws Exception {
		Logger.info(this, "*** Importing Structures");
		
		// Get the system user
		User sysUser = null;
		try {
			sysUser = userAPI.getSystemUser();
		} catch (Exception e) {
			Logger.error(this, "Could not get system user", e);
			failed.put("init", "Could not get system user: "+e.getMessage());
			throw new Exception(e);
		}
		
		// Get the structures from the json
		JSONArray structs;
		try {
			structs = stJSON.getJSONArray("structures");
		} catch (Exception e) {
			Logger.error(this, "Could not get structures from json", e);
			failed.put("init", "Could not get structures from json: "+e.getMessage());
			throw new Exception(e);
		}
		
		// Loop through and build the structures
		for(int i = 0; i<structs.length(); i++) {
			JSONObject struct;
			try {
				struct = structs.getJSONObject(i);
			} catch (Exception e) {
				Logger.error(this, "Could not get the json object for this structure", e);
				failed.put("unknown-"+i, "Could get the json object for this structure: "+e.getMessage());
				continue;
			}
			String stVarName;
			try {
				stVarName = struct.getString("varname");
			} catch (Exception e) {
				Logger.error(this, "Could get the VelocityVarName for this structure", e);
				failed.put("unknown-"+i, "Could get the VelocityVarName for this structure: "+e.getMessage());
				continue;
			}
			
			// TODO: Temporary testing remove
			// if(stVarName.contains("UxThursday")) continue;
			// if(stVarName.contains("litl")) continue;
			// if(stVarName.contains("Studios")) continue;
			// if(stVarName.contains("AquentSamples")) continue;
			// if(stVarName.contains("MarketingSal")) continue;
			// if(stVarName.startsWith("VT")) continue;
			// if(stVarName.contains("ContentStream")) continue;
			 
			Structure st = StructureCache.getStructureByVelocityVarName(stVarName);
			
			Logger.debug(this, "Working on structure: "+stVarName);
			
			// If the structure already exists we are going to delete it first
			if(structureExists(st)) {
				Logger.debug(this, "The Structure Exists so deleting it");
				try {
					stAPI.delete(st, sysUser);
				} catch (Exception e) {
					Logger.error(this, "Could not clear out the structure", e);
					failed.put(stVarName, "Could not clear out the structure: "+e.getMessage());
					continue;
				}
			}
			
			// Get the Structure's Host
			Host stHost = null;
			String hostName = "";
			try {
				hostName = struct.getString("host");
				stHost = hostAPI.findByName(hostName, sysUser, false);
			} catch (Exception e) {
				Logger.error(this, "Could not get the structure's Host from the data", e);
				failed.put(stVarName, "Could not get the structure's Host from the data: "+e.getMessage());
				continue;
			}
			if(stHost == null) {
				Logger.error(this, "could not get the structure's host: "+hostName);
				failed.put(stVarName, "Could not get the structure's host: "+hostName);
				continue;
			}
			
			// Clean out Files
			Folder folder;
			try {
				folder = folderAPI.findFolderByPath("/"+stVarName, stHost, sysUser, false);
			} catch (Exception e) {
				Logger.error(this, "could not get the structure's folder", e);
				failed.put(stVarName, "Could not get the structure's folder: "+e.getMessage());
				continue;
			}
			if(folder != null) {
				Logger.debug(this, "The structure's folder exists so we need to delete it.");
				try {
					folderAPI.delete(folder, sysUser, false);
				} catch (Exception e) {
					Logger.error(this, "Could not delete the structure's folder", e);
					failed.put(stVarName, "Could not delete the structure's folder: "+e.getMessage());
					continue;
				}
			}
			
			// Create the Structure
			Logger.debug(this, "Creating the new structure");
			List<Relationship> rels = new ArrayList<Relationship>();
			try {
				rels = createStructure(stVarName, stHost, struct);
			} catch(Exception e) {
				Logger.error(this, "Exception durring createStructure", e);
				continue;
			}
			
			// Import the content
			Logger.debug(this, "Importing the data");
			int count=0;
			try {
				count = importStructure(stVarName, stHost, rels);
			} catch (Exception e) {
				Logger.error(this, "Exception durring importStructure", e);
				continue;
			}
			
			succeeded.put(stVarName, String.valueOf(count));
		}
	}

	private boolean structureExists(Structure st) {
		boolean returnValue = false;
		if (InodeUtils.isSet(st.getInode())) {
			returnValue = true;
		}
		return returnValue;
	}
	
	private List<Relationship> createStructure(String stVarName, Host stHost, JSONObject json) throws Exception {
		Structure st = new Structure();
		st.setDefaultStructure(false);
		st.setFixed(false);
		st.setStructureType(Structure.STRUCTURE_TYPE_CONTENT);
		try {
			st.setDescription(json.getString("description"));
			st.setName(json.getString("name"));
		} catch(Exception e) {
			failed.put(stVarName, "Could not fetch structure details from json: "+e.getMessage());
			throw new Exception(e);
		}
		st.setVelocityVarName(stVarName);
		st.setHost(stHost.getIdentifier());
		
		try {
			StructureFactory.saveStructure(st);
		} catch(Exception e) {
			failed.put(stVarName, "Unable to save the structure: "+e.getMessage());
			throw new Exception(e);
		}
		StructureCache.removeStructure(st);
		StructureCache.addStructure(st);
		String stInode = st.getInode();
		
		JSONArray fs;
		try {
			fs = json.getJSONArray("fields");
		} catch (Exception e) {
			failed.put(stVarName, "Unable to get the fields for this structure: "+e.getMessage());
			throw new Exception(e);
		}
		for(int i = 0; i<fs.length(); i++) {
			JSONObject fjson;
			try {
				fjson = fs.getJSONObject(i);
			} catch (Exception e) {
				failed.put(stVarName, "Unable to get json for this field ("+i+"): "+e.getMessage());
				throw new Exception(e);
			}
			
			String fVarName = "";
			try {
				fVarName = fjson.getString("varname");
			} catch (Exception e) {
				failed.put(stVarName, "Unable to get the VarName for this field ("+i+"): "+e.getMessage());
				throw new Exception(e);
			}
			
			Field field = new Field();
			field.setVelocityVarName(fVarName);
			field.setStructureInode(stInode);
			field.setFixed(false);
			field.setReadOnly(false);
			
			try {
				field.setFieldName(fjson.getString("name"));
				field.setFieldContentlet(fjson.getString("indexName"));
				
				String type = fjson.getString("type");
				field.setFieldType(type);
				if(type.equals("radio") || type.equals("checkbox") || type.equals("select")) {
					field.setValues(fjson.getString("values").replaceAll(",", "\r\n").replaceAll("~#~", ","));
				}
				if(type.equals("custom_field")) {
					field.setValues(fjson.getString("values"));
				}
				if(type.equals("category")) {
					String catId = catAPI.findByKey(fjson.getString("values"), userAPI.getSystemUser(), true).getCategoryId();
					field.setValues(catId);
				}
				
				field.setSortOrder(fjson.getInt("sortOrder"));
				field.setRequired(fjson.getBoolean("required"));
				field.setListed(fjson.getBoolean("listed"));
				
				boolean isIndexed = fjson.getBoolean("indexed");
				if(isIndexed) {
					field.setSearchable(true);
					field.setIndexed(true);
				} else {
					field.setSearchable(false);
					field.setIndexed(false);
				}
			} catch (Exception e) {
				failed.put(stVarName, "Unable to get properties for this field ("+fVarName+") from the json: "+e.getMessage());
				throw new Exception(e);
			}
			
			try {
				FieldFactory.saveField(field);
			} catch (Exception e) {
				failed.put(stVarName, "Unable to save this field ("+fVarName+"): "+e.getMessage());
				throw new Exception(e);
			}
			
		}
		
		// import relationships
		List<Relationship> returnRels = new ArrayList<Relationship>();
		if(json.has("relationships")) {
			JSONArray rels;
			try {
				rels = json.getJSONArray("relationships");
			} catch (Exception e) {
				failed.put(stVarName, "Unable to load relationships: "+e.getMessage());
				throw new Exception(e);
			}
			for(int i=0; i<rels.length(); i++) {
				JSONObject rel;
				try {
					rel = rels.getJSONObject(i);
				} catch (Exception e) {
					failed.put(stVarName, "Unable to load relationship ("+i+"): "+e.getMessage());
					throw new Exception(e);
				}
				
				Relationship r = new Relationship(StructureCache.getStructureByVelocityVarName(rel.getString("parent")), 
						st, rel.getString("parentRelationName"), rel.getString("childRelationName"), rel.getInt("relation"), 
						rel.getBoolean("parentRequired"), rel.getBoolean("childRequired"));
				r.setFixed(false);
				
				try {
					RelationshipFactory.saveRelationship(r);
				} catch (Exception e) {
					failed.put(stVarName, "Unable to create relationship ("+i+"): "+e.getMessage());
					throw new Exception(e);
				}
				returnRels.add(r);
			}
		}
		
		return returnRels;
	}
	
	@SuppressWarnings({ "rawtypes", "unchecked" })
	private int importStructure(String stVarName, Host stHost, List<Relationship> rels) throws Exception {
		String stInode = StructureCache.getStructureByVelocityVarName(stVarName).getInode();
		
		// build a field map for easy lookup
        Map<String,Field> fieldMap=new HashMap<String,Field>();
        for(Field f : FieldsCache.getFieldsByStructureInode(stInode)) {
            fieldMap.put(f.getVelocityVarName(), f);
        }
		
        // Get the data from our old system
		String url = OLDHOST+"/JSONContent/?type=json&limit=1000&offset=0&q=";
		try {
			url = url + URLEncoder.encode("+structureName:"+stVarName+" +live:true +deleted:false", "UTF-8");
		} catch (Exception e) {
			failed.put(stVarName, "Unable to encode the url to fetch the data: "+e.getMessage());
			throw new Exception(e);
		}
		
		Logger.debug(this, "The url we are fetching: "+url);
		
		DefaultHttpClient client = new DefaultHttpClient();
		HttpGet method = new HttpGet(url);
		
		HttpResponse response;
		try {
			response = client.execute(method);
		} catch (Exception e) {
			Logger.error(this, "Unable to fetch the data ("+url+")", e);
			failed.put(stVarName, "Unable to fetch the data ("+url+"): "+e.getMessage());
			throw new Exception(e);
		}
		
		InputStream inStream = response.getEntity().getContent();
		StringWriter writer = new StringWriter();
		IOUtils.copy(inStream, writer, "UTF-8");
		String dataString = writer.toString();
		
		JSONObject data;
		JSONArray contentlets;
		try {
			data = new JSONObject(dataString);
			contentlets = data.getJSONArray("contentlets");
		} catch (Exception e) {
			failed.put(stVarName, "Unable to get the data from the json: "+e.getMessage());
			throw new Exception(e);
		}
		
		// Loop through each contentlet
		int count = 0;
		for(int i = 0; i<contentlets.length(); i++) {
			Logger.info(this, "Working on "+stVarName+" contentlet #"+(i+1)+" OF "+contentlets.length());
			
			JSONObject con;
			try {
				con = contentlets.getJSONObject(i);
			} catch (Exception e) {
				failed.put(stVarName, "Unable to get a single contentlet JSONObject from the json: "+e.getMessage());
    			throw new Exception(e);
			}
			
			Logger.debug(this, "Working on: "+con.toString(2));
			
			Contentlet contentlet=new Contentlet();
			contentlet.setStructureInode(stInode);
			contentlet.setHost(stHost.getIdentifier());
			
			long languageId;
			try {
				languageId = con.getLong("languageId");
			}  catch (Exception e) {
				failed.put(stVarName, "Unable to get a single contentlet JSONObject from the json: "+e.getMessage());
    			throw new Exception(e);
			}
			contentlet.setLanguageId(languageId);
			
			JSONArray keys = con.names();
			for(int j = 0; j<keys.length(); j++) {
				String key;
				String value;
				try {
					key = keys.getString(j);
					value = con.getString(key);
				} catch (Exception e) {
					failed.put(stVarName, "Unable to get a key or value from the json <textarea>"+con.toString(2)+"</textarea>: "+e.getMessage());
        			throw new Exception(e);
				}
				
				Logger.debug(this, j+"="+key+"="+value);
				
				Field f = fieldMap.get(key);
				if(f == null) continue;
				
				if(f.getFieldType().equals(FieldType.HOST_OR_FOLDER.toString())) {
					// We already set the Host based on the structure - nothing to see here
				} else if(f.getFieldType().equals(FieldType.CATEGORY.toString())) {
                    // handled later - nothing to see here
				} else if(f.getFieldType().equals(FieldType.BINARY.toString())) {
					String fileurl = OLDHOST+value+"?force_download=true";
					Logger.debug(this, "fileurl="+fileurl);
            		HttpGet filemethod = new HttpGet(fileurl);
            		
            		InputStream fileStream;
            		HttpResponse fileResponse;
            		try {
            			fileResponse = client.execute(filemethod);
            			fileStream = fileResponse.getEntity().getContent();
            		} catch(Exception e) {
            			failed.put(stVarName, "Unable to fetch the file ("+url+") for field "+key+": "+e.getMessage());
            			throw new Exception(e);
            		}
            		
            		// The json feed from dotCMS returns a value for all binary fields even if they are not set
            		// If you try to download the binary file that does not exist it returns a 500
            		// Hopefully a 500 here always means that the file does not exist and there are no real issues to deal with
            		if(fileResponse.getStatusLine().getStatusCode() != 500) {
	            		String fileName = getFileName(fileResponse);
	            		Logger.debug(this, "Filename="+fileName);
	            		
	            		if(! UtilMethods.isSet(fileName)) {
	            			failed.put(stVarName, "Unable to get a filename for field "+key+" from Content-Disposition");
	            			throw new Exception("No Filename for Binary field");
	            		}
	            		
	            		File tmpFile = new File("/tmp/"+fileName);
	            		try {
							FileUtils.copyInputStreamToFile(fileStream, tmpFile);
						} catch (Exception e) {
							failed.put(stVarName, "Unable to save a temp file for field "+key+": "+e.getMessage());
	            			throw new Exception(e);
						}
	            		
	            		contentlet.setBinary(key, tmpFile);
            		} else {
            			Logger.warn(this, "500 response on a binary field - "+key);
            		}
            		EntityUtils.consumeQuietly(fileResponse.getEntity());
                } else if((f.getFieldType().equals(FieldType.FILE.toString()) 
                		|| f.getFieldType().equals(FieldType.IMAGE.toString()))
                		&& UtilMethods.isSet(value)) {
                	String fileurl = OLDHOST+"/dotAsset/"+value+"?force_download=true";
                	Logger.debug(this, "fileurl="+fileurl);
            		HttpGet filemethod = new HttpGet(fileurl);
            		
            		InputStream fileStream;
            		HttpResponse fileResponse;
            		try {
            			fileResponse = client.execute(filemethod);
            			fileStream = fileResponse.getEntity().getContent();
            		} catch(Exception e) {
            			failed.put(stVarName, "Unable to fetch the file ("+url+") for field "+key+": "+e.getMessage());
            			throw new Exception(e);
            		}
            		
            		String fileName = getFileName(fileResponse);
            		Logger.debug(this, "Filename="+fileName);
            		
            		if(! UtilMethods.isSet(fileName)) {
            			failed.put(stVarName, "Unable to get a filename for field "+key+" from Content-Disposition");
            			throw new Exception("No Filename for File field");
            		}
            		
            		File tmpFile = new File("/tmp/"+fileName);
            		try {
						FileUtils.copyInputStreamToFile(fileStream, tmpFile);
					} catch (Exception e) {
						failed.put(stVarName, "Unable to save a temp file for field "+key+": "+e.getMessage());
            			throw new Exception(e);
					}
            		
            		Folder folder = null;
            		
            		try {
	            		String folderPath = "/"+stVarName+"/"+key;
	            		folder = folderAPI.findFolderByPath(folderPath, stHost, userAPI.getSystemUser(), false);
	            		if(folder == null || !UtilMethods.isSet(folder.getInode())) {
	            			folder = folderAPI.createFolders(folderPath, stHost, userAPI.getSystemUser(), false);
	            		}
            		} catch(Exception e) {
            			failed.put(stVarName, "Unable to get a folder to save a file for the field "+key+": "+e.getMessage());
            			throw new Exception(e);
            		}
            		
            		Logger.debug(this, "Folder="+folder.getPath());
            		
            		Contentlet c;
            		if(fileAPI.fileNameExists(stHost, folder, fileName, "")) {
            			// Assuming here that if the file already exists that it can point to the same file
            			try {
            				Identifier fileAsset = idAPI.find(stHost, folder.getPath()+fileName);
            				c = conAPI.findContentletByIdentifier(fileAsset.getId(), true, languageId, userAPI.getSystemUser(), false);
            			} catch (Exception e) {
							failed.put(stVarName, "Unable to get the existing fileAsset for a file that already exists for the field "+key+": "+e.getMessage());
	            			throw new Exception(e);
						}
            		} else {
	            		Structure faSt = StructureFactory.getStructureByVelocityVarName("FileAsset");
	            		
	            		Map m = new HashMap();
	            		m.put("stInode", faSt.getInode());
	            		m.put("host", stHost.getIdentifier());
	            		m.put("folder", folder.getInode());
	            		m.put("languageId", languageId);
	            		m.put("sortOrder", new Long(0));
	            		m.put("title", fileName);
	            		m.put("fileName", fileName);
	            		m.put("fileAsset", tmpFile);
	            		c = new Contentlet(m);
	            		c.setIdentifier("");
	            
	            		try {
							c = conAPI.checkin(c, userAPI.getSystemUser(), false);
							conAPI.publish(c, userAPI.getSystemUser(), false);
						} catch (Exception e) {
							failed.put(stVarName, "Unable to save a file contentlet for the field "+key+": "+e.getMessage());
	            			throw new Exception(e);
						}
            		}
            		
            		contentlet.setStringProperty(key, c.getIdentifier());
            		
            		EntityUtils.consumeQuietly(fileResponse.getEntity());
                } else if(f.getFieldContentlet().contains("integer")) {
                	int v = Integer.valueOf(value);
                	conAPI.setContentletProperty(contentlet, f, v);
                } else if(f.getFieldContentlet().contains("float")) {
                	float v = Float.valueOf(value);
                	conAPI.setContentletProperty(contentlet, f, v);
                } else {
                    conAPI.setContentletProperty(contentlet, f, value);
                }
			}
			
			// Get the old identifier and inode for relationship and category mapping
			String oldIdentifier;
			String oldInode;
			try {
				oldIdentifier = con.getString("identifier");
				oldInode = con.getString("inode");
			} catch (Exception e) {
				failed.put(stVarName, "Unable to get the old identifier for this contentlet: "+e.getMessage());
    			throw new Exception(e);
			}
			
			// Deal with relationships
			Map<Relationship, List<Contentlet>> cRels = new HashMap<Relationship, List<Contentlet>>();
			if(rels.size() > 0) {
				for(Relationship rel : rels) {
					String relName = rel.getParentRelationName()+"-"+rel.getChildRelationName();
					String relQuery = "+structureName:"+rel.getParentStructure().getVelocityVarName()+
							" +"+relName+":"+oldIdentifier+" +live:true +deleted:false";
					String relurl = OLDHOST+"/JSONContent/?type=json&limit=1000&offset=0&q=";
					
					Logger.debug(this, "the query is: "+relQuery);
					
					try {
						relurl = relurl + URLEncoder.encode(relQuery, "UTF-8");
					} catch (Exception e) {
						failed.put(stVarName, "Unable to encode the url to fetch the relationships: "+e.getMessage());
						throw new Exception(e);
					}
					
					Logger.debug(this, "The relurl is "+relurl);
					
					HttpGet relmethod = new HttpGet(relurl);
					
					HttpResponse relresponse;
					try {
						relresponse = client.execute(relmethod);
					} catch (Exception e) {
						failed.put(stVarName, "Unable to fetch the relationships data from ("+relName+") from ("+relurl+"): "+e.getMessage());
						throw new Exception(e);
					}
					
					InputStream relinStream = relresponse.getEntity().getContent();
					StringWriter relwriter = new StringWriter();
					IOUtils.copy(relinStream, relwriter, "UTF-8");
					String reldataString = relwriter.toString();
										
					JSONObject relData;
					JSONArray relCons;
					try {
						relData = new JSONObject(reldataString);
						relCons = relData.getJSONArray("contentlets");
					} catch (Exception e) {
						failed.put(stVarName, "Unable to get the relationships for ("+relName+") from the json: "+e.getMessage());
						throw new Exception(e);
					}
					
					Logger.debug(this, "The Related Cons: "+relCons.toString(2));
					
					ArrayList<Contentlet> parents = new ArrayList<Contentlet>();
					for(int j=0; j<relCons.length(); j++) {
						try {
							JSONObject relCon = relCons.getJSONObject(j);
							String parentId = identifierMap.get(relCon.getString("identifier"));
							Contentlet parentCon = conAPI.findContentletByIdentifier(parentId, true, languageId, userAPI.getSystemUser(), false);
							parents.add(parentCon);
						} catch (Exception e) {
							failed.put(stVarName, "Unable to get a parent for the relationship ("+relName+") from the json ("+j+"): "+e.getMessage());
							throw new Exception(e);
						}
					}
					cRels.put(rel, parents);
				}
			}
			
			// Deal with categories
			List<Category> cats = new ArrayList<Category>();
			for(Field f: fieldMap.values()) {
				if(f.getFieldType().equals(FieldType.CATEGORY.toString())) {
					String caturl = OLDHOST+CATPAGE+"?id="+oldInode+"&f="+f.getVelocityVarName();
					Logger.debug(this, "caturl="+caturl);
            		HttpGet filemethod = new HttpGet(caturl);
            		
            		InputStream catStream;
            		HttpResponse catResponse;
            		StringWriter catwriter;
            		try {
            			catResponse = client.execute(filemethod);
            			catStream = catResponse.getEntity().getContent();
            			catwriter = new StringWriter();
                		IOUtils.copy(catStream, catwriter, "UTF-8");
            		} catch(Exception e) {
            			failed.put(stVarName, "Unable to fetch the categories ("+url+") for field "+f.getVelocityVarName()+": "+e.getMessage());
            			throw new Exception(e);
            		}
            		
            		String catsData = catwriter.toString().trim();
            		
            		JSONObject catsJson;
            		JSONArray categories;
            		try {
            			catsJson = new JSONObject(catsData);
            			categories = catsJson.getJSONArray("cats");
            		} catch (Exception e) {
            			failed.put(stVarName, "Unable to get the categories from the json: "+e.getMessage()+"<textarea>"+catsData+"</textarea>");
            			throw new Exception(e);
            		}
            		
            		for(int j=0; j<categories.length(); j++) {
            			JSONObject catJson;
            			try {
            				catJson = categories.getJSONObject(j);
            			} catch (Exception e) {
                			failed.put(stVarName, "Unable to get a category from the json: "+e.getMessage());
                			throw new Exception(e);
                		}
            			
            			String key;
            			try {
            				key = catJson.getString("key");
            			} catch (Exception e) {
                			failed.put(stVarName, "Unable to get a category key from the json: "+e.getMessage());
                			throw new Exception(e);
                		}
            			
            			Logger.debug(this, "Found the category: "+key+" for the field: "+f.getVelocityVarName());
            			
            			Category cat;
            			try {
            				contentlet.setStringProperty(f.getVelocityVarName(), key);
            				cat = catAPI.findByKey(key, userAPI.getSystemUser(), false);
            			} catch (Exception e) {
                			failed.put(stVarName, "Unable to set the category field's value in the contentlet or find the category key in the system: "+e.getMessage());
                			throw new Exception(e);
            			}
            			
            			cats.add(cat);
            		}
				}
			}
			
			Logger.debug(this, "Categories: "+cats);
			
			try {
				contentlet = conAPI.checkin(contentlet, cRels, cats, new ArrayList<Permission>(), userAPI.getSystemUser(), false);
				conAPI.publish(contentlet, userAPI.getSystemUser(), false);
			} catch (Exception e) {
				failed.put(stVarName, "Unable to checkin and publish Contentlet: "+e.getMessage()+"<br /><textarea>"+con.toString(2)+"<textarea>");
    			throw new Exception(e);
			}
			
			identifierMap.put(oldIdentifier, contentlet.getIdentifier());
						
			count++;
			
			Logger.info(this, "Finished contentlet #"+(i+1)+" OF "+contentlets.length());
			
		}
		
		return count;
	}
	
	@SuppressWarnings("rawtypes")
	private String getFileName(HttpResponse fileResponse) {
		String fileName = "";
		Header cdh = fileResponse.getFirstHeader("Content-Disposition");
		if(cdh != null) {
    		String cd = cdh.getValue();
    		if(cd != null && cd.toLowerCase().startsWith("attachment")) {
    			ParameterParser parser = new ParameterParser();
    			parser.setLowerCaseNames(true);
    			Map params = parser.parse(cd, ';');
    			if (params.containsKey("filename")) {
                    fileName = (String) params.get("filename");
                    if (fileName != null) {
                        fileName = fileName.trim();
                    }
                }
    		}
		}
		return fileName;
	}

}