gistfile1.txt

package io.github.czxttkl.stocks.old;

import io.github.czxttkl.stocks.auxi.JS;
import io.github.czxttkl.stocks.auxi.Stock;
import io.github.czxttkl.stocks.auxi.StockGlobal;
import io.github.czxttkl.util.UtilJava;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.util.Arrays;

import com.csvreader.CsvWriter;
import com.google.gson.Gson;
import com.google.gson.stream.MalformedJsonException;

public class DataDownTaskXUEQIU implements Runnable {
	private final static int DOWNLOAD_FROM_START = 11;
	private final static int DOWNLOAD_FROM_END = 22;

	private final String code;
	private boolean stop = false;
	private int tries = 0;
	private String jsFilePath;
	private String csvFilePath;
	private File jsFile;
	private String first100Bytes = null;
	private URL mUrl;
	private int download_status = DOWNLOAD_FROM_START;
	private boolean rangeDownSupport = true;

	public DataDownTaskXUEQIU(String code) throws IOException {
		this.code = code;
		jsFilePath = StockGlobal.STOCK_RAW_PATH + code + StockGlobal.STOCK_RAW_FILE_SUFFIX;
		csvFilePath = StockGlobal.STOCK_PROCESSED_PATH + code + StockGlobal.STOCK_PROCESSED_FILE_SUFFIX;
		jsFile = new File(jsFilePath);
		if (!jsFile.exists())
			jsFile.createNewFile();
		String url = constructUrl();
		mUrl = new URL(url);
	}

	@Override
	public void run() {
		while (!stop) {
			try {
				tries++;
				System.out.println(Thread.currentThread().getName() + ": try " + tries + " th times :" + code + " starting with" + jsFile.length());

				long serverFileLength = getFileLength(mUrl);
				if (serverFileLength == jsFile.length())
					break;

				if (jsFile.length() > 100 && rangeDownSupport) {
					while (first100Bytes == null || first100Bytes.length() != 100) {
						InputStream input = getInputStreamFromUrl(0, 99);
						if (rangeDownSupport) {
							first100Bytes = UtilJava.inputStreamToString(input, 100);
							String fileFirst100Bytes = UtilJava.fileToString(jsFile, 100);
							if (first100Bytes.equals(fileFirst100Bytes)) {
								System.out.println("first 100 bytes equal");
								download_status = DOWNLOAD_FROM_END;
							}
						} else
							break;
					}
				}

				InputStream input = null;
				switch (download_status) {
				case DOWNLOAD_FROM_END:
					// long fileLength = getFileLength(mUrl);
					long startPos = jsFile.length() - 100;
					System.out.println("start download from file end:" + startPos);
					input = getInputStreamFromUrl(startPos);
					writeInputStreamToFile(input, startPos);
					System.out.println("end download from file end");
					break;
				case DOWNLOAD_FROM_START:
					System.out.println("start download from file start");
					input = getInputStreamFromUrl(0);
					writeInputStreamToFile(input, 0);
					System.out.println("end download from file start");
					break;
				default:
					System.err.println("status undefined;");
					stop = true;
					break;
				}

				input.close();
				if (convertJSToCsv()) {
					System.out.println(Thread.currentThread().getName() + " finished:" + mUrl.toString() + " file size:" + jsFile.length());
					stop = true;
				}

			} catch (IOException e) {
				System.err.println(Thread.currentThread().getName() + ":Download failed:" + e.getMessage() + ":" + mUrl.toString());
				// For the stocks who are obsolete
				if (e.getMessage().contains("HTTP response code: 400 for"))
					break;
			}
		}
	}

	public long getFileLength(URL url) {
		int nFileLength = -1;
		try {
			HttpURLConnection httpConnection = (HttpURLConnection) url.openConnection();
			httpConnection.setRequestProperty(StockGlobal.USER_AGENT, StockGlobal.USER_AGENT_VALUE);
			httpConnection.setConnectTimeout(5000);
			httpConnection.setReadTimeout(25000);
			int responseCode = httpConnection.getResponseCode();
			if (responseCode >= 400) {
				return -2; // -2 represent access is error
			}
			String sHeader;
			for (int i = 1;; i++) {
				sHeader = httpConnection.getHeaderFieldKey(i);
				if (sHeader != null) {
					if (sHeader.equals("Content-Length")) {
						nFileLength = Integer.parseInt(httpConnection.getHeaderField(sHeader));
						break;
					}
				} else
					break;
			}
		} catch (IOException e) {
			e.printStackTrace();
		} catch (Exception e) {
			e.printStackTrace();
		}
		return nFileLength;
	}

	private String constructUrl() {
		String url;
		if (code.startsWith("6") || code.startsWith("9")) {
			url = StockGlobal.XUEQIU_URL_PREFIX_SH + code + StockGlobal.XUEQIU_URL_SUFFIX;
		} else {
			url = StockGlobal.XUEQIU_URL_PREFIX_SZ + code + StockGlobal.XUEQIU_URL_SUFFIX;
		}
		return url;
	}

	private InputStream getInputStreamFromUrl(long startPos) throws IOException {
		HttpURLConnection huc = (HttpURLConnection) mUrl.openConnection();
		huc.setConnectTimeout(5000);
		huc.setReadTimeout(300000);
		huc.setRequestMethod("GET");
		huc.setRequestProperty(StockGlobal.USER_AGENT, StockGlobal.USER_AGENT_VALUE);
		String sProperty = "bytes=" + startPos + "-";
		huc.setRequestProperty("RANGE", sProperty);
		int code = huc.getResponseCode();
		if (code == 200)
			rangeDownSupport = false;
		return huc.getInputStream();
	}

	private InputStream getInputStreamFromUrl(long startPos, long endPos) throws IOException {
		HttpURLConnection huc = (HttpURLConnection) mUrl.openConnection();
		huc.setConnectTimeout(5000);
		huc.setReadTimeout(300000);
		huc.setRequestMethod("GET");
		huc.setRequestProperty(StockGlobal.USER_AGENT, StockGlobal.USER_AGENT_VALUE);
		String sProperty = "bytes=" + startPos + "-" + endPos;
		huc.setRequestProperty("RANGE", sProperty);
		int code = huc.getResponseCode();
		if (code == 200)
			rangeDownSupport = false;
		return huc.getInputStream();
	}

	/**
	 * Write input to file. Return true if write successfully.
	 * 
	 * @param input
	 * @param filePath
	 * @return
	 * @throws IOException
	 */
	private boolean convertJSToCsv() throws IOException {
		CsvWriter mCsvWriter = new CsvWriter(csvFilePath);
		Gson gson = new Gson();
		JS mJS = null;
		try {
			mJS = gson.fromJson(new FileReader(jsFile), JS.class);
		} catch (Exception e) {
			// catch GSON Exception
			System.err.println(Thread.currentThread().getName() + ":GSON Convert failed:" + e.getMessage() + ":" + code);
			return false;
		}

		if (mJS.chartlist != null) {
			for (Stock stock : mJS.chartlist) {
				mCsvWriter.writeRecord(stock.toStringArray());
			}
			mCsvWriter.flush();
		}
		return true;
	}

	private void writeInputStreamToFile(InputStream is, long startPos) throws IOException {
		RandomAccessFile raf = new RandomAccessFile(jsFile, "rw");
		raf.seek(startPos);
		byte[] tmp = new byte[2048];
		int offset = is.read(tmp);
		while (offset != -1) {
			raf.write(tmp, 0, offset);
			// System.out.println("read:" + offset + " bytes. file length:" + jsFile.length() + ":" + new String(tmp,0,offset));
			offset = is.read(tmp);
		}
		raf.close();
	}
}