nlMovieFetcher.java

package extensions;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.net.URLDecoder;
import java.net.Socket;
import java.text.DateFormat;
import java.util.Calendar;
import java.util.TimeZone;
import java.util.Collections;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import dareka.Main;
import dareka.common.CloseUtil;
import dareka.common.DiskFreeSpace;
import dareka.common.Logger;
import dareka.common.Pair;
import dareka.extensions.Extension2;
import dareka.extensions.ExtensionManager;
import dareka.extensions.RequestFilter;
import dareka.extensions.Rewriter;
import dareka.processor.HttpRequestHeader;
import dareka.processor.HttpResponseHeader;
import dareka.processor.Resource;
import dareka.processor.URLResource;
import dareka.processor.StringResource;
import dareka.processor.TransferListener;
import dareka.processor.Processor;
import dareka.processor.impl.Cache;
import dareka.processor.impl.NicoCachingTitleRetriever;
import dareka.processor.util.GetThumbInfoUtil;

/*
 * とりあえず動画をキャッシュして後で見るよ
 */
public class nlMovieFetcher implements Extension2, Processor, Rewriter, RequestFilter,
		TransferListener, Runnable {
	
	public static final int REVISION = 110411;
	public static final String VER_STRING = "nlMovieFetcher_"+REVISION+"(v0.6)";
	public static final String LOG_PREFIX = "fetch";
	public static final String PROP_DEBUG = "movieFetchDebug";
	
	private static final String[] PROCESSOR_SUPPORTED_METHODS = new String[] { "GET", "POST" };
	private static final Pattern PROCESSOR_SUPPORTED_PATTERN = Pattern.compile(
			"^http://www\\.nicovideo\\.jp/cache/fetch\\?" +
			"((?:(status|rm|cancel)=)?(?:\\w{2}\\d+|all)|list|json|storage|updatelow)(&.+|)$");
	private static final Pattern REWRITER_SUPPORTED_PATTERN = Pattern.compile(
			"^http://flapi\\.nicovideo\\.jp/api/getflv(.*)$");
	
	private static final Map<String, String> fetchResults =
		Collections.synchronizedMap(new java.util.LinkedHashMap<String, String>());
	private static final Pair<File, Long> fetchResultsFileInfo =
			new Pair<File, Long>(new File("local/fetched.json"), 0L);
	private static volatile boolean fetchResultsUpdated;
	
	private static volatile String storageCache;
	private static final Pair<File, Long> storageFileInfo =
			new Pair<File, Long>(new File("local/fetched.storage"), 0L);
	private static volatile boolean storageUpdated;
	
	private static final String FETCH_URI = "fetch-uri";
	private static final Charset FILE_CHARSET = getCharset("movieFetchCharset");
	private static final StringResource resultOK = ExtUtil.responseText("OK");
	private static final StringResource resultNG = ExtUtil.responseText("NG");
	
	private static ExtUtil util;
	private static String user_session;
	
	private String fetchId;
	private HttpRequestHeader fetchRequestHeader;
	private ScheduledFuture<?> future;
	private volatile boolean canceled;
	private long contentLength, transferedLength, completedTime;
	private int premiumWait;
	
	private void debug2(String format, Object...args) {
		if(Boolean.getBoolean("movieFetchDebug2")) util.debug(format, args); }
	private void debugWithId(String mes){ util.debug("%s %s", this.fetchId, mes); }
	
	// 引数付きコンストラクタを定義した場合は引数無しコンストラクタも必要
	public nlMovieFetcher() {
		this.fetchId = "#global#";
	}
	
	// Extension2 interface
	public void registerExtensions(ExtensionManager mgr) {
		final int REQUIRED_REVISION = 100909;
		if (ExtUtil.REVISION < REQUIRED_REVISION) {
			Logger.warning(LOG_PREFIX +
					": ERROR!!! ExtUtil_"+REQUIRED_REVISION+" OR LATER IS REQUIRED.");
		} else {
			if (util == null) {
				util = new ExtUtil(this, LOG_PREFIX, PROP_DEBUG);
				util.loadFile(fetchResultsFileInfo, "loadJSON", FILE_CHARSET);
				util.execute("taker");
			}
			mgr.registerProcessor(this);
			mgr.registerRewriter(this);
			mgr.registerRequestFilter(this);
		}
	}
	
	public String getVersionString() {
		return VER_STRING;
	}
	
	// Processor Interface
	public String[] getSupportedMethods() {
		return PROCESSOR_SUPPORTED_METHODS;
	}
	
	public Pattern getSupportedURLAsPattern() {
		return PROCESSOR_SUPPORTED_PATTERN;
	}
	
	public String getSupportedURLAsString() {
		return null;
	}
	
	public Resource onRequest(HttpRequestHeader requestHeader, Socket browser)
			throws IOException {
		if (!Boolean.getBoolean("cacheThumbnail")) {
			util.info("must do 'cacheThumbnail=true' for using this extension.");
		} else {
			String method = requestHeader.getMethod();
			String fetch_uri = requestHeader.getURI();
			util.debug("%s %s", method, fetch_uri);
			Matcher m = PROCESSOR_SUPPORTED_PATTERN.matcher(fetch_uri);
			if (m.matches()) {
				String param = m.group(1);
				requestHeader.setParameter(FETCH_URI, fetch_uri);
				if (method.equals("GET")) {
					return procRequestGET(param, requestHeader);
				} else if (method.equals("POST")) {
					InputStream in = browser.getInputStream();
					byte[] buf = new byte[(int)requestHeader.getContentLength()];
					for (int readed = 0; readed < buf.length;)
						readed += in.read(buf, readed, buf.length - readed);
					String postBody = new String(buf);
					debug2(postBody);
					return procRequestPOST(param, postBody);
				}
			}
		}
		return StringResource.getNotFound();
	}
	
	private StringResource procRequestGET(String param, HttpRequestHeader requestHeader) {
		StringResource result = resultNG;
		if (param.equals("all")) {
			// DO NOTHING
		} else if (param.startsWith("status=")) {
			result = method_status(param.substring(7));
		} else if (param.startsWith("rm=")) {
			result = method_remove(param.substring(3), requestHeader);
		} else if (param.startsWith("cancel=")) {
			result = method_cancel(param.substring(7));
		} else if (param.equals("list")) {
			result = method_list();
		} else if (param.equals("json")) {
			result = method_json();
		} else if (param.equals("storage")) {
			result = method_storage_get();
		} else {
			result = method_fetch(param, requestHeader);
		}
		return result;
	}
	
	private StringResource procRequestPOST(String param, String postBody) {
		StringResource result = resultNG;
		if (param.equals("storage")) {
			result = method_storage_post(postBody);
		}
		return result;
	}
	
	// Rewriter Interface
	public Pattern getRewriterSupportedURLAsPattern() {
		return REWRITER_SUPPORTED_PATTERN;
	}
	
	public String onMatch(Matcher match, HttpResponseHeader responseHeader, String content)
			throws IOException {
		if (Boolean.getBoolean("movieFetchAutoWatched")) {
			Matcher m = GETFLV_URL_PATTERN.matcher(content);
			if (m.find()) {
				String smid = Cache.id2Smid(m.group(5));
				if (smid != null && fetchResults.containsKey(smid)) {
					fetchResults.remove(smid);
					fetchResultsUpdated = true;
					util.debug("%s results auto removed.", smid);
				}
			}
		}
		// 情報取得のみで書き換えはしない
		return content;
	}
	
	// RequestFilter interface
	public int onRequest(HttpRequestHeader requestHeader) throws IOException {
		String uri = requestHeader.getURI();
		if (uri.equals("http://ext.nicovideo.jp/thumb_watch")) {
			overwriteUserSession(requestHeader);
		} else if (!uri.startsWith("http://www")) {
			int pos = uri.indexOf(".nicovideo.jp/cache/fetch?");
			if (pos > 0) {
				requestHeader.setURI("http://www".concat(uri.substring(pos)));
				requestHeader.setMessageHeader("Host", "www.nicovideo.jp");
			}
		}
		return RequestFilter.OK;
	}
	
	// TransferListener interface
	public void onResponseHeader(HttpResponseHeader responseHeader) {
		// selectTransferTo(,,,false)を使っているのでここは呼ばれない
		// contentLengthはfetch()の中で設定
	}
	
	public void onTransferBegin(OutputStream receiverOut) {
		transferedLength = 0L;
		if(contentLength > 0L) setStatusMessage("0%", true);
	}
	
	public void onTransferring(byte[] buf, int length) {
		transferedLength += (long)length;
		if (contentLength > 0L) {
			int done = (int)(transferedLength * 100L / contentLength);
			setStatusMessage(done + "%", true);
		}
	}
	
	public void onTransferEnd(boolean completed) {
		if (contentLength > 0L) {
			setStatusMessage(completed ? "done." : "FAILED!", true);
		}
		debugWithId("content=" + contentLength + ", transfered=" + transferedLength);
	}
	
	private String statusMessage = "not initialized.";
	private String getStatusMessage() {
		return statusMessage;
	}
	
	private void setStatusMessage(String status) {
		setStatusMessage(status, false);
	}
	private void setStatusMessage(String status,  boolean prefix) {
		statusMessage = prefix ? "fetching... ".concat(status) : status;
	}
	
	private void infoStatusMessage(String status) {
		if(status != null) setStatusMessage(status);
		util.info(fetchId + " " + statusMessage);
	}
	
	// ExtUtil callback
	public void onMinutes() {
		expireFetchRequests();
		onShutdown();
	}
	
	public void onShutdown() {
		if (fetchResultsUpdated) {
			fetchResultsUpdated = false;
			util.saveFile(fetchResultsFileInfo, createJSON(), true, FILE_CHARSET);
		}
		if (storageUpdated) {
			storageUpdated = false;
			util.saveFile(storageFileInfo, storageCache, true, FILE_CHARSET);
		}
	}
	
	private static Charset getCharset(String propertyName) {
		Charset charset = Charset.defaultCharset();
		String charsetName = System.getProperty(propertyName);
		if (charsetName != null && !charsetName.equals("")) {
			charset = Charset.forName(charsetName);
		}
		return charset;
	}
	
	private static final int MAX_FETCH = ExtUtil.getInteger("movieFetchMax", 5, 1, 5);
	private static final Semaphore sem = new Semaphore(MAX_FETCH, true);
	private static int fetchCount() { return MAX_FETCH - sem.availablePermits(); }
	private static final ScheduledExecutorService sched =
			Executors.newScheduledThreadPool(MAX_FETCH * 2);
	private static final Random rand = new Random();
	private static final AtomicInteger accessRestriction = new AtomicInteger();
	private static final LinkedBlockingQueue<nlMovieFetcher> requestQueue =
			new LinkedBlockingQueue<nlMovieFetcher>();
	private static final Map<String, nlMovieFetcher> fetchRequests =
			Collections.synchronizedMap(new java.util.LinkedHashMap<String, nlMovieFetcher>());
	
	// ステータス文字列オブジェクト(比較が必要なものだけ)
	private static final String STATUS_FAILED    = "failed.";
	private static final String STATUS_RESERVED  = "reserved.";
	private static final String STATUS_CANCELED  = "canceled.";
	private static final String STATUS_NOT_FOUND = "not found.";
	private static final String STATUS_NO_DISK   = "no disk free space.";
	
	private int maxRetry = ExtUtil.getInteger("movieFetchRetry", 5, 0, 9);
	private String status = STATUS_FAILED;
	
	// コンストラクタ(各タスク用)
	private nlMovieFetcher(String id, HttpRequestHeader requestHeader) {
		this.fetchId = id;
		this.fetchRequestHeader = requestHeader;
		setStatusMessage("wait", true);
	}
	
	// fetch処理は別スレッドで実行
	public void run() {
		debugWithId("task started.");
		if(updatelow()) return;
		
		String smid = id2smid(fetchId, null);
		for (int retry = 0; retry <= maxRetry; retry++) {
			if (!acquireSemaphore()) {
				status = STATUS_CANCELED; break; // FAILURE
			}
			captureUserSession(fetchRequestHeader.getMessageHeader("Cookie"),
					System.getProperty("movieFetchCaptureUserID", "").trim());
			overwriteUserSession(fetchRequestHeader);
			try {
				URLResource r = getWatchPageResource();
				if(r == null) break; // FAILURE
				HttpResponseHeader header = r.getResponseHeader(null, null);
				byte[] body = r.getResponseBody();
				if (header != null && body != null) {
					String content = new String(body, ExtUtil.getCharset(header));
					if (header.getStatusCode() == 200) {
						String authflag = header.getMessageHeader("x-niconico-authflag");
						if (authflag == null || authflag.equals("0")) {
							debugWithId("unauthorized, trying to login...");
							if (login(fetchRequestHeader)) {
								continue; // ログイン成功ならもう一度
							} else {
								status = "unauthorized."; break;
							}
						}
						if (smid.matches("\\d+")) {
							// コミュニティの場合はgetthumbinfoで取得出来ないので
							// 改めてwatchページから取得する
							smid = id2smid(fetchId, content);
						}
						procSuccess(header, smid, content);
						if (Cache.getType(smid) != null)
							break; // SUCCEEDED
					} else {
						procError(header.getStatusCode(), content);
					}
				}
			} catch(Exception e) {
				Logger.error(e);
			} finally {
				sem.release();
			}
			if (retry < maxRetry) {
				if (!waitForRetry(smid, retry)) {
					status = STATUS_CANCELED; break;
				}
			}
		}
		if (Cache.getType(smid) == null) {
			updateFetchResult(smid, fetchId + " " + status);
			if (status == STATUS_RESERVED) {
				return; // 予約の場合はもう一度実行するのでここで終了
			}
			infoStatusMessage(status);
		}
		completedTime = System.currentTimeMillis();
		debugWithId("task completed.");
	}
	
	private URLResource getWatchPageResource() throws IOException {
		String watch_url = "http://www.nicovideo.jp/watch/".concat(fetchId);
		URLResource r = new URLResource(watch_url);
		r.setFollowRedirects(false); // リダイレクトを自前で処理する
		HttpResponseHeader responseHeader = r.getResponseHeader(
				null, ExtUtil.setURL(fetchRequestHeader, watch_url, false));
		if (responseHeader != null && responseHeader.getStatusCode() == 302) {
			String location = responseHeader.getMessageHeader("Location");
			if (location != null) {
				if (location.contains("/ppv_video/")) {
					status = "ppv_video.";
					return null; // FAILURE
				}
				if (location.startsWith("/")) {
					watch_url = "http://www.nicovideo.jp".concat(location);
				} else {
					watch_url = location;
				}
				r = new URLResource(watch_url);
				responseHeader = r.getResponseHeader(
						null, ExtUtil.setURL(fetchRequestHeader, watch_url, false));
			} else {
				status = "redirect error.";
				return null; // FAILURE
			}
		}
		return r; // レスポンスヘッダ受信済
	}
	
	private void procSuccess(
			HttpResponseHeader responseHeader, String smid, String content) {
		accessRestriction.set(0); // 連続アクセス規制解除
		if (setNicohistoryCookie(responseHeader)) {
			String result = STATUS_FAILED;
			String getflvContent = getflv(content);
			if (getflvContent != null) {
				// 先にnl本体にタイトルを認識させておく(公式動画では必須)
				String title = recognizeTitle(smid, content);
				result = fetch(getflvContent, fetchRequestHeader);
				if (result.startsWith("http://")) {
					updateFetchResult(smid, title); // とりあえず結果を保存
					waitForComplete(smid);
				} else if (result != STATUS_NOT_FOUND) {
					maxRetry = 0;
				}
			}
			status = result;
		} else {
			maxRetry = 0;
			status = "cookie error.";
		}
	}
	
	private void procError(int statusCode, String content) {
		if (statusCode == 403) {
			if (content.contains("連続アクセス")) {
				accessRestriction.incrementAndGet();
			} else if (content.contains("非表示")) {
				status = "currently closed.";	maxRetry = 0;
			} else if (content.contains("視聴権限")) {
				status = "member only.";		maxRetry = 0;
			} else if (content.contains("削除")) {
				status = "deleted.";			maxRetry = 0;
			} else {
				status = "forbidden.";
			}
		} else if (statusCode == 404) {
			if (content.contains("<span id=\"deleted_message_")) {
				status = "deleted.";
			} else {
				status = "not found.";
			}
			maxRetry = 0; // 404の場合はリトライしても無駄っぽいので終了
		}
	}
	
	private static final Pattern NICOHISTORY_PATTERN = Pattern.compile(
			"(nicohistory=[^;]+)");
	private boolean setNicohistoryCookie(HttpResponseHeader responseHeader) {
		String set_cookie = responseHeader.getMessageHeader("Set-Cookie");
		if (set_cookie != null) {
			Matcher m = NICOHISTORY_PATTERN.matcher(set_cookie);
			if (m.find()) {
				String cookie = fetchRequestHeader.getMessageHeader("Cookie");
				if (cookie == null) {
					cookie = m.group(1);
				} else {
					if (cookie.indexOf("nicohistory=") < 0) {
						cookie += "; " + m.group(1);
					} else {
						cookie = cookie.replaceFirst("nicohistory=[^;]+", m.group(1));
					}
				}
				fetchRequestHeader.setMessageHeader("Cookie", cookie);
				return true;
			}
		}
		util.info("cannot get nicohistory cookie.");
		debug2("%s", responseHeader);
		return false;
	}
	
	private static final Pattern WATCH_URL_V_PATTERN = Pattern.compile(
			"^http://www\\.nicovideo\\.jp/watch/(\\w{2}\\d+)");
	private static final Pattern WATCHPAGE_V_PATTERN = Pattern.compile(
			"\\.addVariable\\(\"v\",\\s*\"(\\w{2}\\d+)");
	private static final Pattern WATCHPAGE_FLVPLAYER_PATTERN = Pattern.compile(
			"new\\s+SWFObject\\(\"(http://[^\"]+)\",\\s*\"flvplayer\",");
	private static final Pattern WATCHPAGE_MOVIETYPE_PATTERN = Pattern.compile(
			"\\.addVariable\\(\"movie_type\",\\s*\"([^\"]+)");
	
	private String getflv(String content) {
		String getflv_url = "http://flapi.nicovideo.jp/api/getflv";
		String v = fetchId, post_string = null;
		long ts = System.currentTimeMillis();
		if (content == null) {
			Matcher m = WATCH_URL_V_PATTERN.matcher(fetchRequestHeader.getURI());
			if (m.find()) {
				v = m.group(1);
			}
			getflv_url += "/" + v + "?as3=1&ts=" + ts;
		} else {
			Matcher m = WATCHPAGE_V_PATTERN.matcher(content);
			if (m.find()) {
				v = m.group(1);
			}
			post_string = "ts=" + ts + "&v=" + v;
			m = WATCHPAGE_FLVPLAYER_PATTERN.matcher(content);
			if (m.find()) {
				fetchRequestHeader.setURI(m.group(1)); // Referer
			}
			m = WATCHPAGE_MOVIETYPE_PATTERN.matcher(content);
			if (m.find() && m.group(1).equals("swf")) {
				post_string += "&as3=1";
			}
		}
		String getflvContent = ExtUtil.http_get(
				ExtUtil.setURL(fetchRequestHeader, getflv_url, true), post_string);
		if (getflvContent != null) {
			try { // nl本体にgetflvの内容を書き換えさせる(vid2cid対策)
				if (post_string != null) {
					getflv_url += "?" + post_string;
				}
				getflvContent = Main.getRewriterProcessor().stringRewriter(
						getflv_url, getflvContent, fetchRequestHeader, null);
			} catch (IOException e) {
				Logger.error(e);
			}
			if (getflvContent != null && getflvContent.contains("is_premium=1")) {
				premiumWait = Integer.getInteger("movieFetchPremiumWait", 0);
			}
		}
		debug2("requestHeader:\n%spostString: %s\n\ngetflvContent: %s\n",
				fetchRequestHeader, post_string, getflvContent);
		return getflvContent;
	}
	
	private boolean acquireSemaphore() {
		try {
			if (this.canceled) {
				throw new InterruptedException();
			}
			sem.acquire();
		} catch(InterruptedException e) {
			debugWithId("interrupted.");
			return false;
		}
		return true;
	}
	
	private boolean waitForRetry(String smid, int retryCount) {
		String reason = "failed";
		int wait_count = 10 * (retryCount + 1);
		int n = accessRestriction.get();
		if (n > 0) {
			reason = "forbidden";
			wait_count = wait_count * 30 + rand.nextInt(n) * n + n;
		}
		util.info(smid+" "+reason+", waiting "+wait_count+" seconds for next retry...");
		return secWait(wait_count);
	}
	
	private void waitForComplete(String smid) {
		// 割り込みをクリアしておく
		Thread.interrupted();
		
		// 少し待たないとキャッシュファイルが出来ていない事があるので…
		do { secWait(2); } while (Cache.getDLFlag(smid));
		
		if (premiumWait > 0) {
			int wait = rand.nextInt(premiumWait) + 1;
			debugWithId("premium waiting " + wait + " sec.");
			secWait(wait);
		}
	}
	
	private boolean secWait(int sec) {
		if (sec > 0) {
			try {
				Thread.sleep(sec * 1000);
			} catch(InterruptedException e) {
				debugWithId("interrupted.");
				return false;
			}
		}
		return true;
	}
	
	private void updateFetchResult(String smid, String title) {
		if (!smid.matches("[a-z]{2}\\d+"))
			return;
		if (fetchRequestHeader != null &&
				fetchRequestHeader.getParameter(FETCH_URI).contains("&result=0"))
			return;
		fetchResults.remove(smid); // 一度削除してリストの最後にする
		fetchResults.put(smid, title);
		fetchResultsUpdated = true;
	}
	
	// キューからfetcherを取り出して実行
	private static int continuousRequestCount = 0;
	public void taker() {
		try {
			for (;;) {
				nlMovieFetcher fetcher = requestQueue.take(); // BLOCK
				long delay = 0;
				if (!Boolean.getBoolean("movieFetchNoDelay")) {
					int wait_count = fetchCount() + 1;
					// 連投を確認するためにしばらく待つ
					util.debug("%s initial waiting %d sec.",
							fetcher.fetchId, wait_count);
					Thread.sleep(wait_count * 1000);
					delay = continuousRequestCount++ * wait_count * 1500;
					if (requestQueue.size() == 0) {
						continuousRequestCount = 0;
					}
				}
				scheduleFetchRequest(fetcher, delay);
			}
		} catch (InterruptedException e) {
			Logger.error(e);
		}
	}
	
	// updatelow.rb相当の機能
	private static final Pattern TEMPLIST_JSON_PATTERN = Pattern.compile(
			"^\"([a-z]{2}\\d+)(?:low)?\":\\[\"([^\"]*)\",");
	private static final int MIN_INTERVAL = 10000;
	private boolean updatelow() {
		if (!canceled && fetchId.equals("updatelow")) {
			infoStatusMessage("started.");
			long delay = 0;
			for (String id : Cache.getId2File().keySet()) {
				if (id.endsWith("low")) {
					String title = new Cache(id).getTitle();
					if (title.matches("削除\\]$")) continue;
					scheduleFetchRequest(id.substring(0, id.length() - 3), delay);
					delay += MIN_INTERVAL + rand.nextInt(MIN_INTERVAL * 2);
				}
			}
			// 一時ファイルは直接取れないのでJSONで取得
			for (String line : Cache.getTempListAsJson(false).split("\n")) {
				Matcher m = TEMPLIST_JSON_PATTERN.matcher(line);
				if (m.lookingAt()) {
					scheduleFetchRequest(m.group(1), delay);
					delay += MIN_INTERVAL + rand.nextInt(MIN_INTERVAL);
				}
			}
			// 削除以外でfetchに失敗したものも再取得
			for (Map.Entry<String,String> e : fetchResults.entrySet()) {
				String smid = e.getKey(), reason = e.getValue();
				if (Cache.getType(smid) == null && reason.indexOf("deleted") < 0) {
					scheduleFetchRequest(smid, delay);
					delay += MIN_INTERVAL + rand.nextInt(MIN_INTERVAL * 3);
				}
			}
			completedTime = System.currentTimeMillis();
			return true; // "updatelow"を処理した
		}
		if (Boolean.getBoolean("movieFetchUpdateCache") &&
				!fetchRequests.containsKey("updatelow")) {
			reserveFetchRequest("updatelow");
		}
		return false;
	}
	
	private StringResource method_remove(String id, HttpRequestHeader requestHeader) {
		boolean result = true;
		if (id.equals("all")) {
			synchronized (fetchResultsFileInfo) {
				fetchResults.clear();
				ExtUtil.renameToBak(fetchResultsFileInfo.first);
			}
		} else {
			result = fetchResults.remove(id2smid(id, null)) != null;
		}
		if (result) {
			fetchResultsUpdated = true;
			util.debug("%s results removed.", id);
			return resultOK;
		}
		return resultNG;
	}
	
	private StringResource method_cancel(String id) {
		if (id.equals("all")) {
			for (nlMovieFetcher fetcher : fetchRequests.values()) {
				cancelFetchRequest(fetcher);
			}
			return resultOK;
		} else {
			nlMovieFetcher fetcher = fetchRequests.get(id);
			if (fetcher != null) {
				cancelFetchRequest(fetcher);
				return resultOK;
			}
		}
		return resultNG;
	}
	
	private StringResource method_list() {
		return ExtUtil.responseText(
				"var fetchedList = "+createJSON()+";", "text/javascript");
	}
	
	private StringResource method_json() {
		return ExtUtil.responseText(createJSON(), "application/json");
	}
	
	private StringResource method_storage_get() {
		if (storageCache == null || !storageUpdated) {
			util.loadFile(storageFileInfo, "loadStorage", FILE_CHARSET);
		}
		return ExtUtil.responseText(storageCache, "application/json");
	}
	
	public void loadStorage(BufferedReader br) throws IOException {
		if (br != null) {
			long len = storageFileInfo.first.length();
			if (len > Integer.MAX_VALUE) {
				util.warn("'%s' is too large (%,d MB), skip.",
						storageFileInfo.first, len / (1024 * 1024));
				return;
			}
			StringBuilder sb = new StringBuilder((int)len);
			String line;
			while ((line = br.readLine()) != null) {
				sb.append(line);
			}
			storageCache = sb.toString();
		} else {
			storageCache = "{}";
		}
	}
	
	private StringResource method_storage_post(String storageNew) {
		String storageOld = storageCache;
		storageCache = storageNew;
		if (storageOld == null || storageOld.length() != storageNew.length() ||
				!storageOld.equals(storageNew)) {
			storageUpdated = true;
		}
		return resultOK;
	}
	
	private StringResource method_status(String id) {
		if (id.equals("all")) {
			StringBuilder sb = new StringBuilder(fetchRequests.size() * 40);
			for (Map.Entry<String, nlMovieFetcher> e : fetchRequests.entrySet()) {
				// <id>: <statusMessage>CRLF
				sb.append(e.getKey());
				sb.append(": ");
				sb.append(e.getValue().getStatusMessage());
				sb.append("\r\n");
			}
			return ExtUtil.responseText(sb.toString());
		} else {
			nlMovieFetcher fetcher = fetchRequests.get(id);
			if (fetcher != null) {
				String message = fetcher.getStatusMessage();
				if (message != null) {
					return ExtUtil.responseText(message);
				}
			}
		}
		return StringResource.getNotFound();
	}
	
	private StringResource method_fetch(String id, HttpRequestHeader requestHeader) {
		String smid = id2smid(id, null);
		if (id.equals("updatelow")) {
			nlMovieFetcher fetcher = fetchRequests.remove(id);
			if (fetcher != null && fetcher.future != null)
				fetcher.future.cancel(false);
		}
		if (ExtUtil.hasRemoved(smid)) {
			return infoStringResourceOK(smid, "hasRemoved!");
		}
		String thumbinfo = null;
		boolean ng_title = Boolean.getBoolean("movieFetchUseNGtitle");
		boolean ng_user  = Boolean.getBoolean("movieFetchUseNGuser");
		if (ng_title || ng_user) {
			try {
				thumbinfo = GetThumbInfoUtil.get(smid);
			} catch (IOException e) {}
		}
		if (ng_title && thumbinfo != null) {
			Matcher m = THUMBINFO_TITLE_PATTERN.matcher(thumbinfo);
			if (m.find() && ExtUtil.matchNGtitle(m.group(1)) != null)
				return infoStringResourceOK(smid, "NGtitle!!");
		}
		if (ng_user && thumbinfo != null) {
			Matcher m = THUMBINFO_USERID_PATTERN.matcher(thumbinfo);
			if (m.find() && ExtUtil.isNGuserId(m.group(1)))
				return infoStringResourceOK(smid, "NGuser!!");
		}
//		Cache fetched = new Cache(smid);
//		if (fetched.exists()) {
		if (Cache.getType(smid) != null) {
			return infoStringResourceOK(smid, "already cached.");
		} else if (Cache.getDLFlag(smid)) {
			return infoStringResourceOK(smid, "now caching.");
		}
		nlMovieFetcher fetcher = addFetchRequest(id, requestHeader);
		if (fetcher == null) {
			return ExtUtil.responseText("OK already requested.");
		} else if (requestQueue.offer(fetcher)) {
			return ExtUtil.responseText("OK fetch accepted.");
		}
		return resultNG;
	}
	
	private StringResource infoStringResourceOK(String smid, String message) {
		util.info(smid + " " + message);
		return ExtUtil.responseText("OK ".concat(message));
	}
	
	private boolean checkDiskFreeSpace() {
		long freeSpace = DiskFreeSpace.get(Cache.getCacheDir());
		long needed = Long.getLong("needFreeSpace", 100) * 1024 * 1024;
		if (needed < 0 || freeSpace < needed)
			return false;
		return true;
	}
	
	private nlMovieFetcher addFetchRequest(String id, HttpRequestHeader requestHeader) {
		nlMovieFetcher fetcher = fetchRequests.get(id);
/*		if (fetcher != null && fetcher.completedTime > 0) {
			// 完了リクエストなら直ぐに削除して再度リクエストを受け付ける
			// ここに来るのはキャッシュが無い場合なので再度実行して問題無いはず
			fetchRequests.remove(id);
			fetcher = null;
		}
		if (fetcher == null) {
			fetcher = new nlMovieFetcher(id, requestHeader);
			fetchRequests.put(id, fetcher);
		} else {
			util.debug("%s request found.", id);
		}*/
		if (fetcher != null) {
			util.debug("%s request found.", id);
			if (fetcher.completedTime == 0L) {
				return null; // 完了していなければ追加しない
			} else {
				// 完了リクエストなら直ぐに削除して再度リクエストを受け付ける
				// ここに来るのはキャッシュが無い場合なので再度実行して問題無いはず
				fetchRequests.remove(id);
				util.debug("%s request removed.", id);
			}
		}
		fetcher = new nlMovieFetcher(id, requestHeader);
		fetchRequests.put(id, fetcher);
		return fetcher;
	}
	
	private static final long REQUEST_EXPIRED = 60 * 1000; // 60秒
	private void expireFetchRequests() {
		long current = System.currentTimeMillis();
		Set<Map.Entry<String,nlMovieFetcher>> s = fetchRequests.entrySet();
		for (Map.Entry<String,nlMovieFetcher> e : s) {
			long completed = e.getValue().completedTime;
			if (completed != 0 && current - completed > REQUEST_EXPIRED) {
				if (s.remove(e)) {
					util.debug("%s request expired.", e.getKey());
				} else {
					util.warn("%s request expired failed.", e.getKey());
				}
			}
		}
	}
	
	private void reserveFetchRequest(String id) {
		reserveFetchRequest(addFetchRequest(id, fetchRequestHeader));
	}
	private void reserveFetchRequest(nlMovieFetcher fetcher) {
		Calendar c = Calendar.getInstance(TimeZone.getTimeZone("JST"));
		int hour = c.get(Calendar.HOUR_OF_DAY);
		if (hour >= 2) c.add(Calendar.DAY_OF_WEEK, 1); // 翌日
		c.set(Calendar.HOUR_OF_DAY, 2); // 午前2時(エコノミー時間帯明け)
		c.set(Calendar.MINUTE, 0);
		if (!Boolean.getBoolean("movieFetchNoDelay")) {
			// 時間を2時〜5時の間に進める
			c.set(Calendar.SECOND, 0);
			c.add(Calendar.SECOND, rand.nextInt(3 * 3600));
		}
		fetcher.setStatusMessage(STATUS_RESERVED);
		long delay = c.getTimeInMillis() - System.currentTimeMillis();
		scheduleFetchRequest(fetcher, delay);
		DateFormat df = DateFormat.getDateTimeInstance(DateFormat.LONG, DateFormat.LONG);
		util.info(fetcher.fetchId+" reserved at "+df.format(c.getTime())+".");
	}
	
	private void scheduleFetchRequest(String id, long delay) {
		scheduleFetchRequest(addFetchRequest(id, fetchRequestHeader), delay);
	}
	private void scheduleFetchRequest(nlMovieFetcher fetcher, long delay) {
		if(fetcher.canceled || delay < 0) delay = 0;
		util.debug("%s scheduled (%,dms delay).", fetcher.fetchId, delay);
		fetcher.future = sched.schedule(fetcher, delay, TimeUnit.MILLISECONDS);
	}
	
	private void cancelFetchRequest(nlMovieFetcher fetcher) {
		fetcher.canceled = true;
		if (fetcher.future != null) {
			if (fetcher.future.getDelay(TimeUnit.MILLISECONDS) > 0) {
				fetcher.setStatusMessage(STATUS_CANCELED);
				fetcher.completedTime = System.currentTimeMillis();
			}
			fetcher.future.cancel(true);
		}
	}
	
	private static final Pattern JSON_LINE_PATTERN = Pattern.compile(
			"^\"([a-z]{2}\\d+)\":\\s*\"(.*)\",?$");
	public void loadJSON(BufferedReader br) throws IOException {
		fetchResults.clear();
		if (br != null) {
			String line;
			while ((line = br.readLine()) != null) {
				Matcher m = JSON_LINE_PATTERN.matcher(line);
				if (m.matches()) {
					fetchResults.put(m.group(1), m.group(2).replace("\\", ""));
				}
			}
		}
	}
	
	private String createJSON(boolean stripCRLF) {
		// タイトルの最大値が99バイトらしいので1行平均60文字程度で初期化
		StringBuilder sb = new StringBuilder(fetchResults.size() * 60);
		sb.append(stripCRLF ? "{" : "{\r\n");
		if (!fetchResults.isEmpty()) {
			for (Map.Entry<String,String> e : fetchResults.entrySet()) {
				// "<smid>": "<title>",CRLF
				sb.append("\"");
				sb.append(e.getKey());
				sb.append("\": \"");
				sb.append(e.getValue().replace("\"", "\\\""));
				sb.append(stripCRLF ? "\"," : "\",\r\n");
			}
			sb.deleteCharAt(sb.lastIndexOf(","));
		}
		sb.append("}");
		return sb.toString();
	}
	
	private String createJSON() {
		return createJSON(false);
	}
	
	private static final Pattern WATCHPAGE_VIDEOID_PATTERN = Pattern.compile(
			"(?s)var Video = \\{\\s*v:\\s*'(\\d+)',\\s*id:\\s*'([a-z]{2}\\d+)',");
	private static final Pattern THUMBINFO_TITLE_PATTERN = Pattern.compile(
			"<title>([^<]+)</title>");
	private static final Pattern THUMBINFO_VIDEOID_PATTERN = Pattern.compile(
			"<video_id>([a-z]{2}\\d+)</video_id>");
	private static final Pattern THUMBINFO_USERID_PATTERN = Pattern.compile(
			"<user_id>(\\d+)</user_id>");
	
	private String id2smid(String id, String content) {
		String smid = id;
		// smidがthread_idの場合はvideo_idを取得
		if (id.matches("\\d+")) {
			if (content != null) { // watchページ
				Matcher m = WATCHPAGE_VIDEOID_PATTERN.matcher(content);
				if (m.find() && id.equals(m.group(1)))
					return m.group(2);
			}
			try {
				String thumbinfo = GetThumbInfoUtil.get(id);
				if (thumbinfo != null) {
					Matcher m = THUMBINFO_VIDEOID_PATTERN.matcher(thumbinfo);
					if(m.find()) return m.group(1);
				}
			} catch (IOException e) {}
		}
		return smid;
	}
	
	private String recognizeTitle(String smid, String content) {
		String title = NicoCachingTitleRetriever.getTitleFromResponse(content);
		if (title == null) {
			try {
				content = GetThumbInfoUtil.get(smid);
				title = NicoCachingTitleRetriever.getTitleFromXml(content);
			} catch (IOException e) {}
		}
		if (title != null) {
			// nl本体を通すと不要なRewriterを通るので自力でタイトルを設定する
			NicoCachingTitleRetriever.putTitleCache(smid, title);
		} else {
			return "(no title)";
		}
		return title;
	}
	
	// url(1),host(2),path(3),key(4),id(5)
	private static final Pattern GETFLV_URL_PATTERN = Pattern.compile(
			"(?<=^|&)url=(http%3A%2F%2F([^&]+)%2F([^&]+)%3F([^&]+)%3D(\\d+)\\.[^&]+)(?=&|$)");
	private static final Pattern GETFLV_DELETED_PATTERN = Pattern.compile(
			"(?<=^|&)(deleted=\\d)(?=&|$)");
	
	private String fetch(String getflvContent, HttpRequestHeader requestHeader) {
		Matcher m = GETFLV_DELETED_PATTERN.matcher(getflvContent);
		if (m.find()) {
			return m.group(1);	// 動画は削除済
		}
		if (!checkDiskFreeSpace()) {
			return STATUS_NO_DISK;
		}
		m = GETFLV_URL_PATTERN.matcher(getflvContent);
		if (!m.find()) {
			util.info("cannot get movie_url.");
			util.debug(getflvContent);
			return STATUS_FAILED;
		}
		OutputStream out = null;
		try {
			String movie_url = URLDecoder.decode(m.group(1), "UTF-8");
			if (Boolean.getBoolean("movieFetchNoEconomy") && movie_url.endsWith("low")) {
				reserveFetchRequest(this);
				return STATUS_RESERVED;
			}
			util.info(movie_url);
			URLResource r = new URLResource(movie_url);
			r.setProxyMyself(); // nl本体を経由してキャッシュさせる
			HttpResponseHeader responseHeader = r.getResponseHeader(
					null, ExtUtil.setURL(requestHeader, movie_url, false));
			if (responseHeader != null) {
				debug2("requestHeader:\n%sresponseHeader: %s",
						requestHeader, responseHeader);
				int statusCode = responseHeader.getStatusCode();
				if (statusCode == 200) {
					if (util.isDebug() && fetchCount() == 1) {
						out = new FileOutputStream("FETCHED");
					} else {
						//out = new dareka.common.NullOutputStream();
						out = getNullOutputStream();
					}
					// selectTransferToで本体のみ転送するとonResponseHeader()が
					// 呼ばれないのでここで初期化する
					contentLength = responseHeader.getContentLength();
					transferedLength = 0L;
					r.addTransferListener(this);
					r.selectTransferTo(null, out, requestHeader, false);
					return movie_url;
				}
				if(statusCode == 404) return STATUS_NOT_FOUND;
			}
		} catch (IOException e) {
			Logger.error(e);
		} finally {
			CloseUtil.close(out);
		}
		return STATUS_FAILED;
	}
	
	// 互換性維持のため、しばらくは面倒な処理を行う
	private OutputStream getNullOutputStream() throws IOException {
		try {
			return (OutputStream)java.lang.ClassLoader.getSystemClassLoader()
				.loadClass("dareka.common.NullOutputStream").newInstance();
		} catch (Exception e) {
			//WindowsならNUL,それ以外のOSなら/dev/nullがあるはず
			return new FileOutputStream(new File(
					File.separatorChar == '\\' ? "NUL" : "/dev/null"));
		}
	}
	
	private static final Pattern USER_SESSION_PATTERN = Pattern.compile(
			"user_session=(user_session_(\\d+)_\\d+)");
	
	private void overwriteUserSession(HttpRequestHeader requestHeader) {
		if (user_session != null) {
			String cookie = requestHeader.getMessageHeader("Cookie");
			if (cookie != null && cookie.contains("user_session=")) {
				Matcher m = USER_SESSION_PATTERN.matcher(cookie);
				if (!m.find() || m.group(1).equals(user_session))
					return;
				cookie = cookie.replaceFirst("user_session_\\d+_\\d+", user_session);
				debugWithId(m.group(1).concat(" replaced."));
			} else {
				cookie = "user_session=".concat(user_session);
			}
			requestHeader.setMessageHeader("Cookie", cookie);
			debugWithId(user_session.concat(" used."));
		}
	}
	
	private static boolean captureUserSession(String content, String user_id) {
		if (content != null) {
			if (user_id.isEmpty()) {
				if (user_session != null && System.getProperty("mail") == null) {
					util.info(user_session.concat(" removed."));
					user_session = null;
				}
			} else {
				Matcher m = USER_SESSION_PATTERN.matcher(content);
				if (m.find() && !m.group(1).equals(user_session) &&
						(m.group(2).equals(user_id) || "*".equals(user_id))) {
					user_session = m.group(1);
					util.info(user_session.concat(" captured."));
				}
			}
		}
		return user_session != null;
	}
	
	// 同時ログインを防ぐためにstatic synchronizedメソッドにする
	private static final String LOGIN_URL = "https://secure.nicovideo.jp/secure/login?site=niconico";
	private static synchronized boolean login(HttpRequestHeader requestHeader) {
		user_session = null;
		String mail = System.getProperty("mail");
		String password = System.getProperty("password");
		if(mail != null && password != null) {
			String postString = "mail=" + mail + "&password=" + password;
			try {
				URLResource r = new URLResource(LOGIN_URL);
				ExtUtil.setURL(requestHeader, LOGIN_URL, false);
				requestHeader.removeMessageHeader("Cookie");
				requestHeader.setMethod("POST");
				HttpResponseHeader res = r.getResponseHeader(
						new ByteArrayInputStream(postString.getBytes()), requestHeader);
				String cookie = res != null ? res.getMessageHeader("Set-Cookie") : null;
				if (captureUserSession(cookie, "*")) {
					util.info(mail.concat(" login succeeded."));
					return true;
				}
			} catch (IOException e) {
				if(util.isDebug()) Logger.error(e);
				util.warn(mail.concat(" login failed."));
			}
		}
		return false;
	}
	
}