hitomi.la crawler

hitomi.js

import React from 'react';
import client from 'cheerio-httpcli';
import fs from 'fs-extra';
import download from 'images-downloader';
import update from 'react-addons-update';
import archiver from 'archiver';
import axios from 'axios';
import Alert from 'react-s-alert';
import {Grid, Cell, Paper, Divider, Toolbar} from 'react-md';
import ReactDragList from 'react-drag-list';
import DateFormat from 'date-format';
import electron from 'electron';

import {connect} from 'react-redux';

import CrawlItemList from '../components/CrawlItemList';
import SearchBar from '../components/SearchBar';
import OptionSet from '../components/OptionSet';
import DownloadList from '../components/DownloadList';

const number_of_frontends = 2;

const subdomain_from_galleryid = (g) => {
	return String.fromCharCode(97 + (g % number_of_frontends));
}

const subdomain_from_url = (url, base) => {
	var retval = 'a';
	if (base) {
		retval = base;
	}

	var r = /\/(\d+)\//;
	var m = r.exec(url);
	var g;
	if (m) {
		g = parseInt(m[1]);
	}
	if (g) {
		retval = subdomain_from_galleryid(g) + retval;
	}

	return retval;
}

const url_from_url = (url, base) => {
	return url.replace(/\/\/..?\.hitomi\.la\//, '//' + subdomain_from_url(url, base) + '.hitomi.la/');
}

const promiseImgUrl = (code) => {
	return new Promise((resolve, reject) => {
		try {
			client.fetch("https://hitomi.la/galleryblock/" + code + ".html", {}, (err, $, res, body) => {
				resolve($('.dj-img1').children('img').attr('src'));
			});
		} catch (e) {
			console.log(e);
			reject(e);
		}
	});
}

const promiseFileRead = (fileName) => {
	return new Promise((resolve, reject) => {
		try {
			let cacheFolder = electron.remote.app.getAppPath() + "/../cache";
			fs.readFile(cacheFolder + fileName, 'utf8', (err, data) => {
				if (err)
					throw err;
				resolve(JSON.parse(data));
			});
		} catch (e) {
			reject(fileName);
		}
	});
}

const promiseGetGalleries = (galleriesNumber) => {
	return new Promise((resolve, reject) => {
		axios.get("https://ltn.hitomi.la/galleries" + galleriesNumber + ".json").then((resp) => {
			let cacheFolder = electron.remote.app.getAppPath() + "/../cache";
			if (!fs.existsSync(cacheFolder)) {
				fs.mkdirSync(cacheFolder);
			}
			let galleriesFile = fs.createWriteStream(cacheFolder + "/galleries" + galleriesNumber);
			galleriesFile.write(JSON.stringify(resp.data));
			galleriesFile.end();
			console.log("compliteGalleries:" + galleriesNumber);
			resolve(resp.data);
		}).catch((err) => {
			setTimeout(() => {
				promiseGetGalleries(galleriesNumber);
			}, 1000);
		});
	});
}

class Crawler extends React.Component {
	constructor(props) {
		super(props);
		this.state = {
			tags: [],
			galleries: [],
			searchList: [],
			cacheSearchList: [],
			cacheSearchMode: false,
			searchUrl: "",
			nowPage: 0,
			endPage: 0,
			searchResult: 0,
			selectedTagList: [],
			downloadList: [],
			pureTags: []
		};
	}

	componentWillMount() {
		if (localStorage.getItem("tagListTime") != null && localStorage.getItem("tagListTime") > (new Date().getTime() - (1000 * 60 * 60 * 24))) {
			console.log("setupTagCache");
			promiseFileRead("/tags").then((resp) => {
				let pureTagsList = [];
				for (let i = 0; i < resp.length; i++) {
					if (resp[i].category == "tag" || resp[i].category == "female" || resp[i].category == "male") {
						pureTagsList.push(resp[i]);
					}
				}
				this.setState({
					tags: update(this.state.tags, {$push: resp}),
					pureTags: update(this.state.pureTags, {$push: pureTagsList})
				});
			}).catch((e) => {
				console.log(e);
			});
		}
		if (localStorage.getItem("galleriesListTime") != null && localStorage.getItem("galleriesListTime") > (new Date().getTime() - (1000 * 60 * 60 * 24))) {
			console.log("setupGalleriesCache");
			let promiseList = [];
			for (let i = 0; i < 20; i++) {
				promiseList.push(promiseFileRead("/galleries" + i));
			}
			Promise.all(promiseList).then((resp) => {
				let tempGalleries = [];
				for (let i = 0; i < resp.length; i++) {
					tempGalleries = tempGalleries.concat(resp[i]);
				}
				return tempGalleries;
			}).then((resp) => {
				this.setState({
					galleries: update(this.state.galleries, {$push: resp})
				});
				Alert.success("캐싱파일 로딩 성공", {
					position: 'top-right',
					timeout: 5000
				});
			}).catch((e) => {
				console.log(e);
			});
		}
	}

	componentDidMount() {
		if (localStorage.getItem("tagListTime") == null || localStorage.getItem("tagListTime") <= (new Date().getTime() - (1000 * 60 * 60 * 24))) {
			console.log("tag Loading");
			axios.get("https://ltn.hitomi.la/tags.json").then((resp) => {
				let tagTypeNames = Object.keys(resp.data);
				let parseTagsList = [];
				let pureTagsList = [];
				for (let i = 0; i < tagTypeNames.length; i++) {
					if (tagTypeNames[i] != "language") {
						for (let j = 0; j < resp.data[tagTypeNames[i]].length; j++) {
							parseTagsList.push({
								category: tagTypeNames[i],
								tagname: resp.data[tagTypeNames[i]][j].s,
								tagamount: resp.data[tagTypeNames[i]][j].t,
								label: tagTypeNames[i] + ":" + resp.data[tagTypeNames[i]][j].s + " (" + resp.data[tagTypeNames[i]][j].t + ")",
								value: tagTypeNames[i] + ":" + resp.data[tagTypeNames[i]][j].s,
								tagno: parseTagsList.length + 1
							});
							if (tagTypeNames[i] == "tag" || tagTypeNames[i] == "female" || tagTypeNames[i] == "male") {
								pureTagsList.push({
									category: tagTypeNames[i],
									tagname: resp.data[tagTypeNames[i]][j].s,
									tagamount: resp.data[tagTypeNames[i]][j].t,
									label: tagTypeNames[i] + ":" + resp.data[tagTypeNames[i]][j].s + " (" + resp.data[tagTypeNames[i]][j].t + ")",
									value: tagTypeNames[i] + ":" + resp.data[tagTypeNames[i]][j].s,
									tagno: parseTagsList.length + 1
								});
							}
						}
					}
				}
				this.setState({
					tags: update(this.state.tags, {$push: parseTagsList}),
					pureTags: update(this.state.pureTags, {$push: pureTagsList})
				});
				try {
					let cacheFolder = electron.remote.app.getAppPath() + "/../cache";
					if (!fs.existsSync(cacheFolder)) {
						fs.mkdirSync(cacheFolder);
					}
					let tagsFile = fs.createWriteStream(cacheFolder + "/tags");
					tagsFile.write(JSON.stringify(parseTagsList));
					tagsFile.end();
					localStorage.setItem("tagListTime", new Date().getTime());
					console.log("tagSaved");
				} catch (e) {
					console.log(e);
				}
			});
		}
		if (localStorage.getItem("galleriesListTime") == null || localStorage.getItem("galleriesListTime") <= (new Date().getTime() - (1000 * 60 * 60 * 24))) {
			console.log("galleries Loading");
			let promiseList = [];
			let galleriesList = [];
			for (let i = 0; i < 20; i++) {
				promiseList.push(promiseGetGalleries(i));
			}
			Promise.all(promiseList).then((resp) => {
				for (let i = 0; i < resp.length; i++) {
					galleriesList = galleriesList.concat(resp[i]);
				}
				return galleriesList;
			}).then((resp) => {
				this.setState({
					galleries: update(this.state.galleries, {$push: resp})
				});
				localStorage.setItem("galleriesListTime", new Date().getTime());
				Alert.success("캐싱파일 저장 성공", {
					position: 'top-right',
					timeout: 5000
				});
			}).catch((e) => {
				console.log(e);
			});
		}
	}

	_insertDownloadList(title, path, page, code) {
		let no = this.state.downloadList.length;
		let item = {
			no: no,
			title: title,
			path: path,
			page: page,
			code: code,
			status: 0,
			date: new Date()
		}
		this.setState({
			downloadList: update(this.state.downloadList, {$push: [item]})
		});
		return no;
	}

	_updateStatusDownloadList(no, status) {
		this.setState({
			downloadList: update(this.state.downloadList, {
				[no]: {
					status: {
						$set: status
					}
				}
			})
		});
	}

	_download(code, filename) {
		let _this = this;
		let downloadFileNameFormat = this.props.Options.downloadFileNameFormat;

		let downloadPath = this.props.Options.downloadFolderPath;
		let downloadImagePath = downloadPath + "/" + code;
		let downloadArchivePath = downloadPath + "/archive";
		let downloadFinalPath = downloadArchivePath;
		let deleteOption = this.props.Options.downloadAfterRemove;
		if (this.props.Options.downloadDailySeparation) {
			downloadFinalPath = downloadArchivePath + "/" + DateFormat.asString('yyMMdd', new Date());
		}

		if (!fs.existsSync(downloadPath)) {
			fs.mkdirSync(downloadPath);
		}
		if (!fs.existsSync(downloadImagePath)) {
			fs.mkdirSync(downloadImagePath);
		}
		if (!fs.existsSync(downloadArchivePath)) {
			fs.mkdirSync(downloadArchivePath);
		}
		if (!fs.existsSync(downloadFinalPath)) {
			fs.mkdirSync(downloadFinalPath);
		}
		let regExp = /[\/?:|*<>\\\"]/gi
		let stripFilename = filename.replace(regExp, "");
		client.fetch('https://hitomi.la/reader/' + code + '.html', function(err, $, res, body) {
			let mangaImageFileList = [];
			$(".img-url").each((i, item) => {
				mangaImageFileList.push("https:" + url_from_url($(item).text()));
			});
			let downloadNo = _this._insertDownloadList(stripFilename, downloadFinalPath, mangaImageFileList.length, code);
			download.images(mangaImageFileList, downloadImagePath + "/").then((result) => {
				var output = fs.createWriteStream(downloadFinalPath + "/" + stripFilename + ".zip");
				var archive = archiver('zip', {
					zlib: {
						level: 9
					} // Sets the compression level.
				});
				archive.pipe(output);
				output.on('close', function() {
					console.log(archive.pointer() + ' total bytes');
					if (deleteOption) {
						fs.remove(downloadImagePath, err => {
							if (err) {
								console.log(err);
							}
						});
					}
					_this._updateStatusDownloadList(downloadNo, 2);
				});
				for (let i = 0; i < result.length; i++) {
					archive.file(result[i].filename, {
						name: result[i].filename.substr(result[i].filename.lastIndexOf("/") + 1)
					});
				}
				return archive;
			}).then((result) => {
				_this._updateStatusDownloadList(downloadNo, 1);
				console.log("All file Download Done!");
				result.finalize();
			}).catch(error => console.log("downloaded error", error))
		});
	}

	_getNextPage() {
		if (this.state.cacheSearchMode) {
			let promiseList = []
			let nextPageList = this.state.cacheSearchList.slice((25 * (this.state.nowPage + 1)), (this.state.cacheSearchList.length >= (25 * (this.state.nowPage + 2)))
				? 25 * (this.state.nowPage + 2)
				: this.state.cacheSearchList.length);
			for (let i = 0; i < (nextPageList.length > 25
				? 25
				: nextPageList.length); i++) {
				promiseList.push(promiseImgUrl(nextPageList[i].id));
			}
			Promise.all(promiseList).then((resp) => {
				for (let i = 0; i < resp.length; i++) {
					nextPageList[i].img = resp[i];
				}
			}).then(() => {
				this.setState({
					searchList: update(this.state.searchList, {$push: nextPageList}),
					nowPage: this.state.nowPage + 1
				});
			});
		} else if (this.state.url != "") {
			client.fetch(this.state.searchUrl + ((this.state.nowPage * 1) + 2) + '.html', {}, (err, $, res, body) => {
				if (err == null) {
					let jsonList = [];
					let crawlData = $('.gallery-content').children('div');
					let scriptTagList = $('script');
					let endPage = 0;
					for (let i = 0; i < scriptTagList.length; i++) {
						if (scriptTagList[i].children.length != 0 && scriptTagList[i].children[0].data.indexOf("insert_paging") != -1) {
							endPage = scriptTagList[i].children[0].data.substring(scriptTagList[i].children[0].data.indexOf("1, ") + 3, scriptTagList[i].children[0].data.indexOf(");"));
						}
					}
					for (let i = 0; i < $(crawlData).find('h1').length; i++) {
						let artistList = [];
						let tagList = [];
						let seriesList = [];
						$(crawlData[i]).children('.artist-list').children('ul').children('li').each(function(i, artist) {
							artistList.push($(artist).children('a').text());
						});
						$(crawlData[i]).children('.dj-content').children('table').children('tr').eq(0).find('a').each(function(i, series) {
							seriesList.push($(series).text());
						});
						$(crawlData[i]).children('.dj-content').children('table').children('tr').eq(3).find('a').each(function(i, tag) {
							let category;
							let tagname;
							let originTag = $(tag).text();
							if (originTag.indexOf("♀") != -1) {
								category = "female";
								tagname = originTag.substring(0, originTag.indexOf("♀") - 1);
							} else if (originTag.indexOf("♂") != -1) {
								category = "male";
								tagname = originTag.substring(0, originTag.indexOf("♂") - 1);
							} else {
								category = "tag";
								tagname = originTag;
							}
							tagList.push(category + ":" + tagname);
						});
						let originLanguage = $(crawlData[i]).children('.dj-content').children('table').children('tr').eq(2).find('a').attr('href');
						let resultLanguage = null;
						if (originLanguage != null) {
							resultLanguage = originLanguage.substring(originLanguage.indexOf('-') + 1, originLanguage.lastIndexOf('-'));
						}
						if (this.props.Options.ignoreTagList.length != 0) {
							let ignoreTagList = this.props.Options.ignoreTagList;
							let ignoreExist = true;
							for (let j = 0; j < ignoreTagList.length; j++) {
								if (ignoreTagList[j].category == "female" || ignoreTagList[j].category == "male") {
									if (tagList.indexOf(ignoreTagList[j].label) != -1) {
										ignoreExist = false;
										break;
									}
								} else if (ignoreTagList[j].category == "tag") {
									if (tagList.indexOf(ignoreTagList[j].label) != -1) {
										ignoreExist = false;
										break;
									}
								}
							}
							if (ignoreExist) {
								let item = {
									"img": $(crawlData[i]).find('.dj-img-cont').find('img').attr('src'),
									"n": $(crawlData[i]).children('h1').children('a').text(),
									"id": $(crawlData[i]).children('h1').children('a').attr('href').replace('/galleries/', '').replace('.html', ''),
									"a": artistList,
									"p": seriesList,
									"type": $(crawlData[i]).children('.dj-content').children('table').children('tr').eq(1).find('a').text(),
									"l": resultLanguage,
									"t": tagList
								}
								jsonList.push(item);
							}
						} else {
							let item = {
								"img": $(crawlData[i]).find('.dj-img-cont').find('img').attr('src'),
								"n": $(crawlData[i]).children('h1').children('a').text(),
								"id": $(crawlData[i]).children('h1').children('a').attr('href').replace('/galleries/', '').replace('.html', ''),
								"a": artistList,
								"p": seriesList,
								"type": $(crawlData[i]).children('.dj-content').children('table').children('tr').eq(1).find('a').text(),
								"l": resultLanguage,
								"t": tagList
							}
							jsonList.push(item);
						}
					}
					this.setState({
						nowPage: this.state.nowPage + 1,
						searchList: update(this.state.searchList, {$push: jsonList})
					});
				} else {
					setTimeout(() => {
						this._getNextPage();
					}, 1000);
				}
			});
		}
	}

	_setSearchState(tagList, language, type, selectString) {
		this.setState({
			searchList: update(this.state.searchList, {
				$splice: [
					[0, this.state.searchList.length]
				]
			}),
			cacheSearchList: update(this.state.cacheSearchList, {
				$splice: [
					[0, this.state.cacheSearchList.length]
				]
			}),
			nowPage: 0,
			endPage: 0,
			searchResult: 0,
			searchUrl: ""
		}, () => {
			if (tagList.length > 1 || selectString != "" || (type != "all" && tagList.length > 0)) {
				let tempList = [];
				for (let i = 0; i < this.state.galleries.length; i++) {
					if (this.state.galleries[i].l != null && this.state.galleries[i].l.indexOf((language == "all")
						? ""
						: language) != -1) {
						if (this.state.galleries[i].type != null && this.state.galleries[i].type.indexOf((type == "all")
							? ""
							: type) != -1) {
							if (this.state.galleries[i].n.indexOf(selectString) != -1) {
								let exist = true;
								for (let j = 0; j < tagList.length; j++) {
									if (tagList[j].category == "female" || tagList[j].category == "male") {
										if (this.state.galleries[i].t == null || this.state.galleries[i].t.indexOf(tagList[j].label) == -1) {
											exist = false;
											break;
										}
									} else if (tagList[j].category == "group") {
										if (this.state.galleries[i].g == null || this.state.galleries[i].g.indexOf(tagList[j].tagname) == -1) {
											exist = false;
											break;
										}
									} else if (tagList[j].category == "series") {
										if (this.state.galleries[i].p == null || this.state.galleries[i].p.indexOf(tagList[j].tagname) == -1) {
											exist = false;
											break;
										}
									} else if (tagList[j].category == "artist") {
										if (this.state.galleries[i].a == null || this.state.galleries[i].a.indexOf(tagList[j].tagname) == -1) {
											exist = false;
											break;
										}
									} else if (tagList[j].category == "character") {
										if (this.state.galleries[i].c == null || this.state.galleries[i].c.indexOf(tagList[j].tagname) == -1) {
											exist = false;
											break;
										}
									} else if (tagList[j].category == "tag") {
										if (this.state.galleries[i].t == null || this.state.galleries[i].t.indexOf(tagList[j].tagname) == -1) {
											exist = false;
											break;
										}
									}
								}
								if (exist == true) {
									if (this.props.Options.ignoreTagList.length != 0) {
										let ignoreTagList = this.props.Options.ignoreTagList;
										let ignoreExist = true;
										for (let j = 0; j < ignoreTagList.length; j++) {
											if (ignoreTagList[j].category == "female" || ignoreTagList[j].category == "male") {
												if (this.state.galleries[i].t == null || this.state.galleries[i].t.indexOf(ignoreTagList[j].label) != -1) {
													ignoreExist = false;
													break;
												}
											} else if (ignoreTagList[j].category == "tag") {
												if (this.state.galleries[i].t == null || this.state.galleries[i].t.indexOf(ignoreTagList[j].tagname) != -1) {
													ignoreExist = false;
													break;
												}
											}
										}
										if (ignoreExist) {
											tempList.push(this.state.galleries[i]);
										}
									} else {
										tempList.push(this.state.galleries[i]);
									}
								}
							}
						}
					}
				}
				tempList = tempList.sort((a, b) => {
					return b.id - a.id
				});
				let promiseList = []
				for (let i = 0; i < (tempList.length > 25
					? 25
					: tempList.length); i++) {
					promiseList.push(promiseImgUrl(tempList[i].id));
				}
				Promise.all(promiseList).then((resp) => {
					for (let i = 0; i < resp.length; i++) {
						tempList[i].img = resp[i];
					}
				}).then(() => {
					this.setState({
						searchList: update(this.state.searchList, {
							$push: tempList.slice(0, (tempList.length >= 25)
								? 25
								: tempList.length)
						}),
						cacheSearchList: update(this.state.cacheSearchList, {$push: tempList}),
						cacheSearchMode: true,
						nowPage: 0,
						endPage: Math.floor(tempList.length / 25),
						searchResult: tempList.length
					});
				});
			} else {
				let pagePreUrl = "https://hitomi.la/index-" + language + "-";
				if (type != "all") {
					pagePreUrl = "https://hitomi.la/type/" + type + "-" + language + "-";
				} else if (tagList.length == 1) {
					if (tagList[0].category == "female" || tagList[0].category == "male") {
						pagePreUrl = "https://hitomi.la/tag/" + tagList[0].category + ":" + tagList[0].tagname + "-" + language + "-";
					} else {
						pagePreUrl = "https://hitomi.la/" + tagList[0].category + "/" + tagList[0].tagname + "-" + language + "-";
					}
				}
				client.fetch(pagePreUrl + '1.html', {}, (err, $, res, body) => {
					if (err == null) {
						let jsonList = [];
						let crawlData = $('.gallery-content').children('div');
						let scriptTagList = $('script');
						let endPage = 0;
						for (let i = 0; i < scriptTagList.length; i++) {
							if (scriptTagList[i].children.length != 0 && scriptTagList[i].children[0].data.indexOf("insert_paging") != -1) {
								endPage = scriptTagList[i].children[0].data.substring(scriptTagList[i].children[0].data.indexOf("1, ") + 3, scriptTagList[i].children[0].data.indexOf(");"));
							}
						}
						for (let i = 0; i < $(crawlData).find('h1').length; i++) {
							let artistList = [];
							let tagList = [];
							let seriesList = [];
							$(crawlData[i]).children('.artist-list').children('ul').children('li').each(function(i, artist) {
								artistList.push($(artist).children('a').text());
							});
							$(crawlData[i]).children('.dj-content').children('table').children('tr').eq(0).find('a').each(function(i, series) {
								seriesList.push($(series).text());
							});
							$(crawlData[i]).children('.dj-content').children('table').children('tr').eq(3).find('a').each(function(i, tag) {
								let category;
								let tagname;
								let originTag = $(tag).text();
								if (originTag.indexOf("♀") != -1) {
									category = "female";
									tagname = originTag.substring(0, originTag.indexOf("♀") - 1);
								} else if (originTag.indexOf("♂") != -1) {
									category = "male";
									tagname = originTag.substring(0, originTag.indexOf("♂") - 1);
								} else {
									category = "tag";
									tagname = originTag;
								}
								tagList.push(category + ":" + tagname);
							});
							let originLanguage = $(crawlData[i]).children('.dj-content').children('table').children('tr').eq(2).find('a').attr('href');
							let resultLanguage = null;
							if (originLanguage != null) {
								resultLanguage = originLanguage.substring(originLanguage.indexOf('-') + 1, originLanguage.lastIndexOf('-'));
							}
							if (this.props.Options.ignoreTagList.length != 0) {
								let ignoreTagList = this.props.Options.ignoreTagList;
								let ignoreExist = true;
								for (let j = 0; j < ignoreTagList.length; j++) {
									if (ignoreTagList[j].category == "female" || ignoreTagList[j].category == "male") {
										if (tagList.indexOf(ignoreTagList[j].label) != -1) {
											ignoreExist = false;
											break;
										}
									} else if (ignoreTagList[j].category == "tag") {
										if (tagList.indexOf(ignoreTagList[j].label) != -1) {
											ignoreExist = false;
											break;
										}
									}
								}
								if (ignoreExist) {
									let item = {
										"img": $(crawlData[i]).find('.dj-img-cont').find('img').attr('src'),
										"n": $(crawlData[i]).children('h1').children('a').text(),
										"id": $(crawlData[i]).children('h1').children('a').attr('href').replace('/galleries/', '').replace('.html', ''),
										"a": artistList,
										"p": seriesList,
										"type": $(crawlData[i]).children('.dj-content').children('table').children('tr').eq(1).find('a').text(),
										"l": resultLanguage,
										"t": tagList
									}
									jsonList.push(item);
								}
							} else {
								let item = {
									"img": $(crawlData[i]).find('.dj-img-cont').find('img').attr('src'),
									"n": $(crawlData[i]).children('h1').children('a').text(),
									"id": $(crawlData[i]).children('h1').children('a').attr('href').replace('/galleries/', '').replace('.html', ''),
									"a": artistList,
									"p": seriesList,
									"type": $(crawlData[i]).children('.dj-content').children('table').children('tr').eq(1).find('a').text(),
									"l": resultLanguage,
									"t": tagList
								}
								jsonList.push(item);
							}
						}
						this.setState({
							searchList: update(this.state.searchList, {$push: jsonList}),
							searchUrl: pagePreUrl,
							cacheSearchMode: false,
							nowPage: 0,
							endPage: ((endPage * 1) - 1)
						});
					} else {
						console.log(err);
						setTimeout(() => {
							this._setSearchState(tagList, language, selectString);
						}, 500);
					}
				});
			}
		});
	}

	_searchFilter(list, keyword) {
		if (keyword == "") {
			return [];
		} else {
			let itemList = [];
			for (let i = 0; i < list.length; i++) {
				if (list[i].tagname.indexOf(keyword) != -1) {
					let isContained = false
					for (let j = 0; j < this.state.selectedTagList.length; j++) {
						if (list[i].tagname == this.state.selectedTagList[j].tagname) {
							isContained = true;
							break;
						}
					}
					if (!isContained) {
						itemList.push(list[i]);
					}
				}
			}
			itemList.sort((a, b) => {
				return b.tagamount - a.tagamount
			});
			return itemList.slice(0, 30);
		}
	}

	_insertTag(label, tagname, category) {
		let isExist = false;
		for (let i = 0; i < this.state.selectedTagList.length; i++) {
			if (this.state.selectedTagList[i].label.indexOf(label) != -1) {
				isExist = true;
				break;
			}
		}
		if (!isExist) {
			this.setState({
				selectedTagList: update(this.state.selectedTagList, {
					$push: [
						{
							label: label,
							tagname: tagname,
							category: category
						}
					]
				})
			});
		}
	}

	_deleteTag(key) {
		this.setState({
			selectedTagList: update(this.state.selectedTagList, {
				$splice: [
					[key, 1]
				]
			})
		});
	}

	render() {
		return (
			<div>
				<SearchBar onClick={(tagList, language, type, selectString) => {
					this._setSearchState(tagList, language, type, selectString);
				}} tags={this.state.tags} pureTags={this.state.pureTags} searchResult={this.state.searchResult} searchFilter={(list, keyword) => {
					return (this._searchFilter(list, keyword));
				}} searchTagList={this.state.selectedTagList} insertTag={(label, tagname, category) => {
					this._insertTag(label, tagname, category);
				}} deleteTag={(key) => {
					this._deleteTag(key);
				}}/>
				<DownloadList downloadList={this.state.downloadList}/>
				<Grid className="md-toolbar-relative--prominent">
					<Cell desktopSize={8} desktopOffset={2} tabletSize={8} phoneSize={4}>
						<CrawlItemList dataList={this.state.searchList} nowPage={this.state.nowPage} endPage={this.state.endPage} nextPageLoad={() => {
							this._getNextPage();
						}} onDown={(code, filename) => {
							this._download(code, filename);
						}} insertTag={(label, tagname, category) => {
							this._insertTag(label, tagname, category);
						}}/>
					</Cell>
				</Grid>
			</div>
		);
	}

}

const mapStateToProps = (state) => {
	return {Options: state.OptionReducer};
};

export default connect(mapStateToProps)(Crawler);