Skip to content

Instantly share code, notes, and snippets.

@esterTion

esterTion/0_coreseek_sphinx_index-reindex-collection

Created October 25, 2018 14:49
Show Gist options
  • Save esterTion/a361124b2ed5e66209fd4ec8282976b0 to your computer and use it in GitHub Desktop.
Save esterTion/a361124b2ed5e66209fd4ec8282976b0 to your computer and use it in GitHub Desktop.
Download ZIP
colection of coreseek/sphinx configuration & reindex script on biliplus.com
Raw
cs-index.conf
#MySQL数据源配置，详情请查看：http://www.coreseek.cn/products-install/mysql/
#请先将var/test/documents.sql导入数据库，并配置好以下的MySQL用户密码数据库
#源定义
source avinfo
{
type = mysql
sql_host = <deleted>
sql_user = <deleted>
sql_pass = <deleted>
sql_db = <deleted>
sql_port = <deleted>
sql_query_pre = SET NAMES utf8
sql_query = SELECT /*+ MAX_EXECUTION_TIME(1800000) */ aid, created, tid, mid, video_review, play, title, description, tag FROM cache_av
sql_attr_timestamp = created
sql_attr_uint = tid
sql_attr_uint = mid
sql_attr_uint = video_review
sql_attr_uint = play
}
index avinfo
{
source = avinfo
path = /data/index/temp/avinfo
docinfo = extern
mlock = 0
morphology = none
min_word_len = 1
html_strip = 0
charset_dictpath = /usr/local/coreseek/etc/
charset_type = zh_cn.utf-8
}
source avinfo_delta
{
type = mysql
sql_host = <deleted>
sql_user = <deleted>
sql_pass = <deleted>
sql_db = <deleted>
sql_port = <deleted>
sql_query_pre = SET NAMES utf8
sql_query_pre = REPLACE INTO sphinx_last_ts SELECT 1, ts FROM sphinx_last_ts WHERE id=2
sql_query_pre = REPLACE INTO sphinx_last_ts SELECT 2, MAX(lastupdatets) FROM cache_av
sql_query = SELECT /*+ MAX_EXECUTION_TIME(1800000) */ aid, created, tid, mid, video_review, play, title, description, tag FROM cache_av WHERE lastupdatets > (SELECT ts FROM sphinx_last_ts WHERE id=1)
sql_attr_timestamp = created
sql_attr_uint = tid
sql_attr_uint = mid
sql_attr_uint = video_review
sql_attr_uint = play
}
index avinfo_delta
{
source = avinfo_delta
path = /data/index/temp/avinfo_delta
docinfo = extern
mlock = 0
morphology = none
min_word_len = 1
html_strip = 0
charset_dictpath = /usr/local/coreseek/etc/
charset_type = zh_cn.utf-8
}
#全局index定义
indexer
{
mem_limit = 512M
write_buffer = 8M
max_iops = 50
}
Raw
cs.conf
#MySQL数据源配置，详情请查看：http://www.coreseek.cn/products-install/mysql/
#请先将var/test/documents.sql导入数据库，并配置好以下的MySQL用户密码数据库
#源定义
source avinfo
{
type = mysql
sql_host = <deleted>
sql_user = <deleted>
sql_pass = <deleted>
sql_db = <deleted>
sql_port = <deleted>
sql_query_pre = SET NAMES utf8
sql_query = SELECT /*+ MAX_EXECUTION_TIME(1800000) */ aid, created, tid, mid, video_review, play, title, description, tag FROM cache_av
sql_attr_timestamp = created
sql_attr_uint = tid
sql_attr_uint = mid
sql_attr_uint = video_review
sql_attr_uint = play
}
index avinfo
{
source = avinfo
path = /data/index/avinfo
#path = var/data/avinfo
docinfo = extern
mlock = 0
morphology = none
min_word_len = 1
html_strip = 0
#中文分词配置，详情请查看：http://www.coreseek.cn/products-install/coreseek_mmseg/
#charset_dictpath = /usr/local/mmseg3/etc/ #BSD、Linux环境下设置，/符号结尾
charset_dictpath = /usr/local/coreseek/etc/ #Windows环境下设置，/符号结尾，最好给出绝对路径，例如：C:/usr/local/coreseek/etc/...
charset_type = zh_cn.utf-8
}
source fav
{
type = mysql
sql_host = <deleted>
sql_user = <deleted>
sql_pass = <deleted>
sql_db = <deleted>
sql_port = <deleted>
sql_query_pre = SET NAMES utf8
sql_query = SELECT aid, created, typeid, mid, video_review, play, title, description, keywords FROM bilibili_all
sql_attr_timestamp = created
sql_attr_uint = typeid
sql_attr_uint = mid
sql_attr_uint = video_review
sql_attr_uint = play
}
index fav
{
source = fav
path = /data/index/fav
#path = var/data/fav
docinfo = extern
mlock = 0
morphology = none
min_word_len = 1
html_strip = 0
#中文分词配置，详情请查看：http://www.coreseek.cn/products-install/coreseek_mmseg/
#charset_dictpath = /usr/local/mmseg3/etc/ #BSD、Linux环境下设置，/符号结尾
charset_dictpath = /usr/local/coreseek/etc/ #Windows环境下设置，/符号结尾，最好给出绝对路径，例如：C:/usr/local/coreseek/etc/...
charset_type = zh_cn.utf-8
}
#全局index定义
indexer
{
mem_limit = 128M
}
#searchd服务定义
searchd
{
listen = 9312
read_timeout = 5
max_children = 30
max_matches = 1000
seamless_rotate = 0
preopen_indexes = 0
unlink_old = 1
pid_file = /usr/local/coreseek/var/log/searchd_mysql.pid #请修改为实际使用的绝对路径，例如：/usr/local/coreseek/var/...
log = /usr/local/coreseek/var/log/searchd_mysql.log #请修改为实际使用的绝对路径，例如：/usr/local/coreseek/var/...
query_log = /usr/local/coreseek/var/log/query_mysql.log #请修改为实际使用的绝对路径，例如：/usr/local/coreseek/var/...
binlog_path = #关闭binlog日志
}
Raw
reindex.sh
#!/bin/sh
\cp -f /data/index/avinfo.sp* /data/index/temp/ &
sudo -u mysql /usr/local/coreseek/bin/indexer --config /usr/local/coreseek/etc/cs-index.conf avinfo_delta
wait
chown -R mysql:mysql /data/index/temp
#sudo -u mysql /usr/local/coreseek/bin/indexer --config /usr/local/coreseek/etc/cs-index.conf --merge avinfo avinfo_delta --merge-dst-range deleted 0 0
sudo -u mysql /usr/local/coreseek/bin/indexer --config /usr/local/coreseek/etc/cs-index.conf --merge avinfo avinfo_delta
rm -rf /data/index/temp/avinfo_delta*
touch /data/home/web/task/.searchd_maintaince
killall searchd
mv -f /data/index/temp/* /data/index/
sudo -u mysql /usr/local/coreseek/bin/searchd --config /usr/local/coreseek/etc/cs.conf >/dev/null 2>&1
rm -f /data/home/web/task/.searchd_maintaince
exit
sudo -u mysql /usr/local/coreseek/bin/indexer --config /usr/local/coreseek/etc/cs-index.conf avinfo
touch /data/home/web/task/.searchd_maintaince
killall searchd
mv -f /data/index/temp/* /data/index/
sudo -u mysql /usr/local/coreseek/bin/searchd --config /usr/local/coreseek/etc/cs.conf >/dev/null 2>&1 &
rm -f /data/home/web/task/.searchd_maintaince
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment