Skip to content

Instantly share code, notes, and snippets.

@seanorama

seanorama/README.md

Last active January 2, 2018 11:07
Show Gist options
  • Save seanorama/3860b48c83ec17a2680678e48d95b657 to your computer and use it in GitHub Desktop.
Save seanorama/3860b48c83ec17a2680678e48d95b657 to your computer and use it in GitHub Desktop.
Download ZIP
improve logsearch solr
Raw
README.md

#in progress of writing this

  1. In Ambari, stop LogSearch service

  2. Patch schema files

sudo -iu logsearch
curl -sLO https://gist.github.com/seanorama/3860b48c83ec17a2680678e48d95b657/raw/722fe9e92ad46552521b2df5a5ebd5cd5263c48a/logsearch_audit_logs_managed-schema.patch
curl -sLO https://gist.github.com/seanorama/3860b48c83ec17a2680678e48d95b657/raw/a75fb29916b57bbc07d20ea224e005d34ee40d04/logsearch_hadoop_logs_managed-schema.patch
curl -sLO https://gist.github.com/seanorama/3860b48c83ec17a2680678e48d95b657/raw/a75fb29916b57bbc07d20ea224e005d34ee40d04/logsearch_history_managed-schema.patch

patch -b /etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf/managed-schema logsearch_audit_logs_managed-schema.patch
patch -b /etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf/managed-schema logsearch_hadoop_logs_managed-schema.patch
patch -b /etc/ambari-logsearch-portal/conf/solr_configsets/history/conf/managed-schema logsearch_history_managed-schema.patch
exit
  1. Upload new solr Schema and delete collections
## become infra-solr
sudo -iu infra-solr

## if using kerberos
keytab=/etc/security/keytabs/ambari-infra-solr.service.keytab
kinit -kt ${keytab} $(klist -kt ${keytab}| awk '{print $NF}'|tail -1)

## set infra-solr env
source /etc/ambari-infra-solr/conf/infra-solr-env.sh
export SOLR_ZK_CREDS_AND_ACLS="${SOLR_AUTHENTICATION_OPTS}"

## put new schemas
/usr/lib/ambari-infra-solr/server/scripts/cloud-scripts/zkcli.sh --zkhost "${ZK_HOST}" -cmd putfile /configs/audit_logs/managed-schema /etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf/managed-schema
/usr/lib/ambari-infra-solr/server/scripts/cloud-scripts/zkcli.sh --zkhost "${ZK_HOST}" -cmd putfile /configs/hadoop_logs/managed-schema /etc/ambari-logsearch-portal/conf/solr_configsets/hadoop_logs/conf/managed-schema
/usr/lib/ambari-infra-solr/server/scripts/cloud-scripts/zkcli.sh --zkhost "${ZK_HOST}" -cmd putfile /configs/history/managed-schema /etc/ambari-logsearch-portal/conf/solr_configsets/history/conf/managed-schema

## delete collections
curl -u : --negotiate -i "http://$(hostname -f):8886/solr/admin/collections?action=DELETE&wt=json&indent=on&name=audit_logs"
curl -u : --negotiate -i "http://$(hostname -f):8886/solr/admin/collections?action=DELETE&wt=json&indent=on&name=hadoop_logs"
curl -u : --negotiate -i "http://$(hostname -f):8886/solr/admin/collections?action=DELETE&wt=json&indent=on&name=history"
  1. In Ambari, start LogSearch service
Raw
logsearch_audit_logs_managed-schema.patch
--- /etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf/managed-schema.orig 2017-10-30 18:51:11.000000000 +0000
+++ /etc/ambari-logsearch-portal/conf/solr_configsets/audit_logs/conf/managed-schema 2018-01-01 16:36:28.771676587 +0000
@@ -21,9 +21,9 @@
<fieldType name="binary" class="solr.BinaryField"/>
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
<fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
- <fieldType name="date" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="0"/>
- <fieldType name="double" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="0"/>
- <fieldType name="float" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="date" class="solr.TrieDateField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="ignored" class="solr.StrField" indexed="false" stored="false" multiValued="true"/>
<fieldType name="int" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="0"/>
<fieldType name="key_lower_case" class="solr.TextField" omitNorms="true" sortMissingLast="true" multiValued="false">
@@ -32,7 +32,7 @@
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
- <fieldType name="long" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="long" class="solr.TrieLongField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="n_gram" class="solr.TextField" omitNorms="true" sortMissingLast="true">
<analyzer>
<tokenizer class="solr.NGramTokenizerFactory"/>
@@ -41,10 +41,10 @@
</fieldType>
<fieldType name="random" class="solr.RandomSortField" indexed="true"/>
<fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
- <fieldType name="tdate" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="6"/>
- <fieldType name="tdates" class="solr.TrieDateField" positionIncrementGap="0" multiValued="true" precisionStep="6"/>
- <fieldType name="tdouble" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tdoubles" class="solr.TrieDoubleField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tdate" class="solr.TrieDateField" docValues="true" precisionStep="6" positionIncrementGap="0"/>
+ <fieldType name="tdates" class="solr.TrieDateField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="6"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tdoubles" class="solr.TrieDoubleField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
<fieldType name="text_std_token_lower_case" class="solr.TextField" positionIncrementGap="100" multiValued="true">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
@@ -57,16 +57,16 @@
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
- <fieldType name="tfloat" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tfloats" class="solr.TrieFloatField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
- <fieldType name="tint" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tints" class="solr.TrieIntField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
- <fieldType name="tlong" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tlongs" class="solr.TrieLongField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tfloats" class="solr.TrieFloatField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tint" class="solr.TrieIntField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tints" class="solr.TrieIntField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tlong" class="solr.TrieLongField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tlongs" class="solr.TrieLongField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
<field name="_expire_at_" type="tdate" multiValued="false" stored="true"/>
<field name="_router_field_" type="int" multiValued="false" indexed="false" stored="false"/>
<field name="_ttl_" type="string" multiValued="false" indexed="true" stored="true"/>
- <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="_version_" type="long" indexed="false" stored="true"/>
<field name="access" type="key_lower_case" multiValued="false"/>
<field name="action" type="key_lower_case" multiValued="false"/>
<field name="agent" type="key_lower_case" multiValued="false"/>
Raw
logsearch_hadoop_logs_managed-schema.patch
--- managed-schema.orig 2017-10-30 18:51:11.000000000 +0000
+++ managed-schema 2018-01-01 22:37:38.605319646 +0000
@@ -22,9 +22,9 @@
<fieldType name="binary" class="solr.BinaryField"/>
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
<fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
- <fieldType name="date" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="0"/>
- <fieldType name="double" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="0"/>
- <fieldType name="float" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="date" class="solr.TrieDateField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="ignored" class="solr.StrField" indexed="false" stored="false" multiValued="true"/>
<fieldType name="int" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="0"/>
<fieldType name="ip_address" class="solr.TextField">
@@ -33,13 +33,13 @@
</analyzer>
</fieldType>
<fieldType name="logLevel" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="log_levels"/>
- <fieldType name="long" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="long" class="solr.TrieLongField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="random" class="solr.RandomSortField" indexed="true"/>
<fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
- <fieldType name="tdate" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="6"/>
- <fieldType name="tdates" class="solr.TrieDateField" positionIncrementGap="0" multiValued="true" precisionStep="6"/>
- <fieldType name="tdouble" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tdoubles" class="solr.TrieDoubleField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tdate" class="solr.TrieDateField" docValues="true" precisionStep="6" positionIncrementGap="0"/>
+ <fieldType name="tdates" class="solr.TrieDateField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="6"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tdoubles" class="solr.TrieDoubleField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
<fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
@@ -71,15 +71,15 @@
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
- <fieldType name="tfloat" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tfloats" class="solr.TrieFloatField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
- <fieldType name="tint" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tints" class="solr.TrieIntField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
- <fieldType name="tlong" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tlongs" class="solr.TrieLongField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tfloats" class="solr.TrieFloatField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tint" class="solr.TrieIntField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tints" class="solr.TrieIntField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tlong" class="solr.TrieLongField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tlongs" class="solr.TrieLongField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
<field name="@timestamp" type="tdate" multiValued="false" stored="false"/>
<field name="@version" type="long" multiValued="false" indexed="true" stored="false"/>
- <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="_version_" type="long" indexed="false" stored="true"/>
<field name="_expire_at_" type="tdate" stored="true" multiValued="false"/>
<field name="_ttl_" type="string" indexed="true" stored="true" multiValued="false"/>
<field name="_router_field_" type="int" indexed="false" stored="false" multiValued="false"/>
Raw
logsearch_history_managed-schema.patch
--- managed-schema.orig 2017-10-30 18:51:11.000000000 +0000
+++ managed-schema 2018-01-01 22:41:05.972465160 +0000
@@ -24,9 +24,9 @@
<fieldType name="binary" class="solr.BinaryField"/>
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
<fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
- <fieldType name="date" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="0"/>
- <fieldType name="double" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="0"/>
- <fieldType name="float" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="date" class="solr.TrieDateField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="ignored" class="solr.StrField" indexed="false" stored="false" multiValued="true"/>
<fieldType name="int" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="0"/>
<fieldType name="ip_address" class="solr.TextField">
@@ -34,13 +34,13 @@
<tokenizer class="solr.PathHierarchyTokenizerFactory" replace="." delimiter="."/>
</analyzer>
</fieldType>
- <fieldType name="long" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="0"/>
+ <fieldType name="long" class="solr.TrieLongField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="random" class="solr.RandomSortField" indexed="true"/>
<fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
- <fieldType name="tdate" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="6"/>
- <fieldType name="tdates" class="solr.TrieDateField" positionIncrementGap="0" multiValued="true" precisionStep="6"/>
- <fieldType name="tdouble" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tdoubles" class="solr.TrieDoubleField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tdate" class="solr.TrieDateField" docValues="true" precisionStep="6" positionIncrementGap="0"/>
+ <fieldType name="tdates" class="solr.TrieDateField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="6"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tdoubles" class="solr.TrieDoubleField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
<fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
@@ -72,16 +72,16 @@
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
- <fieldType name="tfloat" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tfloats" class="solr.TrieFloatField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
- <fieldType name="tint" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tints" class="solr.TrieIntField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
- <fieldType name="tlong" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="8"/>
- <fieldType name="tlongs" class="solr.TrieLongField" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tfloats" class="solr.TrieFloatField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tint" class="solr.TrieIntField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tints" class="solr.TrieIntField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
+ <fieldType name="tlong" class="solr.TrieLongField" docValues="true" positionIncrementGap="0" precisionStep="8"/>
+ <fieldType name="tlongs" class="solr.TrieLongField" docValues="true" positionIncrementGap="0" multiValued="true" precisionStep="8"/>
<solrQueryParser defaultOperator="OR"/>
- <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="_version_" type="long" indexed="false" stored="true"/>
<field name="filtername" type="key_lower_case" indexed="true" required="true" stored="true"/>
<field name="id" type="string" required="true"/>
<field name="jsons" type="string"/>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment