gistfile1.txt

[marty@cen-1701 testrunner]$ python testrunner -i couchbase-elasticsearch.ini -c es.conf
/usr/lib64/python2.6/site-packages/Crypto/Util/randpool.py:40: RandomPool_DeprecationWarning: This application uses RandomPool, which is BROKEN in older releases.  See http://www.pycrypto.org/randpool-broken
  RandomPool_DeprecationWarning)
filename:es.conf
[Global Test input params]
{'cluster_name': 'couchbase-elasticsearch', 'num_nodes': 3, 'ini': 'couchbase-elasticsearch.ini', 'conf_file': 'es.conf', 'spec': 'es'}

  ./testrunner -i couchbase-elasticsearch.ini  -t rebalance.rebalanceout.RebalanceOutTests.rebalance_out_with_ops,nodes_init=3,nodes_out=2,replicas=1,items=20000,doc_ops=create;update;delete
[Test Input params] : {'replicas': '1', 'doc_ops': 'create;update;delete', 'items': '20000', 'case_number': 1, 'conf_file': 'es.conf', 'num_nodes': 3, 'cluster_name': 'couchbase-elasticsearch', 'ini': 'couchbase-elasticsearch.ini', 'nodes_out': '2', 'spec': 'es', 'nodes_init': '3'}
rebalance_out_with_ops (rebalance.rebalanceout.RebalanceOutTests) ... 2012-11-19 10:16:44,824 - root - INFO - ==============  basetestcase setup was started for test #1 rebalance_out_with_ops==============
2012-11-19 10:16:44,825 - root - INFO - ==============  basetestcase cleanup was started for test #1 rebalance_out_with_ops ==============
2012-11-19 10:16:44,878 - root - INFO - deleting existing buckets on [ip:10.5.2.12 port:8091 ssh_username:Administrator, ip:10.5.2.7 port:8091 ssh_username:Administrator, ip:10.5.2.6 port:8091 ssh_username:Administrator]
2012-11-19 10:16:54,974 - root - INFO - waiting for ns_server @ 10.5.2.12:8091
2012-11-19 10:16:54,985 - root - INFO - ns_server @ 10.5.2.12:8091 is running
2012-11-19 10:16:54,991 - root - INFO - waiting for ns_server @ 10.5.2.7:8091
2012-11-19 10:16:54,997 - root - INFO - ns_server @ 10.5.2.7:8091 is running
2012-11-19 10:16:55,004 - root - INFO - waiting for ns_server @ 10.5.2.6:8091
2012-11-19 10:16:55,011 - root - INFO - ns_server @ 10.5.2.6:8091 is running
2012-11-19 10:16:55,011 - root - INFO - ==============  basetestcase cleanup was finished for test #1 rebalance_out_with_ops ==============
2012-11-19 10:16:56,028 - root - INFO - adding node 10.5.2.7:8091 to cluster
2012-11-19 10:16:56,029 - root - INFO - adding remote node @10.5.2.7:8091 to this cluster @10.5.2.12:8091
2012-11-19 10:16:57,015 - root - INFO - adding node 10.5.2.6:8091 to cluster
2012-11-19 10:16:57,016 - root - INFO - adding remote node @10.5.2.6:8091 to this cluster @10.5.2.12:8091
2012-11-19 10:16:57,566 - root - INFO - rebalance params : password=password&ejectedNodes=&user=Administrator&knownNodes=ns_1%4010.5.2.6%2Cns_1%4010.5.2.7%2Cns_1%4010.5.2.12
2012-11-19 10:16:57,578 - root - INFO - rebalance operation started
2012-11-19 10:16:57,584 - root - INFO - rebalance percentage : 0 %
2012-11-19 10:17:08,590 - root - INFO - rebalancing was completed with progress: 100% in 11.011493206 sec
2012-11-19 10:17:09,602 - root - INFO - default consistent_view value will be used on server
2012-11-19 10:17:09,602 - root - INFO - settings/web params : username=Administrator&password=password&port=8091
2012-11-19 10:17:09,616 - root - INFO - pools/default params : memoryQuota=2106&username=Administrator&password=password
2012-11-19 10:17:09,627 - root - INFO - default consistent_view value will be used on server
2012-11-19 10:17:09,628 - root - INFO - settings/web params : username=Administrator&password=password&port=8091
2012-11-19 10:17:09,638 - root - INFO - pools/default params : memoryQuota=2106&username=Administrator&password=password
2012-11-19 10:17:09,651 - root - INFO - default consistent_view value will be used on server
2012-11-19 10:17:09,651 - root - INFO - settings/web params : username=Administrator&password=password&port=8091
2012-11-19 10:17:09,665 - root - INFO - pools/default params : memoryQuota=2106&username=Administrator&password=password
2012-11-19 10:17:10,679 - root - INFO - http://10.5.2.12:8091/pools/default/buckets with param: proxyPort=11211&bucketType=membase&authType=sasl&replicaIndex=1&name=default&saslPassword=None&flushEnabled=1&replicaNumber=1&ramQuotaMB=1404
2012-11-19 10:17:10,695 - root - INFO - 0.0157299041748 seconds to create bucket default
2012-11-19 10:17:10,695 - root - INFO - waiting for memcached bucket : default in 10.5.2.12 to accept set ops
2012-11-19 10:17:12,693 - root - INFO - creating direct client 10.5.2.12:11210 default
2012-11-19 10:17:13,127 - root - INFO - creating direct client 10.5.2.6:11210 default
2012-11-19 10:17:13,543 - root - INFO - creating direct client 10.5.2.7:11210 default
2012-11-19 10:17:14,228 - root - INFO - creating direct client 10.5.2.12:11210 default
2012-11-19 10:17:14,670 - root - INFO - creating direct client 10.5.2.6:11210 default
2012-11-19 10:17:15,113 - root - INFO - creating direct client 10.5.2.7:11210 default
2012-11-19 10:17:16,423 - root - INFO - bucket 'default' was created with per node RAM quota: 1404
2012-11-19 10:17:16,424 - root - INFO - ==============  basetestcase setup was finished for test #1 rebalance_out_with_ops ==============
2012-11-19 10:17:16,763 - root - INFO - creating direct client 10.5.2.12:11210 default
2012-11-19 10:17:17,239 - root - INFO - creating direct client 10.5.2.6:11210 default
2012-11-19 10:17:17,653 - root - INFO - creating direct client 10.5.2.7:11210 default
2012-11-19 10:17:18,427 - root - INFO - Batch create documents done #: 0
2012-11-19 10:17:27,461 - root - INFO - Batch create documents done #: 20000
2012-11-19 10:17:27,790 - root - INFO - creating direct client 10.5.2.12:11210 default
2012-11-19 10:17:28,216 - root - INFO - creating direct client 10.5.2.6:11210 default
2012-11-19 10:17:28,449 - root - INFO - rebalance params : password=password&ejectedNodes=ns_1%4010.5.2.6%2Cns_1%4010.5.2.7&user=Administrator&knownNodes=ns_1%4010.5.2.6%2Cns_1%4010.5.2.7%2Cns_1%4010.5.2.12
2012-11-19 10:17:28,461 - root - INFO - rebalance operation started
2012-11-19 10:17:28,561 - root - INFO - rebalance percentage : 0 %
2012-11-19 10:17:28,718 - root - INFO - creating direct client 10.5.2.7:11210 default
2012-11-19 10:17:30,386 - root - INFO - creating direct client 10.5.2.12:11210 default
2012-11-19 10:17:32,255 - root - INFO - creating direct client 10.5.2.6:11210 default
2012-11-19 10:17:33,839 - root - INFO - creating direct client 10.5.2.7:11210 default
2012-11-19 10:17:36,033 - root - INFO - creating direct client 10.5.2.12:11210 default
2012-11-19 10:17:37,641 - root - INFO - creating direct client 10.5.2.6:11210 default
2012-11-19 10:17:38,712 - root - INFO - rebalance percentage : 3.93089254611 %
2012-11-19 10:17:39,365 - root - INFO - creating direct client 10.5.2.7:11210 default
2012-11-19 10:17:48,815 - root - INFO - rebalance percentage : 9.44921276239 %
2012-11-19 10:17:59,381 - root - INFO - rebalance percentage : 15.0405962997 %
2012-11-19 10:18:09,436 - root - INFO - rebalance percentage : 21.3649029854 %
2012-11-19 10:18:19,553 - root - INFO - rebalance percentage : 27.0784403808 %
2012-11-19 10:18:29,871 - root - INFO - rebalance percentage : 32.303362344 %
2012-11-19 10:18:39,998 - root - INFO - rebalance percentage : 37.3350708235 %
2012-11-19 10:18:50,300 - root - INFO - rebalance percentage : 42.8319944355 %
2012-11-19 10:19:00,622 - root - INFO - rebalance percentage : 47.8403026153 %
2012-11-19 10:19:10,662 - root - INFO - rebalance percentage : 52.604303079 %
2012-11-19 10:19:20,854 - root - INFO - rebalance percentage : 57.2461496846 %
2012-11-19 10:19:31,080 - root - INFO - rebalance percentage : 62.2544578644 %
2012-11-19 10:19:41,495 - root - INFO - rebalance percentage : 67.9956891925 %
2012-11-19 10:19:51,696 - root - INFO - rebalance percentage : 74.5919975269 %
2012-11-19 10:20:01,824 - root - INFO - rebalance percentage : 80.6996904291 %
2012-11-19 10:20:12,068 - root - INFO - rebalance percentage : 87.7846141956 %
2012-11-19 10:20:22,368 - root - INFO - rebalance percentage : 95.2359995363 %
2012-11-19 10:20:32,654 - root - INFO - rebalance percentage : 100.0 %
2012-11-19 10:20:42,015 - root - ERROR - socket error while connecting to http://10.5.2.6:8091/pools error [Errno 111] Connection refused 
2012-11-19 10:20:43,259 - root - ERROR - socket error while connecting to http://10.5.2.7:8091/nodes/self error [Errno 111] Connection refused 
2012-11-19 10:20:44,290 - root - INFO - rebalancing was completed with progress: 100.0% in 195.828989029 sec