Created
June 5, 2012 00:24
-
-
Save gordonm/2871674 to your computer and use it in GitHub Desktop.
Salt master not responding
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [root@provision tmp]# date | |
| Mon Jun 4 17:08:07 PDT 2012 | |
| [root@provision tmp]# grep salt-master /var/log/messages | |
| Jun 4 17:05:30 provision kernel: salt-master[5136]: segfault at 20 ip 00007fc3753e1e1c sp 00007fff60f935e0 error 4 in libzmq.so.1.0.0[7fc3753c5000+2e000] | |
| [root@provision tmp]# ps auxww | grep [s]alt | |
| root 5128 0.0 0.0 646044 14780 ? Sl Jun01 0:02 /usr/bin/python /usr/bin/salt-master -d | |
| root 5129 1.8 0.0 256748 13036 ? S Jun01 86:32 /usr/bin/python /usr/bin/salt-master -d | |
| root 5136 0.7 0.0 0 0 ? Z Jun01 37:41 [salt-master] <defunct> | |
| root 5139 0.0 0.0 634384 30500 ? Sl Jun01 0:02 /usr/bin/python /usr/bin/salt-master -d | |
| root 5142 0.0 0.0 635112 32188 ? Sl Jun01 0:02 /usr/bin/python /usr/bin/salt-master -d | |
| root 5145 0.0 0.0 634384 30532 ? Sl Jun01 0:02 /usr/bin/python /usr/bin/salt-master -d | |
| root 5148 0.0 0.0 635024 31120 ? Sl Jun01 0:02 /usr/bin/python /usr/bin/salt-master -d | |
| root 5151 0.0 0.0 634984 30852 ? Sl Jun01 0:02 /usr/bin/python /usr/bin/salt-master -d | |
| [root@provision tmp]# service salt-master status | |
| salt-master (pid 5128) is running... | |
| [root@provision tmp]# ~/debug_salt.sh | |
| Signalling pid 5128... done. | |
| Signalling pid 5129... done. | |
| Signalling pid 5136... done. | |
| Signalling pid 5139... done. | |
| Signalling pid 5142... done. | |
| Signalling pid 5145... done. | |
| Signalling pid 5148... done. | |
| Signalling pid 5151... done. | |
| [root@provision tmp]# cat /tmp/salt-debug-13388549* | |
| ======== Salt Debug Stack Trace ========= | |
| File "/usr/bin/salt-master", line 17, in <module> | |
| main() | |
| File "/usr/bin/salt-master", line 14, in main | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/__init__.py", line 110, in start | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 192, in start | |
| reqserv.run() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 304, in run | |
| self.__bind() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 286, in __bind | |
| zmq.device(zmq.QUEUE, self.clients, self.workers) | |
| ========================================= | |
| ======== Salt Debug Stack Trace ========= | |
| File "/usr/bin/salt-master", line 17, in <module> | |
| main() | |
| File "/usr/bin/salt-master", line 14, in main | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/__init__.py", line 110, in start | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 158, in start | |
| clear_old_jobs_proc.start() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 104, in start | |
| self._popen = Popen(self) | |
| File "/usr/lib64/python2.6/multiprocessing/forking.py", line 99, in __init__ | |
| code = process_obj._bootstrap() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 232, in _bootstra | |
| p | |
| self.run() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 88, in run | |
| self._target(*self._args, **self._kwargs) | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 145, in _clear_ol | |
| d_jobs | |
| time.sleep(60) | |
| ========================================= | |
| ======== Salt Debug Stack Trace ========= | |
| File "/usr/bin/salt-master", line 17, in <module> | |
| main() | |
| File "/usr/bin/salt-master", line 14, in main | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/__init__.py", line 110, in start | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 192, in start | |
| reqserv.run() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 304, in run | |
| self.__bind() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 280, in __bind | |
| proc.start() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 104, in start | |
| self._popen = Popen(self) | |
| File "/usr/lib64/python2.6/multiprocessing/forking.py", line 99, in __init__ | |
| code = process_obj._bootstrap() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 232, in _bootstra | |
| p | |
| self.run() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 399, in run | |
| self.__bind() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 343, in __bind | |
| ret = self.serial.dumps(self._handle_payload(payload)) | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 366, in _handle_p | |
| ayload | |
| 'clear': self._handle_clear}[key](load) | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 373, in _handle_c | |
| lear | |
| return getattr(self.clear_funcs, load['cmd'])(load) | |
| ========================================= | |
| ======== Salt Debug Stack Trace ========= | |
| File "/usr/bin/salt-master", line 17, in <module> | |
| main() | |
| File "/usr/bin/salt-master", line 14, in main | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/__init__.py", line 110, in start | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 192, in start | |
| reqserv.run() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 304, in run | |
| self.__bind() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 280, in __bind | |
| proc.start() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 104, in start | |
| self._popen = Popen(self) | |
| File "/usr/lib64/python2.6/multiprocessing/forking.py", line 99, in __init__ | |
| code = process_obj._bootstrap() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 232, in _bootstra | |
| p | |
| self.run() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 399, in run | |
| self.__bind() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 343, in __bind | |
| ret = self.serial.dumps(self._handle_payload(payload)) | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 366, in _handle_p | |
| ayload | |
| 'clear': self._handle_clear}[key](load) | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 373, in _handle_c | |
| lear | |
| return getattr(self.clear_funcs, load['cmd'])(load) | |
| ========================================= | |
| ======== Salt Debug Stack Trace ========= | |
| File "/usr/bin/salt-master", line 17, in <module> | |
| main() | |
| File "/usr/bin/salt-master", line 14, in main | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/__init__.py", line 110, in start | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 192, in start | |
| reqserv.run() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 304, in run | |
| self.__bind() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 280, in __bind | |
| proc.start() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 104, in start | |
| self._popen = Popen(self) | |
| File "/usr/lib64/python2.6/multiprocessing/forking.py", line 99, in __init__ | |
| code = process_obj._bootstrap() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 232, in _bootstra | |
| p | |
| self.run() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 399, in run | |
| self.__bind() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 343, in __bind | |
| ret = self.serial.dumps(self._handle_payload(payload)) | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 366, in _handle_p | |
| ayload | |
| 'clear': self._handle_clear}[key](load) | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 373, in _handle_c | |
| lear | |
| return getattr(self.clear_funcs, load['cmd'])(load) | |
| ========================================= | |
| ======== Salt Debug Stack Trace ========= | |
| File "/usr/bin/salt-master", line 17, in <module> | |
| main() | |
| File "/usr/bin/salt-master", line 14, in main | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/__init__.py", line 110, in start | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 192, in start | |
| reqserv.run() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 304, in run | |
| self.__bind() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 280, in __bind | |
| proc.start() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 104, in start | |
| self._popen = Popen(self) | |
| File "/usr/lib64/python2.6/multiprocessing/forking.py", line 99, in __init__ | |
| code = process_obj._bootstrap() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 232, in _bootstra | |
| p | |
| self.run() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 399, in run | |
| self.__bind() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 341, in __bind | |
| package = socket.recv() | |
| ========================================= | |
| ======== Salt Debug Stack Trace ========= | |
| File "/usr/bin/salt-master", line 17, in <module> | |
| main() | |
| File "/usr/bin/salt-master", line 14, in main | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/__init__.py", line 110, in start | |
| master.start() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 192, in start | |
| reqserv.run() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 304, in run | |
| self.__bind() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 280, in __bind | |
| proc.start() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 104, in start | |
| self._popen = Popen(self) | |
| File "/usr/lib64/python2.6/multiprocessing/forking.py", line 99, in __init__ | |
| code = process_obj._bootstrap() | |
| File "/usr/lib64/python2.6/multiprocessing/process.py", line 232, in _bootstra | |
| p | |
| self.run() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 399, in run | |
| self.__bind() | |
| File "/usr/lib/python2.6/site-packages/salt/master.py", line 341, in __bind | |
| package = socket.recv() | |
| ========================================= |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment