mpi segfault

gistfile1.txt

d1021358@labx13:~/Documents/mpi$ mpirun -n 4 -machinefile machines.txt a.out 
0.170508
*** glibc detected *** a.out: free(): invalid next size (normal): 0x00000000011cde20 ***
[labx02:29507] *** Process received signal ***
======= Backtrace: =========
[labx02:29507] Signal: Segmentation fault (11)
[labx02:29507] Signal code: Address not mapped (1)
[labx02:29507] Failing at address: 0x3e00000047
/lib/x86_64-linux-gnu/libc.so.6(+0x7eb96)[0x7fd26faedb96]
[labx03:25102] *** Process received signal ***
/usr/lib/openmpi/lib/openmpi/mca_btl_tcp.so(+0x52b6)[0x7fd26ba712b6]
/usr/lib/openmpi/lib/openmpi/mca_btl_tcp.so(+0x2599)[0x7fd26ba6e599]
[labx03:25102] Signal: Segmentation fault (11)
[labx03:25102] Signal code: Address not mapped (1)
[labx03:25102] Failing at address: 0x660000006c
[labx02:29507] [ 0] /lib/x86_64-linux-gnu/libpthread.so.0(+0xfcb0) [0x7f68c3cf4cb0]
/usr/lib/libmpi.so.0(mca_btl_base_close+0x5f)[0x7fd2700bd0af]
/usr/lib/openmpi/lib/openmpi/mca_bml_r2.so(+0x2652)[0x7fd26c73d652]
/usr/lib/openmpi/lib/openmpi/mca_pml_ob1.so(+0x48ce)[0x7fd26cb488ce]
/usr/lib/libmpi.so.0(+0x371db)[0x7fd2700821db]
a.out(main+0x32bc)[0x404260]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xed)[0x7fd26fa9076d]
[labx03:25102] [ 0] /lib/x86_64-linux-gnu/libpthread.so.0(+0xfcb0) [0x7f278a670cb0]
[labx02:29507] [ 1] /usr/lib/openmpi/lib/openmpi/mca_allocator_bucket.so(+0xe68) [0x7f68c1c46e68]
a.out[0x400ee9]
[labx02:29507] [ 2] /usr/lib/openmpi/lib/openmpi/mca_allocator_bucket.so(+0x9a9) [0x7f68c1c469a9]
======= Memory map: ========
[labx03:25102] [ 1] /usr/lib/openmpi/lib/openmpi/mca_allocator_bucket.so(+0xe68) [0x7f27885c2e68]
[labx02:29507] [ 3] /usr/lib/libmpi.so.0(+0x371db) [0x7f68c3f391db]
[labx02:29507] [ 4] a.out(main+0x32bc) [0x404260]
[labx03:25102] [ 2] /usr/lib/openmpi/lib/openmpi/mca_allocator_bucket.so(+0x9a9) [0x7f27885c29a9]
[labx02:29507] [ 5] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xed) [0x7f68c394776d]
[labx03:25102] [ 3] /usr/lib/libmpi.so.0(+0x371db) [0x7f278a8b51db]
[labx02:29507] [ 6] a.out() [0x400ee9]
[labx02:29507] *** End of error message ***
[labx03:25102] [ 4] a.out(main+0x32bc) [0x404260]
[labx03:25102] [ 5] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xed) [0x7f278a2c376d]
00400000-00405000 r-xp 00000000 00:1b 1491256                            /home/d1021358/Documents/mpi/a.out
00604000-00605000 r--p 00004000 00:1b 1491256                            /home/d1021358/Documents/mpi/a.out
00605000-00606000 rw-p 00005000 00:1b 1491256                            /home/d1021358/Documents/mpi/a.out
00e56000-0120f000 rw-p 00000000 00:00 0                                  [heap]
7fd26a81c000-7fd26a829000 r-xp 00000000 08:01 2363385                    /usr/lib/openmpi/lib/openmpi/mca_osc_rdma.so
7fd26a829000-7fd26aa28000 ---p 0000d000 08:01 2363385                    /usr/lib/openmpi/lib/openmpi/mca_osc_rdma.so
7fd26aa28000-7fd26aa29000 r--p 0000c000 08:01 2363385                    /usr/lib/openmpi/lib/openmpi/mca_osc_rdma.so
7fd26aa29000-7fd26aa2a000 rw-p 0000d000 08:01 2363385                    /usr/lib/openmpi/lib/openmpi/mca_osc_rdma.so
7fd26aa2a000-7fd26aa34000 r-xp 00000000 08:01 2363384                    /usr/lib/openmpi/lib/openmpi/mca_osc_pt2pt.so
7fd26aa34000-7fd26ac33000 ---[labx03:25102] [ 6] a.out() [0x400ee9]
[labx03:25102] *** End of error message ***
p 0000a000 08:01 2363384                    /usr/lib/openmpi/lib/openmpi/mca_osc_pt2pt.so
7fd26ac33000-7fd26ac34000 r--p 00009000 08:01 2363384                    /usr/lib/openmpi/lib/openmpi/mca_osc_pt2pt.so
7fd26ac34000-7fd26ac35000 rw-p 0000a000 08:01 2363384                    /usr/lib/openmpi/lib/openmpi/mca_osc_pt2pt.so
7fd26ac35000-7fd26ac4c000 r-xp 00000000 08:01 2363358                    /usr/lib/openmpi/lib/openmpi/mca_coll_tuned.so
7fd26ac4c000-7fd26ae4b000 ---p 00017000 08:01 2363358                    /usr/lib/openmpi/lib/openmpi/mca_coll_tuned.so
7fd26ae4b000-7fd26ae4c000 r--p 00016000 08:01 2363358                    /usr/lib/openmpi/lib/openmpi/mca_coll_tuned.so
7fd26ae4c000-7fd26ae4d000 rw-p 00017000 08:01 2363358                    /usr/lib/openmpi/lib/openmpi/mca_coll_tuned.so
7fd26ae4d000-7fd26ae50000 r-xp 00000000 08:01 2363357                    /usr/lib/openmpi/lib/openmpi/mca_coll_sync.so
7fd26ae50000-7fd26b04f000 ---p 00003000 08:01 2363357                    /usr/lib/openmpi/lib/openmpi/mca_coll_sync.so
7fd26b04f000-7fd26b050000 r--p 00002000 08:01 2363357                    /usr/lib/openmpi/lib/openmpi/mca_coll_sync.so
7fd26b050000-7fd26b051000 rw-p 00003000 08:01 2363357                    /usr/lib/openmpi/lib/openmpi/mca_coll_sync.so
7fd26b257000-7fd26b259000 r-xp 00000000 08:01 2363355                    /usr/lib/openmpi/lib/openmpi/mca_coll_self.so
7fd26b259000-7fd26b458000 ---p 00002000 08:01 2363355                    /usr/lib/openmpi/lib/openmpi/mca_coll_self.so
7fd26b458000-7fd26b459000 r--p 00001000 08:01 2363355                    /usr/lib/openmpi/lib/openmpi/mca_coll_self.so
7fd26b459000-7fd26b45a000 rw-p 00002000 08:01 2363355                    /usr/lib/openmpi/lib/openmpi/mca_coll_self.so
7fd26b45a000-7fd26b45d000 r-xp 00000000 08:01 2363354                    /usr/lib/openmpi/lib/openmpi/mca_coll_inter.so
7fd26b45d000-7fd26b65c000 ---p 00003000 08:01 2363354                    /usr/lib/openmpi/lib/openmpi/mca_coll_inter.so
7fd26b65c000-7fd26b65d000 r--p 00002000 08:01 2363354                    /usr/lib/openmpi/lib/openmpi/mca_coll_inter.so
7fd26b65d000-7fd26b65e000 rw-p 00003000 08:01 2363354                    /usr/lib/openmpi/lib/openmpi/mca_coll_inter.so
7fd26b65e000-7fd26b662000 r-xp 00000000 08:01 2363353                    /usr/lib/openmpi/lib/openmpi/mca_coll_hierarch.so
7fd26b662000-7fd26b861000 ---p 00004000 08:01 2363353                    /usr/lib/openmpi/lib/openmpi/mca_coll_hierarch.so
7fd26b861000-7fd26b862000 r--p 00003000 08:01 2363353                    /usr/lib/openmpi/lib/openmpi/mca_coll_hierarch.so
7fd26b862000-7fd26b863000 rw-p 00004000 08:01 2363353                    /usr/lib/openmpi/lib/openmpi/mca_coll_hierarch.so
7fd26b863000-7fd26b86b000 r-xp 00000000 08:01 2363352                    /usr/lib/openmpi/lib/openmpi/mca_coll_basic.so
7fd26b86b000-7fd26ba6a000 ---p 00008000 08:01 2363352                    /usr/lib/openmpi/lib/openmpi/mca_coll_basic.so
7fd26ba6a000-7fd26ba6b000 r--p 00007000 08:01 2363352                    /usr/lib/openmpi/lib/openmpi/mca_coll_basic.so
7fd26ba6b000-7fd26ba6c000 rw-p 00008000 08:01 2363352                    /usr/lib/openmpi/lib/openmpi/mca_coll_basic.so
7fd26ba6c000-7fd26ba77000 r-xp 00000000 08:01 2363349                    /usr/lib/openmpi/lib/openmpi/mca_btl_tcp.so
7fd26ba77000-7fd26bc76000 ---p 0000b000 08:01 2363349                    /usr/lib/openmpi/lib/openmpi/mca_btl_tcp.so
7fd26bc76000-7fd26bc77000 r--p 0000a000 08:01 2363349                    /usr/lib/openmpi/lib/openmpi/mca_btl_tcp.so
7fd26bc77000-7fd26bc78000 rw-p 0000b000 08:01 2363349                    /usr/lib/openmpi/lib/openmpi/mca_btl_tcp.so
7fd26bc78000-7fd26bcf8000 rw-p 00000000 00:00 0 
7fd26bcf8000-7fd26bcfd000 r-xp 00000000 08:01 2363348                    /usr/lib/openmpi/lib/openmpi/mca_btl_sm.so
7fd26bcfd000-7fd26befc000 ---p 00005000 08:01 2363348                    /usr/lib/openmpi/lib/openmpi/mca_btl_sm.so
7fd26befc000-7fd26befd000 r--p 00004000 08:01 2363348                    /usr/lib/openmpi/lib/openmpi/mca_btl_sm.so
7fd26befd000-7fd26befe000 rw-p 00005000 08:01 2363348                    /usr/lib/openmpi/lib/openmpi/mca_btl_sm.so
7fd26befe000-7fd26bf01000 r-xp 00000000 08:01 2363347                    /usr/lib/openmpi/lib/openmpi/mca_btl_self.so
7fd26bf01000-7fd26c100000 ---p 00003000 08:01 2363347                    /usr/lib/openmpi/lib/openmpi/mca_btl_self.so
7fd26c100000-7fd26c101000 r--p 00002000 08:01 2363347                    /usr/lib/openmpi/lib/openmpi/mca_btl_self.so
7fd26c101000-7fd26c102000 rw-p 00003000 08:01 2363347                    /usr/lib/openmpi/lib/openmpi/mca_btl_self.so
7fd26c525000-7fd26c53a000 r-xp 00000000 08:01 1575583                    /lib/x86_64-linux-gnu/libgcc_s.so.1
7fd26c53a000-7fd26c739000 ---p 00015000 08:01 1575583                    /lib/x86_64-linux-gnu/libgcc_s.so.1
7fd26c739000-7fd26c73a000 r--p 00014000 08:01 1575583                    /lib/x86_64-linux-gnu/libgcc_s.so.1
7fd26c73a000-7fd26c73b000 rw-p 00015000 08:01 1575583                    /lib/x86_64-linux-gnu/libgcc_s.so.1
7fd26c73b000-7fd26c73f000 r-xp 00000000 08:01 2363344                    /usr/lib/openmpi/lib/openmpi/mca_bml_r2.so
7fd26c73f000-7fd26c93e000 ---p 00004000 08:01 2363344                    /usr/lib/openmpi/lib/openmpi/mca_bml_r2.so
7fd26c93e000-7fd26c93f000 r--p 00003000 08:01 2363344                    /usr/lib/openmpi/lib/openmpi/mca_bml_r2.so
7fd26c93f000-7fd26c940000 rw-p 00004000 08:01 2363344                    /usr/lib/openmpi/lib/openmpi/mca_bml_r2.so
7fd26cb44000-7fd26cb57000 r-xp 00000000 08:01 2363393                    /usr/lib/openmpi/lib/openmpi/mca_pml_ob1.so
7fd26cb57000-7fd26cd57000 ---p 00013000 08:01 2363393                    /usr/lib/openmpi/lib/openmpi/mca_pml_ob1.so
7fd26cd57000-7fd26cd58000 r--p 00013000 08:01 2363393                    /usr/lib/openmpi/lib/openmpi/mca_pml_ob1.so
7fd26cd58000-7fd26cd59000 rw-p 00014000 08:01 2363393                    /usr/lib/openmpi/lib/openmpi/mca_pml_ob1.so
7fd26cd73000-7fd26cd74000 r-xp 00000000 08:01 2363409                    /usr/lib/openmpi/lib/openmpi/mca_topo_unity.so
7fd26cd74000-7fd26cf73000 ---p 00001000 08:01 2363409                    /usr/lib/openmpi/lib/openmpi/mca_topo_unity.so
7fd26cf73000-7fd26cf74000 r--p 00000000 08:01 2363409                    /usr/lib/openmpi/lib/openmpi/mca_topo_unity.so
7fd26cf74000-7fd26cf75000 rw-p 00001000 08:01 2363409                    /usr/lib/openmpi/lib/openmpi/mca_topo_unity.so
7fd26cf75000-7fd26cf7a000 r-xp 00000000 08:01 2363362                    /usr/lib/openmpi/lib/openmpi/mca_dpm_orte.so
7fd26cf7a000-7fd26d179000 ---p 00005000 08:01 2363362                    /usr/lib/openmpi/lib/openmpi/mca_dpm_orte.so
7fd26d179000-7fd26d17a000 r--p 00004000 08:01 2363362                    /usr/lib/openmpi/lib/openmpi/mca_dpm_orte.so
7fd26d17a000-7fd26d17b000 rw-p 00005000 08:01 2363362                    /usr/lib/openmpi/lib/openmpi/mca_dpm_orte.so
7fd26d17b000-7fd26d17e000 r-xp 00000000 08:01 2363395                    /usr/lib/openmpi/lib/openmpi/mca_pubsub_orte.so
7fd26d17e000-7fd26d37d000 ---p 00003000 08:01 2363395                    /usr/lib/openmpi/lib/openmpi/mca_pubsub_orte.so
7fd26d37d000-7fd26d37e000 r--p 00002000 08:01 2363395                    /usr/lib/openmpi/lib/openmpi/mca_pubsub_orte.so
7fd26d37e000-7fd26d37f000 rw-p 00003000 08:01 2363395                    /usr/lib/openmpi/lib/openmpi/mca_pubsub_orte.so
7fd26d37f000-7fd26d381000 r-xp 00000000 08:01 2363340                    /usr/lib/openmpi/lib/libmca_common_sm.so.1.0.0
7fd26d381000-7fd26d580000 ---p 00002000 08:01 2363340                    /usr/lib/openmpi/lib/libmca_common_sm.so.1.0.0
7fd26d580000-7fd26d581000 r--p 00001000 08:01 2363340                    /usr/lib/openmpi/lib/libmca_common_sm.so.1.0.0
7fd26d581000-7fd26d582000 rw-p 00002000 08:01 2363340                    /usr/lib/openmpi/lib/libmca_common_sm.so.1.0.0
7fd26d582000-7fd26d584000 r-xp 00000000 08:01 2363380                    /usr/lib/openmpi/lib/openmpi/mca_mpool_sm.so
7fd26d584000-7fd26d783000 ---p 00002000 08:01 2363380                    /usr/lib/openmpi/lib/openmpi/mca_mpool_sm.so
7fd26d783000-7fd26d784000 r--p 00001000 08:01 2363380                    /usr/lib/openmpi/lib/openmpi/mca_mpool_sm.so
7fd26d784000-7fd26d785000 rw-p 00002000 08:01 2363380                    /usr/lib/openmpi/lib/openmpi/mca_mpool_sm.so
7fd26d785000-7fd26d788000 r-xp 00000000 08:01 2363379                    /usr/lib/openmpi/lib/openmpi/mca_mpool_rdma.so
7fd26d788000-7fd26d987000 ---p 00003000 08:01 2363379                    /usr/lib/openmpi/lib/openmpi/mca_mpool_rdma.so
7fd26d987000-7fd26d988000 r--p 00002000 08:01 2363379                    /usr/lib/openmpi/lib/openmpi/mca_mpool_rdma.so
7fd26d988000-7fd26d989000 rw-p 00003000 08:01 2363379                    /usr/lib/openmpi/lib/openmpi/mca_mpool_rdma.so
7fd26d989000-7fd26d98a000 r-xp 00000000 08:01 2363378                    /usr/lib/openmpi/lib/openmpi/mca_mpool_fake.so
7fd26d98a000-7fd26db89000 ---p 00001000 08:01 2363378                    /usr/lib/openmpi/lib/openmpi/mca_mpool_fake.so
7fd26db89000-7fd26db8a000 r--p 00000000 08:01 2363378                    /usr/lib/openmpi/lib/openmpi/mca_mpool_fake.so
7fd26db8a000-7fd26db8b000 rw-p 00001000 08:01 2363378                    /usr/lib/openmpi/lib/openmpi/mca_mpool_fake.so
7fd26db8b000-7fd26db8e000 r-xp 00000000 08:01 2363398                    /usr/lib/openmpi/lib/openmpi/mca_rcache_vma.so
7fd26db8e000-7fd26dd8d000 ---p 00003000 08:01 2363398                    /usr/lib/openmpi/lib/openmpi/mca_rcache_vma.so
7fd26dd8d000-7fd26dd8e000 r--p 00002000 08:01 2363398                    /usr/lib/openmpi/lib/openmpi/mca_rcache_vma.so
7fd26dd8e000-7fd26dd8f000 rw-p 00003000 08:01 2363398                    /usr/lib/openmpi/lib/openmpi/mca_rcache_vma.so
7fd26dd8f000-7fd26dd91000 r-xp 00000000 08:01 2363343                    /usr/lib/openmpi/lib/openmpi/mca_allocator_bucket.so
7fd26dd91000-7fd26df90000 ---p 00002000 08:01 2363343                    /usr/lib/openmpi/lib/openmpi/mca_allocator_bucket.so
7fd26df90000-7fd26df91000 r--p 00001000 08:01 2363343                    /usr/lib/openmpi/lib/openmpi/mca_allocator_bucket.so
7fd26df91000-7fd26df92000 rw-p 00002000 08:01 2363343                    /usr/lib/openmpi/lib/openmpi/mca_allocator_bucket.so
7fd26df92000-7fd26df94000 r-xp 00000000 08:01 2363342                    /usr/lib/openmpi/lib/openmpi/mca_allocator_basic.so
7fd26df94000-7fd26e193000 ---p 00002000 08:01 2363342                    /usr/lib/openmpi/lib/openmpi/mca_allocator_basic.so
7fd26e193000-7fd26e194000 r--p 00001000 08:01 2363342                    /usr/lib/openmpi/lib/openmpi/mca_allocator_basic.so
7fd26e194000-7fd26e195000 rw-p 00002000 08:01 2363342                    /usr/lib/openmpi/lib/openmpi/mca_allocator_basic.so
7fd26e195000-7fd26e199000 r-xp 00000000 08:01 2363405                    /usr/lib/openmpi/lib/openmpi/mca_routed_binomial.so
7fd26e199000-7fd26e398000 ---p 00004000 08:01 2363405                    /usr/lib/openmpi/lib/openmpi/mca_routed_binomial.so
7fd26e398000-7fd26e399000 r--p 00003000 08:01 2363405                    /usr/lib/openmpi/lib/openmpi/mca_routed_binomial.so
7fd26e399000-7fd26e39a000 rw-p 00004000 08:01 2363405                    /usr/lib/openmpi/lib/openmpi/mca_routed_binomial.so
7fd26e39a000-7fd26e3a9000 r-xp 00000000 08:01 2363383                    /usr/lib/openmpi/lib/openmpi/mca_oob_tcp.so
7fd26e3a9000-7fd26e5a8000 ---p 0000f000 08:01 2363383                    /usr/lib/openmpi/lib/openmpi/mca_oob_tcp.so
7fd26e5a8000-7fd26e5a9000 r--p 0000e000 08:01 2363383                    /usr/lib/openmpi/lib/openmpi/mca_oob_tcp.so
7fd26e5a9000-7fd26e5aa000 rw-p 0000f000 08:01 2363383                    /usr/lib/openmpi/lib/openmpi/mca_oob_tcp.so
7fd26e5aa000-7fd26e5af000 r-xp 00000000 08:01 2363404                    /usr/lib/openmpi/lib/openmpi/mca_rml_oob.so
7fd26e5af000-7fd26e7ae000 ---p 00005000 08:01 2363404                    /usr/lib/openmpi/lib/openmpi/mca_rml_oob.so
7fd26e7ae000-7fd26e7af000 r--p 00004000 08:01 2363404                    /usr/lib/openmpi/lib/openmpi/mca_rml_oob.so
7fd26e7af000-7fd26e7b0000 rw-p 00005000 08:01 2363404                    /usr/lib/openmpi/lib/openmpi/mca_rml_oob.so
7fd26e7b0000-7fd26e7b3000 r-xp 00000000 08:01 2363370                    /usr/lib/openmpi/lib/openmpi/mca_grpcomm_bad.so
7fd26e7b3000-7fd26e9b2000 ---p 00003000 08:01 2363370                    /usr/lib/openmpi/lib/openmpi/mca_grpcomm_bad.so
7fd26e9b2000-7fd26e9b3000 r--p 00002000 08:01 2363370                    /usr/lib/openmpi/lib/openmpi/mca_grpcomm_bad.so
7fd26e9b3000-7fd26e9b4000 rw-p 00003000 08:01 2363370                    /usr/lib/openmpi/lib/openmpi/mca_grpcomm_bad.so
7fd26e9b4000-7fd26e9b7000 r-xp 00000000 08:01 2363364                    /usr/lib/openmpi/lib/openmpi/mca_ess_env.so
7fd26e9b7000-7fd26ebb6000 ---p 00003000 08:01 2363364                    /usr/lib/openmpi/lib/openmpi/mca_ess_env.so
7fd26ebb6000-7fd26ebb7000 r--p 00002000 08:01 2363364                    /usr/lib/openmpi/lib/openmpi/mca_ess_env.so
7fd26ebb7000-7fd26ebb8000 rw-p 00003000 08:01 2363364                    /usr/lib/openmpi/lib/openmpi/mca_ess_env.so
7fd26ebb8000-7fd26ebba000 r-xp 00000000 08:01 2363360                    /usr/lib/openmpi/lib/openmpi/mca_crs_none.so
7fd26ebba000-7fd26edb9000 ---p 00002000 08:01 2363360                    /usr/lib/openmpi/lib/openmpi/mca_crs_none.so
7fd26edb9000-7fd26edba000 r--p 00001000 08:01 2363360                    /usr/lib/openmpi/lib/openmpi/mca_crs_none.so
7fd26edba000-7fd26edbb000 rw-p 00002000 08:01 2363360                    /usr/lib/openmpi/lib/openmpi/mca_crs_none.so
7fd26edbb000-7fd26edbc000 r-xp 00000000 08:01 2363350                    /usr/lib/openmpi/lib/openmpi/mca_carto_auto_detect.so
7fd26edbc000-7fd26efbb000 ---p 00001000 08:01 2363350                    /usr/lib/openmpi/lib/openmpi/mca_carto_auto_detect.so
7fd26efbb000-7fd26efbc000 r--p 00000000 08:01 2363350                    /usr/lib/openmpi/lib/openmpi/mca_carto_auto_detect.so
7fd26efbc000-7fd26efbd000 rw-p 00001000 08:01 2363350                    /usr/lib/openmpi/lib/openmpi/mca_carto_auto_detect.so
7fd26efbd000-7fd26efc2000 r-xp 00000000 08:01 2363386                    /usr/lib/openmpi/lib/openmpi/mca_paffinity_linux.so
7fd26efc2000-7fd26f1c1000 ---p 00005000 08:01 2363386                    /usr/lib/openmpi/lib/openmpi/mca_paffinity_linux.so
7fd26f1c1000-7fd26f1c2000 r--p 00004000 08:01 2363386                    /usr/lib/openmpi/lib/openmpi/mca_paffinity_linux.so
7fd26f1c2000-7fd26f1c3000 rw-p 00005000 08:01 2363386                    /usr/lib/openmpi/lib/openmpi/mca_paffinity_linux.so
7fd26f1c3000-7fd26f1c5000 r-xp 00000000 08:01 1581263                    /lib/x86_64-linux-gnu/libutil-2.15.so
7fd26f1c5000-7fd26f3c4000 ---p 00002000 08:01 1581263                    /lib/x86_64-linux-gnu/libutil-2.15.so
7fd26f3c4000-7fd26f3c5000 r--p 00001000 08:01 1581263                    /lib/x86_64-linux-gnu/libutil-2.15.so
7fd26f3c5000-7fd26f3c6000 rw-p 00002000 08:01 1581263                    /lib/x86_64-linux-gnu/libutil-2.15.so
7fd26f3c6000-7fd26f3c8000 r-xp 00000000 08:01 1575642                    /lib/x86_64-linux-gnu/libdl-2.15.so
7fd26f3c8000-7fd26f5c8000 ---p 00002000 08:01 1575642                    /lib/x86_64-linux-gnu/libdl-2.15.so
7fd26f5c8000-7fd26f5c9000 r--p 00002000 08:01 1575642                    /lib/x86_64-linux-gnu/libdl-2.15.so
7fd26f5c9000-7fd26f5ca000 rw-p 00003000 08:01 1575642                    /lib/x86_64-linux-gnu/libdl-2.15.so
7fd26f5ca000-7fd26f61a000 r-xp 00000000 08:01 2363337                    /usr/lib/openmpi/lib/libopen-pal.so.0.0.0
7fd26f61a000-7fd26f81a000 ---p 00050000 08:01 2363337                    /usr/lib/openmpi/lib/libopen-pal.so.0.0.0
7fd26f81a000-7fd26f81b000 r--p 00050000 08:01 2363337                    /usr/lib/openmpi/lib/libopen-pal.so.0.0.0
7fd26f81b000-7fd26f81d000 rw-p 00051000 08:01 2363337                    /usr/lib/openmpi/lib/libopen-pal.so.0.0.0
7fd26f81d000-7fd26f821000 rw-p 00000000 00:00 0 
7fd26f821000-7fd26f86a000 r-xp 00000000 08:01 2363338                    /usr/lib/openmpi/lib/libopen-rte.so.0.0.0
7fd26f86a000-7fd26fa6a000 ---p 00049000 08:01 2363338                    /usr/lib/openmpi/lib/libopen-rte.so.0.0.0
7fd26fa6a000-7fd26fa6b000 r--p 00049000 08:01 2363338                    /usr/lib/openmpi/lib/libopen-rte.so.0.0.0
7fd26fa6b000-7fd26fa6d000 rw-p 0004a000 08:01 2363338                    /usr/lib/openmpi/lib/libopen-rte.so.0.0.0
7fd26fa6d000-7fd26fa6f000 rw-p 00000000 00:00 0 
7fd26fa6f000-7fd26fc24000 r-xp 00000000 08:01 1581130                    /lib/x86_64-linux-gnu/libc-2.15.so
7fd26fc24000-7fd26fe23000 ---p 001b5000 08:01 1581130                    /lib/x86_64-linux-gnu/libc-2.15.so
7fd26fe23000-7fd26fe27000 r--p 001b4000 08:01 1581130                    /lib/x86_64-linux-gnu/libc-2.15.so
7fd26fe27000-7fd26fe29000 rw-p 001b8000 08:01 1581130                    /lib/x86_64-linux-gnu/libc-2.15.so
7fd26fe29000-7fd26fe2e000 rw-p 00000000 00:00 0 
7fd26fe2e000-7fd26fe46000 r-xp 00000000 08:01 1581262                    /lib/x86_64-linux-gnu/libpthread-2.15.so
7fd26fe46000-7fd270045000 ---p 00018000 08:01 1581262                    /lib/x86_64-linux-gnu/libpthread-2.15.so
7fd270045000-7fd270046000 r--p 00017000 08:01 1581262                    /lib/x86_64-linux-gnu/libpthread-2.15.so
7fd270046000-7fd270047000 rw-p 00018000 08:01 1581262                    /lib/x86_64-linux-gnu/libpthread-2.15.so
7fd270047000-7fd27004b000 rw-p 00000000 00:00 0 
7fd27004b000-7fd2700e0000 r-xp 00000000 08:01 2363333                    /usr/lib/openmpi/lib/libmpi.so.0.0.2
7fd2700e0000-7fd2702e0000 ---p 00095000 08:01 2363333                    /usr/lib/openmpi/lib/libmpi.so.0.0.2
7fd2702e0000-7fd2702e1000 r--p 00095000 08:01 2363333                    /usr/lib/openmpi/lib/libmpi.so.0.0.2
7fd2702e1000-7fd2702f2000 rw-p 00096000 08:01 2363333                    /usr/lib/openmpi/lib/libmpi.so.0.0.2
7fd2702f2000-7fd2702fc000 rw-p 00000000 00:00 0 
7fd2702fc000-7fd27031e000 r-xp 00000000 08:01 1575649                    /lib/x86_64-linux-gnu/ld-2.15.so
7fd2704b6000-7fd2704eb000 r--s 00000000 08:01 7616766                    /var/cache/nscd/passwd
7fd2704eb000-7fd2704f0000 rw-p 00000000 00:00 0 
7fd27051b000-7fd27051e000 rw-p 00000000 00:00 0 
7fd27051e000-7fd27051f000 r--p 00022000 08:01 1575649                    /lib/x86_64-linux-gnu/ld-2.15.so
7fd27051f000-7fd270521000 rw-p 00023000 08:01 1575649                    /lib/x86_64-linux-gnu/ld-2.15.so
7fff5a763000-7fff5a784000 rw-p 00000000 00:00 0                          [stack]
7fff5a7b2000-7fff5a7b3000 r-xp 00000000 00:00 0                          [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0                  [vsyscall]
[labx04:27214] *** Process received signal ***
[labx04:27214] Signal: Aborted (6)
[labx04:27214] Signal code:  (-6)
[labx04:27214] [ 0] /lib/x86_64-linux-gnu/libpthread.so.0(+0xfcb0) [0x7fd26fe3dcb0]
[labx04:27214] [ 1] /lib/x86_64-linux-gnu/libc.so.6(gsignal+0x35) [0x7fd26faa5425]
[labx04:27214] [ 2] /lib/x86_64-linux-gnu/libc.so.6(abort+0x17b) [0x7fd26faa8b8b]
[labx04:27214] [ 3] /lib/x86_64-linux-gnu/libc.so.6(+0x7439e) [0x7fd26fae339e]
[labx04:27214] [ 4] /lib/x86_64-linux-gnu/libc.so.6(+0x7eb96) [0x7fd26faedb96]
[labx04:27214] [ 5] /usr/lib/openmpi/lib/openmpi/mca_btl_tcp.so(+0x52b6) [0x7fd26ba712b6]
[labx04:27214] [ 6] /usr/lib/openmpi/lib/openmpi/mca_btl_tcp.so(+0x2599) [0x7fd26ba6e599]
[labx04:27214] [ 7] /usr/lib/libmpi.so.0(mca_btl_base_close+0x5f) [0x7fd2700bd0af]
[labx04:27214] [ 8] /usr/lib/openmpi/lib/openmpi/mca_bml_r2.so(+0x2652) [0x7fd26c73d652]
[labx04:27214] [ 9] /usr/lib/openmpi/lib/openmpi/mca_pml_ob1.so(+0x48ce) [0x7fd26cb488ce]
[labx04:27214] [10] /usr/lib/libmpi.so.0(+0x371db) [0x7fd2700821db]
[labx04:27214] [11] a.out(main+0x32bc) [0x404260]
[labx04:27214] [12] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xed) [0x7fd26fa9076d]
[labx04:27214] [13] a.out() [0x400ee9]
[labx04:27214] *** End of error message ***
--------------------------------------------------------------------------
mpirun noticed that process rank 2 with PID 25102 on node labx03 exited on signal 11 (Segmentation fault).
--------------------------------------------------------------------------