Skip to content

Instantly share code, notes, and snippets.

@yteraoka

yteraoka/repmgrd-double-free-or-corruption.txt

Last active September 28, 2016 13:37
Show Gist options
  • Save yteraoka/cc2410d8296fdde7881955c529738324 to your computer and use it in GitHub Desktop.
Save yteraoka/cc2410d8296fdde7881955c529738324 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
repmgrd-double-free-or-corruption.txt
/var/log/repmgr/repmgr-9.5.log
[2016-09-28 00:43:43] [ERROR] promote command failed. You could check and try it manually.
*** Error in `/usr/pgsql-9.5/bin/repmgrd': double free or corruption (out): 0x00000000008291c0 ***
======= Backtrace: =========
/lib64/libc.so.6(+0x7cfe1)[0x7fced6cf9fe1]
/usr/pgsql-9.5/lib/libpq.so.5(+0xa7df)[0x7fced70487df]
/usr/pgsql-9.5/bin/repmgrd[0x409db3]
/usr/pgsql-9.5/bin/repmgrd[0x40db68]
/usr/pgsql-9.5/bin/repmgrd[0x40304a]
/lib64/libc.so.6(__libc_start_main+0xf5)[0x7fced6c9eb15]
/usr/pgsql-9.5/bin/repmgrd[0x403689]
======= Memory map: ========
00400000-00418000 r-xp 00000000 fd:00 201827076 /usr/pgsql-9.5/bin/repmgrd
00618000-00619000 r--p 00018000 fd:00 201827076 /usr/pgsql-9.5/bin/repmgrd
00619000-00621000 rw-p 00019000 fd:00 201827076 /usr/pgsql-9.5/bin/repmgrd
00621000-00631000 rw-p 00000000 00:00 0
00820000-00841000 rw-p 00000000 00:00 0 [heap]
00841000-0085d000 rw-p 00000000 00:00 0 [heap]
7fcecc000000-7fcecc021000 rw-p 00000000 00:00 0
7fcecc021000-7fced0000000 ---p 00000000 00:00 0
7fced28c0000-7fced28d5000 r-xp 00000000 fd:00 201326729 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7fced28d5000-7fced2ad4000 ---p 00015000 fd:00 201326729 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7fced2ad4000-7fced2ad5000 r--p 00014000 fd:00 201326729 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7fced2ad5000-7fced2ad6000 rw-p 00015000 fd:00 201326729 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
7fced2ad6000-7fced2ae1000 r-xp 00000000 fd:00 201367802 /usr/lib64/libnss_files-2.17.so
7fced2ae1000-7fced2ce0000 ---p 0000b000 fd:00 201367802 /usr/lib64/libnss_files-2.17.so
7fced2ce0000-7fced2ce1000 r--p 0000a000 fd:00 201367802 /usr/lib64/libnss_files-2.17.so
7fced2ce1000-7fced2ce2000 rw-p 0000b000 fd:00 201367802 /usr/lib64/libnss_files-2.17.so
7fced2ce2000-7fced2ce8000 rw-p 00000000 00:00 0
7fced2ce8000-7fced2cea000 r-xp 00000000 fd:00 201359602 /usr/lib64/libfreebl3.so
7fced2cea000-7fced2ee9000 ---p 00002000 fd:00 201359602 /usr/lib64/libfreebl3.so
7fced2ee9000-7fced2eea000 r--p 00001000 fd:00 201359602 /usr/lib64/libfreebl3.so
7fced2eea000-7fced2eeb000 rw-p 00002000 fd:00 201359602 /usr/lib64/libfreebl3.so
7fced2eeb000-7fced2f0f000 r-xp 00000000 fd:00 201398549 /usr/lib64/liblzma.so.5.0.99
7fced2f0f000-7fced310e000 ---p 00024000 fd:00 201398549 /usr/lib64/liblzma.so.5.0.99
7fced310e000-7fced310f000 r--p 00023000 fd:00 201398549 /usr/lib64/liblzma.so.5.0.99
7fced310f000-7fced3110000 rw-p 00024000 fd:00 201398549 /usr/lib64/liblzma.so.5.0.99
7fced3110000-7fced3170000 r-xp 00000000 fd:00 201398581 /usr/lib64/libpcre.so.1.2.0
7fced3170000-7fced336f000 ---p 00060000 fd:00 201398581 /usr/lib64/libpcre.so.1.2.0
7fced336f000-7fced3370000 r--p 0005f000 fd:00 201398581 /usr/lib64/libpcre.so.1.2.0
7fced3370000-7fced3371000 rw-p 00060000 fd:00 201398581 /usr/lib64/libpcre.so.1.2.0
7fced3371000-7fced3378000 r-xp 00000000 fd:00 201398534 /usr/lib64/librt-2.17.so
7fced3378000-7fced3577000 ---p 00007000 fd:00 201398534 /usr/lib64/librt-2.17.so
7fced3577000-7fced3578000 r--p 00006000 fd:00 201398534 /usr/lib64/librt-2.17.so
7fced3578000-7fced3579000 rw-p 00007000 fd:00 201398534 /usr/lib64/librt-2.17.so
7fced3579000-7fced3581000 r-xp 00000000 fd:00 201367788 /usr/lib64/libcrypt-2.17.so
7fced3581000-7fced3780000 ---p 00008000 fd:00 201367788 /usr/lib64/libcrypt-2.17.so
7fced3780000-7fced3781000 r--p 00007000 fd:00 201367788 /usr/lib64/libcrypt-2.17.so
7fced3781000-7fced3782000 rw-p 00008000 fd:00 201367788 /usr/lib64/libcrypt-2.17.so
7fced3782000-7fced37b0000 rw-p 00000000 00:00 0
7fced37b0000-7fced37d1000 r-xp 00000000 fd:00 201421056 /usr/lib64/libselinux.so.1
7fced37d1000-7fced39d1000 ---p 00021000 fd:00 201421056 /usr/lib64/libselinux.so.1
7fced39d1000-7fced39d2000 r--p 00021000 fd:00 201421056 /usr/lib64/libselinux.so.1
7fced39d2000-7fced39d3000 rw-p 00022000 fd:00 201421056 /usr/lib64/libselinux.so.1
7fced39d3000-7fced39d5000 rw-p 00000000 00:00 0
7fced39d5000-7fced3a0e000 r-xp 00000000 fd:00 201421077 /usr/lib64/libnspr4.so
7fced3a0e000-7fced3c0e000 ---p 00039000 fd:00 201421077 /usr/lib64/libnspr4.so
7fced3c0e000-7fced3c0f000 r--p 00039000 fd:00 201421077 /usr/lib64/libnspr4.so
7fced3c0f000-7fced3c11000 rw-p 0003a000 fd:00 201421077 /usr/lib64/libnspr4.so
7fced3c11000-7fced3c13000 rw-p 00000000 00:00 0
7fced3c13000-7fced3c17000 r-xp 00000000 fd:00 201421078 /usr/lib64/libplc4.so
7fced3c17000-7fced3e16000 ---p 00004000 fd:00 201421078 /usr/lib64/libplc4.so
7fced3e16000-7fced3e17000 r--p 00003000 fd:00 201421078 /usr/lib64/libplc4.so
7fced3e17000-7fced3e18000 rw-p 00004000 fd:00 201421078 /usr/lib64/libplc4.so
7fced3e18000-7fced3e1b000 r-xp 00000000 fd:00 201421079 /usr/lib64/libplds4.so
7fced3e1b000-7fced401a000 ---p 00003000 fd:00 201421079 /usr/lib64/libplds4.so
7fced401a000-7fced401b000 r--p 00002000 fd:00 201421079 /usr/lib64/libplds4.so
7fced401b000-7fced401c000 rw-p 00003000 fd:00 201421079 /usr/lib64/libplds4.so
7fced401c000-7fced4041000 r-xp 00000000 fd:00 201421080 /usr/lib64/libnssutil3.so
7fced4041000-7fced4241000 ---p 00025000 fd:00 201421080 /usr/lib64/libnssutil3.so
7fced4241000-7fced4247000 r--p 00025000 fd:00 201421080 /usr/lib64/libnssutil3.so
7fced4247000-7fced4248000 rw-p 0002b000 fd:00 201421080 /usr/lib64/libnssutil3.so
7fced4248000-7fced4366000 r-xp 00000000 fd:00 201650439 /usr/lib64/libnss3.so
7fced4366000-7fced4565000 ---p 0011e000 fd:00 201650439 /usr/lib64/libnss3.so
7fced4565000-7fced456a000 r--p 0011d000 fd:00 201650439 /usr/lib64/libnss3.so
7fced456a000-7fced456c000 rw-p 00122000 fd:00 201650439 /usr/lib64/libnss3.so
7fced456c000-7fced456e000 rw-p 00000000 00:00 0
7fced456e000-7fced4592000 r-xp 00000000 fd:00 201650441 /usr/lib64/libsmime3.so
7fced4592000-7fced4791000 ---p 00024000 fd:00 201650441 /usr/lib64/libsmime3.so
7fced4791000-7fced4794000 r--p 00023000 fd:00 201650441 /usr/lib64/libsmime3.so
7fced4794000-7fced4795000 rw-p 00026000 fd:00 201650441 /usr/lib64/libsmime3.so
7fced4795000-7fced47d2000 r-xp 00000000 fd:00 201650442 /usr/lib64/libssl3.so
7fced47d2000-7fced49d2000 ---p 0003d000 fd:00 201650442 /usr/lib64/libssl3.so
7fced49d2000-7fced49d5000 r--p 0003d000 fd:00 201650442 /usr/lib64/libssl3.so
7fced49d5000-7fced49d6000 rw-p 00040000 fd:00 201650442 /usr/lib64/libssl3.so
7fced49d6000-7fced49d7000 rw-p 00000000 00:00 0
7fced49d7000-7fced49f3000 r-xp 00000000 fd:00 201503344 /usr/lib64/libsasl2.so.3.0.0
7fced49f3000-7fced4bf2000 ---p 0001c000 fd:00 201503344 /usr/lib64/libsasl2.so.3.0.0
7fced4bf2000-7fced4bf3000 r--p 0001b000 fd:00 201503344 /usr/lib64/libsasl2.so.3.0.0
7fced4bf3000-7fced4bf4000 rw-p 0001c000 fd:00 201503344 /usr/lib64/libsasl2.so.3.0.0
7fced4bf4000-7fced4c02000 r-xp 00000000 fd:00 201669062 /usr/lib64/liblber-2.4.so.2.10.3
7fced4c02000-7fced4e01000 ---p 0000e000 fd:00 201669062 /usr/lib64/liblber-2.4.so.2.10.3
7fced4e01000-7fced4e02000 r--p 0000d000 fd:00 201669062 /usr/lib64/liblber-2.4.so.2.10.3
7fced4e02000-7fced4e03000 rw-p 0000e000 fd:00 201669062 /usr/lib64/liblber-2.4.so.2.10.3
7fced4e03000-7fced4e19000 r-xp 00000000 fd:00 201398532 /usr/lib64/libresolv-2.17.so
7fced4e19000-7fced5019000 ---p 00016000 fd:00 201398532 /usr/lib64/libresolv-2.17.so
7fced5019000-7fced501a000 r--p 00016000 fd:00 201398532 /usr/lib64/libresolv-2.17.so
7fced501a000-7fced501b000 rw-p 00017000 fd:00 201398532 /usr/lib64/libresolv-2.17.so
7fced501b000-7fced501d000 rw-p 00000000 00:00 0
7fced501d000-7fced5020000 r-xp 00000000 fd:00 201440331 /usr/lib64/libkeyutils.so.1.5
7fced5020000-7fced521f000 ---p 00003000 fd:00 201440331 /usr/lib64/libkeyutils.so.1.5
7fced521f000-7fced5220000 r--p 00002000 fd:00 201440331 /usr/lib64/libkeyutils.so.1.5
7fced5220000-7fced5221000 rw-p 00003000 fd:00 201440331 /usr/lib64/libkeyutils.so.1.5
7fced5221000-7fced522e000 r-xp 00000000 fd:00 201556353 /usr/lib64/libkrb5support.so.0.1
7fced522e000-7fced542e000 ---p 0000d000 fd:00 201556353 /usr/lib64/libkrb5support.so.0.1
7fced542e000-7fced542f000 r--p 0000d000 fd:00 201556353 /usr/lib64/libkrb5support.so.0.1
7fced542f000-7fced5430000 rw-p 0000e000 fd:00 201556353 /usr/lib64/libkrb5support.so.0.1
7fced5430000-7fced5445000 r-xp 00000000 fd:00 201421059 /usr/lib64/libz.so.1.2.7
7fced5445000-7fced5644000 ---p 00015000 fd:00 201421059 /usr/lib64/libz.so.1.2.7
7fced5644000-7fced5645000 r--p 00014000 fd:00 201421059 /usr/lib64/libz.so.1.2.7
7fced5645000-7fced5646000 rw-p 00015000 fd:00 201421059 /usr/lib64/libz.so.1.2.7
7fced5646000-7fced5649000 r-xp 00000000 fd:00 201367790 /usr/lib64/libdl-2.17.so
7fced5649000-7fced5848000 ---p 00003000 fd:00 201367790 /usr/lib64/libdl-2.17.so
7fced5848000-7fced5849000 r--p 00002000 fd:00 201367790 /usr/lib64/libdl-2.17.so
7fced5849000-7fced584a000 rw-p 00003000 fd:00 201367790 /usr/lib64/libdl-2.17.so
7fced584a000-7fced5879000 r-xp 00000000 fd:00 201556341 /usr/lib64/libk5crypto.so.3.1
7fced5879000-7fced5a78000 ---p 0002f000 fd:00 201556341 /usr/lib64/libk5crypto.so.3.1
7fced5a78000-7fced5a7a000 r--p 0002e000 fd:00 201556341 /usr/lib64/libk5crypto.so.3.1
7fced5a7a000-7fced5a7b000 rw-p 00030000 fd:00 201556341 /usr/lib64/libk5crypto.so.3.1
7fced5a7b000-7fced5a7c000 rw-p 00000000 00:00 0
7fced5a7c000-7fced5a7f000 r-xp 00000000 fd:00 201421062 /usr/lib64/libcom_err.so.2.1
7fced5a7f000-7fced5c7e000 ---p 00003000 fd:00 201421062 /usr/lib64/libcom_err.so.2.1
7fced5c7e000-7fced5c7f000 r--p 00002000 fd:00 201421062 /usr/lib64/libcom_err.so.2.1
7fced5c7f000-7fced5c80000 rw-p 00003000 fd:00 201421062 /usr/lib64/libcom_err.so.2.1
7fced5c80000-7fced5d55000 r-xp 00000000 fd:00 201556351 /usr/lib64/libkrb5.so.3.3
7fced5d55000-7fced5f55000 ---p 000d5000 fd:00 201556351 /usr/lib64/libkrb5.so.3.3
7fced5f55000-7fced5f62000 r--p 000d5000 fd:00 201556351 /usr/lib64/libkrb5.so.3.3
7fced5f62000-7fced5f65000 rw-p 000e2000 fd:00 201556351 /usr/lib64/libkrb5.so.3.3
7fced5f65000-7fced5f7b000 r-xp 00000000 fd:00 201398530 /usr/lib64/libpthread-2.17.so
7fced5f7b000-7fced617b000 ---p 00016000 fd:00 201398530 /usr/lib64/libpthread-2.17.so
7fced617b000-7fced617c000 r--p 00016000 fd:00 201398530 /usr/lib64/libpthread-2.17.so
7fced617c000-7fced617d000 rw-p 00017000 fd:00 201398530 /usr/lib64/libpthread-2.17.so
7fced617d000-7fced6181000 rw-p 00000000 00:00 0
7fced6181000-7fced61d7000 r-xp 00000000 fd:00 201669066 /usr/lib64/libldap_r-2.4.so.2.10.3
7fced61d7000-7fced63d7000 ---p 00056000 fd:00 201669066 /usr/lib64/libldap_r-2.4.so.2.10.3
7fced63d7000-7fced63da000 r--p 00056000 fd:00 201669066 /usr/lib64/libldap_r-2.4.so.2.10.3
7fced63da000-7fced63db000 rw-p 00059000 fd:00 201669066 /usr/lib64/libldap_r-2.4.so.2.10.3
7fced63db000-7fced63dd000 rw-p 00000000 00:00 0
7fced63dd000-7fced6426000 r-xp 00000000 fd:00 201556337 /usr/lib64/libgssapi_krb5.so.2.2
7fced6426000-7fced6626000 ---p 00049000 fd:00 201556337 /usr/lib64/libgssapi_krb5.so.2.2
7fced6626000-7fced6627000 r--p 00049000 fd:00 201556337 /usr/lib64/libgssapi_krb5.so.2.2
7fced6627000-7fced6629000 rw-p 0004a000 fd:00 201556337 /usr/lib64/libgssapi_krb5.so.2.2
7fced6629000-7fced67e6000 r-xp 00000000 fd:00 201556360 /usr/lib64/libcrypto.so.1.0.1e
7fced67e6000-7fced69e6000 ---p 001bd000 fd:00 201556360 /usr/lib64/libcrypto.so.1.0.1e
7fced69e6000-7fced6a00000 r--p 001bd000 fd:00 201556360 /usr/lib64/libcrypto.so.1.0.1e
7fced6a00000-7fced6a0c000 rw-p 001d7000 fd:00 201556360 /usr/lib64/libcrypto.so.1.0.1e
7fced6a0c000-7fced6a10000 rw-p 00000000 00:00 0
7fced6a10000-7fced6a73000 r-xp 00000000 fd:00 201556362 /usr/lib64/libssl.so.1.0.1e
7fced6a73000-7fced6c72000 ---p 00063000 fd:00 201556362 /usr/lib64/libssl.so.1.0.1e
7fced6c72000-7fced6c76000 r--p 00062000 fd:00 201556362 /usr/lib64/libssl.so.1.0.1e
7fced6c76000-7fced6c7d000 rw-p 00066000 fd:00 201556362 /usr/lib64/libssl.so.1.0.1e
7fced6c7d000-7fced6e33000 r-xp 00000000 fd:00 201367784 /usr/lib64/libc-2.17.so
7fced6e33000-7fced7033000 ---p 001b6000 fd:00 201367784 /usr/lib64/libc-2.17.so
7fced7033000-7fced7037000 r--p 001b6000 fd:00 201367784 /usr/lib64/libc-2.17.so
7fced7037000-7fced7039000 rw-p 001ba000 fd:00 201367784 /usr/lib64/libc-2.17.so
7fced7039000-7fced703e000 rw-p 00000000 00:00 0
7fced703e000-7fced706b000 r-xp 00000000 fd:00 134482475 /usr/pgsql-9.5/lib/libpq.so.5.8
7fced706b000-7fced726a000 ---p 0002d000 fd:00 134482475 /usr/pgsql-9.5/lib/libpq.so.5.8
7fced726a000-7fced726d000 r--p 0002c000 fd:00 134482475 /usr/pgsql-9.5/lib/libpq.so.5.8
7fced726d000-7fced726e000 rw-p 0002f000 fd:00 134482475 /usr/pgsql-9.5/lib/libpq.so.5.8
7fced726e000-7fced728f000 r-xp 00000000 fd:00 201367777 /usr/lib64/ld-2.17.so
7fced7478000-7fced7488000 rw-p 00000000 00:00 0
7fced748c000-7fced748e000 rw-p 00000000 00:00 0
7fced748e000-7fced748f000 rw-p 00000000 00:00 0
7fced748f000-7fced7490000 r--p 00021000 fd:00 201367777 /usr/lib64/ld-2.17.so
7fced7490000-7fced7491000 rw-p 00022000 fd:00 201367777 /usr/lib64/ld-2.17.so
7fced7491000-7fced7492000 rw-p 00000000 00:00 0
7ffea038a000-7ffea03bf000 rw-p 00000000 00:00 0 [stack]
7ffea03e6000-7ffea03e8000 r-xp 00000000 00:00 0 [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
Raw
software-versions.txt
-bash-4.2$ uname -a
Linux db2 3.10.0-327.el7.x86_64 #1 SMP Thu Nov 19 22:10:57 UTC 2015 x86_64 x86_64 x86_64 GNU/Linux
-bash-4.2$ rpm -q glibc
glibc-2.17-105.el7.x86_64
-bash-4.2$ rpm -qa | grep postgresql
postgresql95-libs-9.5.4-2PGDG.rhel7.x86_64
postgresql95-server-9.5.4-2PGDG.rhel7.x86_64
postgresql95-9.5.4-2PGDG.rhel7.x86_64
-bash-4.2$ rpm -qa | grep repmgr
repmgr95-3.1.3-1.rhel7.x86_64
-bash-4.2$
-bash-4.2$ ldd /usr/pgsql-9.5/bin/repmgrd
linux-vdso.so.1 => (0x00007ffe2aba5000)
libpq.so.5 => /usr/pgsql-9.5/lib/libpq.so.5 (0x00007f80112a0000)
libc.so.6 => /lib64/libc.so.6 (0x00007f8010ed8000)
libssl.so.10 => /lib64/libssl.so.10 (0x00007f8010c6b000)
libcrypto.so.10 => /lib64/libcrypto.so.10 (0x00007f8010884000)
libgssapi_krb5.so.2 => /lib64/libgssapi_krb5.so.2 (0x00007f8010637000)
libldap_r-2.4.so.2 => /lib64/libldap_r-2.4.so.2 (0x00007f80103db000)
libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f80101bf000)
/lib64/ld-linux-x86-64.so.2 (0x00007f80114d1000)
libkrb5.so.3 => /lib64/libkrb5.so.3 (0x00007f800fed9000)
libcom_err.so.2 => /lib64/libcom_err.so.2 (0x00007f800fcd5000)
libk5crypto.so.3 => /lib64/libk5crypto.so.3 (0x00007f800faa3000)
libdl.so.2 => /lib64/libdl.so.2 (0x00007f800f89e000)
libz.so.1 => /lib64/libz.so.1 (0x00007f800f688000)
libkrb5support.so.0 => /lib64/libkrb5support.so.0 (0x00007f800f479000)
libkeyutils.so.1 => /lib64/libkeyutils.so.1 (0x00007f800f274000)
libresolv.so.2 => /lib64/libresolv.so.2 (0x00007f800f05a000)
liblber-2.4.so.2 => /lib64/liblber-2.4.so.2 (0x00007f800ee4b000)
libsasl2.so.3 => /lib64/libsasl2.so.3 (0x00007f800ec2d000)
libssl3.so => /lib64/libssl3.so (0x00007f800e9eb000)
libsmime3.so => /lib64/libsmime3.so (0x00007f800e7c4000)
libnss3.so => /lib64/libnss3.so (0x00007f800e49d000)
libnssutil3.so => /lib64/libnssutil3.so (0x00007f800e271000)
libplds4.so => /lib64/libplds4.so (0x00007f800e06d000)
libplc4.so => /lib64/libplc4.so (0x00007f800de67000)
libnspr4.so => /lib64/libnspr4.so (0x00007f800dc29000)
libselinux.so.1 => /lib64/libselinux.so.1 (0x00007f800da03000)
libcrypt.so.1 => /lib64/libcrypt.so.1 (0x00007f800d7cc000)
librt.so.1 => /lib64/librt.so.1 (0x00007f800d5c3000)
libpcre.so.1 => /lib64/libpcre.so.1 (0x00007f800d362000)
liblzma.so.5 => /lib64/liblzma.so.5 (0x00007f800d13d000)
libfreebl3.so => /lib64/libfreebl3.so (0x00007f800cf39000)
-bash-4.2$
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment