Do: AMANDA_CFLAGS="-g"
before you configure
Add the following lines to amanda.conf:
 debug_auth 1
 debug_protocol 1
 debug_dumper 1

Send me the Backtrace and the dumper.*.debug if it still crash.

Jean-Louis

Bruce Thompson wrote:

Hi all,

I've been having troubles with Amanda on my new box (an eMachines desktop running Fedora Core 6).

I have four machines being backed up to vdisks with about 500Gb total being backed up with two clients active at the time of the crash. All three are running builds of 2.5.2.

I keep getting these strange crashes. I run amdump manually using screen to multiplex a status dump (amstatus every 30 seconds) in one virtual terminal with amdump running in the other. What happens is that amdump crashes so hard that it wipes out both virtual terminals and the screen process! I never saw anything substantial in the logs, just the dump went away very suddenly.

Yesterday I ran amdump inside script and captured at lest part of what was going on:

[amanda@chekov ~]$ amdump daily

*** glibc detected *** dumper2: double free or corruption (!prev): 0x085888c0 ***

======= Backtrace: =========
/lib/libc.so.6[0x9a209d]
/lib/libc.so.6(cfree+0x90)[0x9a56f0]
/usr/local/lib/libamanda-2.5.2.so(sec_tcp_conn_put+0x275)[0x59ab0f]
/usr/local/lib/libamanda-2.5.2.so(tcpma_stream_close+0xb1)[0x59890a]
/usr/local/lib/libamanda-2.5.2.so(security_stream_close+0x81)[0x597023]
/usr/local/lib/libamanda-2.5.2.so(sec_close+0x70)[0x59711c]
/usr/local/lib/libamanda-2.5.2.so(security_close+0x97)[0x596e72]
/usr/local/lib/libamanda-2.5.2.so[0x595f6c]
/usr/local/lib/libamanda-2.5.2.so[0x595ae3]
/usr/local/lib/libamanda-2.5.2.so[0x581af3]
/usr/local/lib/libamanda-2.5.2.so[0x595bc7]
/usr/local/lib/libamanda-2.5.2.so[0x58f2bf]
/usr/local/lib/libamanda-2.5.2.so(event_loop+0x19)[0x58e9f9]
/usr/local/lib/libamanda-2.5.2.so(protocol_run+0x1e)[0x595c11]
dumper2[0x804ee4f]
dumper2[0x804abf8]
/lib/libc.so.6(__libc_start_main+0xdc)[0x951f2c]
dumper2[0x8049a61]
======= Memory map: ========

00309000-00325000 r-xp 00000000 03:45 7048297 /usr/local/lib/libamserver-2.5.2.so 00325000-00326000 rwxp 0001c000 03:45 7048297 /usr/local/lib/libamserver-2.5.2.so

00326000-00327000 rwxp 00326000 00:00 0
00404000-00405000 r-xp 00404000 00:00 0          [vdso]

00578000-005aa000 r-xp 00000000 03:45 7048299 /usr/local/lib/libamanda-2.5.2.so 005aa000-005ad000 rwxp 00031000 03:45 7048299 /usr/local/lib/libamanda-2.5.2.so

005ad000-005d3000 rwxp 005ad000 00:00 0
00666000-00679000 r-xp 00000000 03:03 58532      /lib/libnsl-2.5.so
00679000-0067a000 r-xp 00012000 03:03 58532      /lib/libnsl-2.5.so
0067a000-0067b000 rwxp 00013000 03:03 58532      /lib/libnsl-2.5.so
0067b000-0067d000 rwxp 0067b000 00:00 0
00684000-00693000 r-xp 00000000 03:03 58550      /lib/libresolv-2.5.so
00693000-00694000 r-xp 0000e000 03:03 58550      /lib/libresolv-2.5.so
00694000-00695000 rwxp 0000f000 03:03 58550      /lib/libresolv-2.5.so
00695000-00697000 rwxp 00695000 00:00 0
0091f000-00938000 r-xp 00000000 03:03 58499      /lib/ld-2.5.so
00938000-00939000 r-xp 00018000 03:03 58499      /lib/ld-2.5.so
00939000-0093a000 rwxp 00019000 03:03 58499      /lib/ld-2.5.so
0093c000-00a73000 r-xp 00000000 03:03 58515      /lib/libc-2.5.so
00a73000-00a75000 r-xp 00137000 03:03 58515      /lib/libc-2.5.so
00a75000-00a76000 rwxp 00139000 03:03 58515      /lib/libc-2.5.so
00a76000-00a79000 rwxp 00a76000 00:00 0
00a7b000-00aa0000 r-xp 00000000 03:03 58548      /lib/libm-2.5.so
00aa0000-00aa1000 r-xp 00024000 03:03 58548      /lib/libm-2.5.so
00aa1000-00aa2000 rwxp 00025000 03:03 58548      /lib/libm-2.5.so
00bd6000-00bdf000 r-xp 00000000 03:03 58537      /lib/libnss_files-2.5.so
00bdf000-00be0000 r-xp 00008000 03:03 58537      /lib/libnss_files-2.5.so
00be0000-00be1000 rwxp 00009000 03:03 58537      /lib/libnss_files-2.5.so
00d70000-00d74000 r-xp 00000000 03:03 58535      /lib/libnss_dns-2.5.so
00d74000-00d75000 r-xp 00003000 03:03 58535      /lib/libnss_dns-2.5.so
00d75000-00d76000 rwxp 00004000 03:03 58535      /lib/libnss_dns-2.5.so

00ee3000-00eee000 r-xp 00000000 03:45 7048295 /usr/local/lib/libamtape-2.5.2.so 00eee000-00eef000 rwxp 0000b000 03:45 7048295 /usr/local/lib/libamtape-2.5.2.so 029e3000-029ee000 r-xp 00000000 03:03 58568 /lib/libgcc_s-4.1.1-20070105.so.1 029ee000-029ef000 rwxp 0000a000 03:03 58568 /lib/libgcc_s-4.1.1-20070105.so.1

02fcb000-02fce000 r-xp 00000000 03:03 58579      /lib/libtermcap.so.2.0.8
02fce000-02fcf000 rwxp 00002000 03:03 58579      /lib/libtermcap.so.2.0.8

02fd1000-03000000 r-xp 00000000 03:06 1760450 /usr/lib/libreadline.so.5.1 03000000-03004000 rwxp 0002f000 03:06 1760450 /usr/lib/libreadline.so.5.1

03004000-03005000 rwxp 03004000 00:00 0

08048000-08050000 r-xp 00000000 03:45 917531 /usr/local/libexec/dumper 08050000-08051000 rw-p 00008000 03:45 917531 /usr/local/libexec/dumper

08051000-08052000 rw-p 08051000 00:00 0
08579000-0859a000 rw-p 08579000 00:00 0
b7e00000-b7e21000 rw-p b7e00000 00:00 0
b7e21000-b7f00000 ---p b7e21000 00:00 0
b7f42000-b7f45000 rw-p b7f42000 00:00 0
b7f53000-b7f56000 rw-p b7f53000 00:00 0
bf851000-bf866000 rw-p bf851000 00:00 0          [stack]

*** glibc detected *** dumper0: double free or corruption (top): 0x08aa4258 ***

======= Backtrace: =========
/lib/libc.so.6[0x9a209d]
/lib/libc.so.6(cfree+0x90)[0x9a56f0]
/usr/local/lib/libamanda-2.5.2.so(sec_tcp_conn_put+0x275)[0x436b0f]
/usr/local/lib/libamanda-2.5.2.so(tcpma_stream_close+0xb1)[0x43490a]
/usr/local/lib/libamanda-2.5.2.so(security_stream_close+0x81)[0x433023]
/usr/local/lib/libamanda-2.5.2.so(sec_close+0x70)[0x43311c]
/usr/local/lib/libamanda-2.5.2.so(security_close+0x97)[0x432e72]
/usr/local/lib/libamanda-2.5.2.so[0x431f6c]
/usr/local/lib/libamanda-2.5.2.so[0x431ae3]
/usr/local/lib/libamanda-2.5.2.so[0x41daf3]
/usr/local/lib/libamanda-2.5.2.so[0x431bc7]
/usr/local/lib/libamanda-2.5.2.so[0x42b2bf]
/usr/local/lib/libamanda-2.5.2.so(event_loop+0x19)[0x42a9f9]
/usr/local/lib/libamanda-2.5.2.so(protocol_run+0x1e)[0x431c11]
dumper0[0x804ee4f]
dumper0[0x804abf8]
/lib/libc.so.6(__libc_start_main+0xdc)[0x951f2c]
dumper0[0x8049a61]
======= Memory map: ========
00110000-00111000 r-xp 00110000 00:00 0          [vdso]

0015c000-00178000 r-xp 00000000 03:45 7048297 /usr/local/lib/libamserver-2.5.2.so 00178000-00179000 rwxp 0001c000 03:45 7048297 /usr/local/lib/libamserver-2.5.2.so

00179000-0017a000 rwxp 00179000 00:00 0

0028a000-00295000 r-xp 00000000 03:45 7048295 /usr/local/lib/libamtape-2.5.2.so 00295000-00296000 rwxp 0000b000 03:45 7048295 /usr/local/lib/libamtape-2.5.2.so 00414000-00446000 r-xp 00000000 03:45 7048299 /usr/local/lib/libamanda-2.5.2.so 00446000-00449000 rwxp 00031000 03:45 7048299 /usr/local/lib/libamanda-2.5.2.so

00449000-0046f000 rwxp 00449000 00:00 0
00666000-00679000 r-xp 00000000 03:03 58532      /lib/libnsl-2.5.so
00679000-0067a000 r-xp 00012000 03:03 58532      /lib/libnsl-2.5.so
0067a000-0067b000 rwxp 00013000 03:03 58532      /lib/libnsl-2.5.so
0067b000-0067d000 rwxp 0067b000 00:00 0
00684000-00693000 r-xp 00000000 03:03 58550      /lib/libresolv-2.5.so
00693000-00694000 r-xp 0000e000 03:03 58550      /lib/libresolv-2.5.so
00694000-00695000 rwxp 0000f000 03:03 58550      /lib/libresolv-2.5.so
00695000-00697000 rwxp 00695000 00:00 0
006b4000-006bd000 r-xp 00000000 03:03 58537      /lib/libnss_files-2.5.so
006bd000-006be000 r-xp 00008000 03:03 58537      /lib/libnss_files-2.5.so
006be000-006bf000 rwxp 00009000 03:03 58537      /lib/libnss_files-2.5.so
008da000-008de000 r-xp 00000000 03:03 58535      /lib/libnss_dns-2.5.so
008de000-008df000 r-xp 00003000 03:03 58535      /lib/libnss_dns-2.5.so
008df000-008e0000 rwxp 00004000 03:03 58535      /lib/libnss_dns-2.5.so
0091f000-00938000 r-xp 00000000 03:03 58499      /lib/ld-2.5.so
00938000-00939000 r-xp 00018000 03:03 58499      /lib/ld-2.5.so
00939000-0093a000 rwxp 00019000 03:03 58499      /lib/ld-2.5.so
0093c000-00a73000 r-xp 00000000 03:03 58515      /lib/libc-2.5.so
00a73000-00a75000 r-xp 00137000 03:03 58515      /lib/libc-2.5.so
00a75000-00a76000 rwxp 00139000 03:03 58515      /lib/libc-2.5.so
00a76000-00a79000 rwxp 00a76000 00:00 0
00a7b000-00aa0000 r-xp 00000000 03:03 58548      /lib/libm-2.5.so
00aa0000-00aa1000 r-xp 00024000 03:03 58548      /lib/libm-2.5.so
00aa1000-00aa2000 rwxp 00025000 03:03 58548      /lib/libm-2.5.so

029e3000-029ee000 r-xp 00000000 03:03 58568 /lib/libgcc_s-4.1.1-20070105.so.1 029ee000-029ef000 rwxp 0000a000 03:03 58568 /lib/libgcc_s-4.1.1-20070105.so.1

02fcb000-02fce000 r-xp 00000000 03:03 58579      /lib/libtermcap.so.2.0.8
02fce000-02fcf000 rwxp 00002000 03:03 58579      /lib/libtermcap.so.2.0.8

02fd1000-03000000 r-xp 00000000 03:06 1760450 /usr/lib/libreadline.so.5.1 03000000-03004000 rwxp 0002f000 03:06 1760450 /usr/lib/libreadline.so.5.1

03004000-03005000 rwxp 03004000 00:00 0
08048000-08050000 r-xp 00000000 03:45 917531     /usr/local/libexec/du

It chops off because, again, it wiped everything out.

I do not see any core files lying around, and I have /tmp/amanda preserved for both the server and the two clients being backed up at the time this happened. Please let me know if I should send them as well.

Any ideas/fixes/debugging tips would be greatly appreciated!

Cheers,
Bruce.


--This message has been scanned for viruses and
dangerous content by MailScanner, and is
believed to be clean.