[root@llm11 ~]# cd /proc/
[root@llm11 proc]# ls -l vmcore
-r-------- 1 root root 3849938296 Jul 3 16:48 vmcore
[root@llm11 proc]# crash /home/murali/linux-2.6.17.2/System.map /home/murali/linux-2.6.17.2/vmlinux vmcore
crash 4.0-2.24
Copyright (C) 2002, 2003, 2004, 2005, 2006 Red Hat, Inc.
Copyright (C) 2004, 2005, 2006 IBM Corporation
Copyright (C) 1999-2006 Hewlett-Packard Co
Copyright (C) 2005 Fujitsu Limited
Copyright (C) 2005 NEC Corporation
Copyright (C) 1999, 2002 Silicon Graphics, Inc.
Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc.
This program is free software, covered by the GNU General Public License,
and you are welcome to change it and/or distribute copies of it under
certain conditions. Enter "help copying" to see the conditions.
This program has absolutely no warranty. Enter "help warranty" for details.
GNU gdb 6.1
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "i686-pc-linux-gnu"...
please wait... (gathering module symbol data)WARNING: cannot access module kallsyms
SYSTEM MAP: /home/murali/linux-2.6.17.2/System.map
DEBUG KERNEL: /home/murali/linux-2.6.17.2/vmlinux (2.6.17.2)
DUMPFILE: vmcore
CPUS: 8
DATE: Mon Jul 3 16:46:40 2006
UPTIME: 00:04:01
LOAD AVERAGE: 0.08, 0.17, 0.08
TASKS: 163
NODENAME: llm11.in.ibm.com
RELEASE: 2.6.17.2
VERSION: #7 SMP PREEMPT Mon Jul 3 15:34:28 IST 2006
MACHINE: i686 (2488 Mhz)
MEMORY: 4.6 GB
PANIC: "Oops: 0002 [#1]" (check log for details)
PID: 0
COMMAND: "swapper"
TASK: c7945570 (1 of 8) [THREAD_INFO: f7f44000]
CPU: 4
STATE: TASK_RUNNING (PANIC)
crash> bt
PID: 0 TASK: c7945570 CPU: 4 COMMAND: "swapper"
#0 [f7f45df8] crash_kexec at c01408fd
#1 [f7f45e4c] die at c010455a
#2 [f7f45e90] do_page_fault at c0116ab7
#3 [f7f45efc] error_code (via page_fault) at c0103c35
EAX: 00000000 EBX: e0000000 ECX: c7923bdc EDX: 00000001 EBP: 00000000
DS: 007b ESI: c05890bc ES: 007b EDI: c05890a0
CS: 0060 EIP: c03bb4fc ERR: ffffffff EFLAGS: 00010046
#4 [f7f45f30] cpoint_default_exit at c03bb4fc
#5 [f7f45f38] INT_HARDWARE_ENTRY_hook at c0145d1a
#6 [f7f45f64] do_IRQ at c0105a2e
#7 [f7f45f70] common_interrupt at c0103ad9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: f7f44000 EBP: c05ce2c0
DS: 007b ESI: c05ce2a0 ES: 007b EDI: f7f44000
CS: 0060 EIP: c0100d9e ERR: ffffff00 EFLAGS: 00000246
#8 [f7f45fa4] default_idle at c0100d9e
#9 [f7f45fa8] cpu_idle at c0100e40
crash> dis c0103c35
0xc0103c35 <error_code+77>: call *%edi
crash> dis c03bb4fc
0xc03bb4fc <cpoint_default_exit+92>: mov %eax,0x0
crash> dis cpoint_default_exit
0xc03bb4a0 <cpoint_default_exit>: sub $0x4,%esp
0xc03bb4a3 <cpoint_default_exit+3>: mov 0x8(%esp),%eax
0xc03bb4a7 <cpoint_default_exit+7>: call 0xc03bb1e0 <cpoint_countdown>
0xc03bb4ac <cpoint_default_exit+12>: cmp $0x8,%eax
0xc03bb4af <cpoint_default_exit+15>: ja 0xc03bb4c4 <cpoint_default_exit+36>
0xc03bb4b1 <cpoint_default_exit+17>: jmp *0xc048b7d4(,%eax,4)
0xc03bb4b8 <cpoint_default_exit+24>: movl $0xc04d95e0,(%esp)
0xc03bb4bf <cpoint_default_exit+31>: call 0xc0120d20 <printk>
0xc03bb4c4 <cpoint_default_exit+36>: pop %ecx
0xc03bb4c5 <cpoint_default_exit+37>: xor %eax,%eax
0xc03bb4c7 <cpoint_default_exit+39>: ret
0xc03bb4c8 <cpoint_default_exit+40>: movl $0xc04d9600,(%esp)
0xc03bb4cf <cpoint_default_exit+47>: call 0xc0120d20 <printk>
0xc03bb4d4 <cpoint_default_exit+52>: jmp 0xc03bb4c4 <cpoint_default_exit+36>
0xc03bb4d6 <cpoint_default_exit+54>: movl $0xc04c6a15,(%esp)
0xc03bb4dd <cpoint_default_exit+61>: call 0xc0120260 <panic>
0xc03bb4e2 <cpoint_default_exit+66>: movl $0xc04c6a1e,(%esp)
0xc03bb4e9 <cpoint_default_exit+73>: call 0xc0120d20 <printk>
0xc03bb4ee <cpoint_default_exit+78>: ud2a
0xc03bb4f0 <cpoint_default_exit+80>: xor $0x1,%al
0xc03bb4f2 <cpoint_default_exit+82>: sub %ch,0x4c(%edx)
0xc03bb4f5 <cpoint_default_exit+85>: rcrb $0xc0,0x31(%ecx)
0xc03bb4f9 <cpoint_default_exit+89>: ret
0xc03bb4fa <cpoint_default_exit+90>: xor %eax,%eax
0xc03bb4fc <cpoint_default_exit+92>: mov %eax,0x0
0xc03bb501 <cpoint_default_exit+97>: xor %eax,%eax
0xc03bb503 <cpoint_default_exit+99>: pop %ecx
0xc03bb504 <cpoint_default_exit+100>: ret
0xc03bb505 <cpoint_default_exit+101>: movl $0xc04c6a3c,(%esp)
0xc03bb50c <cpoint_default_exit+108>: call 0xc0120d20 <printk>
0xc03bb511 <cpoint_default_exit+113>: jmp 0xc03bb511 <cpoint_default_exit+113>
0xc03bb513 <cpoint_default_exit+115>: movl $0x0,(%esp)
0xc03bb51a <cpoint_default_exit+122>: call 0xc03bb450 <recursive_loop>
0xc03bb51f <cpoint_default_exit+127>: pop %ecx
0xc03bb520 <cpoint_default_exit+128>: xor %eax,%eax
0xc03bb522 <cpoint_default_exit+130>: ret
0xc03bb523 <cpoint_default_exit+131>: lea 0x0(%esi),%esi
0xc03bb529 <cpoint_default_exit+137>: lea 0x0(%edi),%edi
crash>
crash> bt -a
PID: 0 TASK: c04fd4e0 CPU: 0 COMMAND: "swapper"
#0 [c058ff2c] crash_nmi_callback at c0113e9c
#1 [c058ff84] do_nmi at c010533d
#2 [c058ff94] nmi at c0103cd9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c058e000 EBP: c05ce2c0
DS: 007b ESI: c05ce2a0 ES: 007b EDI: c058e000
CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246
#3 [c058ffc8] default_idle at c0100d9e
#4 [c058ffcc] cpu_idle at c0100e40
PID: 0 TASK: c7940530 CPU: 1 COMMAND: "swapper"
#0 [c7961f08] crash_nmi_callback at c0113e9c
#1 [c7961f60] do_nmi at c010533d
#2 [c7961f70] nmi at c0103cd9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c7960000 EBP: c05ce2c0
DS: 007b ESI: c05ce2a0 ES: 007b EDI: c7960000
CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246
#3 [c7961fa4] default_idle at c0100d9e
#4 [c7961fa8] cpu_idle at c0100e40
PID: 0 TASK: c7940030 CPU: 2 COMMAND: "swapper"
#0 [f7f03f08] crash_nmi_callback at c0113e9c
#1 [f7f03f60] do_nmi at c010533d
#2 [f7f03f70] nmi at c0103cd9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: f7f02000 EBP: c05ce2c0
DS: 007b ESI: c05ce2a0 ES: 007b EDI: f7f02000
CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246
#3 [f7f03fa4] default_idle at c0100d9e
#4 [f7f03fa8] cpu_idle at c0100e40
PID: 0 TASK: c7945a70 CPU: 3 COMMAND: "swapper"
#0 [f7f23f08] crash_nmi_callback at c0113e9c
#1 [f7f23f60] do_nmi at c010533d
#2 [f7f23f70] nmi at c0103cd9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: f7f22000 EBP: c05ce2c0
DS: 007b ESI: c05ce2a0 ES: 007b EDI: f7f22000
CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246
#3 [f7f23fa4] default_idle at c0100d9e
#4 [f7f23fa8] cpu_idle at c0100e40
PID: 0 TASK: c7945570 CPU: 4 COMMAND: "swapper"
#0 [f7f45df8] crash_kexec at c01408fd
#1 [f7f45e4c] die at c010455a
#2 [f7f45e90] do_page_fault at c0116ab7
#3 [f7f45efc] error_code (via page_fault) at c0103c35
EAX: 00000000 EBX: e0000000 ECX: c7923bdc EDX: 00000001 EBP: 00000000
DS: 007b ESI: c05890bc ES: 007b EDI: c05890a0
CS: 0060 EIP: c03bb4fc ERR: ffffffff EFLAGS: 00010046
#4 [f7f45f30] cpoint_default_exit at c03bb4fc
#5 [f7f45f38] INT_HARDWARE_ENTRY_hook at c0145d1a
#6 [f7f45f64] do_IRQ at c0105a2e
#7 [f7f45f70] common_interrupt at c0103ad9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: f7f44000 EBP: c05ce2c0
DS: 007b ESI: c05ce2a0 ES: 007b EDI: f7f44000
CS: 0060 EIP: c0100d9e ERR: ffffff00 EFLAGS: 00000246
#8 [f7f45fa4] default_idle at c0100d9e
#9 [f7f45fa8] cpu_idle at c0100e40
PID: 0 TASK: c7945070 CPU: 5 COMMAND: "swapper"
#0 [f7f65f08] crash_nmi_callback at c0113e9c
#1 [f7f65f60] do_nmi at c010533d
#2 [f7f65f70] nmi at c0103cd9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: f7f64000 EBP: c05ce2c0
DS: 007b ESI: c05ce2a0 ES: 007b EDI: f7f64000
CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246
#3 [f7f65fa4] default_idle at c0100d9e
#4 [f7f65fa8] cpu_idle at c0100e40
PID: 0 TASK: c7947ab0 CPU: 6 COMMAND: "swapper"
#0 [c7987f08] crash_nmi_callback at c0113e9c
#1 [c7987f60] do_nmi at c010533d
#2 [c7987f70] nmi at c0103cd9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c7986000 EBP: c05ce2c0
DS: 007b ESI: c05ce2a0 ES: 007b EDI: c7986000
CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246
#3 [c7987fa4] default_idle at c0100d9e
#4 [c7987fa8] cpu_idle at c0100e40
PID: 0 TASK: c79475b0 CPU: 7 COMMAND: "swapper"
#0 [c79a7f08] crash_nmi_callback at c0113e9c
#1 [c79a7f60] do_nmi at c010533d
#2 [c79a7f70] nmi at c0103cd9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c79a6000 EBP: c05ce2c0
DS: 007b ESI: c05ce2a0 ES: 007b EDI: c79a6000
CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246
#3 [c79a7fa4] default_idle at c0100d9e
#4 [c79a7fa8] cpu_idle at c0100e40
crash>
crash>
crash>
crash>
crash>
crash> files
PID: 0 TASK: c7945570 CPU: 4 COMMAND: "swapper"
ROOT: / CWD: /
No open files
crash> mod
MODULE NAME SIZE OBJECT FILE
f8d76220 aic7xxx 181652 (not loaded)
crash> runq
RUNQUEUES[0]: c7817580
ACTIVE PRIO_ARRAY: c7817a3c
EXPIRED PRIO_ARRAY: c78175c4
RUNQUEUES[1]: c781f580
ACTIVE PRIO_ARRAY: c781f5c4
EXPIRED PRIO_ARRAY: c781fa3c
RUNQUEUES[2]: c7827580
ACTIVE PRIO_ARRAY: c7827a3c
EXPIRED PRIO_ARRAY: c78275c4
RUNQUEUES[3]: c782f580
ACTIVE PRIO_ARRAY: c782fa3c
EXPIRED PRIO_ARRAY: c782f5c4
RUNQUEUES[4]: c7837580
ACTIVE PRIO_ARRAY: c78375c4
[115] PID: 3513 TASK: c7a3c030 CPU: 4 COMMAND: "klogd"
EXPIRED PRIO_ARRAY: c7837a3c
RUNQUEUES[5]: c783f580
ACTIVE PRIO_ARRAY: c783f5c4
EXPIRED PRIO_ARRAY: c783fa3c
RUNQUEUES[6]: c7847580
ACTIVE PRIO_ARRAY: c78475c4
EXPIRED PRIO_ARRAY: c7847a3c
RUNQUEUES[7]: c784f580
ACTIVE PRIO_ARRAY: c784fa3c
EXPIRED PRIO_ARRAY: c784f5c4
crash>