[root@llm11 ~]# cd /proc/
[root@llm11 proc]# ls -l vmcore
-r-------- 1 root root 3849938296 Jul 4 13:15 vmcore
[root@llm11 proc]# crash /home/murali/linux-2.6.17.2/System.map /home/murali/linux-2.6.17.2/vmlinux vmcore
crash 4.0-2.24
Copyright (C) 2002, 2003, 2004, 2005, 2006 Red Hat, Inc.
Copyright (C) 2004, 2005, 2006 IBM Corporation
Copyright (C) 1999-2006 Hewlett-Packard Co
Copyright (C) 2005 Fujitsu Limited
Copyright (C) 2005 NEC Corporation
Copyright (C) 1999, 2002 Silicon Graphics, Inc.
Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc.
This program is free software, covered by the GNU General Public License,
and you are welcome to change it and/or distribute copies of it under
certain conditions. Enter "help copying" to see the conditions.
This program has absolutely no warranty. Enter "help warranty" for details.
GNU gdb 6.1
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "i686-pc-linux-gnu"...
please wait... (gathering module symbol data)
WARNING: cannot access vmalloc'd module memory
SYSTEM MAP: /home/murali/linux-2.6.17.2/System.map
DEBUG KERNEL: /home/murali/linux-2.6.17.2/vmlinux (2.6.17.2)
DUMPFILE: vmcore
CPUS: 8
DATE: Tue Jul 4 13:02:23 2006
UPTIME: 00:02:53
LOAD AVERAGE: 1.72, 0.52, 0.19
TASKS: 167
NODENAME: llm11.in.ibm.com
RELEASE: 2.6.17.2
VERSION: #9 SMP PREEMPT Mon Jul 3 17:59:41 IST 2006
MACHINE: i686 (2489 Mhz)
MEMORY: 4.6 GB
PANIC: "SysRq : Trigger a crashdump"
PID: 0
COMMAND: "swapper"
TASK: c885ca70 (1 of 8) [THREAD_INFO: c8886000]
CPU: 6
STATE: TASK_RUNNING (SYSRQ)
crash> bt
PID: 0 TASK: c885ca70 CPU: 6 COMMAND: "swapper"
#0 [c8887d7c] crash_kexec at c0142ced
#1 [c8887dd0] __handle_sysrq at c02e84a5
#2 [c8887dfc] handle_sysrq at c02e8580
#3 [c8887e10] kbd_event at c02e2a25
#4 [c8887e34] input_event at c03b8c66
#5 [c8887e5c] atkbd_report_key at c03bc4f6
#6 [c8887e78] atkbd_interrupt at c03bc8d2
#7 [c8887eb4] serio_interrupt at c03b67c1
#8 [c8887edc] i8042_interrupt at c03b7382
#9 [c8887f14] INT_HW_IRQ_EN_hook at c0147fae
#10 [c8887f38] INT_HARDWARE_ENTRY_hook at c014807f
#11 [c8887f64] do_IRQ at c0105a2e
#12 [c8887f70] common_interrupt at c0103ad9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c8886000 EBP: c05de2c0
DS: 007b ESI: c05de2a0 ES: 007b EDI: c8886000
CS: 0060 EIP: c0100d9e ERR: ffffff01 EFLAGS: 00000246
#13 [c8887fa4] default_idle at c0100d9e
#14 [c8887fa8] cpu_idle at c0100e40
crash> bt -a
PID: 5115 TASK: f7699030 CPU: 0 COMMAND: "sshd"
#0 [f7fa9d9c] crash_nmi_callback at c0113f0c
#1 [f7fa9df4] do_nmi at c010533d
#2 [f7fa9e04] nmi at c0103cd9
EAX: 00000001 EBX: c050c370 ECX: 00000000 EDX: 00000001 EBP: 00000000
DS: 007b ESI: f7fa8000 ES: 007b EDI: 00000000
CS: 0060 EIP: c04868bc ERR: 00000001 EFLAGS: 00000202
#3 [f7fa9e38] _spin_lock at c04868bc
#4 [f7fa9e48] flush_tlb_others at c010f97c
#5 [f7fa9e60] flush_tlb_mm at c010fb00
#6 [f7fa9e78] dup_mm at c011e31f
#7 [f7fa9eb8] copy_mm at c011e456
#8 [f7fa9ec4] copy_process at c011eef5
#9 [f7fa9f28] do_fork at c011fa14
#10 [f7fa9f9c] sys_clone at c0101ba7
#11 [f7fa9fb8] sysenter_entry at c01030a4
EAX: 00000078 EBX: 01200011 ECX: 00000000 EDX: 00000000
DS: 007b ESI: 00000000 ES: 007b EDI: b7c89708
SS: 007b ESP: bfce5988 EBP: bfce5a08
CS: 0073 EIP: ffffe410 ERR: 00000078 EFLAGS: 00000282
PID: 27 TASK: c8864a70 CPU: 1 COMMAND: "events/1"
#0 [f7e93e14] crash_nmi_callback at c0113f0c
#1 [f7e93e6c] do_nmi at c010533d
#2 [f7e93e7c] nmi at c0103cd9
EAX: 00000006 EBX: 00000007 ECX: 000000fb EDX: 000c08fb EBP: 00000000
DS: 007b ESI: 00000001 ES: 007b EDI: c8833640
CS: 0060 EIP: c010fdd2 ERR: 00000006 EFLAGS: 00000297
#3 [f7e93eb0] smp_call_function at c010fdd2
#4 [f7e93ef0] on_each_cpu at c0127078
#5 [f7e93f10] mce_work_fn at c010bd52
#6 [f7e93f24] run_workqueue at c0132615
#7 [f7e93f54] worker_thread at c01327e3
#8 [f7e93fbc] kthread at c01364cb
#9 [f7e93fe8] kernel_thread_helper at c01011f3
PID: 0 TASK: c8855030 CPU: 2 COMMAND: "swapper"
#0 [f7f03f08] crash_nmi_callback at c0113f0c
#1 [f7f03f60] do_nmi at c010533d
#2 [f7f03f70] nmi at c0103cd9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: f7f02000 EBP: c05de2c0
DS: 007b ESI: c05de2a0 ES: 007b EDI: f7f02000
CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246
#3 [f7f03fa4] default_idle at c0100d9e
#4 [f7f03fa8] cpu_idle at c0100e40
PID: 0 TASK: c885aa70 CPU: 3 COMMAND: "swapper"
#0 [f7f25f08] crash_nmi_callback at c0113f0c
#1 [f7f25f60] do_nmi at c010533d
#2 [f7f25f70] nmi at c0103cd9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: f7f24000 EBP: c05de2c0
DS: 007b ESI: c05de2a0 ES: 007b EDI: f7f24000
CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246
#3 [f7f25fa4] default_idle at c0100d9e
#4 [f7f25fa8] cpu_idle at c0100e40
PID: 3941 TASK: f7699a70 CPU: 4 COMMAND: "db2fmcd"
#0 [f59e1de4] crash_nmi_callback at c0113f0c
#1 [f59e1e3c] do_nmi at c010533d
#2 [f59e1e4c] nmi at c0103cd9
EAX: 000008fc EBX: f7f05b80 ECX: 00000001 EDX: 00000004 EBP: c86b079c
DS: 007b ESI: bfc33a0c ES: 007b EDI: bfc33a0c
CS: 0060 EIP: c010f9b5 ERR: 000008fc EFLAGS: 00000202
#3 [f59e1e80] flush_tlb_others at c010f9b5
#4 [f59e1e98] flush_tlb_page at c010fbd0
#5 [f59e1eb4] do_wp_page at c015991f
#6 [f59e1f0c] __handle_mm_fault at c015ae8c
#7 [f59e1f4c] do_page_fault at c011694b
#8 [f59e1fb8] error_code (via page_fault) at c0103c35
EAX: 00000047 EBX: 43987ff4 ECX: b7cdf001 EDX: 00000047
DS: 007b ESI: 00000000 ES: 007b EDI: bfc33a0c
SS: 007b ESP: bfc2ed88 EBP: bfc2edac
CS: 0073 EIP: 438b5ddd ERR: ffffffff EFLAGS: 00010212
PID: 0 TASK: c885a030 CPU: 5 COMMAND: "swapper"
#0 [f7f67e80] crash_nmi_callback at c0113f0c
#1 [f7f67ed8] do_nmi at c010533d
#2 [f7f67ee8] nmi at c0103cd9
EAX: 0000000d EBX: c0598f68 ECX: c050cb54 EDX: 00000001 EBP: c05de2c0
DS: 007b ESI: 00000001 ES: 007b EDI: c05dfe00
CS: 0060 EIP: c03c45d1 ERR: 0000000d EFLAGS: 00000082
#3 [f7f67f1c] cpoint_default_exit at c03c45d1
#4 [f7f67f24] INT_TASKLET_ENTRY_hook at c0126cca
#5 [f7f67f3c] __do_softirq at c0126785
#6 [f7f67f5c] do_softirq at c012685e
#7 [f7f67f68] irq_exit at c012694f
#8 [f7f67f70] apic_timer_interrupt at c0103b67
#9 [f7f67fa8] cpu_idle at c0100e40
PID: 0 TASK: c885ca70 CPU: 6 COMMAND: "swapper"
#0 [c8887d7c] crash_kexec at c0142ced
#1 [c8887dd0] __handle_sysrq at c02e84a5
#2 [c8887dfc] handle_sysrq at c02e8580
#3 [c8887e10] kbd_event at c02e2a25
#4 [c8887e34] input_event at c03b8c66
#5 [c8887e5c] atkbd_report_key at c03bc4f6
#6 [c8887e78] atkbd_interrupt at c03bc8d2
#7 [c8887eb4] serio_interrupt at c03b67c1
#8 [c8887edc] i8042_interrupt at c03b7382
#9 [c8887f14] INT_HW_IRQ_EN_hook at c0147fae
#10 [c8887f38] INT_HARDWARE_ENTRY_hook at c014807f
#11 [c8887f64] do_IRQ at c0105a2e
#12 [c8887f70] common_interrupt at c0103ad9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c8886000 EBP: c05de2c0
DS: 007b ESI: c05de2a0 ES: 007b EDI: c8886000
CS: 0060 EIP: c0100d9e ERR: ffffff01 EFLAGS: 00000246
#13 [c8887fa4] default_idle at c0100d9e
#14 [c8887fa8] cpu_idle at c0100e40
PID: 0 TASK: c885c550 CPU: 7 COMMAND: "swapper"
#0 [c88a9f08] crash_nmi_callback at c0113f0c
#1 [c88a9f60] do_nmi at c010533d
#2 [c88a9f70] nmi at c0103cd9
EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c88a8000 EBP: c05de2c0
DS: 007b ESI: c05de2a0 ES: 007b EDI: c88a8000
CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246
#3 [c88a9fa4] default_idle at c0100d9e
#4 [c88a9fa8] cpu_idle at c0100e40
crash>
crash>
crash>
crash>
crash>
crash>
crash>
crash>
crash>
crash> runq
RUNQUEUES[0]: c8726640
ACTIVE PRIO_ARRAY: c8726b08
[110] PID: 26 TASK: f7e4b030 CPU: 0 COMMAND: "events/0"
[115] PID: 4695 TASK: f55ad550 CPU: 0 COMMAND: "sshd"
[117] PID: 5115 TASK: f7699030 CPU: 0 COMMAND: "sshd"
EXPIRED PRIO_ARRAY: c8726690
RUNQUEUES[1]: c872e640
ACTIVE PRIO_ARRAY: c872e690
[ 0] PID: 7 TASK: c88dc550 CPU: 1 COMMAND: "watchdog/1"
EXPIRED PRIO_ARRAY: c872eb08
[110] PID: 27 TASK: c8864a70 CPU: 1 COMMAND: "events/1"
RUNQUEUES[2]: c8736640
ACTIVE PRIO_ARRAY: c8736b08
EXPIRED PRIO_ARRAY: c8736690
RUNQUEUES[3]: c873e640
ACTIVE PRIO_ARRAY: c873e690
EXPIRED PRIO_ARRAY: c873eb08
RUNQUEUES[4]: c8746640
ACTIVE PRIO_ARRAY: c8746690
[ 0] PID: 16 TASK: f7e12550 CPU: 4 COMMAND: "watchdog/4"
PID: 14 TASK: f7e05030 CPU: 4 COMMAND: "migration/4"
[110] PID: 30 TASK: f7fef550 CPU: 4 COMMAND: "events/4"
EXPIRED PRIO_ARRAY: c8746b08
[117] PID: 3941 TASK: f7699a70 CPU: 4 COMMAND: "db2fmcd"
RUNQUEUES[5]: c874e640
ACTIVE PRIO_ARRAY: c874eb08
[110] PID: 3188 TASK: f7faba70 CPU: 5 COMMAND: "kjournald"
[115] PID: 3552 TASK: f7da3030 CPU: 5 COMMAND: "klogd"
[116] PID: 5114 TASK: c8b04a70 CPU: 5 COMMAND: "bash"
PID: 4662 TASK: c8b23a70 CPU: 5 COMMAND: "bash"
EXPIRED PRIO_ARRAY: c874e690
RUNQUEUES[6]: c8756640
ACTIVE PRIO_ARRAY: c8756b08
EXPIRED PRIO_ARRAY: c8756690
RUNQUEUES[7]: c875e640
ACTIVE PRIO_ARRAY: c875eb08
EXPIRED PRIO_ARRAY: c875e690
crash>
crash>
crash>
crash>
crash>
crash> files
PID: 0 TASK: c885ca70 CPU: 6 COMMAND: "swapper"
ROOT: / CWD: /
No open files
crash> mod
mod: cannot access vmalloc'd module memory
crash>