[root@llm11 dtt_tools]# ./ttutils set -p INT_TASKLET_ENTRY -t loop -c 100 [root@llm11 dtt_tools]# ./ttutils ls id crash type crash point name count location 1 none INT_HARDWARE_ENTRY 0 kern 2 loop INT_TASKLET_ENTRY 11 kern 3 none FS_DEVRW 0 kern 4 none MEM_SWAPOUT 0 kern 5 none TASKLET 0 kern 6 none TIMERADD 0 kern 8 none INT_HW_IRQ_EN 0 kern 101 none SCSI_DISPATCH_CMD 0 kern [crash point set] name:INT_TASKLET_ENTRY type:loop count:50 BUG: soft lockup detected on CPU#3! softlockup_tick+0xba/0xe0 update_process_times+0x85/0xc0 smp_apic_timer_interrupt+0x5b/0x70 cpoint_default_exit+0x71/0x90 smp_apic_timer_interrupt+0x5b/0x70 cpoint_default_exit+0x71/0x90 [root@llm11 ~]# apic_timer_interrupt+0x1c/0x24 cpoint_default_exit+0x71/0 INT_TASKLET_ENTRY_hook+0x8e/0xbd __do_softirq+0x77/0x100 do_softirq+0x53/0x60 irq_exit+0x44/0x50 default_idle+0x0/0x60 apic_timer_interrupt+0x1c/0x24 default_idle+0x0/0x60 default_idle+0x2e/0x60 cpu_idle+0x52/0xa0 [root@llm11 ~]# TRIG_LOOP BUG: soft lockup detected on CPU#3! softlockup_tick+0xba/0xe0 update_process_times+0x85/0xc0 smp_apic_timer_interrupt+0x5b/0x70 cpoint_default_exit+0x71/0x90 apic_timer_interrupt+0x1c/0x24 cpoint_default_exit+0x71/0x90 INT_TASKLET_ENTRY_hook+0x8e/0xbd __do_softirq+0x77/0x100 do_softirq+0x53/0x60 irq_exit+0x44/0x50 default_idle+0x0/0x60 apic_timer_interrupt+0x1c/0x24 default_idle+0x0/0x60 default_idle+0x2e/0x60 cpu_idle+0x52/0xa0 [root@llm11 ~]# cd /proc/ [root@llm11 proc]# ls -l vmcore -r-------- 1 root root 3849938296 Jul 4 15:25 vmcore [root@llm11 proc]# crash /home/murali/linux-2.6.17.2/System.map /home/murali/linux-2.6.17.2/vmlinux vmcore crash 4.0-2.24 Copyright (C) 2002, 2003, 2004, 2005, 2006 Red Hat, Inc. Copyright (C) 2004, 2005, 2006 IBM Corporation Copyright (C) 1999-2006 Hewlett-Packard Co Copyright (C) 2005 Fujitsu Limited Copyright (C) 2005 NEC Corporation Copyright (C) 1999, 2002 Silicon Graphics, Inc. Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc. This program is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Enter "help copying" to see the conditions. This program has absolutely no warranty. Enter "help warranty" for details. GNU gdb 6.1 Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "i686-pc-linux-gnu"... please wait... (gathering module symbol data) WARNING: cannot access vmalloc'd module memory SYSTEM MAP: /home/murali/linux-2.6.17.2/System.map DEBUG KERNEL: /home/murali/linux-2.6.17.2/vmlinux (2.6.17.2) DUMPFILE: vmcore CPUS: 8 DATE: Tue Jul 4 15:24:00 2006 UPTIME: 00:05:41 LOAD AVERAGE: 7.47, 3.10, 1.18 TASKS: 165 NODENAME: llm11.in.ibm.com RELEASE: 2.6.17.2 VERSION: #9 SMP PREEMPT Mon Jul 3 17:59:41 IST 2006 MACHINE: i686 (2489 Mhz) MEMORY: 4.6 GB PANIC: "SysRq : Trigger a crashdump" PID: 0 COMMAND: "swapper" TASK: c8855550 (1 of 8) [THREAD_INFO: c8862000] CPU: 1 STATE: TASK_RUNNING (SYSRQ) crash> bt PID: 0 TASK: c8855550 CPU: 1 COMMAND: "swapper" #0 [c8863d7c] crash_kexec at c0142ced #1 [c8863dd0] __handle_sysrq at c02e84a5 #2 [c8863dfc] handle_sysrq at c02e8580 #3 [c8863e10] kbd_event at c02e2a25 #4 [c8863e34] input_event at c03b8c66 #5 [c8863e5c] atkbd_report_key at c03bc4f6 #6 [c8863e78] atkbd_interrupt at c03bc8d2 #7 [c8863eb4] serio_interrupt at c03b67c1 #8 [c8863edc] i8042_interrupt at c03b7382 #9 [c8863f14] INT_HW_IRQ_EN_hook at c0147fae #10 [c8863f38] INT_HARDWARE_ENTRY_hook at c014807f #11 [c8863f64] do_IRQ at c0105a2e #12 [c8863f70] common_interrupt at c0103ad9 EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c8862000 EBP: c05de2c0 DS: 007b ESI: c05de2a0 ES: 007b EDI: c8862000 CS: 0060 EIP: c0100d9e ERR: ffffff01 EFLAGS: 00000246 #13 [c8863fa4] default_idle at c0100d9e #14 [c8863fa8] cpu_idle at c0100e40 crash> bt -a PID: 0 TASK: c050a540 CPU: 0 COMMAND: "swapper" #0 [c059ff2c] crash_nmi_callback at c0113f0c #1 [c059ff84] do_nmi at c010533d #2 [c059ff94] nmi at c0103cd9 EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c059e000 EBP: c05de2c0 DS: 007b ESI: c05de2a0 ES: 007b EDI: c059e000 CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246 #3 [c059ffc8] default_idle at c0100d9e #4 [c059ffcc] cpu_idle at c0100e40 PID: 0 TASK: c8855550 CPU: 1 COMMAND: "swapper" #0 [c8863d7c] crash_kexec at c0142ced #1 [c8863dd0] __handle_sysrq at c02e84a5 #2 [c8863dfc] handle_sysrq at c02e8580 #3 [c8863e10] kbd_event at c02e2a25 #4 [c8863e34] input_event at c03b8c66 #5 [c8863e5c] atkbd_report_key at c03bc4f6 #6 [c8863e78] atkbd_interrupt at c03bc8d2 #7 [c8863eb4] serio_interrupt at c03b67c1 #8 [c8863edc] i8042_interrupt at c03b7382 #9 [c8863f14] INT_HW_IRQ_EN_hook at c0147fae #10 [c8863f38] INT_HARDWARE_ENTRY_hook at c014807f #11 [c8863f64] do_IRQ at c0105a2e #12 [c8863f70] common_interrupt at c0103ad9 EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c8862000 EBP: c05de2c0 DS: 007b ESI: c05de2a0 ES: 007b EDI: c8862000 CS: 0060 EIP: c0100d9e ERR: ffffff01 EFLAGS: 00000246 #13 [c8863fa4] default_idle at c0100d9e #14 [c8863fa8] cpu_idle at c0100e40 PID: 0 TASK: c8855030 CPU: 2 COMMAND: "swapper" #0 [f7f03f08] crash_nmi_callback at c0113f0c #1 [f7f03f60] do_nmi at c010533d #2 [f7f03f70] nmi at c0103cd9 EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: f7f02000 EBP: c05de2c0 DS: 007b ESI: c05de2a0 ES: 007b EDI: f7f02000 CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246 #3 [f7f03fa4] default_idle at c0100d9e #4 [f7f03fa8] cpu_idle at c0100e40 PID: 0 TASK: c885aa70 CPU: 3 COMMAND: "swapper" #0 [f7f25e80] crash_nmi_callback at c0113f0c #1 [f7f25ed8] do_nmi at c010533d #2 [f7f25ee8] nmi at c0103cd9 EAX: 0000000d EBX: c873f180 ECX: c050cb54 EDX: 00000001 EBP: c05de2c0 DS: 007b ESI: 00000000 ES: 007b EDI: c05dfda0 CS: 0060 EIP: c03c45d1 ERR: 0000000d EFLAGS: 00000282 #3 [f7f25f1c] cpoint_default_exit at c03c45d1 #4 [f7f25f24] INT_TASKLET_ENTRY_hook at c0126cab #5 [f7f25f3c] __do_softirq at c0126785 #6 [f7f25f5c] do_softirq at c012685e #7 [f7f25f68] irq_exit at c012694f #8 [f7f25f70] apic_timer_interrupt at c0103b67 #9 [f7f25fa8] cpu_idle at c0100e40 PID: 5062 TASK: f7d94a70 CPU: 4 COMMAND: "bash" #0 [f489be38] crash_nmi_callback at c0113f0c #1 [f489be90] do_nmi at c010533d #2 [f489bea0] nmi at c0103cd9 EAX: 0000000d EBX: f7703958 ECX: c050cb54 EDX: 00000001 EBP: c05de2c0 DS: 007b ESI: 00000000 ES: 007b EDI: c05dfda0 CS: 0060 EIP: c03c45d1 ERR: 0000000d EFLAGS: 00000296 #3 [f489bed4] cpoint_default_exit at c03c45d1 #4 [f489bedc] INT_TASKLET_ENTRY_hook at c0126cab #5 [f489bef4] __do_softirq at c0126785 #6 [f489bf14] do_softirq at c012685e #7 [f489bf20] irq_exit at c012694f #8 [f489bf28] do_IRQ at c0105a33 #9 [f489bf34] common_interrupt at c0103ad9 EAX: 00000002 EBX: f70ff3fc ECX: 00000001 EDX: 00000010 EBP: 00000010 DS: 007b ESI: 00000002 ES: 007b EDI: f70fc0a4 CS: 0060 EIP: c01a61f0 ERR: ffffff04 EFLAGS: 00000287 #10 [f489bf68] dnotify_parent at c01a61f0 #11 [f489bf90] sys_write at c016dd0c #12 [f489bfb8] sysenter_entry at c01030a4 EAX: 00000004 EBX: 00000002 ECX: b7f6a000 EDX: 00000010 DS: 007b ESI: 00000010 ES: 007b EDI: b7f6a000 SS: 007b ESP: bf9f6adc EBP: bf9f6b08 CS: 0073 EIP: ffffe410 ERR: 00000004 EFLAGS: 00000246 PID: 0 TASK: c885a030 CPU: 5 COMMAND: "swapper" #0 [f7f67f08] crash_nmi_callback at c0113f0c #1 [f7f67f60] do_nmi at c010533d #2 [f7f67f70] nmi at c0103cd9 EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: f7f66000 EBP: c05de2c0 DS: 007b ESI: c05de2a0 ES: 007b EDI: f7f66000 CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246 #3 [f7f67fa4] default_idle at c0100d9e #4 [f7f67fa8] cpu_idle at c0100e40 PID: 0 TASK: c885ca70 CPU: 6 COMMAND: "swapper" #0 [c8887f08] crash_nmi_callback at c0113f0c #1 [c8887f60] do_nmi at c010533d #2 [c8887f70] nmi at c0103cd9 EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c8886000 EBP: c05de2c0 DS: 007b ESI: c05de2a0 ES: 007b EDI: c8886000 CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246 #3 [c8887fa4] default_idle at c0100d9e #4 [c8887fa8] cpu_idle at c0100e40 PID: 0 TASK: c885c550 CPU: 7 COMMAND: "swapper" #0 [c88a9f08] crash_nmi_callback at c0113f0c #1 [c88a9f60] do_nmi at c010533d #2 [c88a9f70] nmi at c0103cd9 EAX: 00000000 EBX: c0100d70 ECX: 00000001 EDX: c88a8000 EBP: c05de2c0 DS: 007b ESI: c05de2a0 ES: 007b EDI: c88a8000 CS: 0060 EIP: c0100d9e ERR: 00000000 EFLAGS: 00000246 #3 [c88a9fa4] default_idle at c0100d9e #4 [c88a9fa8] cpu_idle at c0100e40 crash> crash> crash> runq RUNQUEUES[0]: c8726640 ACTIVE PRIO_ARRAY: c8726690 EXPIRED PRIO_ARRAY: c8726b08 RUNQUEUES[1]: c872e640 ACTIVE PRIO_ARRAY: c872eb08 [110] PID: 27 TASK: c8864a70 CPU: 1 COMMAND: "events/1" EXPIRED PRIO_ARRAY: c872e690 RUNQUEUES[2]: c8736640 ACTIVE PRIO_ARRAY: c8736690 EXPIRED PRIO_ARRAY: c8736b08 RUNQUEUES[3]: c873e640 ACTIVE PRIO_ARRAY: c873eb08 [ 0] PID: 13 TASK: f7e05550 CPU: 3 COMMAND: "watchdog/3" EXPIRED PRIO_ARRAY: c873e690 RUNQUEUES[4]: c8746640 ACTIVE PRIO_ARRAY: c8746690 [ 0] PID: 16 TASK: f7e12550 CPU: 4 COMMAND: "watchdog/4" PID: 14 TASK: f7e05030 CPU: 4 COMMAND: "migration/4" EXPIRED PRIO_ARRAY: c8746b08 [116] PID: 5062 TASK: f7d94a70 CPU: 4 COMMAND: "bash" RUNQUEUES[5]: c874e640 ACTIVE PRIO_ARRAY: c874e690 EXPIRED PRIO_ARRAY: c874eb08 RUNQUEUES[6]: c8756640 ACTIVE PRIO_ARRAY: c8756b08 EXPIRED PRIO_ARRAY: c8756690 RUNQUEUES[7]: c875e640 ACTIVE PRIO_ARRAY: c875eb08 EXPIRED PRIO_ARRAY: c875e690 crash> crash> mod mod: cannot access vmalloc'd module memory crash> files PID: 0 TASK: c8855550 CPU: 1 COMMAND: "swapper" ROOT: / CWD: / No open files crash>