A 'simple lock timeout panic' issue in TruCluster KCH subsystem 

 Typical stack trace for simple lock timeout is as follows:
     1 panic
     2 simple_lock_fault
     3 simple_lock_time_violation
     4 kch_lock_chain
     5 kch_insert_set
     6 kch_join_attach_internal
     7 kch_join_set
     8 drd_kgs_create_server_join_start
     9 drd_kgs_start_proposal
    10 drd_handle_one_event
    11 drd_handle_events
    12 drd_event_thread 

crash> t
>  0 stop_secondary_cpu(do_lwc = 0x0) ["../../../../src/kernel/arch/alpha/cpu.c":1416, 0xffffffff0049ee70]
   1 panic(s = 0xffffffff00986cb8 = "event_timeout: panic request") ["../../../../src/kernel/bsd/subr_prf.c":1509, 0xffffffff00083930]
   2 event_timeout(func = 0xffffffff00083df0, arg = 0xfffffc00152162f8, timeout = 0x0) ["../../../../src/kernel/arch/alpha/cpu.c":2430, 0xffffffff004a06cc]
   3 printf(fmt = 0x5) ["../../../../src/kernel/bsd/subr_prf.c":1018, 0xffffffff00082898]
   4 panic(s = 0xffffffff008ee0a0 = "simple_lock: time limit exceeded") ["../../../../src/kernel/bsd/subr_prf.c":1566, 0xffffffff00083a88]
   5 simple_lock_fault(slp = 0xfffffe0404f805b0, state = 0xc00000e000709eed, caller = 0xffffffff00709eec, arg = (nil), fmt = 0xffffffff00a955e8 = "\n%s\n\n", error = 0xffffffff008ee0a0 = "simple_lock: time limit exceeded") ["../../../../src/kernel/kern/lock.c":2874, 0xffffffff000a9830]
   6 simple_lock_time_violation(slp = (unallocated - symbol optimized away), state = (unallocated - symbol optimized away), caller = (unallocated - symbol optimized away)) ["../../../../src/kernel/kern/lock.c":3060, 0xffffffff000a9dbc]
   7 kch_lock_chain(hval = 0x0) ["../../../../src/kernel/kch/kch_obj.c":834, 0xffffffff00709eec]
   8 kch_insert_set(set = 0xfffffc029e764f00) ["../../../../src/kernel/kch/kch_obj.c":872, 0xffffffff00709fc4]
   9 kch_rcv_start_join_req(node = 0x1, msg = 0xfffffc01c59e6908, msglen = 0x1, resp = 0xfffffc02e2376240) ["../../../../src/kernel/kch/kch_server.c":564, 0xffffffff0070b280]
  10 kch_msg(Bogus_Node = (unallocated - symbol optimized away), kch_msg_buf = 0xfffffc01c59e6908, kch_msg_buf__len = 0x230) ["../../../../src/kernel/kch/kch_msg.c":483, 0xffffffff00711e00]
  11 svr_kch_msg(handle = 0xfffffc0190703d80) ["./tnc_gen/icssvr_kch_gen.c":25, 0xffffffff007117b8]
  12 icssvr_daemon_from_pool(void_prio = (nil)) ["../../../../src/kernel/tnc_common/tnc_ics/ics_svr_mgmt.c":803, 0xffffffff0076ef70]

