Hi Team,
Today we observed one server node of JAVA only Instance is using High CPU continuously for more than 6 hrs and still continuing. I have gone through SAP Notes 743191, 710154 and analysed the Thread Dump and found Thread ID which is causing high CPU usage. But from the Thread Dump Viewer I am not able to conclude the exact issue and further action. Can some of you help in providing the info on this Thread ID and also provide any link which has information about how to troubleshoot from Thread Dump Viewer. Below is the complete stack of the Thread which is causing high CPU usage.
SAPEngine_System_Thread[impl:5]_79" tid=t@4706447360 runnable cpu=11779004.12 ms allocated memory=69036588800 B file i/o:11277950/1972 B net i/o:306254/7092983 B files opened:9 socks opened:0 nid=0x3920157
at com.sap.tc.logging.reader.LocalBuffer.readPrevious()Lcom.sap.tc.logging.interfaces.IMessageBuilder;(LocalBuffer.java:281)
at com.sap.tc.logging.reader.FileReadLog.readPrevious(I)Lcom.sap.tc.logging.interfaces.ILoggingResultSet;(FileReadLog.java:404)
at com.sap.tc.logging.reader.ReadableLog.readLatestMsgRecords(I)Lcom.sap.tc.logging.interfaces.ILoggingResultSet;(ReadableLog.java:531)
at com.sapmarkets.bam.logcontroller.AbstractBAMLogReader$4.doRead()Lcom.sap.tc.logging.interfaces.ILoggingResultSet;(AbstractBAMLogReader.java:408)
at com.sapmarkets.bam.logcontroller.AbstractBAMLogReader$LogRecordReader.getRecords()Lcom.sap.tc.logging.interfaces.ILoggingResultSet;(AbstractBAMLogReader.java:262)
at com.sapmarkets.bam.logcontroller.AbstractBAMLogReader$LogRecordReader.readAndConvertRecords()Ljava.util.List;(AbstractBAMLogReader.java:204)
at com.sapmarkets.bam.logcontroller.AbstractBAMLogReader.readLatest()Ljava.util.List;(AbstractBAMLogReader.java:412)
at com.sapmarkets.bam.logcontroller.BAMLogReaderRecordSource$1.doRead()Ljava.util.List;(BAMLogReaderRecordSource.java:431)
at com.sapmarkets.bam.logcontroller.BAMLogReaderRecordSource.read(ILcom.sapmarkets.bam.logcontroller.BAMLogReaderRecordSource$ReadHelper;)Lcom.sapmarkets.bam.logcontroller.LogQueryResultset;(BAMLogReaderRecordSource.java:498)
at com.sapmarkets.bam.logcontroller.BAMLogReaderRecordSource.getLatest(I)Lcom.sapmarkets.bam.logcontroller.LogQueryResultset;(BAMLogReaderRecordSource.java:443)
at com.sapmarkets.bam.application.RecordBuffer.fillBuffer()V(RecordBuffer.java:264)
at com.sapmarkets.bam.application.RecordBuffer.init(II)V(RecordBuffer.java:106)
at com.sapmarkets.bam.application.MergeEngine.initRecordBuffers(I)V(MergeEngine.java:167)
at com.sapmarkets.bam.application.MergeEngine.getLatest(I)Lcom.sapmarkets.bam.logcontroller.LogQueryResultset;(MergeEngine.java:186)
at com.sapmarkets.bam.logcontroller.LogQueryBrowser.getLatest(I)Lcom.sapmarkets.bam.logcontroller.LogQueryResultset;(LogQueryBrowser.java:107)
at com.sapmarkets.bam.logcontroller.jmx.LogControllerFacade.getLatest(Lcom.sapmarkets.bam.logcontroller.LogQuerySession;I)Lcom.sapmarkets.bam.logcontroller.LogQueryResultset;(LogControllerFacade.java:406)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Ljava.lang.reflect.Method;Ljava.lang.Object;[Ljava.lang.Object;)Ljava.lang.Object;(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Ljava.lang.Object;[Ljava.lang.Object;)Ljava.lang.Object;(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Ljava.lang.Object;[Ljava.lang.Object;)Ljava.lang.Object;(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Ljava.lang.Object;[Ljava.lang.Object;)Ljava.lang.Object;(Method.java:331)
at com.sap.pj.jmx.introspect.DefaultMBeanInvoker.invoke(Ljava.lang.Object;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(DefaultMBeanInvoker.java:58)
at javax.management.StandardMBean.invoke(Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(StandardMBean.java:286)
at com.sap.pj.jmx.server.MBeanServerImpl.invoke(Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(MBeanServerImpl.java:944)
at com.sap.pj.jmx.server.interceptor.MBeanServerWrapperInterceptor.invoke(Lcom.sap.pj.jmx.server.interceptor.InvocationContext;Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(MBeanServerWrapperInterceptor.java:288)
at com.sap.engine.services.jmx.CompletionInterceptor.invoke(Lcom.sap.pj.jmx.server.interceptor.InvocationContext;Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(CompletionInterceptor.java:471)
at com.sap.pj.jmx.server.interceptor.BasicMBeanServerInterceptor.invoke(Lcom.sap.pj.jmx.server.interceptor.InvocationContext;Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(BasicMBeanServerInterceptor.java:277)
at com.sap.jmx.provider.ProviderInterceptor.invoke(Lcom.sap.pj.jmx.server.interceptor.InvocationContext;Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(ProviderInterceptor.java:258)
at com.sap.engine.services.jmx.RedirectInterceptor.invoke(Lcom.sap.pj.jmx.server.interceptor.InvocationContext;Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(RedirectInterceptor.java:340)
at com.sap.pj.jmx.server.interceptor.MBeanServerInterceptorChain.invoke(Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(MBeanServerInterceptorChain.java:330)
at com.sap.engine.services.jmx.MBeanServerInvoker.invokeMbs(Lcom.sap.engine.services.jmx.RequestMessage;)Lcom.sap.engine.services.jmx.ResponseMessage;(MBeanServerInvoker.java:131)
at com.sap.engine.services.jmx.JmxServiceConnectorServer.receiveWait(II[BII)Lcom.sap.engine.frame.cluster.message.MessageAnswer;(JmxServiceConnectorServer.java:172)
at com.sap.engine.core.service630.context.cluster.message.MessageListenerWrapper.process(ILcom.sap.engine.core.cluster.MessageObject;)V(MessageListenerWrapper.java:81)
at com.sap.engine.core.cluster.impl6.lazy.UnorderedChannel$MessageRunner.run()V(UnorderedChannel.java:54)
at com.sap.engine.frame.core.thread.Task.run()Ljava.lang.Object;(Task.java:64)
at com.sap.engine.core.thread.impl5.SingleThread.execute()V(SingleThread.java:83)
at com.sap.engine.core.thread.impl5.SingleThread.run()V(SingleThread.java:156)
"SAPEngine_System_Thread[impl:5]_115" tid=t@4707524608 runnable cpu=675962.95 ms allocated memory=61531848040 B file i/o:11385651/32 B net i/o:348350/9850991 B files opened:8 socks opened:0 nid=0x5640039
at com.sap.tc.logging.reader.LocalBuffer.previousLine()[B(LocalBuffer.java:530)
at com.sap.tc.logging.reader.LocalBuffer.readPrevious()Lcom.sap.tc.logging.interfaces.IMessageBuilder;(LocalBuffer.java:281)
at com.sap.tc.logging.reader.FileReadLog.readPrevious(I)Lcom.sap.tc.logging.interfaces.ILoggingResultSet;(FileReadLog.java:404)
at com.sap.tc.logging.reader.ReadableLog.readLatestMsgRecords(I)Lcom.sap.tc.logging.interfaces.ILoggingResultSet;(ReadableLog.java:531)
at com.sapmarkets.bam.logcontroller.AbstractBAMLogReader$4.doRead()Lcom.sap.tc.logging.interfaces.ILoggingResultSet;(AbstractBAMLogReader.java:408)
at com.sapmarkets.bam.logcontroller.AbstractBAMLogReader$LogRecordReader.getRecords()Lcom.sap.tc.logging.interfaces.ILoggingResultSet;(AbstractBAMLogReader.java:262)
at com.sapmarkets.bam.logcontroller.AbstractBAMLogReader$LogRecordReader.readAndConvertRecords()Ljava.util.List;(AbstractBAMLogReader.java:204)
at com.sapmarkets.bam.logcontroller.AbstractBAMLogReader.readLatest()Ljava.util.List;(AbstractBAMLogReader.java:412)
at com.sapmarkets.bam.logcontroller.BAMLogReaderRecordSource$1.doRead()Ljava.util.List;(BAMLogReaderRecordSource.java:431)
at com.sapmarkets.bam.logcontroller.BAMLogReaderRecordSource.read(ILcom.sapmarkets.bam.logcontroller.BAMLogReaderRecordSource$ReadHelper;)Lcom.sapmarkets.bam.logcontroller.LogQueryResultset;(BAMLogReaderRecordSource.java:498)
at com.sapmarkets.bam.logcontroller.BAMLogReaderRecordSource.getLatest(I)Lcom.sapmarkets.bam.logcontroller.LogQueryResultset;(BAMLogReaderRecordSource.java:443)
at com.sapmarkets.bam.application.RecordBuffer.fillBuffer()V(RecordBuffer.java:264)
at com.sapmarkets.bam.application.RecordBuffer.init(II)V(RecordBuffer.java:106)
at com.sapmarkets.bam.application.MergeEngine.initRecordBuffers(I)V(MergeEngine.java:167)
at com.sapmarkets.bam.application.MergeEngine.getLatest(I)Lcom.sapmarkets.bam.logcontroller.LogQueryResultset;(MergeEngine.java:186)
at com.sapmarkets.bam.logcontroller.LogQueryBrowser.getLatest(I)Lcom.sapmarkets.bam.logcontroller.LogQueryResultset;(LogQueryBrowser.java:107)
at com.sapmarkets.bam.logcontroller.jmx.LogControllerFacade.getLatest(Lcom.sapmarkets.bam.logcontroller.LogQuerySession;I)Lcom.sapmarkets.bam.logcontroller.LogQueryResultset;(LogControllerFacade.java:406)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Ljava.lang.reflect.Method;Ljava.lang.Object;[Ljava.lang.Object;)Ljava.lang.Object;(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Ljava.lang.Object;[Ljava.lang.Object;)Ljava.lang.Object;(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Ljava.lang.Object;[Ljava.lang.Object;)Ljava.lang.Object;(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Ljava.lang.Object;[Ljava.lang.Object;)Ljava.lang.Object;(Method.java:331)
at com.sap.pj.jmx.introspect.DefaultMBeanInvoker.invoke(Ljava.lang.Object;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(DefaultMBeanInvoker.java:58)
at javax.management.StandardMBean.invoke(Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(StandardMBean.java:286)
at com.sap.pj.jmx.server.MBeanServerImpl.invoke(Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(MBeanServerImpl.java:944)
at com.sap.pj.jmx.server.interceptor.MBeanServerWrapperInterceptor.invoke(Lcom.sap.pj.jmx.server.interceptor.InvocationContext;Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(MBeanServerWrapperInterceptor.java:288)
at com.sap.engine.services.jmx.CompletionInterceptor.invoke(Lcom.sap.pj.jmx.server.interceptor.InvocationContext;Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(CompletionInterceptor.java:471)
at com.sap.pj.jmx.server.interceptor.BasicMBeanServerInterceptor.invoke(Lcom.sap.pj.jmx.server.interceptor.InvocationContext;Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(BasicMBeanServerInterceptor.java:277)
at com.sap.jmx.provider.ProviderInterceptor.invoke(Lcom.sap.pj.jmx.server.interceptor.InvocationContext;Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(ProviderInterceptor.java:258)
at com.sap.engine.services.jmx.RedirectInterceptor.invoke(Lcom.sap.pj.jmx.server.interceptor.InvocationContext;Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(RedirectInterceptor.java:340)
at com.sap.pj.jmx.server.interceptor.MBeanServerInterceptorChain.invoke(Ljavax.management.ObjectName;Ljava.lang.String;[Ljava.lang.Object;[Ljava.lang.String;)Ljava.lang.Object;(MBeanServerInterceptorChain.java:330)
at com.sap.engine.services.jmx.MBeanServerInvoker.invokeMbs(Lcom.sap.engine.services.jmx.RequestMessage;)Lcom.sap.engine.services.jmx.ResponseMessage;(MBeanServerInvoker.java:131)
at com.sap.engine.services.jmx.JmxServiceConnectorServer.receiveWait(II[BII)Lcom.sap.engine.frame.cluster.message.MessageAnswer;(JmxServiceConnectorServer.java:172)
at com.sap.engine.core.service630.context.cluster.message.MessageListenerWrapper.process(ILcom.sap.engine.core.cluster.MessageObject;)V(MessageListenerWrapper.java:81)
at com.sap.engine.core.cluster.impl6.lazy.UnorderedChannel$MessageRunner.run()V(UnorderedChannel.java:54)
at com.sap.engine.frame.core.thread.Task.run()Ljava.lang.Object;(Task.java:64)
at com.sap.engine.core.thread.impl5.SingleThread.execute()V(SingleThread.java:83)
at com.sap.engine.core.thread.impl5.SingleThread.run()V(SingleThread.java:156)