1.

Open files by user:

=====================
[root@iaqa94-ug-1 ~]# for f in $(sed</etc/passwd 's/:.*$//g'); do ( echo -n $f ' ';
lsof -u $f 2>/dev/null | wc -l ); done | grep -v ' 0$'
root 23856
dbus 44
polkitd 55
ntp 27
db2inst1 400
db2fenc1 57
dsadm 398
[root@iaqa94-ug-1 ~]#

Observation: Too many files are opened by 'root' user

2. Check WebSphere port

[root@iaqa94-ug-1 ~]# ps -ef | grep WebSphere

root 4372 23035 0 21:58 pts/1 00:00:00 grep --color=auto WebSphere
root 11451 1 32 Jun03 ? 19:58:39
/opt/IBM/WebSphere/AppServer/java/8.0/bin/java
-Dosgi.install.area=/opt/IBM/WebSphere/AppServer -Dwas.status.socket=38370
-Dosgi.configuration.area=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/servers/
server1/configuration
-Dosgi.framework.extensions=com.ibm.cds,com.ibm.ws.eclipse.adaptors
-Xshareclasses:name=webspherev9_8.0_64_%g,nonFatal
-Dcom.ibm.xtq.processor.overrideSecureProcessing=true -Xcheck:dump
-Djava.security.properties=/opt/IBM/WebSphere/AppServer/properties/java.security
-Djava.security.policy=/opt/IBM/WebSphere/AppServer/properties/java.policy
-Dcom.ibm.CORBA.ORBPropertyFilePath=/opt/IBM/WebSphere/AppServer/properties
-Xbootclasspath/p:/opt/IBM/WebSphere/AppServer/java/8.0/jre/lib/ibmorb.jar
-classpath
/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/properties:/opt/IBM/WebSphere/AppS
erver/properties:/opt/IBM/WebSphere/AppServer/lib/startup.jar:/opt/IBM/WebSphere/Ap
pServer/lib/bootstrap.jar:/opt/IBM/WebSphere/AppServer/lib/jsf-
nls.jar:/opt/IBM/WebSphere/AppServer/lib/lmproxy.jar:/opt/IBM/WebSphere/AppServer/l
ib/urlprotocols.jar:/opt/IBM/WebSphere/AppServer/deploytool/itp/batchboot.jar:/opt/
IBM/WebSphere/AppServer/deploytool/itp/batch2.jar:/opt/IBM/WebSphere/AppServer/java
/8.0/lib/tools.jar -Dibm.websphere.internalClassAccessMode=allow -verbose:gc
-Xverbosegclog:/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/logs/server1/verbos
egc.%seq.log,10,7000 -Xms1280m -Xmx4096m -Xcompressedrefs -Xscmaxaot4M -Xscmx60M
-Dws.ext.dirs=/opt/IBM/WebSphere/AppServer/java/8.0/lib:/opt/IBM/WebSphere/AppServe
r/profiles/InfoSphere/classes:/opt/IBM/WebSphere/AppServer/classes:/opt/IBM/WebSphe
re/AppServer/lib:/opt/IBM/WebSphere/AppServer/installedChannels:/opt/IBM/WebSphere/
AppServer/lib/ext:/opt/IBM/WebSphere/AppServer/web/help:/opt/IBM/WebSphere/AppServe
r/deploytool/itp/plugins/com.ibm.etools.ejbdeploy/runtime
-Dderby.system.home=/opt/IBM/WebSphere/AppServer/derby
-Dcom.ibm.itp.location=/opt/IBM/WebSphere/AppServer/bin
-Djava.util.logging.configureByServer=true
-Duser.install.root=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere
-Djava.ext.dirs=/opt/IBM/WebSphere/AppServer/tivoli/tam:/opt/IBM/WebSphere/AppServe
r/javaext:/opt/IBM/WebSphere/AppServer/java/8.0/jre/lib/ext
-Djavax.management.builder.initial=com.ibm.ws.management.PlatformMBeanServerBuilder
-Dpython.cachedir=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/temp/cachedir
-Dwas.install.root=/opt/IBM/WebSphere/AppServer
-Djava.util.logging.manager=com.ibm.ws.bootstrap.WsLogManager
-Dserver.root=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere
-Dcom.ibm.security.jgss.debug=off -Dcom.ibm.security.krb5.Krb5Debug=off
-DKC_HOME=/opt/IBM/WebSphere/AppServer/systemApps/isclite.ear/isclite.war/help
-Duser.language=en -Duser.country=US
-Dorg.apache.security.jacc.EJBMethodPermission.methodInterfaces=MessageEndpoint
-Dcom.ibm.ws.cdi.enableImplicitBeanArchives=false
-Dcom.ibm.wsspi.security.web.webAuthReq=persisting -Xdisableexplicitgc
-Djava.awt.headless=true
-Djava.library.path=/opt/IBM/WebSphere/AppServer/lib/native/linux/x86_64/:/opt/IBM/
WebSphere/AppServer/java/8.0/jre/lib/amd64/compressedrefs:/opt/IBM/WebSphere/AppSer
ver/java/8.0/jre/lib/amd64:/opt/IBM/WebSphere/AppServer/bin:/opt/IBM/WebSphere/AppS
erver/nulldllsdir:/usr/lib64:/usr/lib:
-Djava.endorsed.dirs=/opt/IBM/WebSphere/AppServer/endorsed_apis:/opt/IBM/WebSphere/
AppServer/java/8.0/jre/lib/endorsed
-Djava.security.auth.login.config=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/
properties/wsjaas.conf
-Djava.security.policy=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/properties/
server.policy com.ibm.wsspi.bootstrap.WSPreLauncher -nosplash -application
com.ibm.ws.bootstrap.WSLauncher com.ibm.ws.runtime.WsServer
/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/config iaqa94-ug-1Node01Cell
iaqa94-ug-1Node01 server1
root 29141 1 18 May28 ? 1-15:13:19
/opt/IBM/WebSphere/AppServer/java/8.0/bin/java
-Dosgi.install.area=/opt/IBM/WebSphere/AppServer -Dwas.status.socket=43455
-Dosgi.configuration.area=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/servers/
server1/configuration
-Dosgi.framework.extensions=com.ibm.cds,com.ibm.ws.eclipse.adaptors
-Xshareclasses:name=webspherev9_8.0_64_%g,nonFatal
-Dcom.ibm.xtq.processor.overrideSecureProcessing=true -Xcheck:dump
-Djava.security.properties=/opt/IBM/WebSphere/AppServer/properties/java.security
-Djava.security.policy=/opt/IBM/WebSphere/AppServer/properties/java.policy
-Dcom.ibm.CORBA.ORBPropertyFilePath=/opt/IBM/WebSphere/AppServer/properties
-Xbootclasspath/p:/opt/IBM/WebSphere/AppServer/java/8.0/jre/lib/ibmorb.jar
-classpath
/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/properties:/opt/IBM/WebSphere/AppS
erver/properties:/opt/IBM/WebSphere/AppServer/lib/startup.jar:/opt/IBM/WebSphere/Ap
pServer/lib/bootstrap.jar:/opt/IBM/WebSphere/AppServer/lib/jsf-
nls.jar:/opt/IBM/WebSphere/AppServer/lib/lmproxy.jar:/opt/IBM/WebSphere/AppServer/l
ib/urlprotocols.jar:/opt/IBM/WebSphere/AppServer/deploytool/itp/batchboot.jar:/opt/
IBM/WebSphere/AppServer/deploytool/itp/batch2.jar:/opt/IBM/WebSphere/AppServer/java
/8.0/lib/tools.jar -Dibm.websphere.internalClassAccessMode=allow -verbose:gc
-Xverbosegclog:/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/logs/server1/verbos
egc.%seq.log,10,7000 -Xms1280m -Xmx4096m -Xcompressedrefs -Xscmaxaot4M -Xscmx60M
-Dws.ext.dirs=/opt/IBM/WebSphere/AppServer/java/8.0/lib:/opt/IBM/WebSphere/AppServe
r/profiles/InfoSphere/classes:/opt/IBM/WebSphere/AppServer/classes:/opt/IBM/WebSphe
re/AppServer/lib:/opt/IBM/WebSpher
/AppServer/installedChannels:/opt/IBM/WebSphere/AppServer/lib/ext:/opt/IBM/WebSpher
e/AppServer/web/help:/opt/IBM/WebSphere/AppServer/deploytool/itp/plugins/com.ibm.et
ools.ejbdeploy/runtime -Dderby.system.home=/opt/IBM/WebSphere/AppServer/derby
-Dcom.ibm.itp.location=/opt/IBM/WebSphere/AppServer/bin
-Djava.util.logging.configureByServer=true
-Duser.install.root=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere
-Djava.ext.dirs=/opt/IBM/WebSphere/AppServer/tivoli/tam:/opt/IBM/WebSphere/AppServe
r/javaext:/opt/IBM/WebSphere/AppServer/java/8.0/jre/lib/ext
-Djavax.management.builder.initial=com.ibm.ws.management.PlatformMBeanServerBuilder
-Dpython.cachedir=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/temp/cachedir
-Dwas.install.root=/opt/IBM/WebSphere/AppServer
-Djava.util.logging.manager=com.ibm.ws.bootstrap.WsLogManager
-Dserver.root=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere
-Dcom.ibm.security.jgss.debug=off -Dcom.ibm.security.krb5.Krb5Debug=off
-DKC_HOME=/opt/IBM/WebSphere/AppServer/systemApps/isclite.ear/isclite.war/help
-Duser.language=en -Duser.country=US
-Dorg.apache.security.jacc.EJBMethodPermission.methodInterfaces=MessageEndpoint
-Dcom.ibm.ws.cdi.enableImplicitBeanArchives=false
-Dcom.ibm.wsspi.security.web.webAuthReq=persisting -Xdisableexplicitgc
-Djava.awt.headless=true
-Djava.library.path=/opt/IBM/WebSphere/AppServer/lib/native/linux/x86_64/:/opt/IBM/
WebSphere/AppServer/java/8.0/jre/lib/amd64/compressedrefs:/opt/IBM/WebSphere/AppSer
ver/java/8.0/jre/lib/amd64:/opt/IBM/WebSphere/AppServer/bin:/opt/IBM/WebSphere/AppS
erver/nulldllsdir:/usr/lib64:/usr/lib:
-Djava.endorsed.dirs=/opt/IBM/WebSphere/AppServer/endorsed_apis:/opt/IBM/WebSphere/
AppServer/java/8.0/jre/lib/endorsed
-Djava.security.auth.login.config=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/
properties/wsjaas.conf
-Djava.security.policy=/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/properties/
server.policy com.ibm.wsspi.bootstrap.WSPreLauncher -nosplash -application
com.ibm.ws.bootstrap.WSLauncher com.ibm.ws.runtime.WsServer
/opt/IBM/WebSphere/AppServer/profiles/InfoSphere/config iaqa94-ug-1Node01Cell
iaqa94-ug-1Node01 server1
[root@iaqa94-ug-1 ~]#

Observation: Two WebSphere instances are running (Strange !!!!!!!!!!) But it is

okay

3. Limits set for websphere processes

[root@iaqa94-ug-1 ~]# cat /proc/11451/limits

Limit Soft Limit Hard Limit Units
Max cpu time unlimited unlimited seconds
Max file size unlimited unlimited bytes
Max data size unlimited unlimited bytes
Max stack size 8388608 unlimited bytes
Max core file size 0 unlimited bytes
Max resident set unlimited unlimited bytes
Max processes 127955 127955 processes
Max open files 10240 10240 files
Max locked memory 65536 65536 bytes
Max address space unlimited unlimited bytes
Max file locks unlimited unlimited locks
Max pending signals 127955 127955 signals
Max msgqueue size 819200 819200 bytes
Max nice priority 0 0
Max realtime priority 0 0
Max realtime timeout unlimited unlimited us
[root@iaqa94-ug-1 ~]# cat /proc/29141/limits
Limit Soft Limit Hard Limit Units
Max cpu time unlimited unlimited seconds
Max file size unlimited unlimited bytes
Max data size unlimited unlimited bytes
Max stack size 8388608 unlimited bytes
Max core file size 0 unlimited bytes
Max resident set unlimited unlimited bytes
Max processes 127955 127955 processes
Max open files 10240 10240 files
Max locked memory 65536 65536 bytes
Max address space unlimited unlimited bytes
Max file locks unlimited unlimited locks
Max pending signals 127955 127955 signals
Max msgqueue size 819200 819200 bytes
Max nice priority 0 0
Max realtime priority 0 0
Max realtime timeout unlimited unlimited us
[root@iaqa94-ug-1 ~]#

Observation: Max open files 10240 10240

files for both, which is too low and needs a WbeSpere restart so that it can
retrieve latest config from limits.conf.

4. Open file cound for both WebSphere processes

[root@iaqa94-ug-1 ~]# lsof -p 11451 | wc -l

10290
[root@iaqa94-ug-1 ~]# lsof -p 29141 | wc -l
10323
[root@iaqa94-ug-1 ~]#

5.lsof output for first WebSphere process (only last few lines):

java 11451 root *209r FIFO 0,9 0t0 61141455 pipe

java 11451 root *210w FIFO 0,9 0t0 61141455 pipe
java 11451 root *211u a_inode 0,10 0 6415
[eventpoll]
java 11451 root *212u IPv4 123849509 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37370->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *213r FIFO 0,9 0t0 61226184 pipe
java 11451 root *214u a_inode 0,10 0 6415
[eventpoll]
java 11451 root *215u IPv4 123855104 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37382->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *216u IPv4 123856913 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37372->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *217u IPv4 123855973 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37396->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *218u IPv4 123855327 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37448->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *219w FIFO 0,9 0t0 61226184 pipe
java 11451 root *220u a_inode 0,10 0 6415
[eventpoll]
java 11451 root *221r FIFO 0,9 0t0 61233147 pipe
java 11451 root *222u IPv4 123849656 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37454->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *223u IPv4 123855385 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37462->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *224w FIFO 0,9 0t0 61233147 pipe
java 11451 root *225u a_inode 0,10 0 6415
[eventpoll]
java 11451 root *226u a_inode 0,10 0 6415
[eventpoll]
java 11451 root *227u IPv4 123849686 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37464->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *228u IPv4 123857242 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37496->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *229u IPv4 123855545 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37518->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *230u IPv4 123854222 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37398->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *231u IPv4 123852675 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37574->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *232u IPv4 123855197 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37400->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *233u IPv4 123854243 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37402->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 11451 root *234r FIFO 0,9 0t0 61256986 pipe
java 11451 root *235w FIFO 0,9 0t0 61256986 pipe
java 11451 root *236u a_inode 0,10 0 6415
[eventpoll]
java 11451 root *237u IPv4 123849610 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:37416->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)

Observation: These open files are KAFKA connection related Ref:

https://issues.apache.org/jira/browse/KAFKA-3317

6. lsof output for second WebSphere process (only last few lines):

java 29141 root *211r FIFO 0,9 0t0 12845342 pipe

java 29141 root *212u a_inode 0,10 0 6415
[eventpoll]
java 29141 root *213w FIFO 0,9 0t0 12845342 pipe
java 29141 root *214u a_inode 0,10 0 6415
[eventpoll]
java 29141 root *215u IPv4 123823917 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:35626->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 29141 root *216u a_inode 0,10 0 6415
[eventpoll]
java 29141 root *217u a_inode 0,10 0 6415
[eventpoll]
java 29141 root *218u IPv4 17041300 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:55666->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 29141 root *219u a_inode 0,10 0 6415
[eventpoll]
java 29141 root *220r FIFO 0,9 0t0 17095836 pipe
java 29141 root *221r FIFO 0,9 0t0 15796807 pipe
java 29141 root *222w FIFO 0,9 0t0 15796807 pipe
java 29141 root *223u IPv4 123794800 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:59064->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 29141 root *224u a_inode 0,10 0 6415
[eventpoll]
java 29141 root *225w FIFO 0,9 0t0 12853376 pipe
java 29141 root *226r FIFO 0,9 0t0 12848668 pipe
java 29141 root *227w FIFO 0,9 0t0 12848668 pipe
java 29141 root *228u a_inode 0,10 0 6415
[eventpoll]
java 29141 root *229u a_inode 0,10 0 6415
[eventpoll]
java 29141 root *230w FIFO 0,9 0t0 17095836 pipe
java 29141 root *231r FIFO 0,9 0t0 12849782 pipe
java 29141 root *232r REG 253,0 1050431 12471005
/opt/IBM/WebSphere/AppServer/plugins/com.ibm.ws.wlm.jar
java 29141 root *233w FIFO 0,9 0t0 12849782 pipe
java 29141 root *234u a_inode 0,10 0 6415
[eventpoll]
java 29141 root *235u a_inode 0,10 0 6415
[eventpoll]
java 29141 root *236w FIFO 0,9 0t0 15795891 pipe
java 29141 root *237u a_inode 0,10 0 6415
[eventpoll]
java 29141 root *238u IPv4 123794801 0t0 TCP iaqa94-ug-
1.fyre.ibm.com:59066->iaqa94-ug-2.fyre.ibm.com:XmlIpcRegSvc (ESTABLISHED)
java 29141 root *239u a_inode 0,10 0 6415
[eventpoll]

Observation: These open files are KAFKA connection related Ref:

https://issues.apache.org/jira/browse/KAFKA-3317

Need to check on how we are closing the Kafka connections and also if there still
any open issue / leak in Kafka component.

7. Next steps:

Make sure we have correct configuration of nofiles for each major process like
WebSphere, dsrpcd etc.

Run Load & Stress tests and see how these open files are changing per user and per
process for every 3 hours.