Large concurrent user performance issues for Apache + mod_jk + GlassFish v3.1 clusters
- by user10035
I am running a java ee 6 ear application on a GlassFish v3.1 ( 2 clusters with 2 instances each) load balanced by an Apache v2.2 with mod_jk - all on the same server (Windows Server 2003 R2, Intel Xeon CPU x5670 @2.93Ghz, 6GB RAM, 2 cpus).
The web application is accessed by around ~100 users. When they all try to access it at the same time every morning ~8am, the response is very slow while trying to access the main jsf home page.
Apart from that I have seen the CPU usage spike upto 99% by the httpd process during the day frequently and I start seeing errors in the mod_jk.log file.
[Wed Jun 08 08:25:43 2011] [9380:8216] [info] ajp_process_callback::jk_ajp_common.c (1885): Writing to client aborted or client network problems
[Wed Jun 08 08:25:43 2011] [9380:8216] [info] ajp_service::jk_ajp_common.c (2543): (myAppLocalInstance4) sending request to tomcat failed (unrecoverable), because of client write error (attempt=1)
Any suggestions on how I can go about improving this?
Apache configuration is mostly the default as shown below
ServerRoot "C:/Program Files/Apache Software Foundation/Apache2.2"
Listen 80
LoadModule actions_module modules/mod_actions.so
LoadModule alias_module modules/mod_alias.so
LoadModule asis_module modules/mod_asis.so
LoadModule auth_basic_module modules/mod_auth_basic.so
LoadModule authn_default_module modules/mod_authn_default.so
LoadModule authn_file_module modules/mod_authn_file.so
LoadModule authz_default_module modules/mod_authz_default.so
LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule authz_user_module modules/mod_authz_user.so
LoadModule autoindex_module modules/mod_autoindex.so
LoadModule cgi_module modules/mod_cgi.so
LoadModule dir_module modules/mod_dir.so
LoadModule env_module modules/mod_env.so
LoadModule include_module modules/mod_include.so
LoadModule isapi_module modules/mod_isapi.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule mime_module modules/mod_mime.so
LoadModule negotiation_module modules/mod_negotiation.so
LoadModule setenvif_module modules/mod_setenvif.so
<IfModule !mpm_netware_module>
<IfModule !mpm_winnt_module>
User daemon
Group daemon
</IfModule>
</IfModule>
DocumentRoot "C:/Program Files/Apache Software Foundation/Apache2.2/htdocs"
<Directory />
Options FollowSymLinks
AllowOverride None
Order deny,allow
Deny from all
</Directory>
<Directory "C:/Program Files/Apache Software Foundation/Apache2.2/htdocs">
Options Indexes FollowSymLinks
AllowOverride None
Order allow,deny
Allow from all
</Directory>
<IfModule dir_module>
DirectoryIndex index.html
</IfModule>
<FilesMatch "^\.ht">
Order allow,deny
Deny from all
Satisfy All
</FilesMatch>
ErrorLog "logs/error.log"
LogLevel warn
<IfModule log_config_module>
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
LogFormat "%h %l %u %t \"%r\" %>s %b" common
<IfModule logio_module>
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
</IfModule>
CustomLog "logs/access.log" common
</IfModule>
<IfModule alias_module>
ScriptAlias /cgi-bin/ "C:/Program Files/Apache Software Foundation/Apache2.2/cgi-bin/"
</IfModule>
<Directory "C:/Program Files/Apache Software Foundation/Apache2.2/cgi-bin">
AllowOverride None
Options None
Order allow,deny
Allow from all
</Directory>
DefaultType text/plain
<IfModule mime_module>
TypesConfig conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
</IfModule>
Include conf/extra/httpd-mpm.conf
<IfModule ssl_module>
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
</IfModule>
LoadModule jk_module modules/mod_jk.so
JkWorkersFile conf/workers.properties
JkLogFile logs/mod_jk.log
JkLogLevel info
JkLogStampFormat "[%a %b %d %H:%M:%S %Y] "
JkOptions +ForwardKeySize +ForwardURICompat -ForwardDirectories
JkRequestLogFormat "%w %V %T"
JkMount /myApp/* loadbalancerLocal
JkMount /myAppRemote/* loadbalancerRemote
JkMount /myApp loadbalancerLocal
JkMount /myAppRemote loadbalancerRemote
The workers.properties config file is:
worker.list=loadbalancerLocal,loadbalancerRemote
worker.myAppLocalInstance1.type=ajp13
worker.myAppLocalInstance1.host=localhost
worker.myAppLocalInstance1.port=8109
worker.myAppLocalInstance1.lbfactor=1
worker.myAppLocalInstance1.socket_keepalive=1
worker.myAppLocalInstance1.socket_timeout=1000
worker.myAppLocalInstance2.type=ajp13
worker.myAppLocalInstance2.host=localhost
worker.myAppLocalInstance2.port=8209
worker.myAppLocalInstance2.lbfactor=1
worker.myAppLocalInstance2.socket_keepalive=1
worker.myAppLocalInstance2.socket_timeout=1000
worker.myAppLocalInstance3.type=ajp13
worker.myAppLocalInstance3.host=localhost
worker.myAppLocalInstance3.port=8309
worker.myAppLocalInstance3.lbfactor=1
worker.myAppLocalInstance3.socket_keepalive=1
worker.myAppLocalInstance3.socket_timeout=1000
worker.myAppLocalInstance4.type=ajp13
worker.myAppLocalInstance4.host=localhost
worker.myAppLocalInstance4.port=8409
worker.myAppLocalInstance4.lbfactor=1
worker.myAppLocalInstance4.socket_keepalive=1
worker.myAppLocalInstance4.socket_timeout=1000
worker.myAppRemoteInstance1.type=ajp13
worker.myAppRemoteInstance1.host=localhost
worker.myAppRemoteInstance1.port=8509
worker.myAppRemoteInstance1.lbfactor=1
worker.myAppRemoteInstance1.socket_keepalive=1
worker.myAppRemoteInstance1.socket_timeout=1000
worker.myAppRemoteInstance2.type=ajp13
worker.myAppRemoteInstance2.host=localhost
worker.myAppRemoteInstance2.port=8609
worker.myAppRemoteInstance2.lbfactor=1
worker.myAppRemoteInstance2.socket_keepalive=1
worker.myAppRemoteInstance2.socket_timeout=1000
worker.myAppRemoteInstance3.type=ajp13
worker.myAppRemoteInstance3.host=localhost
worker.myAppRemoteInstance3.port=8709
worker.myAppRemoteInstance3.lbfactor=1
worker.myAppRemoteInstance3.socket_keepalive=1
worker.myAppRemoteInstance3.socket_timeout=1000
worker.myAppRemoteInstance4.type=ajp13
worker.myAppRemoteInstance4.host=localhost
worker.myAppRemoteInstance4.port=8809
worker.myAppRemoteInstance4.lbfactor=1
worker.myAppRemoteInstance4.socket_keepalive=1
worker.myAppRemoteInstance4.socket_timeout=1000
worker.loadbalancerLocal.type=lb
worker.loadbalancerLocal.sticky_session=True
worker.loadbalancerLocal.balance_workers=myAppLocalInstance1,myAppLocalInstance2,myAppLocalInstance3,myAppLocalInstance4
worker.loadbalancerRemote.type=lb
worker.loadbalancerRemote.balance_workers=myAppRemoteInstance1,myAppRemoteInstance2,myAppRemoteInstance3,myAppRemoteInstance4
worker.loadbalancerRemote.sticky_session=True
