Where is all the memory being consumed?
- by Mark L
Hello,
I have a Dell R300 Ubuntu 9.10 box with 4GB of memory. All I'm running on there is haproxy, nagios and postfix yet there is ~2.7GB of memory being consumed. I've run ps and I can't get the sums to add up. Could anyone shed any light on where all the memory is being used?
Cheers,
Mark
$ sudo free -m
total used free shared buffers cached
Mem: 3957 2746 1211 0 169 2320
-/+ buffers/cache: 256 3701
Swap: 6212 0 6212
Sorry for pasting all of ps' output but I'm keen to get to the bottom of this.
$ sudo ps aux
[sudo] password for mark:
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 1 0.0 0.0 19320 1656 ? Ss May20 0:05 /sbin/init
root 2 0.0 0.0 0 0 ? S< May20 0:00 [kthreadd]
root 3 0.0 0.0 0 0 ? S< May20 0:00 [migration/0]
root 4 0.0 0.0 0 0 ? S< May20 0:16 [ksoftirqd/0]
root 5 0.0 0.0 0 0 ? S< May20 0:00 [watchdog/0]
root 6 0.0 0.0 0 0 ? S< May20 0:03 [migration/1]
root 7 0.0 0.0 0 0 ? S< May20 3:10 [ksoftirqd/1]
root 8 0.0 0.0 0 0 ? S< May20 0:00 [watchdog/1]
root 9 0.0 0.0 0 0 ? S< May20 0:00 [migration/2]
root 10 0.0 0.0 0 0 ? S< May20 0:19 [ksoftirqd/2]
root 11 0.0 0.0 0 0 ? S< May20 0:00 [watchdog/2]
root 12 0.0 0.0 0 0 ? S< May20 0:01 [migration/3]
root 13 0.0 0.0 0 0 ? S< May20 0:41 [ksoftirqd/3]
root 14 0.0 0.0 0 0 ? S< May20 0:00 [watchdog/3]
root 15 0.0 0.0 0 0 ? S< May20 0:03 [events/0]
root 16 0.0 0.0 0 0 ? S< May20 0:10 [events/1]
root 17 0.0 0.0 0 0 ? S< May20 0:08 [events/2]
root 18 0.0 0.0 0 0 ? S< May20 0:08 [events/3]
root 19 0.0 0.0 0 0 ? S< May20 0:00 [cpuset]
root 20 0.0 0.0 0 0 ? S< May20 0:00 [khelper]
root 21 0.0 0.0 0 0 ? S< May20 0:00 [netns]
root 22 0.0 0.0 0 0 ? S< May20 0:00 [async/mgr]
root 23 0.0 0.0 0 0 ? S< May20 0:00 [kintegrityd/0]
root 24 0.0 0.0 0 0 ? S< May20 0:00 [kintegrityd/1]
root 25 0.0 0.0 0 0 ? S< May20 0:00 [kintegrityd/2]
root 26 0.0 0.0 0 0 ? S< May20 0:00 [kintegrityd/3]
root 27 0.0 0.0 0 0 ? S< May20 0:00 [kblockd/0]
root 28 0.0 0.0 0 0 ? S< May20 0:01 [kblockd/1]
root 29 0.0 0.0 0 0 ? S< May20 0:04 [kblockd/2]
root 30 0.0 0.0 0 0 ? S< May20 0:02 [kblockd/3]
root 31 0.0 0.0 0 0 ? S< May20 0:00 [kacpid]
root 32 0.0 0.0 0 0 ? S< May20 0:00 [kacpi_notify]
root 33 0.0 0.0 0 0 ? S< May20 0:00 [kacpi_hotplug]
root 34 0.0 0.0 0 0 ? S< May20 0:00 [ata/0]
root 35 0.0 0.0 0 0 ? S< May20 0:00 [ata/1]
root 36 0.0 0.0 0 0 ? S< May20 0:00 [ata/2]
root 37 0.0 0.0 0 0 ? S< May20 0:00 [ata/3]
root 38 0.0 0.0 0 0 ? S< May20 0:00 [ata_aux]
root 39 0.0 0.0 0 0 ? S< May20 0:00 [ksuspend_usbd]
root 40 0.0 0.0 0 0 ? S< May20 0:00 [khubd]
root 41 0.0 0.0 0 0 ? S< May20 0:00 [kseriod]
root 42 0.0 0.0 0 0 ? S< May20 0:00 [kmmcd]
root 43 0.0 0.0 0 0 ? S< May20 0:00 [bluetooth]
root 44 0.0 0.0 0 0 ? S May20 0:00 [khungtaskd]
root 45 0.0 0.0 0 0 ? S May20 0:00 [pdflush]
root 46 0.0 0.0 0 0 ? S May20 0:09 [pdflush]
root 47 0.0 0.0 0 0 ? S< May20 0:00 [kswapd0]
root 48 0.0 0.0 0 0 ? S< May20 0:00 [aio/0]
root 49 0.0 0.0 0 0 ? S< May20 0:00 [aio/1]
root 50 0.0 0.0 0 0 ? S< May20 0:00 [aio/2]
root 51 0.0 0.0 0 0 ? S< May20 0:00 [aio/3]
root 52 0.0 0.0 0 0 ? S< May20 0:00 [ecryptfs-kthrea]
root 53 0.0 0.0 0 0 ? S< May20 0:00 [crypto/0]
root 54 0.0 0.0 0 0 ? S< May20 0:00 [crypto/1]
root 55 0.0 0.0 0 0 ? S< May20 0:00 [crypto/2]
root 56 0.0 0.0 0 0 ? S< May20 0:00 [crypto/3]
root 70 0.0 0.0 0 0 ? S< May20 0:00 [scsi_eh_0]
root 71 0.0 0.0 0 0 ? S< May20 0:00 [scsi_eh_1]
root 74 0.0 0.0 0 0 ? S< May20 0:00 [scsi_eh_2]
root 75 0.0 0.0 0 0 ? S< May20 0:00 [scsi_eh_3]
root 82 0.0 0.0 0 0 ? S< May20 0:00 [kstriped]
root 83 0.0 0.0 0 0 ? S< May20 0:00 [kmpathd/0]
root 84 0.0 0.0 0 0 ? S< May20 0:00 [kmpathd/1]
root 85 0.0 0.0 0 0 ? S< May20 0:00 [kmpathd/2]
root 86 0.0 0.0 0 0 ? S< May20 0:00 [kmpathd/3]
root 87 0.0 0.0 0 0 ? S< May20 0:00 [kmpath_handlerd]
root 88 0.0 0.0 0 0 ? S< May20 0:00 [ksnapd]
root 89 0.0 0.0 0 0 ? S< May20 0:00 [kondemand/0]
root 90 0.0 0.0 0 0 ? S< May20 0:00 [kondemand/1]
root 91 0.0 0.0 0 0 ? S< May20 0:00 [kondemand/2]
root 92 0.0 0.0 0 0 ? S< May20 0:00 [kondemand/3]
root 93 0.0 0.0 0 0 ? S< May20 0:00 [kconservative/0]
root 94 0.0 0.0 0 0 ? S< May20 0:00 [kconservative/1]
root 95 0.0 0.0 0 0 ? S< May20 0:00 [kconservative/2]
root 96 0.0 0.0 0 0 ? S< May20 0:00 [kconservative/3]
root 97 0.0 0.0 0 0 ? S< May20 0:00 [krfcommd]
root 315 0.0 0.0 0 0 ? S< May20 0:09 [mpt_poll_0]
root 317 0.0 0.0 0 0 ? S< May20 0:00 [mpt/0]
root 547 0.0 0.0 0 0 ? S< May20 0:00 [scsi_eh_4]
root 587 0.0 0.0 0 0 ? S< May20 0:11 [kjournald2]
root 636 0.0 0.0 12748 860 ? S May20 0:00 upstart-udev-bridge --daemon
root 657 0.0 0.0 17064 924 ? S<s May20 0:00 udevd --daemon
root 666 0.0 0.0 8192 612 ? Ss May20 0:00 dd bs=1 if=/proc/kmsg of=/var/run/rsyslog/kmsg
root 774 0.0 0.0 17060 888 ? S< May20 0:00 udevd --daemon
root 775 0.0 0.0 17060 888 ? S< May20 0:00 udevd --daemon
syslog 825 0.0 0.0 191696 1988 ? Sl May20 0:31 rsyslogd -c4
root 839 0.0 0.0 0 0 ? S< May20 0:00 [edac-poller]
root 870 0.0 0.0 0 0 ? S< May20 0:00 [kpsmoused]
root 1006 0.0 0.0 5988 604 tty4 Ss+ May20 0:00 /sbin/getty -8 38400 tty4
root 1008 0.0 0.0 5988 604 tty5 Ss+ May20 0:00 /sbin/getty -8 38400 tty5
root 1015 0.0 0.0 5988 604 tty2 Ss+ May20 0:00 /sbin/getty -8 38400 tty2
root 1016 0.0 0.0 5988 608 tty3 Ss+ May20 0:00 /sbin/getty -8 38400 tty3
root 1018 0.0 0.0 5988 604 tty6 Ss+ May20 0:00 /sbin/getty -8 38400 tty6
daemon 1025 0.0 0.0 16512 472 ? Ss May20 0:00 atd
root 1026 0.0 0.0 18708 1000 ? Ss May20 0:03 cron
root 1052 0.0 0.0 49072 1252 ? Ss May20 0:25 /usr/sbin/sshd
root 1084 0.0 0.0 5988 604 tty1 Ss+ May20 0:00 /sbin/getty -8 38400 tty1
root 6320 0.0 0.0 19440 956 ? Ss May21 0:00 /usr/sbin/xinetd -pidfile /var/run/xinetd.pid -stayalive -inetd_compat -inetd_ipv6
nagios 8197 0.0 0.0 27452 1696 ? SNs May21 2:57 /usr/sbin/nagios3 -d /etc/nagios3/nagios.cfg
root 10882 0.1 0.0 70280 3104 ? Ss 10:30 0:00 sshd: mark [priv]
mark 10934 0.0 0.0 70432 1776 ? S 10:30 0:00 sshd: mark@pts/0
mark 10935 1.4 0.1 21572 4336 pts/0 Ss 10:30 0:00 -bash
root 10953 1.0 0.0 15164 1136 pts/0 R+ 10:30 0:00 ps aux
haproxy 12738 0.0 0.0 17208 992 ? Ss Jun08 0:49 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg
root 23953 0.0 0.0 37012 2192 ? Ss Jun04 0:03 /usr/lib/postfix/master
postfix 23955 0.0 0.0 39232 2356 ? S Jun04 0:00 qmgr -l -t fifo -u
postfix 32603 0.0 0.0 39072 2132 ? S 09:05 0:00 pickup -l -t fifo -u -c
Here's meminfo:
$ cat /proc/meminfo
MemTotal: 4052852 kB
MemFree: 1240488 kB
Buffers: 173172 kB
Cached: 2376420 kB
SwapCached: 0 kB
Active: 1479288 kB
Inactive: 1081876 kB
Active(anon): 11792 kB
Inactive(anon): 0 kB
Active(file): 1467496 kB
Inactive(file): 1081876 kB
Unevictable: 0 kB
Mlocked: 0 kB
SwapTotal: 6361700 kB
SwapFree: 6361700 kB
Dirty: 44 kB
Writeback: 0 kB
AnonPages: 11568 kB
Mapped: 5844 kB
Slab: 155032 kB
SReclaimable: 145804 kB
SUnreclaim: 9228 kB
PageTables: 1592 kB
NFS_Unstable: 0 kB
Bounce: 0 kB
WritebackTmp: 0 kB
CommitLimit: 8388124 kB
Committed_AS: 51732 kB
VmallocTotal: 34359738367 kB
VmallocUsed: 282604 kB
VmallocChunk: 34359453499 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 2048 kB
DirectMap4k: 6784 kB
DirectMap2M: 4182016 kB
Here's slabinfo:
$ cat /proc/slabinfo
slabinfo - version: 2.1
# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab> : tunables <limit> <batchcount> <sharedfactor> : slabdata <active_slabs> <num_slabs> <sharedavail>
ip6_dst_cache 50 50 320 25 2 : tunables 0 0 0 : slabdata 2 2 0
UDPLITEv6 0 0 960 17 4 : tunables 0 0 0 : slabdata 0 0 0
UDPv6 68 68 960 17 4 : tunables 0 0 0 : slabdata 4 4 0
tw_sock_TCPv6 0 0 320 25 2 : tunables 0 0 0 : slabdata 0 0 0
TCPv6 72 72 1792 18 8 : tunables 0 0 0 : slabdata 4 4 0
dm_raid1_read_record 0 0 1064 30 8 : tunables 0 0 0 : slabdata 0 0 0
kcopyd_job 0 0 368 22 2 : tunables 0 0 0 : slabdata 0 0 0
dm_uevent 0 0 2608 12 8 : tunables 0 0 0 : slabdata 0 0 0
dm_rq_target_io 0 0 376 21 2 : tunables 0 0 0 : slabdata 0 0 0
uhci_urb_priv 0 0 56 73 1 : tunables 0 0 0 : slabdata 0 0 0
cfq_queue 0 0 168 24 1 : tunables 0 0 0 : slabdata 0 0 0
mqueue_inode_cache 18 18 896 18 4 : tunables 0 0 0 : slabdata 1 1 0
fuse_request 0 0 632 25 4 : tunables 0 0 0 : slabdata 0 0 0
fuse_inode 0 0 768 21 4 : tunables 0 0 0 : slabdata 0 0 0
ecryptfs_inode_cache 0 0 1024 16 4 : tunables 0 0 0 : slabdata 0 0 0
hugetlbfs_inode_cache 26 26 608 26 4 : tunables 0 0 0 : slabdata 1 1 0
journal_handle 680 680 24 170 1 : tunables 0 0 0 : slabdata 4 4 0
journal_head 144 144 112 36 1 : tunables 0 0 0 : slabdata 4 4 0
revoke_table 256 256 16 256 1 : tunables 0 0 0 : slabdata 1 1 0
revoke_record 512 512 32 128 1 : tunables 0 0 0 : slabdata 4 4 0
ext4_inode_cache 53306 53424 888 18 4 : tunables 0 0 0 : slabdata 2968 2968 0
ext4_free_block_extents 292 292 56 73 1 : tunables 0 0 0 : slabdata 4 4 0
ext4_alloc_context 112 112 144 28 1 : tunables 0 0 0 : slabdata 4 4 0
ext4_prealloc_space 156 156 104 39 1 : tunables 0 0 0 : slabdata 4 4 0
ext4_system_zone 0 0 40 102 1 : tunables 0 0 0 : slabdata 0 0 0
ext2_inode_cache 0 0 776 21 4 : tunables 0 0 0 : slabdata 0 0 0
ext3_inode_cache 0 0 784 20 4 : tunables 0 0 0 : slabdata 0 0 0
ext3_xattr 0 0 88 46 1 : tunables 0 0 0 : slabdata 0 0 0
dquot 0 0 256 16 1 : tunables 0 0 0 : slabdata 0 0 0
shmem_inode_cache 606 620 800 20 4 : tunables 0 0 0 : slabdata 31 31 0
pid_namespace 0 0 2112 15 8 : tunables 0 0 0 : slabdata 0 0 0
UDP-Lite 0 0 832 19 4 : tunables 0 0 0 : slabdata 0 0 0
RAW 183 210 768 21 4 : tunables 0 0 0 : slabdata 10 10 0
UDP 76 76 832 19 4 : tunables 0 0 0 : slabdata 4 4 0
tw_sock_TCP 80 80 256 16 1 : tunables 0 0 0 : slabdata 5 5 0
TCP 81 114 1664 19 8 : tunables 0 0 0 : slabdata 6 6 0
blkdev_integrity 144 144 112 36 1 : tunables 0 0 0 : slabdata 4 4 0
blkdev_queue 64 64 2024 16 8 : tunables 0 0 0 : slabdata 4 4 0
blkdev_requests 120 120 336 24 2 : tunables 0 0 0 : slabdata 5 5 0
fsnotify_event 156 156 104 39 1 : tunables 0 0 0 : slabdata 4 4 0
bip-256 7 7 4224 7 8 : tunables 0 0 0 : slabdata 1 1 0
bip-128 0 0 2176 15 8 : tunables 0 0 0 : slabdata 0 0 0
bip-64 0 0 1152 28 8 : tunables 0 0 0 : slabdata 0 0 0
bip-16 84 84 384 21 2 : tunables 0 0 0 : slabdata 4 4 0
sock_inode_cache 224 276 704 23 4 : tunables 0 0 0 : slabdata 12 12 0
file_lock_cache 88 88 184 22 1 : tunables 0 0 0 : slabdata 4 4 0
net_namespace 0 0 1920 17 8 : tunables 0 0 0 : slabdata 0 0 0
Acpi-ParseExt 640 672 72 56 1 : tunables 0 0 0 : slabdata 12 12 0
taskstats 48 48 328 24 2 : tunables 0 0 0 : slabdata 2 2 0
proc_inode_cache 1613 1750 640 25 4 : tunables 0 0 0 : slabdata 70 70 0
sigqueue 100 100 160 25 1 : tunables 0 0 0 : slabdata 4 4 0
radix_tree_node 22443 22475 560 29 4 : tunables 0 0 0 : slabdata 775 775 0
bdev_cache 72 72 896 18 4 : tunables 0 0 0 : slabdata 4 4 0
sysfs_dir_cache 9866 9894 80 51 1 : tunables 0 0 0 : slabdata 194 194 0
inode_cache 2268 2268 592 27 4 : tunables 0 0 0 : slabdata 84 84 0
dentry 285907 286062 192 21 1 : tunables 0 0 0 : slabdata 13622 13622 0
buffer_head 256447 257472 112 36 1 : tunables 0 0 0 : slabdata 7152 7152 0
vm_area_struct 1469 1541 176 23 1 : tunables 0 0 0 : slabdata 67 67 0
mm_struct 82 95 832 19 4 : tunables 0 0 0 : slabdata 5 5 0
files_cache 104 161 704 23 4 : tunables 0 0 0 : slabdata 7 7 0
signal_cache 163 187 960 17 4 : tunables 0 0 0 : slabdata 11 11 0
sighand_cache 145 165 2112 15 8 : tunables 0 0 0 : slabdata 11 11 0
task_xstate 118 140 576 28 4 : tunables 0 0 0 : slabdata 5 5 0
task_struct 128 165 5808 5 8 : tunables 0 0 0 : slabdata 33 33 0
anon_vma 731 896 32 128 1 : tunables 0 0 0 : slabdata 7 7 0
shared_policy_node 85 85 48 85 1 : tunables 0 0 0 : slabdata 1 1 0
numa_policy 170 170 24 170 1 : tunables 0 0 0 : slabdata 1 1 0
idr_layer_cache 240 240 544 30 4 : tunables 0 0 0 : slabdata 8 8 0
kmalloc-8192 27 32 8192 4 8 : tunables 0 0 0 : slabdata 8 8 0
kmalloc-4096 291 344 4096 8 8 : tunables 0 0 0 : slabdata 43 43 0
kmalloc-2048 225 240 2048 16 8 : tunables 0 0 0 : slabdata 15 15 0
kmalloc-1024 366 432 1024 16 4 : tunables 0 0 0 : slabdata 27 27 0
kmalloc-512 536 544 512 16 2 : tunables 0 0 0 : slabdata 34 34 0
kmalloc-256 406 528 256 16 1 : tunables 0 0 0 : slabdata 33 33 0
kmalloc-128 503 576 128 32 1 : tunables 0 0 0 : slabdata 18 18 0
kmalloc-64 3467 3712 64 64 1 : tunables 0 0 0 : slabdata 58 58 0
kmalloc-32 1520 1920 32 128 1 : tunables 0 0 0 : slabdata 15 15 0
kmalloc-16 3547 3840 16 256 1 : tunables 0 0 0 : slabdata 15 15 0
kmalloc-8 4607 4608 8 512 1 : tunables 0 0 0 : slabdata 9 9 0
kmalloc-192 4620 5313 192 21 1 : tunables 0 0 0 : slabdata 253 253 0
kmalloc-96 1780 1848 96 42 1 : tunables 0 0 0 : slabdata 44 44 0
kmem_cache_node 0 0 64 64 1 : tunables 0 0 0 : slabdata 0 0 0
