Highload

Keepalive and buffers
cat /proc/sys/net/ipv4/tcp_keepalive_time
echo 60 > /proc/sys/net/ipv4/tcp_keepalive_time

cat /proc/sys/net/ipv4/tcp_fin_timeout

cat /proc/sys/net/ipv4/tcp_tw_recycle
echo 1 > /proc/sys/net/ipv4/tcp_tw_recycle

sysctl net.ipv4.tcp_tw_reuse=1

# Protect Against TCP Time-Wait
net.ipv4.tcp_rfc1337 = 1
# Decrease the time default value for tcp_fin_timeout connection
net.ipv4.tcp_fin_timeout = 15
# Decrease the time default value for connections to keep alive
net.ipv4.tcp_keepalive_time = 60
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.tcp_keepalive_intvl = 15

# Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks
net.ipv4.tcp_max_tw_buckets = 1440000
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1

# Increase number of incoming connections
net.core.somaxconn = 100000

# Increase number of incoming connections backlog
net.core.netdev_max_backlog = 65536

# Increase the maximum amount of option memory buffers
net.core.optmem_max = 25165824
SMP Affinity
#!/bin/bash 

ncpus=`grep -ciw ^processor /proc/cpuinfo`
test "$ncpus" -gt 1 || exit 1

n=0
for irq in `cat /proc/interrupts | grep eth | awk '{print $1}' | sed s/\://g`
do
    f="/proc/irq/$irq/smp_affinity"
    test -r "$f" || continue
    cpu=$[$ncpus - ($n % $ncpus) - 1]
    if [ $cpu -ge 0 ]
            then
                mask=`printf %x $[2 ** $cpu]`
                echo "Assign SMP affinity: eth$n, irq $irq, cpu $cpu, mask 0x$mask"
                echo "$mask" > "$f"
                let n+=1
    fi
done
Gateway tweaks
# sysctl
net.netfilter.nf_conntrack_tcp_timeout_established=86400
net.netfilter.nf_conntrack_max=1548576
net.nf_conntrack_max=1548576

net.ipv4.neigh.default.gc_thresh1 = 1024
net.ipv4.neigh.default.gc_thresh2 = 2048
net.ipv4.neigh.default.gc_thresh3 = 4096


# conntrack
yum install conntrack-tools
# diag
sysctl net.netfilter.nf_conntrack_count
conntrack -L
conntrack -L |awk '{if ($5 ~ /src/) print $5; else if ($4 ~ /src/) print $4}' | sed "s/src=/ /g" | sort | uniq -c | sort -n | tail -n15

etc
# disable conntrack where not needed
*raw 
-A PREROUTING -j NOTRACK 
COMMIT

# enhance queue
ifconfig eth0 txqueuelen 10000

# boost bufers
ethtool -G eth0 rx 1024

# if conntrack needed boost this
net.ipv4.netfilter.ip_conntrack_max 
/sys/module/ip_conntrack/parameters/hashsize 
# and decrease this 
net.ipv4.netfilter.ip_conntrack_icmp_timeout
net.ipv4.netfilter.ip_conntrack_udp_timeout_stream
net.ipv4.netfilter.ip_conntrack_udp_timeout
net.ipv4.netfilter.ip_conntrack_tcp_timeout_close
net.ipv4.netfilter.ip_conntrack_tcp_timeout_time_wait
net.ipv4.netfilter.ip_conntrack_tcp_timeout_last_ack
net.ipv4.netfilter.ip_conntrack_tcp_timeout_close_wait
net.ipv4.netfilter.ip_conntrack_tcp_timeout_fin_wait
net.ipv4.netfilter.ip_conntrack_tcp_timeout_established
net.ipv4.netfilter.ip_conntrack_tcp_timeout_syn_recv
net.ipv4.netfilter.ip_conntrack_tcp_timeout_syn_sent