nginx配置不好,并发性很差

笑天居士 发布于 2014/07/01 09:34
阅读 23K+
收藏 5
linux服务器:
16G内存,2个4核CPU

linux内核配置:

# /sbin/sysctl -p
net.ipv4.ip_forward = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
error: "net.bridge.bridge-nf-call-ip6tables" is an unknown key
error: "net.bridge.bridge-nf-call-iptables" is an unknown key
error: "net.bridge.bridge-nf-call-arptables" is an unknown key
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
net.ipv4.ip_forward = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
net.ipv4.tcp_max_tw_buckets = 6000
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_rmem = 4096        87380    4194304
net.ipv4.tcp_wmem = 4096        16384    4194304
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.core.netdev_max_backlog = 262144
net.core.somaxconn = 262144
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_fin_timeout = 1
net.ipv4.tcp_keepalive_time = 30
net.ipv4.ip_local_port_range = 1024    65000

nginx配置:
#user  nobody;
worker_processes  8;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;

worker_rlimit_nofile 102400;

events {
    use epoll;
    worker_connections  102400;
    multi_accept on;
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    server_names_hash_bucket_size 128;
    client_header_buffer_size 2k;
    large_client_header_buffers 4 4k;
    client_max_body_size 8m;

    open_file_cache max=204800 inactive=20s;
    open_file_cache_min_uses 1;
    open_file_cache_valid 30s;

    sendfile        on;
    tcp_nopush      on;
    tcp_nodelay     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    lua_package_path "/usr/local/lua-resty-gearman/lib/?.lua;;";

    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;
        location / {
            root   /home/jfy/web/;
            index  index.html index.htm;
        }

    }
}

在另一个机器(位于不同位置,通过一条专线连接),ping结果如下 :
PING 172.16.18.114 (172.16.18.114) 56(84) bytes of data.
64 bytes from 172.16.18.114: icmp_seq=0 ttl=64 time=6.52 ms
64 bytes from 172.16.18.114: icmp_seq=1 ttl=64 time=2.91 ms
64 bytes from 172.16.18.114: icmp_seq=2 ttl=64 time=2.92 ms

ab结果性能很差:
# ./ab -n 500000 -c 2000 http://172.16.18.114/index.html
This is ApacheBench, Version 2.3 <$Revision: 655654 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking 172.16.18.114 (be patient)
apr_socket_recv: Connection reset by peer (104)
Total of 5631 requests completed

并发2000都抗不住?为啥?

换一个内网的机器,在8个worker的情况下,2000并发也不行
./ab -n 500000 -c 2000 http://172.16.18.114/index.html
This is ApacheBench, Version 2.3 <$Revision: 655654 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking 172.16.18.114 (be patient)
Completed 50000 requests
Completed 100000 requests
Completed 150000 requests
Completed 200000 requests
Completed 250000 requests
Completed 300000 requests
Completed 350000 requests
Completed 400000 requests
Completed 450000 requests
apr_socket_recv: Connection reset by peer (104)
Total of 478366 requests completed

top查看CPU内核的负载很不均匀:
top - 10:13:18 up 18:02,  1 user,  load average: 0.00, 0.00, 0.00
Tasks: 250 total,   1 running, 249 sleeping,   0 stopped,   0 zombie
Cpu0  :  0.0%us,  0.0%sy,  0.0%ni, 89.6%id,  0.0%wa,  0.0%hi, 10.4%si,  0.0%st
Cpu1  :  7.1%us, 14.8%sy,  0.0%ni, 78.1%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Cpu2  :  4.0%us,  6.4%sy,  0.0%ni, 89.6%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Cpu3  :  0.0%us,  0.0%sy,  0.0%ni,100.0%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Cpu4  :  0.0%us,  0.3%sy,  0.0%ni, 99.0%id,  0.7%wa,  0.0%hi,  0.0%si,  0.0%st
Cpu5  :  5.0%us, 11.6%sy,  0.0%ni, 83.4%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Cpu6  :  1.3%us,  3.3%sy,  0.0%ni, 95.3%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Cpu7  :  1.7%us,  2.3%sy,  0.0%ni, 96.0%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Mem:  16424784k total,  1216772k used, 15208012k free,    85600k buffers
Swap:  8191992k total,        0k used,  8191992k free,   430128k cached

这是为什么呢?

应该怎样配置呢?初学nginx,按网上的方法进行配置的,是不是有其它参数要配置,请大神帮忙,谢谢!

几过几次折腾,发现设置成一个worker反而性能最高,但是并发2000还是有错误的请求,而且CPU和内存跑不满

加载中
0
Alex_Hark
Alex_Hark

哥  NGINX是单进程的你不知道吗   你开两千个woker来回切能块吗   每次上下文切换 和 进程本身都有资源开销  

肯定快不了   单核就开一个 worker 双核开两个worker

笑天居士
笑天居士
兄弟,应该说worker是单进程的,但是我的机器是8核的,起8个worker,每个worker绑定到一个核心上,这个8个worker一起处理accep请求是可以的 即使我开一个worker,CPU和内存也没跑满
0
南湖船老大
南湖船老大

引用来自“shiyanhk”的评论

哥  NGINX是单进程的你不知道吗   你开两千个woker来回切能块吗   每次上下文切换 和 进程本身都有资源开销  

肯定快不了   单核就开一个 worker 双核开两个worker

楼主就是8核啊。。。

另外 worker_cpu_affinity,这个不建议设置,这是个坑。尤其是如果你用的tengine,它有这个东西,是因为这是针对淘宝的硬件特别优化的,其他人不能这么来

0
笑天居士
笑天居士

就以一个worker来算,8核心CPU,16G内存,并发2000都有失败的情况,那还谈何nginx高并发和C10K

一定是配置方面有问题

0
匿名t3a
匿名t3a

net.ipv4.tcp_syncookies = 0

禁用Syncookies 试试,因为在高并发的情况下,内核会认为系统受到了SYN flood攻击,会发送cookies(possible SYN flooding on port 80. Sending cookies),这样会减慢影响请求的速度。

0
eechen
eechen
并发写100,然后分别在本地和远程执行ab进行压力测试,观察每秒请求数(RPS)和每秒传输数据量(吞吐量):
ab -c100 -n10000 http://127.0.0.1/index.html
ab -c100 -n10000 http://172.16.18.114/index.html

其中远程使用ab压力测试可能受限于网络设备的吞吐量,Nginx的RPS和吞吐量可能都不会太高.

另外Linux上CPU亲缘性绑定是完全可用的,比如taskset命令就可以绑定指定进程到对应的CPU核心.Nginx提供了更便捷的方式,在配置里提供了指令worker_cpu_affinity用于进行绑定.

0
许鹏
许鹏

apr_socket_recv: Connection reset by peer (104)

这个是 ab 开不了那么多并发数吧

0
kleen
kleen

引用来自“许鹏”的评论

apr_socket_recv: Connection reset by peer (104)

这个是 ab 开不了那么多并发数吧

哪里配置错误了吧。仔细检查检查
kleen
kleen
这问题问题很纠结呢
0
二的基本算合格
二的基本算合格

首先,你检测一下

ulimit -a



是不是open files的数量不够,如果没问题,检查你的内核参数,用我这个试试:

net.ipv4.ip_local_port_range = 1024 65000
net.core.rmem_max=16777216
net.core.wmem_max=16777216
net.ipv4.tcp_rmem=4096 87380 16777216
net.ipv4.tcp_wmem=4096 65536 16777216
net.ipv4.tcp_fin_timeout = 10
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_window_scaling = 0
net.ipv4.tcp_sack = 0
net.core.netdev_max_backlog = 30000
net.ipv4.tcp_no_metrics_save=1
net.core.somaxconn = 262144
net.ipv4.tcp_syncookies = 0
net.ipv4.tcp_max_orphans = 262144
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_syn_retries = 2



你这个硬件配置,Nginx随便配置一下能近10万的TPS,不可能性能这么渣的。。。

0
笑天居士
笑天居士

[root@localhost logs]# /sbin/sysctl -p
net.ipv4.ip_forward = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 0
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
net.ipv4.ip_local_port_range = 1024 65000
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216
net.ipv4.tcp_fin_timeout = 10
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_window_scaling = 0
net.ipv4.tcp_sack = 0
net.core.netdev_max_backlog = 30000
net.ipv4.tcp_no_metrics_save = 1
net.core.somaxconn = 262144
net.ipv4.tcp_syncookies = 0
net.ipv4.tcp_max_orphans = 262144
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_syn_retries = 2

[root@localhost logs]# ulimit -n
102400

[root@localhost logs]# ../sbin/nginx -s stop
[root@localhost logs]# ../sbin/nginx
[root@localhost logs]# ps -ef|grep nginx
root     59707     1  0 07:22 ?        00:00:00 nginx: master process ../sbin/nginx
nobody   59708 59707  0 07:22 ?        00:00:00 nginx: worker process



ab测试结果:

Server Software:        nginx/1.7.2
Server Hostname:        172.16.18.114
Server Port:            80


Document Path:          /index.html
Document Length:        61 bytes


Concurrency Level:      2000
Time taken for tests:   81.722 seconds
Complete requests:      500000
Failed requests:        0
Write errors:           0
Keep-Alive requests:    496000
Total transferred:      147980000 bytes
HTML transferred:       30500000 bytes
Requests per second:    6118.34 [#/sec] (mean)
Time per request:       326.886 [ms] (mean)
Time per request:       0.163 [ms] (mean, across all concurrent requests)
Transfer rate:          1768.34 [Kbytes/sec] received

还是一样差,为啥呢?

eechen
eechen
我在本地测试,i5-3230M的笔记本CPU,64位Ubuntu 14.04,默认sysctl配置,没有调优,RPS都差不多20K,你双路四核心的RPS才6K多,肯定有问题.
OSCHINA
登录后可查看更多优质内容
返回顶部
顶部