nginx从外部机压测tps逐渐下降,从nginx本机上压测就一直很稳定

乱注册 发布于 2017/08/14 09:48
阅读 918
收藏 0

操作系统:CentOS Linux release 7.2.1511 (Core) 

CPU:Intel Xeon E312xx (Sandy Bridge) 8核

内存:16G

我这边在linux上搭了一套nginx环境,对我们的web服务进行压力测试。在nginx本机上测试的时候,tps一直比较稳定,但是在内网上的另一台机器上测试的时候,tps却随着时间的加长一直往下掉,压力测试工具用的是Apache Jmeter.外机和本机机器硬件配置和操作系统都是一样的。

linux内核配置:

fs.file-max = 999999
net.ipv4.tcp_fin_timeout = 30
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.rp_filter = 1
net.ipv4.tcp_syncookies = 0
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_keepalive_time = 300
net.core.netdev_max_backlog = 8096
net.ipv4.ip_local_port_range = 1024 65500
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_max_syn_backlog = 3240000
net.core.somaxconn = 65535
net.ipv4.tcp_max_tw_buckets = 6000
net.ipv4.tcp_max_orphans = 262144
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216
net.ipv4.tcp_congestion_control = cubic

nginx.conf配置:

user  root;
worker_rlimit_nofile 409600;
worker_processes 8;
error_log  logs/error.log;
events {
    use epoll;
    worker_connections  204800;
    multi_accept on;
    accept_mutex off;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                     '$upstream_addr $upstream_response_time $request_time '
                     '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  logs/access.log  main;
    #access_log off;
    #error_log logs/error.log crit;
    sendfile        on;
    tcp_nopush     on;
    server_tokens off;
    tcp_nodelay on;
    keepalive_timeout  300;
    keepalive_requests 200000;
    client_body_timeout 3;
    client_header_timeout 3;
    send_timeout 3;     
    gzip        on;
    gzip_min_length 1k;
    gzip_http_version 1.0;
    gzip_buffers 30  64k;
    gzip_comp_level 9;
    gzip_proxied expired no-cache no-store private auth;
    gzip_types text/plain text/css text/javascript application/json application/x-javascript text/xml application/xml;
    gzip_vary on;
        
    proxy_temp_path /tmp/temp_dir;
    proxy_cache_path /tmp/cache levels=1:2 keys_zone=cache_one:10m inactive=1d max_size=1500M;
    upstream test2 {
       sticky;
        server 19.14.132.94:80 max_fails=2 fail_timeout=3s;
        server 19.14.132.95:80 max_fails=2 fail_timeout=3s;
        server 19.14.132.96:80 max_fails=2 fail_timeout=3s;
        keepalive 2000;
    }
    upstream test1 {
        sticky;
        server 19.14.132.91:9001;
        server 19.14.132.93:9001;
        check interval=3000 rise=2 fall=3 timeout=30000 type=http;
        check_http_send "GET / HTTP/1.0\r\nconnection: keep-alive\r\n\r\n";
        check_http_expect_alive http_2xx http_3xx http_4xx;
    }
    server {
        listen       80 reuseport;
        server_name  localhost;
        charset utf-8;
        client_max_body_size    30m;
        client_body_buffer_size 2048k;
        proxy_connect_timeout   30;
        proxy_send_timeout      3;
        proxy_read_timeout      3;
        proxy_buffer_size       64k;
        proxy_buffers           12 64k;
        proxy_busy_buffers_size 64k;
        proxy_temp_file_write_size 128k;
        proxy_max_temp_file_size 30m;
        proxy_intercept_errors off;
        proxy_ignore_client_abort on;
        add_header X-Cache $upstream_cache_status;  
        location ~* "\.(jpg|jpeg|png|gif|html|css|js)$" {  
               proxy_pass http://test2;  
               proxy_redirect off;  
               proxy_set_header Host $host;  
               proxy_cache cache_one;  
               proxy_cache_valid 200 302 1h;  
               proxy_cache_valid 301 1d;  
               proxy_cache_valid any 1d;  
               expires 1d;  
       }  
       location / {
                proxy_set_header    X-Real-IP           $remote_addr;
                proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
                proxy_set_header    Host                $http_host;
                proxy_set_header    X-NginX-Proxy       true;
                proxy_set_header    Connection          "";
                proxy_set_header X-Forwarded-Server $host;
                proxy_set_header X-Forwarded-Host $host;
                proxy_http_version  1.1;
                proxy_set_header Connection "";
                proxy_pass http://test2;
        }
        
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }
}

nginx本机压测结果如下:
Creating summariser <summary>
Created the tree successfully using app_index.jmx
Starting the test @ Mon Aug 14 09:17:25 CST 2017 (1502673445948)
Waiting for possible Shutdown/StopTestNow/Heapdump message on port 4445
summary +  18585 in 00:00:04 = 4709.8/s Avg:     3 Min:     0 Max:   165 Err:     0 (0.00%) Active: 32 Started: 32 Finished: 0
summary + 374702 in 00:00:30 = 12490.1/s Avg:    12 Min:     0 Max:   343 Err:     0 (0.00%) Active: 280 Started: 280 Finished: 0
summary = 393287 in 00:00:34 = 11585.7/s Avg:    11 Min:     0 Max:   343 Err:     0 (0.00%)
summary + 371924 in 00:00:30 = 12397.5/s Avg:    31 Min:     0 Max:   515 Err:     0 (0.00%) Active: 521 Started: 521 Finished: 0
summary = 765211 in 00:01:04 = 11966.5/s Avg:    21 Min:     0 Max:   515 Err:     0 (0.00%)
summary + 363929 in 00:00:30 = 12131.0/s Avg:    49 Min:     0 Max:   691 Err:     0 (0.00%) Active: 749 Started: 749 Finished: 0
summary = 1129140 in 00:01:34 = 12019.0/s Avg:    30 Min:     0 Max:   691 Err:     0 (0.00%)
summary + 347118 in 00:00:30 = 11570.6/s Avg:    70 Min:     0 Max:  1088 Err:     0 (0.00%) Active: 987 Started: 987 Finished: 0
summary = 1476258 in 00:02:04 = 11910.5/s Avg:    39 Min:     0 Max:  1088 Err:     0 (0.00%)
summary + 364144 in 00:00:30 = 12138.1/s Avg:    75 Min:     0 Max:   986 Err:     0 (0.00%) Active: 1000 Started: 1000 Finished: 0
summary = 1840402 in 00:02:34 = 11954.9/s Avg:    46 Min:     0 Max:  1088 Err:     0 (0.00%)
summary + 367700 in 00:00:30 = 12256.7/s Avg:    77 Min:     0 Max:   736 Err:     0 (0.00%) Active: 1000 Started: 1000 Finished: 0
summary = 2208102 in 00:03:04 = 12004.1/s Avg:    51 Min:     0 Max:  1088 Err:     0 (0.00%)
summary + 363088 in 00:00:30 = 12102.9/s Avg:    77 Min:     0 Max:  1045 Err:     0 (0.00%) Active: 1000 Started: 1000 Finished: 0
summary = 2571190 in 00:03:34 = 12017.9/s Avg:    55 Min:     0 Max:  1088 Err:     0 (0.00%)
summary + 366629 in 00:00:30 = 12221.0/s Avg:    78 Min:     0 Max:  1139 Err:     0 (0.00%) Active: 1000 Started: 1000 Finished: 0
summary = 2937819 in 00:04:04 = 12042.9/s Avg:    58 Min:     0 Max:  1139 Err:     0 (0.00%)
summary + 370899 in 00:00:30 = 12363.3/s Avg:    76 Min:     0 Max:   881 Err:     0 (0.00%) Active: 1000 Started: 1000 Finished: 0
summary = 3308718 in 00:04:34 = 12078.0/s Avg:    60 Min:     0 Max:  1139 Err:     0 (0.00%)
summary + 316285 in 00:00:26 = 11975.5/s Avg:    75 Min:     0 Max:   943 Err:     0 (0.00%) Active: 0 Started: 1000 Finished: 1000
summary = 3625003 in 00:05:00 = 12069.0/s Avg:    61 Min:     0 Max:  1139 Err:     0 (0.00%)
Tidying up ...    @ Mon Aug 14 09:22:26 CST 2017 (1502673746411)
... end of run

外机是一台内网机器,ping值如下:

PING 19.14.132.97 (19.14.132.97) 56(84) bytes of data.
64 bytes from 19.14.132.97: icmp_seq=1 ttl=64 time=0.748 ms
64 bytes from 19.14.132.97: icmp_seq=2 ttl=64 time=0.402 ms
64 bytes from 19.14.132.97: icmp_seq=3 ttl=64 time=0.260 ms
64 bytes from 19.14.132.97: icmp_seq=4 ttl=64 time=0.306 ms
64 bytes from 19.14.132.97: icmp_seq=5 ttl=64 time=0.276 ms
^C
--- 19.14.132.97 ping statistics ---
5 packets transmitted, 5 received, 0% packet loss, time 4000ms
rtt min/avg/max/mdev = 0.260/0.398/0.748/0.182 ms

外机测试结果如下:

Creating summariser <summary>
Created the tree successfully using app_index.jmx
Starting the test @ Mon Aug 14 09:36:24 CST 2017 (1502674584119)
Waiting for possible Shutdown/StopTestNow/Heapdump message on port 4445
summary +  45958 in 00:00:06 = 7955.3/s Avg:     2 Min:     0 Max:   169 Err:     0 (0.00%) Active: 47 Started: 47 Finished: 0
summary + 575562 in 00:00:30 = 19185.4/s Avg:     8 Min:     0 Max:   304 Err:     0 (0.00%) Active: 295 Started: 295 Finished: 0
summary = 621520 in 00:00:36 = 17372.1/s Avg:     8 Min:     0 Max:   304 Err:     0 (0.00%)
summary + 456777 in 00:00:30 = 15225.9/s Avg:    26 Min:     0 Max:  2456 Err:     0 (0.00%) Active: 543 Started: 543 Finished: 0
summary = 1078297 in 00:01:06 = 16393.2/s Avg:    16 Min:     0 Max:  2456 Err:     0 (0.00%)
summary + 290589 in 00:00:30 = 9686.3/s Avg:    67 Min:     0 Max:  3656 Err:     0 (0.00%) Active: 791 Started: 791 Finished: 0
summary = 1368886 in 00:01:36 = 14292.4/s Avg:    27 Min:     0 Max:  3656 Err:     0 (0.00%)
summary + 277274 in 00:00:30 = 9242.5/s Avg:    98 Min:     0 Max:  7675 Err:     0 (0.00%) Active: 1000 Started: 1000 Finished: 0
summary = 1646160 in 00:02:06 = 13087.9/s Avg:    39 Min:     0 Max:  7675 Err:     0 (0.00%)
summary + 270881 in 00:00:30 = 9029.4/s Avg:   110 Min:     0 Max:  7289 Err:     1 (0.00%) Active: 1000 Started: 1000 Finished: 0
summary = 1917041 in 00:02:36 = 12306.3/s Avg:    49 Min:     0 Max:  7675 Err:     1 (0.00%)
summary + 260875 in 00:00:30 = 8695.8/s Avg:   114 Min:     0 Max:  7286 Err:     0 (0.00%) Active: 1000 Started: 1000 Finished: 0
summary = 2177916 in 00:03:06 = 11723.3/s Avg:    57 Min:     0 Max:  7675 Err:     1 (0.00%)
summary + 253937 in 00:00:30 = 8464.3/s Avg:   115 Min:     0 Max: 15100 Err:     1 (0.00%) Active: 1000 Started: 1000 Finished: 0
summary = 2431853 in 00:03:36 = 11270.2/s Avg:    63 Min:     0 Max: 15100 Err:     2 (0.00%)
summary + 261147 in 00:00:30 = 8705.2/s Avg:   115 Min:     0 Max:  7804 Err:     4 (0.00%) Active: 1000 Started: 1000 Finished: 0
summary = 2693000 in 00:04:06 = 10957.1/s Avg:    68 Min:     0 Max: 15100 Err:     6 (0.00%)
summary + 277159 in 00:00:30 = 9238.6/s Avg:   107 Min:     0 Max:  7458 Err:     1 (0.00%) Active: 1000 Started: 1000 Finished: 0
summary = 2970159 in 00:04:36 = 10770.1/s Avg:    71 Min:     0 Max: 15100 Err:     7 (0.00%)
summary + 191763 in 00:00:28 = 6781.3/s Avg:   127 Min:     0 Max: 17832 Err:    10 (0.01%) Active: 0 Started: 1000 Finished: 1000
summary = 3161922 in 00:05:04 = 10399.1/s Avg:    75 Min:     0 Max: 17832 Err:    17 (0.00%)
Tidying up ...    @ Mon Aug 14 09:41:28 CST 2017 (1502674888279)
... end of run

不知道为什么会这样,还请高手指点一下

加载中
0
C
CallMain
是不是局域网的网络问题
乱注册
确实是网络的问题。应该是带宽不够,最后去掉几张大一点的图片,tps就上来了。但是有点疑惑的是当时我用iperf测试带宽的时候却发现带宽是远远超过压测时的流量值的。
返回顶部
顶部