nginx日志每日生成单个文件
nginx日志分割是nginx本身自带，不需重新编译。
直接在server中添加即可。
log_format main '$remote_addr $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '$http_user_agent $http_x_forwarded_for $request_time $upstream_response_time $upstream_addr $upstream_status'; #定义一个日志格式main，此main可自己随机定义，必须与一下main一致 if ($time_iso8601 ~ "^(\d{4})-(\d{2})-(\d{2})") { set $year $1; set $month $2; set $day $3; } #用于日志分割时间：年月日 access_log /var/logs/xxxx/access/xxxxx_xx_access_$year-$month-$day.log main; #日志存放位置及生成名称。 main名称与日志格式名称一致。log_format main'$remote_addr $remote_user [$time_local] "$request" ''$status $body_bytes_sent "$http_referer" ''$http_user_agent $http_x_forwarded_for $request_time $upstream_response_time $upstream_addr $upstream_status';# 从 ISO8601 时间提取年-月-日map$time_iso8601$log_date{"~^(\d{4})-(\d{2})-(\d{2})"$1-$2-$3;}access_log /usr/local/nginx/nginx/logs/access_$log_date.log main;nginx js/css…开启文件缓存
location ~ .*\.(js|css|ico|gif|jpg|jpeg|png|bmp|swf)$ { expires 7d; access_log off; proxy_pass http://wwwserv; #如果图片和网站放在一起，必须加这个转向 } location ~ .*\.(gif|jpg|jpeg|png|bmp|swf)$ location ~匹配的意思以后面结尾的缓存30不记录 {undefined expires 30d; access_log off; } location ~ .*\.(js|css)$ js/css结尾缓存12个小时并且不记录访问日志 {undefined expires 12h; access_log off; }日志分割命令：
split -l 行数 -b 大小
日志分析过滤 ：
$7 “页面索引” $9 状态码 $10 流量 $11 url
***访问量：*** 统计ip前十： awk '{print $1}' xaa |sort |uniq -c |sort -nr | head -10 awk '{a[$1]++}END{for(i in a)print a[i],i|"sort -k1 -nr|head -n10"}' access.log 统计单日： grep "05/Nov/2019" xaa |awk '{print $1}' |sort |uniq -c |sort -nr | head -10 统计某个特定时间之后：awk '($4>"05/Nov/2019"){print $1}' xaa |sort |uniq -c |sort -nr | head -10 小时单位里ip连接数最多的10个时段：awk -vFS="[:]" '{gsub("-.*","",$1);num[$2" "$1]++}END{for(i in num)print i,num[i]}' log_file | sort -n -k 3 -r | head -10 统计访问最多的前10 awk '{a[$7]++}END{for(i in a)print a[i],i|"sort -k1 -nr|head -n10"}' access.log ***流量：*** 统计单日流量： grep "05/Nov/2019" xaa | awk '{sum+=$10}END{print sum}' 统计网站流量（G) ：cat access.log |awk '{sum+=$10} END {print sum/1024/1024/1024}' ***资源：*** 指定资源统计： awk '($7~/\.html$/){print $1 " " $7 " " $9}' xaa##第七列已html结尾的 将每个IP访问的页面数进行从小到大排序：awk '{++S[$1]} END {for (a in S) print S[a],a}' log_file | sort -n ***状态码*** 按状态码： awk '($9>"200"){print $1 " " $7 " " $9}' xaa 统计状态码： awk '{print $9}' xaa |sort |uniq -c |sort -nr | head -10 统计访问状态码为404的IP及出现次数 awk '{if($9~/404/)a[$1" "$9]++}END{for(i in a)print i,a[i]}' access.lognginx配合时间任务、自动封堵ip shell
crontab -e * * * * * cd /usr/local/nginx/logs/ && sh ip_test.sh 每一分钟运行一次#不能把别人IP一直封着吧，这里就清除掉了 echo""> /usr/local/nginx/conf/blockip.conf #前面最开始编写的统计数据功能 ip_list=$(awk'{print $1}' access.log | sort | uniq -cd | awk '{if($1>60)print $0}')#判断这个变量是否为空 if test -z"$ip_list"then #为空写入 11.log中，并重新启动ngnix echo"为空">> /usr/local/nginx/logs/11.log /usr/local/nginx/sbin/nginx -s reload else #如果不为空 前面加上 deny格式和ip写入blockip.conf中 echo"deny"$ip_list > /usr/local/nginx/conf/blockip.conf #因为前面携带了行数，所有我们需要去除掉前面的行数，写入后在读取一次 ip_list2=$(awk'{print $3}' /usr/local/nginx/conf/blockip.conf)#最后再把读取出来的值，在次写入到blockip.conf中 echo"deny"$ip_list2";"> /usr/local/nginx/conf/blockip.conf #重启ngnix /usr/local/nginx/sbin/nginx -s reload #清空之前的日志，从最新的开始截取 echo""> /usr/local/nginx/logs/access.log fi日志json格式
log_format json_analytics escape=json '{' '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution '"connection": "$connection", ' # connection serial number '"connection_requests": "$connection_requests", ' # number of requests made in connection '"pid": "$pid", ' # process pid '"request_id": "$request_id", ' # the unique request id '"request_length": "$request_length", ' # request length (including headers and body) '"remote_addr": "$remote_addr", ' # client IP '"remote_user": "$remote_user", ' # client HTTP username '"remote_port": "$remote_port", ' # client port '"time_local": "$time_local", ' '"time_iso8601": "$time_iso8601", ' # local time in the ISO 8601 standard format '"request": "$request", ' # full path no arguments if the request '"request_uri": "$request_uri", ' # full path and arguments if the request '"args": "$args", ' # args '"status": "$status", ' # response status code '"body_bytes_sent": "$body_bytes_sent", ' # the number of body bytes exclude headers sent to a client '"bytes_sent": "$bytes_sent", ' # the number of bytes sent to a client '"http_referer": "$http_referer", ' # HTTP referer '"http_user_agent": "$http_user_agent", ' # user agent '"http_x_forwarded_for": "$http_x_forwarded_for", ' # http_x_forwarded_for '"http_host": "$http_host", ' # the request Host: header '"server_name": "$server_name", ' # the name of the vhost serving the request '"request_time": "$request_time", ' # request processing time in seconds with msec resolution '"upstream": "$upstream_addr", ' # upstream backend server for proxied requests '"upstream_connect_time": "$upstream_connect_time", ' # upstream handshake time incl. TLS '"upstream_header_time": "$upstream_header_time", ' # time spent receiving upstream headers '"upstream_response_time": "$upstream_response_time", ' # time spend receiving upstream body '"upstream_response_length": "$upstream_response_length", ' # upstream response length '"upstream_cache_status": "$upstream_cache_status", ' # cache HIT/MISS where applicable '"ssl_protocol": "$ssl_protocol", ' # TLS protocol '"ssl_cipher": "$ssl_cipher", ' # TLS cipher '"scheme": "$scheme", ' # http or https '"request_method": "$request_method", ' # request method '"server_protocol": "$server_protocol", ' # request protocol, like HTTP/1.1 or HTTP/2.0 '"pipe": "$pipe", ' # "p" if request was pipelined, "." otherwise '"gzip_ratio": "$gzip_ratio", ' '"http_cf_ray": "$http_cf_ray",' '"geoip_country_code": "$geoip_country_code"' '}';json格式简化版
log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"' '$request_time $upstream_addr $upstream_response_time'; log_format json '{"@timestamp":"$time_iso8601",' '"clientip":"$remote_addr",' '"clientip2":"$proxy_protocol_addr",' '"responsetime":$request_time,' '"upstreamtime":"$upstream_response_time",' '"upstreamhost":"$upstream_addr",' '"domain":"$host",' '"url":"$uri",' '"referer":"$http_referer",' '"status":"$status"}'; access_log /var/log/nginx/access.log json;