Differences

This shows you the differences between two versions of the page.

--- log_hits [2016/03/10 11:28] – luke7858
+++ log_hits [2024/05/23 07:26] (current) – external edit 127.0.0.1
@@ Line 1: / Line 1: @@
 === Summarising custom logs ===
+The following uses lsof to check all open log files by the web server, (access and error logs).
+\\
+You will need to run this command first to save the logs files as LOGS:
 <sxh bash>
 LOGS=$(lsof -ln | awk '$4 ~ /[0-9]w/ && $5 ~ /REG/ {FILE[$NF]++}END{for (i in FILE) print i}')
 </sxh>
 \\
+=== Browser and robot.txt check ===
+Now you can run the following command to receive an output:
 <sxh bash>
 for log in $(echo "$LOGS" | grep access); do HIT_COUNT=$(grep $(date "+%d/%b/%Y") $log -c); if [[ "$HIT_COUNT" -ge 100 ]]; then echo -e "\n$log - $HIT_COUNT total hits today\n"; grep $(date "+%d/%b/%Y") $log | awk -F \" '{USER[$(NF-1)]++}END{for (i in USER) print USER[i],i}' | sort -n | tail -10 ; fi; done
 </sxh>
-\\
 \\
 Example Output:
@@ Line 24: / Line 27: @@
 Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
 Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2
+</sxh>
+\\
+=== IP Check ===
+<sxh bash>
+for log in $(echo "$LOGS" | grep access); do HIT_COUNT=$(grep $(date "+%d/%b/%Y") $log -c); if [[ "$HIT_COUNT" -ge 100 ]]; then echo -e "\n$log - $HIT_COUNT total hits today\n"; grep $(date "+%d/%b/%Y") $log | awk '{ if ( $2 !~ /^[0-9]/ ) REQ[$1" "$6" "$7]++; if ( $2 ~ /^[0-9]/ ) REQ[$1" "$2" "$7" "$8]++}END{for (i in REQ) print REQ[i],i}' | sort -n | tail -10 ; fi; done
+</sxh>
+\\
+Example output:
+<sxh bash>
+/var/log/nginx/lukeslinuxlessons.co.uk.access.log - 200 total hits today
+180.76.15.157 "GET /
+66.249.78.114 "GET /robots.txt
+66.249.78.121 "POST /wp-admin/admin-ajax.php
+88.119.179.121 "GET /wp-admin/admin-ajax.php?action=revslider_show_image&img=../wp-config.php
+180.76.15.134 "GET /
+94.236.7.190 "GET /favicon.ico
+94.236.7.190 "GET /logrotate/
+94.236.7.190 "POST /wp-admin/admin-ajax.php
+113.190.128.197 "POST /xmlrpc.php
+117.4.251.108 "POST /xmlrpc.php
+</sxh>
+\\
+=== Finding Crawlers ===
+**Note:** You will need to change date range and make sure you run the very first LOG command on this page first.
+<sxh bash>
+LC_ALL=C awk '/21\/Jul\/2016:20:4/ {REQ[FILENAME" "substr($0,index($0,$12))]++}END{for (i in REQ) print REQ[i],i}' $(echo "$LOGS" | grep access) | sort -rn | egrep -i "bot|crawl|spider|slurp" | head -25
+</sxh>
+\\
+=== Accurate number of Apache requests per hour ===
+**Note:** Change date range and log file locationg
+<sxh bash>
+LC_ALL=C awk '/02\/Aug\/2016/ && $0 !~ /(.js|.png|.jpg|.css|.ico) HTTP|.*Monitoring/' /var/log/nginx/exampledomain.com | cut -d[ -f2 | cut -d] -f1 | awk -F: '{print $2":00"}' | sort -n | uniq -c
+</sxh>
+\\
+=== Log requests for Specific date/time range ===
+**Note:** Change date/time and log file location
+<sxh bash>
+LC_ALL=C awk -F \" '/09\/Jun\/2016:(10:[12345]|11:[01])/ && $0 !~ /(.js|.png|.jpg|.css|.ico) HTTP/ {REQ[$2]++}END{for (i in REQ) print REQ[i],i}' /var/log/httpd/lexampledomain.com | sort -rn | head -50
 </sxh>