Differences

This shows you the differences between two versions of the page.

--- log_hits [2016/04/04 08:12] – luke7858
+++ log_hits [2024/05/23 07:26] (current) – external edit 127.0.0.1
@@ Line 1: / Line 1: @@
 === Summarising custom logs ===
+The following uses lsof to check all open log files by the web server, (access and error logs).
+\\
+You will need to run this command first to save the logs files as LOGS:
 <sxh bash>
 LOGS=$(lsof -ln | awk '$4 ~ /[0-9]w/ && $5 ~ /REG/ {FILE[$NF]++}END{for (i in FILE) print i}')
 </sxh>
 \\
+=== Browser and robot.txt check ===
+Now you can run the following command to receive an output:
 <sxh bash>
 for log in $(echo "$LOGS" | grep access); do HIT_COUNT=$(grep $(date "+%d/%b/%Y") $log -c); if [[ "$HIT_COUNT" -ge 100 ]]; then echo -e "\n$log - $HIT_COUNT total hits today\n"; grep $(date "+%d/%b/%Y") $log | awk -F \" '{USER[$(NF-1)]++}END{for (i in USER) print USER[i],i}' | sort -n | tail -10 ; fi; done
 </sxh>
-\\
 \\
 Example Output:
@@ Line 27: / Line 30: @@
 \\
-\\
+=== IP Check ===
 <sxh bash>
 for log in $(echo "$LOGS" | grep access); do HIT_COUNT=$(grep $(date "+%d/%b/%Y") $log -c); if [[ "$HIT_COUNT" -ge 100 ]]; then echo -e "\n$log - $HIT_COUNT total hits today\n"; grep $(date "+%d/%b/%Y") $log | awk '{ if ( $2 !~ /^[0-9]/ ) REQ[$1" "$6" "$7]++; if ( $2 ~ /^[0-9]/ ) REQ[$1" "$2" "$7" "$8]++}END{for (i in REQ) print REQ[i],i}' | sort -n | tail -10 ; fi; done
@@ Line 47: / Line 50: @@
 113.190.128.197 "POST /xmlrpc.php
 117.4.251.108 "POST /xmlrpc.php
+</sxh>
+\\
+=== Finding Crawlers ===
+**Note:** You will need to change date range and make sure you run the very first LOG command on this page first.
+<sxh bash>
+LC_ALL=C awk '/21\/Jul\/2016:20:4/ {REQ[FILENAME" "substr($0,index($0,$12))]++}END{for (i in REQ) print REQ[i],i}' $(echo "$LOGS" | grep access) | sort -rn | egrep -i "bot|crawl|spider|slurp" | head -25
+</sxh>
+\\
+=== Accurate number of Apache requests per hour ===
+**Note:** Change date range and log file locationg
+<sxh bash>
+LC_ALL=C awk '/02\/Aug\/2016/ && $0 !~ /(.js|.png|.jpg|.css|.ico) HTTP|.*Monitoring/' /var/log/nginx/exampledomain.com | cut -d[ -f2 | cut -d] -f1 | awk -F: '{print $2":00"}' | sort -n | uniq -c
+</sxh>
+\\
+=== Log requests for Specific date/time range ===
+**Note:** Change date/time and log file location
+<sxh bash>
+LC_ALL=C awk -F \" '/09\/Jun\/2016:(10:[12345]|11:[01])/ && $0 !~ /(.js|.png|.jpg|.css|.ico) HTTP/ {REQ[$2]++}END{for (i in REQ) print REQ[i],i}' /var/log/httpd/lexampledomain.com | sort -rn | head -50
 </sxh>