File: //bigscoots/wpo/nginx/trafficstats.sh
#!/bin/bash
source /bigscoots/includes/common.sh
PATH=/usr/lib64/ccache:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin:/root/bin
if [ -z "$2" ]
then
DOMAINS=( $(ls -1p /home/nginx/domains/| grep -v bigscoots-staging.com | grep / | sed 's/^\(.*\)\//\1/') )
else
DOMAINS=$2
fi
FINAL_OUTPUT="{}"
if ! VISITOR_HASH=$(curl -s https://www.bigscoots.com/downloads/uniquevisithash 2>/dev/null)
then
bash /bigscoots/general/slack.sh "#wpo-errors" ":warning: $(hostname) - ${serverip} - ${DOMAIN} - Failed to pull the unique visitors hash has using CURL."
exit 1
fi
for DOMAIN in ${DOMAINS[@]}
do
validate_domain_in_path "$DOMAIN" trafficstats.sh
if [ "$1" = "today" ]; then
TODAYSLASH=$(date +"%d/%b/%Y")
YESTERDAYDASH=$(date --date="yesterday" +"%m-%d-%Y")
DOMAIN_DATA="[]" # Reset domain data array for each domain
# Find today's log files
TODAYLOGS=$(find /home/nginx/domains/"${DOMAIN}"/log/ -mtime -1 -name 'access.log*' 2>/dev/null | tr '\n' ' ')
if [ -z "$TODAYLOGS" ]; then
# No logs found for today
DAY_DATA=$(echo '{ "date": "'"$TODAYSLASH"'", "unique_visitors": "0", "verified_bots": "0", "bad_bot_requests_blocked": "0", "malicious_requests_blocked": "0" }')
DOMAIN_DATA=$(echo "$DOMAIN_DATA" | jq --argjson day "$DAY_DATA" '. += [$day]')
else
if [ -f /root/.bigscoots/nginx/includes/abuserbl_block.list ]; then
BLOCKLIST_IPS=$(grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' /root/.bigscoots/nginx/includes/abuserbl_block.list)
fi
if zgrep -h "$TODAYSLASH" ${TODAYLOGS} 2>/dev/null \
| grep -v "$serverip" \
| grep -v -f /bigscoots/wpo/nginx/traffic/verified.bots \
| goaccess \
--log-format=COMBINED \
--enable-panel=VISITORS \
--ignore-status=403 \
--ignore-panel=REQUESTS \
--ignore-panel=REQUESTS_STATIC \
--ignore-panel=NOT_FOUND \
--ignore-panel=HOSTS \
--ignore-panel=OS \
--ignore-panel=BROWSERS \
--ignore-panel=VISIT_TIMES \
--ignore-panel=VIRTUAL_HOSTS \
--ignore-panel=REFERRERS \
--ignore-panel=REFERRING_SITES \
--ignore-panel=KEYPHRASES \
--ignore-panel=STATUS_CODES \
--ignore-panel=REMOTE_USER \
--ignore-panel=CACHE_STATUS \
--ignore-panel=GEO_LOCATION \
--ignore-panel=MIME_TYPE \
--ignore-panel=TLS_TYPE \
--no-column-names \
--no-progress \
--no-parsing-spinner \
- -o /root/.bigscoots/counters/uniquevisitors.json; then
UNIQUEVISITS=$(grep -o "\"unique_visitors\": [0-9].*" /root/.bigscoots/counters/uniquevisitors.json | cut -d ',' -f1 | awk '{print $2}')
VERIFIEDBOTS=$(zgrep -h "$TODAYSLASH" ${TODAYLOGS} 2>/dev/null \
| grep -v "$serverip" \
| grep -f /bigscoots/wpo/nginx/traffic/verified.bots \
| goaccess \
--log-format=COMBINED \
--enable-panel=VISITORS \
--ignore-status=403 \
--ignore-panel=REQUESTS \
--ignore-panel=REQUESTS_STATIC \
--ignore-panel=NOT_FOUND \
--ignore-panel=HOSTS \
--ignore-panel=OS \
--ignore-panel=BROWSERS \
--ignore-panel=VISIT_TIMES \
--ignore-panel=VIRTUAL_HOSTS \
--ignore-panel=REFERRERS \
--ignore-panel=REFERRING_SITES \
--ignore-panel=KEYPHRASES \
--ignore-panel=STATUS_CODES \
--ignore-panel=REMOTE_USER \
--ignore-panel=CACHE_STATUS \
--ignore-panel=GEO_LOCATION \
--ignore-panel=MIME_TYPE \
--ignore-panel=TLS_TYPE \
--no-column-names \
--no-progress \
--no-parsing-spinner \
- -o /root/.bigscoots/counters/verifiedbots.json \
&& grep -o "\"unique_visitors\": [0-9].*" /root/.bigscoots/counters/verifiedbots.json | cut -d ',' -f1 | awk '{print $2}')
BADBOTS_BLOCKED=$(zgrep "$TODAYSLASH" ${TODAYLOGS} 2>/dev/null \
| grep -f /bigscoots/wpo/nginx/traffic/bad.bots \
| grep "\" 403" | wc -l)
if [[ -z "$BLOCKLIST_IPS" ]]; then
MALICIOUS_BLOCKED=0
else
MALICIOUS_BLOCKED=$(zgrep -h "$TODAYSLASH" ${TODAYLOGS} 2>/dev/null | fgrep -f <(echo "$BLOCKLIST_IPS") | wc -l)
fi
DAY_DATA=$(echo '{ "date": "'"$TODAYSLASH"'", "unique_visitors": "'"$UNIQUEVISITS"'", "verified_bots": "'"$VERIFIEDBOTS"'", "bad_bot_requests_blocked": "'"$BADBOTS_BLOCKED"'", "malicious_requests_blocked": "'"$MALICIOUS_BLOCKED"'" }')
DOMAIN_DATA=$(echo "$DOMAIN_DATA" | jq --argjson day "$DAY_DATA" '. += [$day]')
# Cleanup temporary files
rm -f /root/.bigscoots/counters/uniquevisitors.json /root/.bigscoots/counters/verifiedbots.json
else
exit 1
fi
fi
# Add the processed data for this domain to the FINAL_OUTPUT object
FINAL_OUTPUT=$(echo "$FINAL_OUTPUT" | jq --argjson domainData "$DOMAIN_DATA" --arg domain "$DOMAIN" '.[$domain] = $domainData')
#### option yesterday ####
elif [ "$1" = "yesterday" ]; then
YESTERDAYSLASH=$(date --date="yesterday" +"%d/%b/%Y")
YESTERDAYDASH=$(date --date="yesterday" +"%m-%d-%Y")
DOMAIN_DATA="[]" # Reset domain data array for each domain
# Find yesterday's log files
YESTERDAYLOGS=$(find /home/nginx/domains/"${DOMAIN}"/log/ -mtime -2 -name 'access.log*' 2>/dev/null | tr '\n' ' ')
if [ -z "$YESTERDAYLOGS" ]; then
# No logs found for yesterday
DAY_DATA=$(echo '{ "date": "'"$YESTERDAYSLASH"'", "unique_visitors": "0", "verified_bots": "0", "bad_bot_requests_blocked": "0", "malicious_requests_blocked": "0" }')
DOMAIN_DATA=$(echo "$DOMAIN_DATA" | jq --argjson day "$DAY_DATA" '. += [$day]')
else
if [ -f /root/.bigscoots/nginx/includes/abuserbl_block.list ]; then
BLOCKLIST_IPS=$(grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' /root/.bigscoots/nginx/includes/abuserbl_block.list)
fi
if zgrep -h "$YESTERDAYSLASH" ${YESTERDAYLOGS} 2>/dev/null \
| grep -v "$serverip" \
| grep -v -f /bigscoots/wpo/nginx/traffic/verified.bots \
| goaccess \
--log-format=COMBINED \
--enable-panel=VISITORS \
--ignore-status=403 \
--ignore-panel=REQUESTS \
--ignore-panel=REQUESTS_STATIC \
--ignore-panel=NOT_FOUND \
--ignore-panel=HOSTS \
--ignore-panel=OS \
--ignore-panel=BROWSERS \
--ignore-panel=VISIT_TIMES \
--ignore-panel=VIRTUAL_HOSTS \
--ignore-panel=REFERRERS \
--ignore-panel=REFERRING_SITES \
--ignore-panel=KEYPHRASES \
--ignore-panel=STATUS_CODES \
--ignore-panel=REMOTE_USER \
--ignore-panel=CACHE_STATUS \
--ignore-panel=GEO_LOCATION \
--ignore-panel=MIME_TYPE \
--ignore-panel=TLS_TYPE \
--no-column-names \
--no-progress \
--no-parsing-spinner \
- -o /root/.bigscoots/counters/uniquevisitors.json; then
UNIQUEVISITS=$(grep -o "\"unique_visitors\": [0-9].*" /root/.bigscoots/counters/uniquevisitors.json | cut -d ',' -f1 | awk '{print $2}')
VERIFIEDBOTS=$(zgrep -h "$YESTERDAYSLASH" ${YESTERDAYLOGS} 2>/dev/null \
| grep -v "$serverip" \
| grep -f /bigscoots/wpo/nginx/traffic/verified.bots \
| goaccess \
--log-format=COMBINED \
--enable-panel=VISITORS \
--ignore-status=403 \
--ignore-panel=REQUESTS \
--ignore-panel=REQUESTS_STATIC \
--ignore-panel=NOT_FOUND \
--ignore-panel=HOSTS \
--ignore-panel=OS \
--ignore-panel=BROWSERS \
--ignore-panel=VISIT_TIMES \
--ignore-panel=VIRTUAL_HOSTS \
--ignore-panel=REFERRERS \
--ignore-panel=REFERRING_SITES \
--ignore-panel=KEYPHRASES \
--ignore-panel=STATUS_CODES \
--ignore-panel=REMOTE_USER \
--ignore-panel=CACHE_STATUS \
--ignore-panel=GEO_LOCATION \
--ignore-panel=MIME_TYPE \
--ignore-panel=TLS_TYPE \
--no-column-names \
--no-progress \
--no-parsing-spinner \
- -o /root/.bigscoots/counters/verifiedbots.json \
&& grep -o "\"unique_visitors\": [0-9].*" /root/.bigscoots/counters/verifiedbots.json | cut -d ',' -f1 | awk '{print $2}')
BADBOTS_BLOCKED=$(zgrep "$YESTERDAYSLASH" ${YESTERDAYLOGS} 2>/dev/null \
| grep -f /bigscoots/wpo/nginx/traffic/bad.bots \
| grep "\" 403" | wc -l)
if [[ -z "$BLOCKLIST_IPS" ]]; then
MALICIOUS_BLOCKED=0
else
MALICIOUS_BLOCKED=$(zgrep -h "$YESTERDAYSLASH" ${YESTERDAYLOGS} 2>/dev/null | fgrep -f <(echo "$BLOCKLIST_IPS") | wc -l)
fi
DAY_DATA=$(echo '{ "date": "'"$YESTERDAYSLASH"'", "unique_visitors": "'"$UNIQUEVISITS"'", "verified_bots": "'"$VERIFIEDBOTS"'", "bad_bot_requests_blocked": "'"$BADBOTS_BLOCKED"'", "malicious_requests_blocked": "'"$MALICIOUS_BLOCKED"'" }')
DOMAIN_DATA=$(echo "$DOMAIN_DATA" | jq --argjson day "$DAY_DATA" '. += [$day]')
curl -H "X-SCOOTS-HASH=${VISITOR_HASH}" -H "X-SCOOTS-HOOK=true" -s -d "date=${YESTERDAYDASH}&domain=${DOMAIN}&count=${UNIQUEVISITS}&verified_bots=${VERIFIEDBOTS}&bad_bot_requests_blocked=${BADBOTS_BLOCKED}&malicious_requests_blocked=${MALICIOUS_BLOCKED}&serverip=${serverip}&hash=${VISITOR_HASH}" -X POST https://api-dev.bigscoots.com/hook/add-traffic-data
# Cleanup temporary files
rm -f /root/.bigscoots/counters/uniquevisitors.json /root/.bigscoots/counters/verifiedbots.json
else
exit 1
fi
fi
# Add the processed data for this domain to the FINAL_OUTPUT object
FINAL_OUTPUT=$(echo "$FINAL_OUTPUT" | jq --argjson domainData "$DOMAIN_DATA" --arg domain "$DOMAIN" '.[$domain] = $domainData')
elif [ "$1" = "ALL30" ]; then
DAYSAGO="$(seq 2 30)"
DOMAIN_DATA="[]" # Reset domain data array for each domain
for DAY in ${DAYSAGO[@]}; do
DAYSLASH="$(date --date="-$DAY days" +"%d/%b/%Y")"
# Find log files for each specific day in the 30-day range
DAYLOGS=$(find /home/nginx/domains/"${DOMAIN}"/log/ -name "access.log*" -type f -newermt "-$DAY days" ! -newermt "-$((DAY - 1)) days" | tr '\n' ' ')
if [ -z "$DAYLOGS" ]; then
# No logs found for the date
DAY_DATA=$(echo '{ "date": "'"$DAYSLASH"'", "unique_visitors": "0", "verified_bots": "0", "bad_bot_requests_blocked": "0", "malicious_requests_blocked": "0" }')
DOMAIN_DATA=$(echo "$DOMAIN_DATA" | jq --argjson day "$DAY_DATA" '. += [$day]')
else
if [ -f /root/.bigscoots/nginx/includes/abuserbl_block.list ]; then
BLOCKLIST_IPS=$(grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' /root/.bigscoots/nginx/includes/abuserbl_block.list)
fi
if zgrep -h "$DAYSLASH" ${DAYLOGS} 2>/dev/null \
| grep -v "$serverip" \
| grep -v -f /bigscoots/wpo/nginx/traffic/verified.bots \
| goaccess \
--log-format=COMBINED \
--enable-panel=VISITORS \
--ignore-status=403 \
--ignore-panel=REQUESTS \
--ignore-panel=REQUESTS_STATIC \
--ignore-panel=NOT_FOUND \
--ignore-panel=HOSTS \
--ignore-panel=OS \
--ignore-panel=BROWSERS \
--ignore-panel=VISIT_TIMES \
--ignore-panel=VIRTUAL_HOSTS \
--ignore-panel=REFERRERS \
--ignore-panel=REFERRING_SITES \
--ignore-panel=KEYPHRASES \
--ignore-panel=STATUS_CODES \
--ignore-panel=REMOTE_USER \
--ignore-panel=CACHE_STATUS \
--ignore-panel=GEO_LOCATION \
--ignore-panel=MIME_TYPE \
--ignore-panel=TLS_TYPE \
--no-column-names \
--no-progress \
--no-parsing-spinner \
- -o /root/.bigscoots/counters/uniquevisitors.json; then
UNIQUEVISITS=$(grep -o "\"unique_visitors\": [0-9].*" /root/.bigscoots/counters/uniquevisitors.json | cut -d ',' -f1 | awk '{print $2}')
VERIFIEDBOTS=$(zgrep -h "$DAYSLASH" ${DAYLOGS} 2>/dev/null \
| grep -v "$serverip" \
| grep -f /bigscoots/wpo/nginx/traffic/verified.bots \
| goaccess \
--log-format=COMBINED \
--enable-panel=VISITORS \
--ignore-status=403 \
--ignore-panel=REQUESTS \
--ignore-panel=REQUESTS_STATIC \
--ignore-panel=NOT_FOUND \
--ignore-panel=HOSTS \
--ignore-panel=OS \
--ignore-panel=BROWSERS \
--ignore-panel=VISIT_TIMES \
--ignore-panel=VIRTUAL_HOSTS \
--ignore-panel=REFERRERS \
--ignore-panel=REFERRING_SITES \
--ignore-panel=KEYPHRASES \
--ignore-panel=STATUS_CODES \
--ignore-panel=REMOTE_USER \
--ignore-panel=CACHE_STATUS \
--ignore-panel=GEO_LOCATION \
--ignore-panel=MIME_TYPE \
--ignore-panel=TLS_TYPE \
--no-column-names \
--no-progress \
--no-parsing-spinner \
- -o /root/.bigscoots/counters/verifiedbots.json \
&& grep -o "\"unique_visitors\": [0-9].*" /root/.bigscoots/counters/verifiedbots.json | cut -d ',' -f1 | awk '{print $2}')
BADBOTS_BLOCKED=$(zgrep "$DAYSLASH" ${DAYLOGS} 2>/dev/null \
| grep -f /bigscoots/wpo/nginx/traffic/bad.bots \
| grep "\" 403" | wc -l)
if [[ -z "$BLOCKLIST_IPS" ]]; then
MALICIOUS_BLOCKED=0
else
MALICIOUS_BLOCKED=$(zgrep -h "$DAYSLASH" ${DAYLOGS} 2>/dev/null | fgrep -f <(echo "$BLOCKLIST_IPS") | wc -l)
fi
DAY_DATA=$(echo '{ "date": "'"$DAYSLASH"'", "unique_visitors": "'"$UNIQUEVISITS"'", "verified_bots": "'"$VERIFIEDBOTS"'", "bad_bot_requests_blocked": "'"$BADBOTS_BLOCKED"'", "malicious_requests_blocked": "'"$MALICIOUS_BLOCKED"'" }')
DOMAIN_DATA=$(echo "$DOMAIN_DATA" | jq --argjson day "$DAY_DATA" '. += [$day]')
# Cleanup temporary files
rm -f /root/.bigscoots/counters/uniquevisitors.json /root/.bigscoots/counters/verifiedbots.json
else
exit 1
fi
fi
done
# Add the processed data for this domain to the FINAL_OUTPUT object
FINAL_OUTPUT=$(echo "$FINAL_OUTPUT" | jq --argjson domainData "$DOMAIN_DATA" --arg domain "$DOMAIN" '.[$domain] = $domainData')
elif [[ "$1" =~ ^[0-9]{8}$ ]]; then # Ensure $1 is exactly 8 digits, e.g., 06102024
DAY="$1"
DAYSLASH=$(date -d "${DAY:4:4}-${DAY:2:2}-${DAY:0:2}" +"%d/%b/%Y")
DOMAIN_DATA="[]" # Reset domain data array for each domain
# Find all log files for the specified date
DAYLOGS=$(zgrep -l "$DAYSLASH" /home/nginx/domains/${DOMAIN}/log/access.log* 2>/dev/null)
if [ -z "$DAYLOGS" ]; then
# No logs found for the date
DAY_DATA=$(echo '{ "date": "'"$DAYSLASH"'", "unique_visitors": "0", "verified_bots": "0", "bad_bot_requests_blocked": "0", "malicious_requests_blocked": "0" }')
DOMAIN_DATA=$(echo "$DOMAIN_DATA" | jq --argjson day "$DAY_DATA" '. += [$day]')
else
if [ -f /root/.bigscoots/nginx/includes/abuserbl_block.list ]; then
BLOCKLIST_IPS=$(grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' /root/.bigscoots/nginx/includes/abuserbl_block.list)
fi
if zgrep -h "$DAYSLASH" ${DAYLOGS} 2>/dev/null \
| grep -v "$serverip" \
| grep -v -f /bigscoots/wpo/nginx/traffic/verified.bots \
| goaccess \
--log-format=COMBINED \
--enable-panel=VISITORS \
--ignore-status=403 \
--ignore-panel=REQUESTS \
--ignore-panel=REQUESTS_STATIC \
--ignore-panel=NOT_FOUND \
--ignore-panel=HOSTS \
--ignore-panel=OS \
--ignore-panel=BROWSERS \
--ignore-panel=VISIT_TIMES \
--ignore-panel=VIRTUAL_HOSTS \
--ignore-panel=REFERRERS \
--ignore-panel=REFERRING_SITES \
--ignore-panel=KEYPHRASES \
--ignore-panel=STATUS_CODES \
--ignore-panel=REMOTE_USER \
--ignore-panel=CACHE_STATUS \
--ignore-panel=GEO_LOCATION \
--ignore-panel=MIME_TYPE \
--ignore-panel=TLS_TYPE \
--no-column-names \
--no-progress \
--no-parsing-spinner \
- -o /root/.bigscoots/counters/uniquevisitors.json; then
UNIQUEVISITS=$(grep -o "\"unique_visitors\": [0-9].*" /root/.bigscoots/counters/uniquevisitors.json | cut -d ',' -f1 | awk '{print $2}')
VERIFIEDBOTS=$(zgrep -h "$DAYSLASH" ${DAYLOGS} 2>/dev/null \
| grep -v "$serverip" \
| grep -f /bigscoots/wpo/nginx/traffic/verified.bots \
| goaccess \
--log-format=COMBINED \
--enable-panel=VISITORS \
--ignore-status=403 \
--ignore-panel=REQUESTS \
--ignore-panel=REQUESTS_STATIC \
--ignore-panel=NOT_FOUND \
--ignore-panel=HOSTS \
--ignore-panel=OS \
--ignore-panel=BROWSERS \
--ignore-panel=VISIT_TIMES \
--ignore-panel=VIRTUAL_HOSTS \
--ignore-panel=REFERRERS \
--ignore-panel=REFERRING_SITES \
--ignore-panel=KEYPHRASES \
--ignore-panel=STATUS_CODES \
--ignore-panel=REMOTE_USER \
--ignore-panel=CACHE_STATUS \
--ignore-panel=GEO_LOCATION \
--ignore-panel=MIME_TYPE \
--ignore-panel=TLS_TYPE \
--no-column-names \
--no-progress \
--no-parsing-spinner \
- -o /root/.bigscoots/counters/verifiedbots.json \
&& grep -o "\"unique_visitors\": [0-9].*" /root/.bigscoots/counters/verifiedbots.json | cut -d ',' -f1 | awk '{print $2}')
BADBOTS_BLOCKED=$(zgrep "$DAYSLASH" ${DAYLOGS} 2>/dev/null \
| grep -f /bigscoots/wpo/nginx/traffic/bad.bots \
| grep "\" 403" | wc -l)
if [[ -z "$BLOCKLIST_IPS" ]]; then
MALICIOUS_BLOCKED=0
else
MALICIOUS_BLOCKED=$(zgrep -h "$DAYSLASH" ${DAYLOGS} 2>/dev/null | fgrep -f <(echo "$BLOCKLIST_IPS") | wc -l)
fi
DAY_DATA=$(echo '{ "date": "'"$DAYSLASH"'", "unique_visitors": "'"$UNIQUEVISITS"'", "verified_bots": "'"$VERIFIEDBOTS"'", "bad_bot_requests_blocked": "'"$BADBOTS_BLOCKED"'", "malicious_requests_blocked": "'"$MALICIOUS_BLOCKED"'" }')
DOMAIN_DATA=$(echo "$DOMAIN_DATA" | jq --argjson day "$DAY_DATA" '. += [$day]')
# Cleanup temporary files
rm -f /root/.bigscoots/counters/uniquevisitors.json /root/.bigscoots/counters/verifiedbots.json
else
exit 1
fi
fi
# Add the processed data for this domain to the FINAL_OUTPUT object
FINAL_OUTPUT=$(echo "$FINAL_OUTPUT" | jq --argjson domainData "$DOMAIN_DATA" --arg domain "$DOMAIN" '.[$domain] = $domainData')
elif [ "$1" -gt 2 -a "$1" -lt 32 ]; then
DAYSAGO="$(seq 2 $1)"
YESTERDAYDASH=$(date --date="yesterday" +"%m-%d-%Y")
DOMAIN_DATA="[]" # Reset domain data array for each domain
for DAY in ${DAYSAGO[@]}; do
DAYSLASH="$(date --date="-$DAY days" +"%d/%b/%Y")"
DAYDASH="$(date --date="-$DAY days" +"%m-%d-%Y")"
# Gather logs specifically for each day in the range
DAYLOGS=$(find /home/nginx/domains/"${DOMAIN}"/log/ -name "access.log*" -type f -newermt "-$DAY days" ! -newermt "-$((DAY - 1)) days" | tr '\n' ' ')
if [ -z "$DAYLOGS" ]; then
# No logs found for the date
DAY_DATA=$(echo '{ "date": "'"$DAYSLASH"'", "unique_visitors": "0", "verified_bots": "0", "bad_bot_requests_blocked": "0", "malicious_requests_blocked": "0" }')
DOMAIN_DATA=$(echo "$DOMAIN_DATA" | jq --argjson day "$DAY_DATA" '. += [$day]')
else
if [ -f /root/.bigscoots/nginx/includes/abuserbl_block.list ]; then
BLOCKLIST_IPS=$(grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' /root/.bigscoots/nginx/includes/abuserbl_block.list)
fi
if zgrep -h "$DAYSLASH" ${DAYLOGS} 2>/dev/null \
| grep -v "$serverip" \
| grep -v -f /bigscoots/wpo/nginx/traffic/verified.bots \
| goaccess \
--log-format=COMBINED \
--enable-panel=VISITORS \
--ignore-status=403 \
--ignore-panel=REQUESTS \
--ignore-panel=REQUESTS_STATIC \
--ignore-panel=NOT_FOUND \
--ignore-panel=HOSTS \
--ignore-panel=OS \
--ignore-panel=BROWSERS \
--ignore-panel=VISIT_TIMES \
--ignore-panel=VIRTUAL_HOSTS \
--ignore-panel=REFERRERS \
--ignore-panel=REFERRING_SITES \
--ignore-panel=KEYPHRASES \
--ignore-panel=STATUS_CODES \
--ignore-panel=REMOTE_USER \
--ignore-panel=CACHE_STATUS \
--ignore-panel=GEO_LOCATION \
--ignore-panel=MIME_TYPE \
--ignore-panel=TLS_TYPE \
--no-column-names \
--no-progress \
--no-parsing-spinner \
- -o /root/.bigscoots/counters/uniquevisitors.json; then
UNIQUEVISITS=$(grep -o "\"unique_visitors\": [0-9].*" /root/.bigscoots/counters/uniquevisitors.json | cut -d ',' -f1 | awk '{print $2}')
VERIFIEDBOTS=$(zgrep -h "$DAYSLASH" ${DAYLOGS} 2>/dev/null \
| grep -v "$serverip" \
| grep -f /bigscoots/wpo/nginx/traffic/verified.bots \
| goaccess \
--log-format=COMBINED \
--enable-panel=VISITORS \
--ignore-status=403 \
--ignore-panel=REQUESTS \
--ignore-panel=REQUESTS_STATIC \
--ignore-panel=NOT_FOUND \
--ignore-panel=HOSTS \
--ignore-panel=OS \
--ignore-panel=BROWSERS \
--ignore-panel=VISIT_TIMES \
--ignore-panel=VIRTUAL_HOSTS \
--ignore-panel=REFERRERS \
--ignore-panel=REFERRING_SITES \
--ignore-panel=KEYPHRASES \
--ignore-panel=STATUS_CODES \
--ignore-panel=REMOTE_USER \
--ignore-panel=CACHE_STATUS \
--ignore-panel=GEO_LOCATION \
--ignore-panel=MIME_TYPE \
--ignore-panel=TLS_TYPE \
--no-column-names \
--no-progress \
--no-parsing-spinner \
- -o /root/.bigscoots/counters/verifiedbots.json \
&& grep -o "\"unique_visitors\": [0-9].*" /root/.bigscoots/counters/verifiedbots.json | cut -d ',' -f1 | awk '{print $2}')
BADBOTS_BLOCKED=$(zgrep "$DAYSLASH" ${DAYLOGS} 2>/dev/null \
| grep -f /bigscoots/wpo/nginx/traffic/bad.bots \
| grep "\" 403" | wc -l)
if [[ -z "$BLOCKLIST_IPS" ]]; then
MALICIOUS_BLOCKED=0
else
MALICIOUS_BLOCKED=$(zgrep -h "$DAYSLASH" ${DAYLOGS} 2>/dev/null | fgrep -f <(echo "$BLOCKLIST_IPS") | wc -l)
fi
DAY_DATA=$(echo '{ "date": "'"$DAYSLASH"'", "unique_visitors": "'"$UNIQUEVISITS"'", "verified_bots": "'"$VERIFIEDBOTS"'", "bad_bot_requests_blocked": "'"$BADBOTS_BLOCKED"'", "malicious_requests_blocked": "'"$MALICIOUS_BLOCKED"'" }')
DOMAIN_DATA=$(echo "$DOMAIN_DATA" | jq --argjson day "$DAY_DATA" '. += [$day]')
curl -H "X-SCOOTS-HASH=${VISITOR_HASH}" -H "X-SCOOTS-HOOK=true" -s -d "date=${DAYDASH}&domain=${DOMAIN}&count=${UNIQUEVISITS}&verified_bots=${VERIFIEDBOTS}&bad_bot_requests_blocked=${BADBOTS_BLOCKED}&malicious_requests_blocked=${MALICIOUS_BLOCKED}&serverip=${serverip}&hash=${VISITOR_HASH}" -X POST https://api-dev.bigscoots.com/hook/add-traffic-data >/dev/null 2>&1
# Cleanup temporary files
rm -f /root/.bigscoots/counters/uniquevisitors.json /root/.bigscoots/counters/verifiedbots.json
else
exit 1
fi
fi
done
# Add the processed data for this domain to the FINAL_OUTPUT object
FINAL_OUTPUT=$(echo "$FINAL_OUTPUT" | jq --argjson domainData "$DOMAIN_DATA" --arg domain "$DOMAIN" '.[$domain] = $domainData')
fi
unset BYPASSGOACCESS
unset UNIQUEVISITS
done
echo "$FINAL_OUTPUT"