From 90e13f8e8448f1f871f6d8e78fed8ae0ea51a108 Mon Sep 17 00:00:00 2001 From: Christian Pierre MOMON Date: Thu, 12 Nov 2020 00:53:52 +0100 Subject: [PATCH] Added activity report scripts (#4848). --- .../forgechaprilorg-rapport-activite.cron | 2 + rapport_activite/rapport_activite.sh | 202 ++++++++++++++++++ rapport_activite/update_redmine_ticket.sh | 42 ++++ 3 files changed, 246 insertions(+) create mode 100644 rapport_activite/forgechaprilorg-rapport-activite.cron create mode 100755 rapport_activite/rapport_activite.sh create mode 100755 rapport_activite/update_redmine_ticket.sh diff --git a/rapport_activite/forgechaprilorg-rapport-activite.cron b/rapport_activite/forgechaprilorg-rapport-activite.cron new file mode 100644 index 0000000..21b8f09 --- /dev/null +++ b/rapport_activite/forgechaprilorg-rapport-activite.cron @@ -0,0 +1,2 @@ +# +5 0 1 * * root /srv/forge.chapril.org/tools/rapport_activite/update_redmine_ticket.sh > /var/log/forgechaprilorg-cron.log 2>&1 diff --git a/rapport_activite/rapport_activite.sh b/rapport_activite/rapport_activite.sh new file mode 100755 index 0000000..62dff2b --- /dev/null +++ b/rapport_activite/rapport_activite.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# +# Copyright (C) 2020 Christian Pierre Momon +# Copyright (C) 2018, 2019 Didier Clermonté +# Copyright (C) 2018, 2019 Christian Pierre Momon +# +# This file is part of paste.chapril.org. +# +# This script is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +# ============================================ + +LOG_PREFIX="/var/log/apache2/forge.chapril.org/forge.chapril.org" +SERVICE_NAME="forge.chapril.org" +GITEA_REPO_HOME="/srv/gitea/repositories/" + +source "/etc/chagirbot/chagirbot.conf" + +IPV4_PATTERN="[0-9]{1,3}(\.[0-9]{1,3}){3}" +IPV6_PATTERN="[0-9a-f]{0,4}(:[0-9a-f]{0,4}){2,7}" +# +help() +{ + echo "Usage:" + echo " $(basename "$0") -h display help" + echo " $(basename "$0") analyse current month" + echo " $(basename "$0") -p analyse preceding month" + echo " $(basename "$0") month_number year analyse this month" +} + +# +generateHTTPReport() +{ + local monthEnglish="$1" + local year="$2" + local yearMonth="$3" + + echo "=§= Rapport d'activité HTTP" + echo + + local httpRequestCount=$(zgrep "$monthEnglish" $LOG_PREFIX-access.log*|grep $year|wc -l) + echo "Nombre total de requêtes http = $httpRequestCount" + + local trueHttpRequestCount=$(zgrep "$monthEnglish" $LOG_PREFIX-access.log*|grep $year|grep -vi bot|grep -v check_http|wc -l) + echo "Nombre de vraies requêtes http = $trueHttpRequestCount" + + local botRequestCount=$(zgrep "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year"|grep -i bot|grep -c -v check_http) + echo "Nombre de requêtes bot = $botRequestCount" + + local errorRequestCount=$(zgrep "$monthEnglish" $LOG_PREFIX-error.log*|grep -c "$year") + echo "Nombre de requêtes http en erreur = $errorRequestCount" + + local ipv4VisitorCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year"|grep -vi bot|grep -v check_http|egrep "^$IPV4_PATTERN "|cut -d' ' -f1|sort|uniq|wc -l ) + echo "Nombre d'ipv4 ayant visité le service = $ipv4VisitorCount" + + local ipv6VisitorCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year"|grep -vi bot|grep -v check_http|egrep "^$IPV6_PATTERN "|cut -d' ' -f1|sort|uniq|wc -l ) + echo "Nombre d'ipv6 ayant visité le service = $ipv6VisitorCount" + + echo "Nombre total d'ip ayant visité le service = $((ipv4VisitorCount+ipv6VisitorCount))" + + echo +} + +# +generateSpecificReport() +{ + local monthEnglish="$1" + local year="$2" + local yearMonth="$3" + + echo "=§= Rapport d'activité spécifique" + echo + + ipv4UserCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year"|grep -vi bot|grep -v check_http|egrep "^$IPV4_PATTERN " | cut -d' ' -f 1|sort|uniq|wc -l ) + echo "Nombre d'ipv4 utilisatrices du service = $ipv4UserCount" + + ipv6UserCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year"|grep -vi bot|gre\p -v check_http|egrep "^$IPV6_PATTERN " | cut -d' ' -f 1|sort|uniq|wc -l ) + echo "Nombre d'ipv6 utilisatrices du service = $ipv6UserCount" + + echo "Nombre total d'ip utilisatrices du service = $((ipv4UserCount+ipv6UserCount))" + echo + + # https://forge.chapril.org/api/swagger + #$(curl -s --dump-header - "https://forge.chapril.org/api/v1/admin/orgs?limit=1&page1&token=$GITEA_API_KEY" > t) + #$(curl -s "https://forge.chapril.org/api/v1/users/search?limit=1&token=$GITEA_API_KEY" | jq '. | length' > t) + local accountCount=$(curl -s "https://forge.chapril.org/api/v1/users/search?limit=100000&token=$GITEA_API_KEY" | jq '. | length') + echo "Nombre de comptes = $accountCount" + + #$(curl -s "https://forge.chapril.org/api/v1/admin/orgs?limit=100000&token=$GITEA_API_KEY" | jq '. | length' > t) + local organizationCount=$(curl -s "https://forge.chapril.org/api/v1/admin/orgs?limit=100000&token=$GITEA_API_KEY" | jq '. | length') + if [[ -z "$organizationCount" ]]; then + organizationCount=0 + fi + echo "Nombre d'organisations = $organizationCount" + + #$(curl -s "https://forge.chapril.org/api/v1/orgs?limit=100000" > t) + local publicOrganizationCount=$(curl -s "https://forge.chapril.org/api/v1/orgs?limit=100000" | jq '. | length') + if [[ -z "$publicOrganizationCount" ]]; then + publicOrganizationCount=0 + fi + echo "Nombre d'organisations publiques = $publicOrganizationCount" + + local repositoryCount=$(curl -s --dump-header - "https://forge.chapril.org/api/v1/repos/search?limit=1&token=$GITEA_API_KEY" | grep "X-Total-Count" | awk '{ print $2 }' | jq '.') + echo "Nombre de dépôts = $repositoryCount" + + local publicRepositoryCount=$(curl -s --dump-header - "https://forge.chapril.org/api/v1/repos/search?limit=1" | grep "X-Total-Count" | awk '{ print $2 }' | jq '.') + echo "Nombre de dépôts publics = $publicRepositoryCount" + + local repositorySpaceDisk=$(du -sh "$GITEA_REPO_HOME" | head -1 | awk '{ print $1 } ') + echo "Espace disque occupé par les dépôts = $repositorySpaceDisk" + + local directoryCount=$(find "$GITEA_REPO_HOME" -type d | wc -l ) + echo "Nombre de dossiers de l'espace disque = $directoryCount" + + local fileCount=$(find "$GITEA_REPO_HOME" -type f | wc -l ) + echo "Nombre de fichiers de l'espace disque = $fileCount" + + local totalFileCount=$(find "$GITEA_REPO_HOME" | wc -l ) + echo "Nombre total de fichiers de l'espace disque = $totalFileCount" + + echo +} + +# +generateReport() +{ + echo "Rapport d'activité du service $SERVICE_NAME de $monthEnglish $year" + echo + echo "=================================================================" + echo + generateHTTPReport "$1" "$2" "$3" + generateSpecificReport "$1" "$2" "$3" +} + + +# ============================================ +echo +if [ "$#" = 1 ] && [ "$1" = "-h" ]; then + help +elif [ "$#" = 1 ] && [ "$1" = "-p" ]; then + if [ "$(date +%m)" -gt 1 ]; then + precedingMonth="$((10#$(date +%m)-1))" + year="$(date +%Y)" + if [ $precedingMonth -lt 10 ]; then + yearMonth="$year-0$precedingMonth" + else + yearMonth="$year-$precedingMonth" + fi + else [ "$(date +%m)" -eq 1 ] + precedingMonth=12 + year="$(($(date +%Y)-1))" + yearMonth="$year-$precedingMonth" + fi + monthEnglish=$(LANG=en_EN.UTF-8;date -d $year-$precedingMonth-01 +%b) + generateReport "$monthEnglish" "$year" "$yearMonth" +elif [ "$#" -eq 2 ]; then + if ! (let "$1") 2>/dev/null; then + echo -e "Le mois doit être numérique." + help + elif ([ "$1" -lt 1 ] || [ "$1" -gt 12 ]); then + echo -e "Numéro du mois non valable $1." + help + elif ! (let "$2") 2>/dev/null; then + echo -e "L'année doit être numérique." + help + elif [ "$2" -lt 2018 ] || [ "$2" -gt "$(date +%Y)" ]; then + echo -e "Année non valable $2." + help + elif [ "$2" -le 2017 ] || [ "$2" -ge $(($(date +%Y)+1)) ] ; then + echo -e "Pas de données pour ce mois." + help + else + if [ "$1" -lt 10 ]; then + yearMonth="$2-0$1" + else + yearMonth="$2-$1" + fi + monthEnglish=$(LANG=en_EN.UTF-8;date -d "$2-$1-01" +%b) + year="$2" + generateReport "$monthEnglish" "$year" "$yearMonth" + fi +elif [ "$#" -eq 0 ]; then + yearMonth="$(date +%Y-%m)" + monthEnglish=$(LANG=en_EN.UTF-8;date +%b) + year="$(date +%Y)" + generateReport "$monthEnglish" "$year" "$yearMonth" +else + echo -e "Bad parameter." + help +fi diff --git a/rapport_activite/update_redmine_ticket.sh b/rapport_activite/update_redmine_ticket.sh new file mode 100755 index 0000000..9d09877 --- /dev/null +++ b/rapport_activite/update_redmine_ticket.sh @@ -0,0 +1,42 @@ +#! /bin/bash + +SCRIPT="/srv/forge.chapril.org/tools/rapport_activite/rapport_activite.sh" +CONFIG="/etc/chagirbot/chagirbot.conf" +source ${CONFIG} + +# +post_report() +{ + datafile=$(mktemp /tmp/report_stats_$(date +%Y-%m-%d_%H%M)_XXXX.json) + cat < "${datafile}" +{ + "issue": { + "notes": "$(/srv/forge.chapril.org/tools/rapport_activite/rapport_activite.sh | sed -z 's/\n/\\n/g')" + } +} +EOF + + curl -s \ + -H "Content-Type: application/json" \ + -H "X-Redmine-API-Key: ${REDMINE_API_KEY}" \ + -X PUT --data-binary "@${datafile}" \ + "${REDMINE_BASE_URL}/issues/${REDMINE_TICKET_ID}.json" + rm "${datafile}" +} + +# +last_comment_date() +{ + curl -H "X-Redmine-API-Key: ${REDMINE_API_KEY}" -s "${REDMINE_BASE_URL}/issues/${REDMINE_TICKET_ID}.json?include=journals" \ + | jq '.issue.journals | last | .created_on' +} + +# +list() +{ + curl -H "X-Redmine-API-Key: ${REDMINE_API_KEY}" -s "${REDMINE_BASE_URL}/issues/${REDMINE_TICKET_ID}.json?include=journals" \ + | jq '.issue.journals[] | [.user.name, .notes]' +} + +# Main. +post_report