From 9f34f102afeba99e98e2876019f86a65ffed894e Mon Sep 17 00:00:00 2001 From: David Date: Tue, 20 Dec 2022 20:38:32 +0100 Subject: [PATCH] Activation des statistiques nouvelle generation --- statistiques/getStats.py | 8 +- .../{getStats_ng.py => getStats_old.py} | 8 +- statistiques/parseStats.py | 103 +++++++----------- .../{parseStats_ng.py => parseStats_old.py} | 103 +++++++++++------- 4 files changed, 111 insertions(+), 111 deletions(-) rename statistiques/{getStats_ng.py => getStats_old.py} (88%) rename statistiques/{parseStats_ng.py => parseStats_old.py} (63%) diff --git a/statistiques/getStats.py b/statistiques/getStats.py index fab9c8c..85311cf 100755 --- a/statistiques/getStats.py +++ b/statistiques/getStats.py @@ -61,11 +61,9 @@ def runMain(): response = requests.get(api_url,timeout=1) element = datetime.datetime.strptime(response.json()['current_timestamp'],'%Y-%m-%d %H:%M:%S.%f') timestamp = int(time.mktime(element.timetuple())) - with open(f'/{statsPath}/jisti_meet_stats_{timestamp}.csv','w') as fh: - for (k,v) in response.json().items(): - db.dbInsert(timestamp,k,v) - fh.write(f"{k};{v};{timestamp}\n") - db.dbQuery() + for (k,v) in response.json().items(): + db.dbInsert(timestamp,k,v) + #db.dbQuery() # Principal if __name__ == '__main__': diff --git a/statistiques/getStats_ng.py b/statistiques/getStats_old.py similarity index 88% rename from statistiques/getStats_ng.py rename to statistiques/getStats_old.py index 85311cf..fab9c8c 100755 --- a/statistiques/getStats_ng.py +++ b/statistiques/getStats_old.py @@ -61,9 +61,11 @@ def runMain(): response = requests.get(api_url,timeout=1) element = datetime.datetime.strptime(response.json()['current_timestamp'],'%Y-%m-%d %H:%M:%S.%f') timestamp = int(time.mktime(element.timetuple())) - for (k,v) in response.json().items(): - db.dbInsert(timestamp,k,v) - #db.dbQuery() + with open(f'/{statsPath}/jisti_meet_stats_{timestamp}.csv','w') as fh: + for (k,v) in response.json().items(): + db.dbInsert(timestamp,k,v) + fh.write(f"{k};{v};{timestamp}\n") + db.dbQuery() # Principal if __name__ == '__main__': diff --git a/statistiques/parseStats.py b/statistiques/parseStats.py index ae6bf8f..49d13ae 100755 --- a/statistiques/parseStats.py +++ b/statistiques/parseStats.py @@ -14,8 +14,8 @@ STAT_DIR = '/srv/visio.chapril.org/statistiques/' STATS_TOT_FIELDS = ['total_conferences_created','total_failed_conferences','total_conferences_completed','total_conference_seconds','total_bytes_received','total_bytes_sent','total_participants','conferences','videochannels','endpoints_sending_audio',] STATS_FR_TOT_FIELDS = ['conferences creees total','conferences totalement echouees','conferences terminees total','duree totale conferences','total octets reçus','total octets envoyés','total participants','nombre de conferences','canaux video','clients en audio',] -STATS_AVG_FIELDS = ['largest_conference',] -STATS_FR_AVG_FIELDS = ['plus grande conference',] +STATS_MAX_FIELDS = ['largest_conference',] +STATS_FR_MAX_FIELDS = ['plus grande conference',] dbPath = '/srv/visio.chapril.org/statistiques/stats_sqlite.db' dbName = 'jitsi_stats' @@ -35,7 +35,6 @@ class Stats: self.__setStartDate(f'{self.year}-{self.mois}-01 00:00:00') maxDays = monthrange(self.year,self.mois)[1] self.__setEndDate(f'{self.year}-{self.mois}-{maxDays} 23:59:59') - def __setStartDate(self,thisDate): self.startDate = self.__date2timestamp(thisDate) @@ -60,7 +59,6 @@ class Stats: return int(timestamp) def __conv(self,octets,dataType='b'): - if dataType == 'b': unit = 'octets' if int(octets) > 1024: @@ -83,64 +81,43 @@ class Stats: octets = int(octets * 10) / 10 return octets,unit - def parse2(self): - res = self.db.dbQuery(f"""SELECT * FROM {dbName} WHERE timestamp > {self.startDate} AND timestamp < {self.endDate}""") + def parse(self): + res = self.db.dbQuery(f"""SELECT * FROM {dbName} WHERE timestamp > {self.startDate} AND timestamp < {self.endDate} ORDER by id""") consolided = {} + moy_conf_by_day = 0 for line in res: field = line[2] if field in STATS_TOT_FIELDS: - if field in consolided: + if field in consolided and 'total' not in field: consolided[field] = consolided[field] + int(line[3]) else: consolided[field] = int(line[3]) - if field in STATS_AVG_FIELDS: + if field == 'conferences': + moy_conf_by_day += 1 + + if field in STATS_MAX_FIELDS: if field in consolided: if consolided[field] < int(line[3]): consolided[field] = int(line[3]) + for (k,v) in consolided.items(): + if 'bytes' in k: + (v,u) = self.__conv(consolided[k]) + consolided[k] = f"{v} {u}" + if 'seconds' in k: + (v,u) = self.__conv(consolided[k],dataType='t') + consolided.pop(k) + n_k = k.replace('_seconds','') + consolided[n_k] = f"{v} {u}" + + if moy_conf_by_day > 1: + tot = consolided['conferences'] + moy_conf_by_day = int(moy_conf_by_day / 12 + 0.5) + moy = int(tot/moy_conf_by_day + 0.5) + consolided.pop('conferences') + consolided['average conferences by day'] = moy + return consolided - def parse(self): - if len(self.files) <= 0: - return None - for f in self.files: - if '.db' in f: - continue - ts = int(f.split('.')[0].split('_')[3]) - if ts >= self.startDate and ts <= self.endDate: - with open(f"{STAT_DIR}/{f}") as fh: - datas = fh.readlines() - for line in datas: - if line.split(';')[0].lower() in STATS_TOT_FIELDS: - key = line.split(';')[0].lower() - value = int(line.split(';')[1]) - if key in self.consolided_datas: - datas = self.consolided_datas[key] - if datas[0] <= 0: - self.consolided_datas[key][0] = value - else: - if value > datas[1]: - self.consolided_datas[key][1] = value - else: - self.consolided_datas[key] = [value,0] - - if line.split(';')[0].lower() in STATS_AVG_FIELDS: - key = line.split(';')[0].lower() - value = int(line.split(';')[1]) - if key in self.consolided: - if self.consolided[key] < int(value): - self.consolided[key] = int(value) - else: - self.consolided[key] = 0 - - for (k,v) in self.consolided_datas.items(): - self.consolided[k] = v[1] - v[0] - if 'byte' in k: - octets,unit = self.__conv(self.consolided[k]) - self.consolided[k] = f"{octets} {unit}" - elif 'seconds' in k: - octets,unit = self.__conv(self.consolided[k],'t') - self.consolided[k] = f"{octets} {unit}" - return self.consolided class SQLite: def __init__(self): @@ -149,13 +126,12 @@ class SQLite: def __initDb(self): self.__openDb() - self.cursor.execute(''' create table jitsi_stats( + self.cursor.execute('''create table jitsi_stats( id integer primary key autoincrement, timestamp text, key_field text, value_field text - ) - ''') + )''') self.conn.commit() self.__closeDb() @@ -168,10 +144,13 @@ class SQLite: self.conn.close() def dbQuery(self,query='SELECT'): - self.__openDb() - self.cursor.execute(query) - rows = self.cursor.fetchall() - self.__closeDb() + try: + self.__openDb() + self.cursor.execute(query) + rows = self.cursor.fetchall() + self.__closeDb() + except sqlite3.OperationalError: + rows = None return rows def dbInsert(self,ts,k,v): @@ -205,17 +184,9 @@ def runMain(): year = currentDate.year stats = Stats(year,mois) - #res = stats.parse2() - #for (k,v) in res.items(): - # print(f"{k}={v}") - res = stats.parse() for (k,v) in res.items(): - if k in STATS_TOT_FIELDS: - chaine = STATS_FR_TOT_FIELDS[STATS_TOT_FIELDS.index(k)] - elif k in STATS_AVG_FIELDS: - chaine = STATS_FR_AVG_FIELDS[STATS_AVG_FIELDS.index(k)] - print(f"{chaine} : {v}") + print(f"{k}={v}") if __name__ == '__main__': runMain() diff --git a/statistiques/parseStats_ng.py b/statistiques/parseStats_old.py similarity index 63% rename from statistiques/parseStats_ng.py rename to statistiques/parseStats_old.py index 49d13ae..ae6bf8f 100755 --- a/statistiques/parseStats_ng.py +++ b/statistiques/parseStats_old.py @@ -14,8 +14,8 @@ STAT_DIR = '/srv/visio.chapril.org/statistiques/' STATS_TOT_FIELDS = ['total_conferences_created','total_failed_conferences','total_conferences_completed','total_conference_seconds','total_bytes_received','total_bytes_sent','total_participants','conferences','videochannels','endpoints_sending_audio',] STATS_FR_TOT_FIELDS = ['conferences creees total','conferences totalement echouees','conferences terminees total','duree totale conferences','total octets reçus','total octets envoyés','total participants','nombre de conferences','canaux video','clients en audio',] -STATS_MAX_FIELDS = ['largest_conference',] -STATS_FR_MAX_FIELDS = ['plus grande conference',] +STATS_AVG_FIELDS = ['largest_conference',] +STATS_FR_AVG_FIELDS = ['plus grande conference',] dbPath = '/srv/visio.chapril.org/statistiques/stats_sqlite.db' dbName = 'jitsi_stats' @@ -35,6 +35,7 @@ class Stats: self.__setStartDate(f'{self.year}-{self.mois}-01 00:00:00') maxDays = monthrange(self.year,self.mois)[1] self.__setEndDate(f'{self.year}-{self.mois}-{maxDays} 23:59:59') + def __setStartDate(self,thisDate): self.startDate = self.__date2timestamp(thisDate) @@ -59,6 +60,7 @@ class Stats: return int(timestamp) def __conv(self,octets,dataType='b'): + if dataType == 'b': unit = 'octets' if int(octets) > 1024: @@ -81,43 +83,64 @@ class Stats: octets = int(octets * 10) / 10 return octets,unit - def parse(self): - res = self.db.dbQuery(f"""SELECT * FROM {dbName} WHERE timestamp > {self.startDate} AND timestamp < {self.endDate} ORDER by id""") + def parse2(self): + res = self.db.dbQuery(f"""SELECT * FROM {dbName} WHERE timestamp > {self.startDate} AND timestamp < {self.endDate}""") consolided = {} - moy_conf_by_day = 0 for line in res: field = line[2] if field in STATS_TOT_FIELDS: - if field in consolided and 'total' not in field: + if field in consolided: consolided[field] = consolided[field] + int(line[3]) else: consolided[field] = int(line[3]) - if field == 'conferences': - moy_conf_by_day += 1 - - if field in STATS_MAX_FIELDS: + if field in STATS_AVG_FIELDS: if field in consolided: if consolided[field] < int(line[3]): consolided[field] = int(line[3]) - for (k,v) in consolided.items(): - if 'bytes' in k: - (v,u) = self.__conv(consolided[k]) - consolided[k] = f"{v} {u}" - if 'seconds' in k: - (v,u) = self.__conv(consolided[k],dataType='t') - consolided.pop(k) - n_k = k.replace('_seconds','') - consolided[n_k] = f"{v} {u}" - - if moy_conf_by_day > 1: - tot = consolided['conferences'] - moy_conf_by_day = int(moy_conf_by_day / 12 + 0.5) - moy = int(tot/moy_conf_by_day + 0.5) - consolided.pop('conferences') - consolided['average conferences by day'] = moy - return consolided + def parse(self): + if len(self.files) <= 0: + return None + for f in self.files: + if '.db' in f: + continue + ts = int(f.split('.')[0].split('_')[3]) + if ts >= self.startDate and ts <= self.endDate: + with open(f"{STAT_DIR}/{f}") as fh: + datas = fh.readlines() + for line in datas: + if line.split(';')[0].lower() in STATS_TOT_FIELDS: + key = line.split(';')[0].lower() + value = int(line.split(';')[1]) + if key in self.consolided_datas: + datas = self.consolided_datas[key] + if datas[0] <= 0: + self.consolided_datas[key][0] = value + else: + if value > datas[1]: + self.consolided_datas[key][1] = value + else: + self.consolided_datas[key] = [value,0] + + if line.split(';')[0].lower() in STATS_AVG_FIELDS: + key = line.split(';')[0].lower() + value = int(line.split(';')[1]) + if key in self.consolided: + if self.consolided[key] < int(value): + self.consolided[key] = int(value) + else: + self.consolided[key] = 0 + + for (k,v) in self.consolided_datas.items(): + self.consolided[k] = v[1] - v[0] + if 'byte' in k: + octets,unit = self.__conv(self.consolided[k]) + self.consolided[k] = f"{octets} {unit}" + elif 'seconds' in k: + octets,unit = self.__conv(self.consolided[k],'t') + self.consolided[k] = f"{octets} {unit}" + return self.consolided class SQLite: def __init__(self): @@ -126,12 +149,13 @@ class SQLite: def __initDb(self): self.__openDb() - self.cursor.execute('''create table jitsi_stats( + self.cursor.execute(''' create table jitsi_stats( id integer primary key autoincrement, timestamp text, key_field text, value_field text - )''') + ) + ''') self.conn.commit() self.__closeDb() @@ -144,13 +168,10 @@ class SQLite: self.conn.close() def dbQuery(self,query='SELECT'): - try: - self.__openDb() - self.cursor.execute(query) - rows = self.cursor.fetchall() - self.__closeDb() - except sqlite3.OperationalError: - rows = None + self.__openDb() + self.cursor.execute(query) + rows = self.cursor.fetchall() + self.__closeDb() return rows def dbInsert(self,ts,k,v): @@ -184,9 +205,17 @@ def runMain(): year = currentDate.year stats = Stats(year,mois) + #res = stats.parse2() + #for (k,v) in res.items(): + # print(f"{k}={v}") + res = stats.parse() for (k,v) in res.items(): - print(f"{k}={v}") + if k in STATS_TOT_FIELDS: + chaine = STATS_FR_TOT_FIELDS[STATS_TOT_FIELDS.index(k)] + elif k in STATS_AVG_FIELDS: + chaine = STATS_FR_AVG_FIELDS[STATS_AVG_FIELDS.index(k)] + print(f"{chaine} : {v}") if __name__ == '__main__': runMain()