296 lines
8.1 KiB
Java
296 lines
8.1 KiB
Java
/*
|
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
|
*
|
|
* This file is part of Logar, simple tool to manage http log files.
|
|
*
|
|
* Logar is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU Affero General Public License as
|
|
* published by the Free Software Foundation, either version 3 of the
|
|
* License, or (at your option) any later version.
|
|
*
|
|
* Logar is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU Affero General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License
|
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
package fr.devinsy.logar.app.log;
|
|
|
|
import java.io.File;
|
|
import java.io.FileNotFoundException;
|
|
import java.io.FileOutputStream;
|
|
import java.io.IOException;
|
|
import java.io.PrintWriter;
|
|
import java.time.LocalDateTime;
|
|
import java.time.format.DateTimeFormatter;
|
|
import java.time.format.DateTimeParseException;
|
|
import java.util.Locale;
|
|
import java.util.regex.Matcher;
|
|
import java.util.regex.Pattern;
|
|
import java.util.zip.GZIPOutputStream;
|
|
|
|
import org.apache.commons.io.IOUtils;
|
|
import org.april.logar.util.LineIterator;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import fr.devinsy.cmdexec.CmdExecException;
|
|
import fr.devinsy.cmdexec.CmdExecUtils;
|
|
|
|
/**
|
|
* The Class NginxAccessLogParser.
|
|
*/
|
|
public final class LogUtils
|
|
{
|
|
private static Logger logger = LoggerFactory.getLogger(LogUtils.class);
|
|
|
|
public static Pattern nginxAccessLogLinePatternFull = Pattern.compile(
|
|
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>\\S+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");
|
|
|
|
public static Pattern NGINX_ACCESSLOG_LINE_PATTERN = Pattern.compile("^(?<remoteAddress>[a-fA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] .*$");
|
|
public static Pattern NGINX_ERRORLOG_LINE_PATTERN = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
|
|
|
|
/**
|
|
* Instantiates a new nginx access log parser.
|
|
*/
|
|
private LogUtils()
|
|
{
|
|
}
|
|
|
|
/**
|
|
* Load.
|
|
*
|
|
* @param file
|
|
* the file
|
|
* @return the logs
|
|
* @throws IOException
|
|
*/
|
|
public static Logs loadAccessLog(final File file) throws IOException
|
|
{
|
|
Logs result;
|
|
|
|
result = new Logs();
|
|
|
|
LineIterator iterator = new LineIterator(file);
|
|
while (iterator.hasNext())
|
|
{
|
|
String line = iterator.next();
|
|
Log log = parseAccessLog(line);
|
|
result.add(log);
|
|
}
|
|
|
|
//
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Load error log.
|
|
*
|
|
* @param file
|
|
* the file
|
|
* @return the logs
|
|
* @throws IOException
|
|
* Signals that an I/O exception has occurred.
|
|
*/
|
|
public static Logs loadErrorLog(final File file) throws IOException
|
|
{
|
|
Logs result;
|
|
|
|
result = new Logs();
|
|
|
|
LineIterator iterator = new LineIterator(file);
|
|
while (iterator.hasNext())
|
|
{
|
|
String line = iterator.next();
|
|
Log log = parseErrorLog(line);
|
|
result.add(log);
|
|
}
|
|
|
|
//
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Load log file.
|
|
*
|
|
* @param file
|
|
* the file
|
|
* @return the logs
|
|
* @throws IOException
|
|
* Signals that an I/O exception has occurred.
|
|
*/
|
|
public static Logs loadLogFile(final File file) throws IOException
|
|
{
|
|
Logs result;
|
|
|
|
if (file == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter.");
|
|
}
|
|
else
|
|
{
|
|
if (file.getName().contains("access"))
|
|
{
|
|
result = loadAccessLog(file);
|
|
}
|
|
else if (file.getName().contains("error"))
|
|
{
|
|
result = loadErrorLog(file);
|
|
}
|
|
else
|
|
{
|
|
throw new IllegalArgumentException("Bad named file (missing access or error).");
|
|
}
|
|
}
|
|
|
|
//
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* From access log.
|
|
*
|
|
* @param line
|
|
* the line
|
|
* @return the log
|
|
*/
|
|
public static Log parseAccessLog(final String line)
|
|
{
|
|
Log result;
|
|
|
|
try
|
|
{
|
|
Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line);
|
|
if (matcher.matches())
|
|
{
|
|
String value = matcher.group("time");
|
|
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
|
|
|
|
String ip = matcher.group("remoteAddress").trim();
|
|
String user = matcher.group("remoteUser").trim();
|
|
|
|
result = new Log(line, date, ip, user);
|
|
}
|
|
else
|
|
{
|
|
throw new IllegalArgumentException("Bad line format: " + line);
|
|
}
|
|
}
|
|
catch (DateTimeParseException exception)
|
|
{
|
|
throw new IllegalArgumentException("Bad line format (date): " + line);
|
|
}
|
|
|
|
//
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* From error log.
|
|
*
|
|
* @param line
|
|
* the line
|
|
* @return the log
|
|
*/
|
|
public static Log parseErrorLog(final String line)
|
|
{
|
|
Log result;
|
|
|
|
try
|
|
{
|
|
Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line);
|
|
if (matcher.matches())
|
|
{
|
|
String value = matcher.group("time");
|
|
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
|
|
|
|
result = new Log(line, date);
|
|
}
|
|
else
|
|
{
|
|
throw new IllegalArgumentException("Bad line format: " + line);
|
|
}
|
|
}
|
|
catch (DateTimeParseException exception)
|
|
{
|
|
throw new IllegalArgumentException("Bad line format (date): " + line);
|
|
}
|
|
|
|
//
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Save access log.
|
|
*
|
|
* @param source
|
|
* the source
|
|
* @throws IOException
|
|
* @throws FileNotFoundException
|
|
*/
|
|
public static void saveLogs(final File target, final Logs logs) throws FileNotFoundException, IOException
|
|
{
|
|
PrintWriter out = null;
|
|
try
|
|
{
|
|
if (target.getName().endsWith(".gz"))
|
|
{
|
|
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
|
|
}
|
|
else
|
|
{
|
|
out = new PrintWriter(new FileOutputStream(target));
|
|
}
|
|
|
|
for (Log log : logs)
|
|
{
|
|
out.println(log.getLine());
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
IOUtils.closeQuietly(out);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Sort access log.
|
|
*
|
|
* @param target
|
|
* the target
|
|
* @throws IOException
|
|
*/
|
|
public static void sortLogFile(final File file) throws IOException
|
|
{
|
|
File workFile = new File(file.getParent(), file.getName() + ".tmp");
|
|
|
|
Logs logs = loadLogFile(file);
|
|
logs.sortByDatetime();
|
|
saveLogs(workFile, logs);
|
|
|
|
File backup = new File(file.getParentFile(), file.getName() + ".bak");
|
|
if (file.renameTo(backup))
|
|
{
|
|
if (!workFile.renameTo(file))
|
|
{
|
|
backup.renameTo(file);
|
|
}
|
|
}
|
|
|
|
// Check.
|
|
try
|
|
{
|
|
String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum \"");
|
|
System.out.print(out);
|
|
out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + ".bak | sort | sha1sum \"");
|
|
System.out.println(out);
|
|
}
|
|
catch (CmdExecException exception)
|
|
{
|
|
exception.printStackTrace();
|
|
}
|
|
}
|
|
}
|