885 lines
31 KiB
Java
885 lines
31 KiB
Java
/*
|
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
|
*
|
|
* This file is part of Logar, simple tool to manage http log files.
|
|
*
|
|
* Logar is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU Affero General Public License as
|
|
* published by the Free Software Foundation, either version 3 of the
|
|
* License, or (at your option) any later version.
|
|
*
|
|
* Logar is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU Affero General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License
|
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
package fr.devinsy.logar.app;
|
|
|
|
import java.io.File;
|
|
import java.io.FileOutputStream;
|
|
import java.io.IOException;
|
|
import java.io.PrintWriter;
|
|
import java.time.LocalDateTime;
|
|
import java.time.YearMonth;
|
|
import java.time.format.DateTimeFormatter;
|
|
import java.time.format.DateTimeParseException;
|
|
import java.util.Locale;
|
|
import java.util.regex.Matcher;
|
|
import java.util.regex.Pattern;
|
|
import java.util.zip.GZIPOutputStream;
|
|
|
|
import org.apache.commons.io.IOUtils;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
import org.april.logar.util.Files;
|
|
import org.april.logar.util.FilesUtils;
|
|
import org.april.logar.util.LineIterator;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import fr.devinsy.logar.app.anonymizer.Anonymizer;
|
|
import fr.devinsy.logar.app.log.LogUtils;
|
|
|
|
/**
|
|
* The Class Logar.
|
|
*/
|
|
public final class Logar
|
|
{
|
|
private static Logger logger = LoggerFactory.getLogger(Logar.class);
|
|
|
|
public static Pattern nginxAccessLogLinePattern = Pattern.compile("^\\S+ - [^\\[]+ \\[(?<time>[^\\]]+)\\] .*$");
|
|
public static Pattern nginxErrorLogLinePattern = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
|
|
|
|
/**
|
|
* Instantiates a new log tool.
|
|
*/
|
|
private Logar()
|
|
{
|
|
}
|
|
|
|
/**
|
|
* Anonymize.
|
|
*
|
|
* @param source
|
|
* the source
|
|
* @param target
|
|
* the target
|
|
*/
|
|
public static void anonymize(final File source)
|
|
{
|
|
anonymize(source, null);
|
|
}
|
|
|
|
/**
|
|
* Anonymize.
|
|
*
|
|
* @param source
|
|
* the source
|
|
* @param target
|
|
* the target
|
|
*/
|
|
public static void anonymize(final File source, final File mapFile)
|
|
{
|
|
if (source == null)
|
|
{
|
|
throw new IllegalArgumentException("Null source file.");
|
|
}
|
|
else if (!source.exists())
|
|
{
|
|
throw new IllegalArgumentException("Source file does not exist.");
|
|
}
|
|
else
|
|
{
|
|
Anonymizer anonymizer = new Anonymizer();
|
|
anonymizer.loadMapTable(mapFile);
|
|
System.out.println("Table size=" + anonymizer.getMapTable().size());
|
|
|
|
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeContaining("-anon.log");
|
|
logger.info("file count={}", files.size());
|
|
for (File file : files)
|
|
{
|
|
if (file.getName().contains("access"))
|
|
{
|
|
anonymizer.anonymizeAccessFile(file);
|
|
}
|
|
else if (file.getName().contains("error"))
|
|
{
|
|
//
|
|
}
|
|
}
|
|
|
|
if (mapFile != null)
|
|
{
|
|
System.out.println("Table size=" + anonymizer.getMapTable().size());
|
|
anonymizer.SaveMapTable(mapFile);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Archive.
|
|
*
|
|
* @param source
|
|
* the source
|
|
* @param target
|
|
* the target
|
|
* @throws IOException
|
|
*/
|
|
public static void archive(final File source, final File target) throws IOException
|
|
{
|
|
archiveAccessFiles(source, target);
|
|
archiveErrorFiles(source, target);
|
|
}
|
|
|
|
/**
|
|
* Test archive access files.
|
|
*
|
|
* @param source
|
|
* the source
|
|
* @throws IOException
|
|
*/
|
|
public static void archiveAccessFiles(final File source, final File target) throws IOException
|
|
{
|
|
if (source == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter [source]");
|
|
}
|
|
else if (!source.isDirectory())
|
|
{
|
|
throw new IllegalArgumentException("Source parameter is not a directory.");
|
|
}
|
|
else if (!target.isDirectory())
|
|
{
|
|
throw new IllegalArgumentException("Target parameter is not a directory.");
|
|
}
|
|
else
|
|
{
|
|
YearMonth targetYearMonth = YearMonth.now().minusMonths(1);
|
|
Stats counter = new Stats();
|
|
|
|
for (File directory : Files.of(source).sortByName())
|
|
{
|
|
if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
|
|
{
|
|
String targetFileName = String.format("%s-access-%s.log.gz", directory.getName(), targetYearMonth.toString());
|
|
File targetDirectory = new File(target, directory.getName());
|
|
targetDirectory.mkdirs();
|
|
File targetFile = new File(targetDirectory, targetFileName);
|
|
logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile());
|
|
PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
|
|
|
|
for (File file : Files.of(directory).sortByName())
|
|
{
|
|
if ((!file.isDirectory()) && (file.getName().contains("access")))
|
|
{
|
|
logger.info(file.getName());
|
|
try
|
|
{
|
|
LineIterator iterator = new LineIterator(file);
|
|
while (iterator.hasNext())
|
|
{
|
|
String line = iterator.next();
|
|
counter.incLineCount();
|
|
|
|
Matcher matcher = nginxAccessLogLinePattern.matcher(line);
|
|
if (matcher.matches())
|
|
{
|
|
String value = matcher.group("time");
|
|
try
|
|
{
|
|
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
|
|
counter.incSuccessLineCount();
|
|
|
|
if (YearMonth.from(date).equals(targetYearMonth))
|
|
{
|
|
out.println(line);
|
|
}
|
|
}
|
|
catch (DateTimeParseException exception)
|
|
{
|
|
System.err.println("Date format problem with [" + line + "]");
|
|
counter.incErrorLineCount();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
System.err.println("Not matching line [" + line + "]");
|
|
counter.incErrorLineCount();
|
|
}
|
|
}
|
|
|
|
counter.incSuccessFileCount();
|
|
}
|
|
catch (IOException exception)
|
|
{
|
|
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
|
|
exception.printStackTrace();
|
|
counter.incErrorFileCount();
|
|
}
|
|
finally
|
|
{
|
|
counter.incFileCount();
|
|
}
|
|
}
|
|
}
|
|
|
|
IOUtils.closeQuietly(out);
|
|
}
|
|
}
|
|
|
|
System.out.println("=====================================================");
|
|
counter.stop();
|
|
System.out.println(counter.toString());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Archive error files.
|
|
*
|
|
* @param source
|
|
* the source
|
|
* @param target
|
|
* the target
|
|
* @throws IOException
|
|
* Signals that an I/O exception has occurred.
|
|
*/
|
|
public static void archiveErrorFiles(final File source, final File target) throws IOException
|
|
{
|
|
if (source == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter [source]");
|
|
}
|
|
else if (!source.isDirectory())
|
|
{
|
|
throw new IllegalArgumentException("Source parameter is not a directory.");
|
|
}
|
|
else if (!target.isDirectory())
|
|
{
|
|
throw new IllegalArgumentException("Target parameter is not a directory.");
|
|
}
|
|
else
|
|
{
|
|
YearMonth targetYearMonth = YearMonth.now().minusMonths(1);
|
|
Stats counter = new Stats();
|
|
counter.start();
|
|
|
|
for (File directory : Files.of(source).sortByName())
|
|
{
|
|
if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
|
|
{
|
|
String targetFileName = String.format("%s-error-%s.log.gz", directory.getName(), targetYearMonth.toString());
|
|
File targetDirectory = new File(target, directory.getName());
|
|
targetDirectory.mkdirs();
|
|
File targetFile = new File(targetDirectory, targetFileName);
|
|
logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile());
|
|
PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
|
|
|
|
for (File file : Files.of(directory).sortByName())
|
|
{
|
|
if ((!file.isDirectory()) && (file.getName().contains("error")))
|
|
{
|
|
// logger.info(file.getName());
|
|
try
|
|
{
|
|
LineIterator iterator = new LineIterator(file);
|
|
while (iterator.hasNext())
|
|
{
|
|
String line = iterator.next();
|
|
counter.incLineCount();
|
|
|
|
Matcher matcher = nginxErrorLogLinePattern.matcher(line);
|
|
if (matcher.matches())
|
|
{
|
|
String value = matcher.group("time");
|
|
try
|
|
{
|
|
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
|
|
counter.incSuccessLineCount();
|
|
|
|
if (YearMonth.from(date).equals(targetYearMonth))
|
|
{
|
|
out.println(line);
|
|
}
|
|
}
|
|
catch (DateTimeParseException exception)
|
|
{
|
|
System.err.println("Date format problem with [" + line + "]");
|
|
counter.incErrorLineCount();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
System.err.println("Not matching line [" + line + "]");
|
|
counter.incErrorLineCount();
|
|
}
|
|
}
|
|
|
|
counter.incSuccessLineCount();
|
|
}
|
|
catch (IOException exception)
|
|
{
|
|
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
|
|
exception.printStackTrace();
|
|
counter.incErrorFileCount();
|
|
}
|
|
finally
|
|
{
|
|
counter.incFileCount();
|
|
}
|
|
}
|
|
}
|
|
|
|
IOUtils.closeQuietly(out);
|
|
}
|
|
}
|
|
|
|
System.out.println("=====================================================");
|
|
counter.stop();
|
|
System.out.println(counter.toString());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check.
|
|
*
|
|
* @param source
|
|
* the source
|
|
*/
|
|
public static void check(final File source)
|
|
{
|
|
if (source != null)
|
|
{
|
|
if (source.isFile())
|
|
{
|
|
if (source.getName().contains("access"))
|
|
{
|
|
checkAccessFiles(source);
|
|
}
|
|
else if (source.getName().contains("error"))
|
|
{
|
|
checkErrorFiles(source);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for (File file : Files.of(source).removeHidden().sortByName())
|
|
{
|
|
if (file.isDirectory())
|
|
{
|
|
check(file);
|
|
}
|
|
else if (StringUtils.endsWithAny(file.getName(), ".log", ".gz", ".1", ".2"))
|
|
{
|
|
check(file);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check access files.
|
|
*
|
|
* @param file
|
|
* the source
|
|
*/
|
|
public static void checkAccessFiles(final File file)
|
|
{
|
|
if (file == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter.");
|
|
}
|
|
else if (!file.isFile())
|
|
{
|
|
throw new IllegalArgumentException("Parameter is not a file.");
|
|
}
|
|
else
|
|
{
|
|
System.out.println("== Check access log for [" + file.getName() + "]");
|
|
|
|
int lineCount = 0;
|
|
int badLineCount = 0;
|
|
try
|
|
{
|
|
LineIterator iterator = new LineIterator(file);
|
|
while (iterator.hasNext())
|
|
{
|
|
String line = iterator.next();
|
|
lineCount += 1;
|
|
|
|
try
|
|
{
|
|
LogUtils.parseAccessLog(line).getDatetime();
|
|
}
|
|
catch (IllegalArgumentException exception)
|
|
{
|
|
System.out.println("Bad format line: " + line);
|
|
badLineCount += 1;
|
|
exception.printStackTrace();
|
|
}
|
|
catch (DateTimeParseException exception)
|
|
{
|
|
System.out.println("Bad datetime format: " + line);
|
|
badLineCount += 1;
|
|
}
|
|
}
|
|
}
|
|
catch (IOException exception)
|
|
{
|
|
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
|
|
exception.printStackTrace();
|
|
}
|
|
|
|
if (badLineCount > 0)
|
|
{
|
|
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check error files.
|
|
*
|
|
* @param file
|
|
* the file
|
|
*/
|
|
public static void checkErrorFiles(final File file)
|
|
{
|
|
if (file == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter.");
|
|
}
|
|
else if (!file.isFile())
|
|
{
|
|
throw new IllegalArgumentException("Parameter is not a file.");
|
|
}
|
|
else
|
|
{
|
|
System.out.println("== Check error log for [" + file.getName() + "]");
|
|
|
|
int lineCount = 0;
|
|
int badLineCount = 0;
|
|
try
|
|
{
|
|
LineIterator iterator = new LineIterator(file);
|
|
while (iterator.hasNext())
|
|
{
|
|
String line = iterator.next();
|
|
lineCount += 1;
|
|
|
|
try
|
|
{
|
|
LogUtils.parseErrorLog(line).getDatetime();
|
|
}
|
|
catch (IllegalArgumentException exception)
|
|
{
|
|
System.out.println("Bad format line: " + line);
|
|
badLineCount += 1;
|
|
}
|
|
catch (DateTimeParseException exception)
|
|
{
|
|
System.out.println("Bad datetime format: " + line);
|
|
badLineCount += 1;
|
|
}
|
|
}
|
|
}
|
|
catch (IOException exception)
|
|
{
|
|
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
|
|
exception.printStackTrace();
|
|
}
|
|
|
|
if (badLineCount > 0)
|
|
{
|
|
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check sort.
|
|
*
|
|
* @param source
|
|
* the source
|
|
* @throws IOException
|
|
*/
|
|
public static void checkSort(final File source) throws IOException
|
|
{
|
|
if (source != null)
|
|
{
|
|
if (source.isFile())
|
|
{
|
|
if (source.getName().contains("access"))
|
|
{
|
|
checkSortForAccessFiles(source);
|
|
}
|
|
else if (source.getName().contains("error"))
|
|
{
|
|
checkSortForErrorFiles(source);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for (File file : Files.of(source).removeHidden().sortByName())
|
|
{
|
|
checkSort(file);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check sort for access files.
|
|
*
|
|
* @param source
|
|
* the source
|
|
* @throws IOException
|
|
*/
|
|
public static void checkSortForAccessFiles(final File file) throws IOException
|
|
{
|
|
if (file == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter [source]");
|
|
}
|
|
else if (!file.isFile())
|
|
{
|
|
throw new IllegalArgumentException("Source parameter is not a file.");
|
|
}
|
|
else
|
|
{
|
|
System.out.println("== Check sort for [" + file.getName() + "]");
|
|
LocalDateTime currentDate = null;
|
|
int lineCount = 0;
|
|
int badLineCount = 0;
|
|
LineIterator iterator = new LineIterator(file);
|
|
while (iterator.hasNext())
|
|
{
|
|
String line = iterator.next();
|
|
lineCount += 1;
|
|
LocalDateTime date = LogUtils.parseAccessLog(line).getDatetime();
|
|
if ((currentDate != null) && (date.isBefore(currentDate)))
|
|
{
|
|
System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString()));
|
|
badLineCount += 1;
|
|
}
|
|
currentDate = date;
|
|
}
|
|
|
|
if (badLineCount > 0)
|
|
{
|
|
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check sort for error files.
|
|
*
|
|
* @param source
|
|
* the source
|
|
* @throws IOException
|
|
*/
|
|
public static void checkSortForErrorFiles(final File file) throws IOException
|
|
{
|
|
if (file == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter [source]");
|
|
}
|
|
else if (!file.isFile())
|
|
{
|
|
throw new IllegalArgumentException("Source parameter is not a file.");
|
|
}
|
|
else
|
|
{
|
|
System.out.println("== Check sort for [" + file.getName() + "]");
|
|
LocalDateTime currentDate = null;
|
|
int lineCount = 0;
|
|
int badLineCount = 0;
|
|
LineIterator iterator = new LineIterator(file);
|
|
while (iterator.hasNext())
|
|
{
|
|
String line = iterator.next();
|
|
lineCount += 1;
|
|
LocalDateTime date = LogUtils.parseErrorLog(line).getDatetime();
|
|
if ((currentDate != null) && (date.isBefore(currentDate)))
|
|
{
|
|
System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString()));
|
|
badLineCount += 1;
|
|
}
|
|
currentDate = date;
|
|
}
|
|
|
|
if (badLineCount > 0)
|
|
{
|
|
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Sort.
|
|
*
|
|
* @param source
|
|
* the source
|
|
* @throws IOException
|
|
* Signals that an I/O exception has occurred.
|
|
*/
|
|
public static void sort(final File source) throws IOException
|
|
{
|
|
if (source != null)
|
|
{
|
|
if (source.isFile())
|
|
{
|
|
if (source.getName().contains("access"))
|
|
{
|
|
sortAccessFiles(source);
|
|
}
|
|
else if (source.getName().contains("error"))
|
|
{
|
|
sortErrorFiles(source);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for (File file : Files.of(source).removeHidden().sortByName())
|
|
{
|
|
sort(file);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Sort access files.
|
|
*
|
|
* @param file
|
|
* the file
|
|
* @throws IOException
|
|
* Signals that an I/O exception has occurred.
|
|
*/
|
|
public static void sortAccessFiles(final File file) throws IOException
|
|
{
|
|
if (file == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter [source]");
|
|
}
|
|
else if (!file.isFile())
|
|
{
|
|
throw new IllegalArgumentException("Source parameter is not a file.");
|
|
}
|
|
else
|
|
{
|
|
System.out.println("== Sort for [" + file.getName() + "]");
|
|
LogUtils.sortAccessLog(file);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Sort error files.
|
|
*
|
|
* @param file
|
|
* the file
|
|
* @throws IOException
|
|
*/
|
|
public static void sortErrorFiles(final File file) throws IOException
|
|
{
|
|
if (file == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter [source]");
|
|
}
|
|
else if (!file.isFile())
|
|
{
|
|
throw new IllegalArgumentException("Source parameter is not a file.");
|
|
}
|
|
else
|
|
{
|
|
System.out.println("== Sort for [" + file.getName() + "]");
|
|
LogUtils.sortErrorLog(file);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Test parsing.
|
|
*
|
|
* @param source
|
|
* the source
|
|
*/
|
|
public static void testParsing(final File source)
|
|
{
|
|
testParsingAccessFiles(source);
|
|
testParsingErrorFiles(source);
|
|
}
|
|
|
|
/**
|
|
* Test parsing access files.
|
|
*
|
|
* @param source
|
|
* the source
|
|
*/
|
|
public static void testParsingAccessFiles(final File source)
|
|
{
|
|
if (source == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter [source]");
|
|
}
|
|
else if (!source.isDirectory())
|
|
{
|
|
throw new IllegalArgumentException("Source parameter is not a directory.");
|
|
}
|
|
else
|
|
{
|
|
System.out.println("== Test access log for [" + source.getName() + "]");
|
|
Stats counter = new Stats();
|
|
|
|
for (File directory : source.listFiles())
|
|
{
|
|
if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
|
|
{
|
|
logger.info("== {}", directory.getName());
|
|
|
|
for (File file : directory.listFiles())
|
|
{
|
|
if ((!file.isDirectory()) && (file.getName().contains("access")))
|
|
{
|
|
// logger.info(file.getName());
|
|
|
|
try
|
|
{
|
|
LineIterator iterator = new LineIterator(file);
|
|
while (iterator.hasNext())
|
|
{
|
|
String line = iterator.next();
|
|
counter.incLineCount();
|
|
|
|
Matcher matcher = nginxAccessLogLinePattern.matcher(line);
|
|
if (matcher.matches())
|
|
{
|
|
String value = matcher.group("time");
|
|
try
|
|
{
|
|
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
|
|
counter.incSuccessLineCount();
|
|
}
|
|
catch (DateTimeParseException exception)
|
|
{
|
|
System.err.println("Date format error with [" + line + "]");
|
|
counter.incErrorLineCount();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
System.err.println("Not matching line [" + line + "]");
|
|
counter.incErrorLineCount();
|
|
}
|
|
}
|
|
|
|
counter.incSuccessFileCount();
|
|
}
|
|
catch (IOException exception)
|
|
{
|
|
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
|
|
exception.printStackTrace();
|
|
counter.incErrorFileCount();
|
|
}
|
|
finally
|
|
{
|
|
counter.incFileCount();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
counter.stop();
|
|
System.out.println(counter.toString());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Test parsing error files.
|
|
*
|
|
* @param source
|
|
* the source
|
|
*/
|
|
public static void testParsingErrorFiles(final File source)
|
|
{
|
|
if (source == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter [source]");
|
|
}
|
|
else if (!source.isDirectory())
|
|
{
|
|
throw new IllegalArgumentException("Source parameter is not a directory.");
|
|
}
|
|
else
|
|
{
|
|
System.out.println("== Test error log for [" + source.getName() + "]");
|
|
Stats counter = new Stats();
|
|
|
|
for (File directory : source.listFiles())
|
|
{
|
|
if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
|
|
{
|
|
logger.info("== {}", directory.getName());
|
|
|
|
for (File file : directory.listFiles())
|
|
{
|
|
if ((!file.isDirectory()) && (file.getName().contains("error")))
|
|
{
|
|
// logger.info(file.getName());
|
|
try
|
|
{
|
|
LineIterator iterator = new LineIterator(file);
|
|
while (iterator.hasNext())
|
|
{
|
|
String line = iterator.next();
|
|
counter.incLineCount();
|
|
|
|
Matcher matcher = nginxErrorLogLinePattern.matcher(line);
|
|
if (matcher.matches())
|
|
{
|
|
String value = matcher.group("time");
|
|
try
|
|
{
|
|
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
|
|
counter.incSuccessLineCount();
|
|
}
|
|
catch (DateTimeParseException exception)
|
|
{
|
|
System.err.println("Date format problem with [" + line + "]");
|
|
counter.incErrorLineCount();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
System.err.println("Not matching line [" + line + "]");
|
|
counter.incErrorLineCount();
|
|
}
|
|
}
|
|
|
|
counter.incSuccessFileCount();
|
|
}
|
|
catch (IOException exception)
|
|
{
|
|
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
|
|
exception.printStackTrace();
|
|
counter.incErrorFileCount();
|
|
}
|
|
finally
|
|
{
|
|
counter.incFileCount();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
System.out.println("=====================================================");
|
|
counter.stop();
|
|
System.out.println(counter.toString());
|
|
}
|
|
}
|
|
}
|