diff --git a/src/fr/devinsy/logar/app/Logar.java b/src/fr/devinsy/logar/app/Logar.java index 1c37dd0..acbb8da 100644 --- a/src/fr/devinsy/logar/app/Logar.java +++ b/src/fr/devinsy/logar/app/Logar.java @@ -40,6 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import fr.devinsy.logar.app.anonymizer.Anonymizer; +import fr.devinsy.logar.app.log.Log; import fr.devinsy.logar.app.log.LogUtils; /** @@ -100,14 +101,7 @@ public final class Logar logger.info("file count={}", files.size()); for (File file : files) { - if (file.getName().contains("access")) - { - anonymizer.anonymize(file); - } - else if (file.getName().contains("error")) - { - // - } + anonymizer.anonymize(file); } if (mapFile != null) @@ -159,74 +153,61 @@ public final class Logar YearMonth targetYearMonth = YearMonth.now().minusMonths(1); Stats counter = new Stats(); - for (File directory : Files.of(source).sortByName()) + for (File directory : Files.of(source).removeHidden().keepDirectoriesOnly().sortByName()) { - if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", ".."))) + String targetFileName = String.format("%s-access-%s.log.gz", directory.getName(), targetYearMonth.toString()); + File targetDirectory = new File(target, directory.getName()); + targetDirectory.mkdirs(); + File targetFile = new File(targetDirectory, targetFileName); + logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile()); + PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile))); + + for (File file : Files.of(directory).sortByName()) { - String targetFileName = String.format("%s-access-%s.log.gz", directory.getName(), targetYearMonth.toString()); - File targetDirectory = new File(target, directory.getName()); - targetDirectory.mkdirs(); - File targetFile = new File(targetDirectory, targetFileName); - logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile()); - PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile))); - - for (File file : Files.of(directory).sortByName()) + if ((!file.isDirectory()) && (file.getName().contains("access"))) { - if ((!file.isDirectory()) && (file.getName().contains("access"))) + logger.info(file.getName()); + try { - logger.info(file.getName()); - try + LineIterator iterator = new LineIterator(file); + while (iterator.hasNext()) { - LineIterator iterator = new LineIterator(file); - while (iterator.hasNext()) + String line = iterator.next(); + counter.incLineCount(); + + try { - String line = iterator.next(); - counter.incLineCount(); + Log log = LogUtils.parseAccessLog(line); + counter.incSuccessLineCount(); - Matcher matcher = nginxAccessLogLinePattern.matcher(line); - if (matcher.matches()) + if (YearMonth.from(log.getDatetime()).equals(targetYearMonth)) { - String value = matcher.group("time"); - try - { - LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)); - counter.incSuccessLineCount(); - - if (YearMonth.from(date).equals(targetYearMonth)) - { - out.println(line); - } - } - catch (DateTimeParseException exception) - { - System.err.println("Date format problem with [" + line + "]"); - counter.incErrorLineCount(); - } - } - else - { - System.err.println("Not matching line [" + line + "]"); - counter.incErrorLineCount(); + out.println(line); } } + catch (IllegalArgumentException exception) + { + System.err.println("Bad line format [" + line + "]"); + counter.incErrorLineCount(); + } + } - counter.incSuccessFileCount(); - } - catch (IOException exception) - { - System.err.println("Error with file [" + file.getAbsolutePath() + "]"); - exception.printStackTrace(); - counter.incErrorFileCount(); - } - finally - { - counter.incFileCount(); - } + counter.incSuccessFileCount(); + } + catch (IOException exception) + { + System.err.println("Error with file [" + file.getAbsolutePath() + "]"); + exception.printStackTrace(); + counter.incErrorFileCount(); + } + finally + { + counter.incFileCount(); } } - - IOUtils.closeQuietly(out); } + + IOUtils.closeQuietly(out); } System.out.println("====================================================="); @@ -265,74 +246,60 @@ public final class Logar Stats counter = new Stats(); counter.start(); - for (File directory : Files.of(source).sortByName()) + for (File directory : Files.of(source).removeHidden().keepDirectoriesOnly().sortByName()) { - if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", ".."))) + String targetFileName = String.format("%s-error-%s.log.gz", directory.getName(), targetYearMonth.toString()); + File targetDirectory = new File(target, directory.getName()); + targetDirectory.mkdirs(); + File targetFile = new File(targetDirectory, targetFileName); + logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile()); + PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile))); + + for (File file : Files.of(directory).sortByName()) { - String targetFileName = String.format("%s-error-%s.log.gz", directory.getName(), targetYearMonth.toString()); - File targetDirectory = new File(target, directory.getName()); - targetDirectory.mkdirs(); - File targetFile = new File(targetDirectory, targetFileName); - logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile()); - PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile))); - - for (File file : Files.of(directory).sortByName()) + if ((!file.isDirectory()) && (file.getName().contains("error"))) { - if ((!file.isDirectory()) && (file.getName().contains("error"))) + // logger.info(file.getName()); + try { - // logger.info(file.getName()); - try + LineIterator iterator = new LineIterator(file); + while (iterator.hasNext()) { - LineIterator iterator = new LineIterator(file); - while (iterator.hasNext()) + String line = iterator.next(); + counter.incLineCount(); + + try { - String line = iterator.next(); - counter.incLineCount(); + Log log = LogUtils.parseErrorLog(line); + counter.incSuccessLineCount(); - Matcher matcher = nginxErrorLogLinePattern.matcher(line); - if (matcher.matches()) + if (YearMonth.from(log.getDatetime()).equals(targetYearMonth)) { - String value = matcher.group("time"); - try - { - LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH)); - counter.incSuccessLineCount(); - - if (YearMonth.from(date).equals(targetYearMonth)) - { - out.println(line); - } - } - catch (DateTimeParseException exception) - { - System.err.println("Date format problem with [" + line + "]"); - counter.incErrorLineCount(); - } - } - else - { - System.err.println("Not matching line [" + line + "]"); - counter.incErrorLineCount(); + out.println(line); } } - - counter.incSuccessLineCount(); - } - catch (IOException exception) - { - System.err.println("Error with file [" + file.getAbsolutePath() + "]"); - exception.printStackTrace(); - counter.incErrorFileCount(); - } - finally - { - counter.incFileCount(); + catch (IllegalArgumentException exception) + { + System.err.println("Bad line format [" + line + "]"); + counter.incErrorLineCount(); + } } + counter.incSuccessFileCount(); + } + catch (IOException exception) + { + System.err.println("Error with file [" + file.getAbsolutePath() + "]"); + exception.printStackTrace(); + counter.incErrorFileCount(); + } + finally + { + counter.incFileCount(); } } - - IOUtils.closeQuietly(out); } + + IOUtils.closeQuietly(out); } System.out.println("====================================================="); @@ -341,51 +308,13 @@ public final class Logar } } - /** - * Check. - * - * @param source - * the source - */ - public static void check(final File source) - { - if (source != null) - { - if (source.isFile()) - { - if (source.getName().contains("access")) - { - checkAccessFiles(source); - } - else if (source.getName().contains("error")) - { - checkErrorFiles(source); - } - } - else - { - for (File file : Files.of(source).removeHidden().sortByName()) - { - if (file.isDirectory()) - { - check(file); - } - else if (StringUtils.endsWithAny(file.getName(), ".log", ".gz", ".1", ".2")) - { - check(file); - } - } - } - } - } - /** * Check access files. * * @param file * the source */ - public static void checkAccessFiles(final File file) + public static void checkLogFile(final File file) { if (file == null) { @@ -398,6 +327,15 @@ public final class Logar else { System.out.println("== Check access log for [" + file.getName() + "]"); + boolean isAccessFile; + if (file.getName().contains("access")) + { + isAccessFile = true; + } + else + { + isAccessFile = false; + } int lineCount = 0; int badLineCount = 0; @@ -411,7 +349,14 @@ public final class Logar try { - LogUtils.parseAccessLog(line).getDatetime(); + if (isAccessFile) + { + LogUtils.parseAccessLog(line).getDatetime(); + } + else + { + LogUtils.parseErrorLog(line).getDatetime(); + } } catch (IllegalArgumentException exception) { @@ -440,55 +385,75 @@ public final class Logar } /** - * Check error files. + * Check. * - * @param file - * the file + * @param source + * the source */ - public static void checkErrorFiles(final File file) + public static void checkLogFiles(final File source) + { + Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz", ".1", ".2", ".3").removeHidden().sortByName(); + + for (File file : files) + { + checkLogFile(file); + } + } + + /** + * Check sort for access files. + * + * @param source + * the source + * @throws IOException + */ + public static void checkSort(final File file) throws IOException { if (file == null) { - throw new IllegalArgumentException("Null parameter."); + throw new IllegalArgumentException("Null parameter [source]"); } else if (!file.isFile()) { - throw new IllegalArgumentException("Parameter is not a file."); + throw new IllegalArgumentException("Source parameter is not a file."); } else { - System.out.println("== Check error log for [" + file.getName() + "]"); + boolean isAccessFile; + if (file.getName().contains("access")) + { + isAccessFile = true; + } + else + { + isAccessFile = false; + } + System.out.println("== Check sort for [" + file.getName() + "]"); + LocalDateTime currentDate = null; int lineCount = 0; int badLineCount = 0; - try + LineIterator iterator = new LineIterator(file); + while (iterator.hasNext()) { - LineIterator iterator = new LineIterator(file); - while (iterator.hasNext()) + String line = iterator.next(); + lineCount += 1; + LocalDateTime date; + if (isAccessFile) { - String line = iterator.next(); - lineCount += 1; - - try - { - LogUtils.parseErrorLog(line).getDatetime(); - } - catch (IllegalArgumentException exception) - { - System.out.println("Bad format line: " + line); - badLineCount += 1; - } - catch (DateTimeParseException exception) - { - System.out.println("Bad datetime format: " + line); - badLineCount += 1; - } + date = LogUtils.parseAccessLog(line).getDatetime(); } - } - catch (IOException exception) - { - System.err.println("Error with file [" + file.getAbsolutePath() + "]"); - exception.printStackTrace(); + else + { + date = LogUtils.parseErrorLog(line).getDatetime(); + } + + if ((currentDate != null) && (date.isBefore(currentDate))) + { + System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString())); + badLineCount += 1; + } + currentDate = date; } if (badLineCount > 0) @@ -505,116 +470,13 @@ public final class Logar * the source * @throws IOException */ - public static void checkSort(final File source) throws IOException + public static void checkSorts(final File source) throws IOException { - if (source != null) - { - if (source.isFile()) - { - if (source.getName().contains("access")) - { - checkSortForAccessFiles(source); - } - else if (source.getName().contains("error")) - { - checkSortForErrorFiles(source); - } - } - else - { - for (File file : Files.of(source).removeHidden().sortByName()) - { - checkSort(file); - } - } - } - } + Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeHidden().sortByName(); - /** - * Check sort for access files. - * - * @param source - * the source - * @throws IOException - */ - public static void checkSortForAccessFiles(final File file) throws IOException - { - if (file == null) + for (File file : files) { - throw new IllegalArgumentException("Null parameter [source]"); - } - else if (!file.isFile()) - { - throw new IllegalArgumentException("Source parameter is not a file."); - } - else - { - System.out.println("== Check sort for [" + file.getName() + "]"); - LocalDateTime currentDate = null; - int lineCount = 0; - int badLineCount = 0; - LineIterator iterator = new LineIterator(file); - while (iterator.hasNext()) - { - String line = iterator.next(); - lineCount += 1; - LocalDateTime date = LogUtils.parseAccessLog(line).getDatetime(); - if ((currentDate != null) && (date.isBefore(currentDate))) - { - System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString())); - badLineCount += 1; - } - currentDate = date; - } - - if (badLineCount > 0) - { - System.out.println("Bad line count: " + badLineCount + "/" + lineCount); - } - } - } - - /** - * Check sort for error files. - * - * @param source - * the source - * @throws IOException - */ - public static void checkSortForErrorFiles(final File file) throws IOException - { - if (file == null) - { - throw new IllegalArgumentException("Null parameter [source]"); - } - else if (!file.isFile()) - { - throw new IllegalArgumentException("Source parameter is not a file."); - } - else - { - System.out.println("== Check sort for [" + file.getName() + "]"); - LocalDateTime currentDate = null; - int lineCount = 0; - int badLineCount = 0; - LineIterator iterator = new LineIterator(file); - while (iterator.hasNext()) - { - String line = iterator.next(); - lineCount += 1; - LocalDateTime date = LogUtils.parseErrorLog(line).getDatetime(); - if ((currentDate != null) && (date.isBefore(currentDate))) - { - System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString())); - badLineCount += 1; - } - currentDate = date; - } - - if (badLineCount > 0) - { - System.out.println("Bad line count: " + badLineCount + "/" + lineCount); - } + checkSort(file); } } @@ -628,75 +490,12 @@ public final class Logar */ public static void sort(final File source) throws IOException { - if (source != null) - { - if (source.isFile()) - { - if (source.getName().contains("access")) - { - sortAccessFiles(source); - } - else if (source.getName().contains("error")) - { - sortErrorFiles(source); - } - } - else - { - for (File file : Files.of(source).removeHidden().sortByName()) - { - sort(file); - } - } - } - } + Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeHidden().sortByName(); - /** - * Sort access files. - * - * @param file - * the file - * @throws IOException - * Signals that an I/O exception has occurred. - */ - public static void sortAccessFiles(final File file) throws IOException - { - if (file == null) - { - throw new IllegalArgumentException("Null parameter [source]"); - } - else if (!file.isFile()) - { - throw new IllegalArgumentException("Source parameter is not a file."); - } - else + for (File file : files) { System.out.println("== Sort for [" + file.getName() + "]"); - LogUtils.sortAccessLog(file); - } - } - - /** - * Sort error files. - * - * @param file - * the file - * @throws IOException - */ - public static void sortErrorFiles(final File file) throws IOException - { - if (file == null) - { - throw new IllegalArgumentException("Null parameter [source]"); - } - else if (!file.isFile()) - { - throw new IllegalArgumentException("Source parameter is not a file."); - } - else - { - System.out.println("== Sort for [" + file.getName() + "]"); - LogUtils.sortErrorLog(file); + LogUtils.sortLogFile(file); } } diff --git a/src/fr/devinsy/logar/app/log/Log.java b/src/fr/devinsy/logar/app/log/Log.java index 758a952..d7c623d 100644 --- a/src/fr/devinsy/logar/app/log/Log.java +++ b/src/fr/devinsy/logar/app/log/Log.java @@ -30,11 +30,17 @@ public final class Log { private static Logger logger = LoggerFactory.getLogger(Log.class); + // Generic attributes. private String line; private LocalDateTime datetime; + + // Specific access log attributes. private String ip; private String user; + // Specific error log attributes. + // private String message; + /** * Instantiates a new log. */ diff --git a/src/fr/devinsy/logar/app/log/LogUtils.java b/src/fr/devinsy/logar/app/log/LogUtils.java index 15437a4..f92e708 100644 --- a/src/fr/devinsy/logar/app/log/LogUtils.java +++ b/src/fr/devinsy/logar/app/log/LogUtils.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -111,6 +112,43 @@ public final class LogUtils return result; } + /** + * Load log file. + * + * @param file + * the file + * @return the logs + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public static Logs loadLogFile(final File file) throws IOException + { + Logs result; + + if (file == null) + { + throw new IllegalArgumentException("Null parameter."); + } + else + { + if (file.getName().contains("access")) + { + result = loadAccessLog(file); + } + else if (file.getName().contains("error")) + { + result = loadErrorLog(file); + } + else + { + throw new IllegalArgumentException("Bad named file (missing access or error)."); + } + } + + // + return result; + } + /** * From access log. * @@ -122,20 +160,27 @@ public final class LogUtils { Log result; - Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line); - if (matcher.matches()) + try { - String value = matcher.group("time"); - LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)); + Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line); + if (matcher.matches()) + { + String value = matcher.group("time"); + LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)); - String ip = matcher.group("remoteAddress").trim(); - String user = matcher.group("remoteUser").trim(); + String ip = matcher.group("remoteAddress").trim(); + String user = matcher.group("remoteUser").trim(); - result = new Log(line, date, ip, user); + result = new Log(line, date, ip, user); + } + else + { + throw new IllegalArgumentException("Bad line format: " + line); + } } - else + catch (DateTimeParseException exception) { - throw new IllegalArgumentException("Bad line format: " + line); + throw new IllegalArgumentException("Bad line format (date): " + line); } // @@ -153,17 +198,24 @@ public final class LogUtils { Log result; - Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line); - if (matcher.matches()) + try { - String value = matcher.group("time"); - LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH)); + Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line); + if (matcher.matches()) + { + String value = matcher.group("time"); + LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH)); - result = new Log(line, date); + result = new Log(line, date); + } + else + { + throw new IllegalArgumentException("Bad line format: " + line); + } } - else + catch (DateTimeParseException exception) { - throw new IllegalArgumentException("Bad line format: " + line); + throw new IllegalArgumentException("Bad line format (date): " + line); } // @@ -183,7 +235,14 @@ public final class LogUtils PrintWriter out = null; try { - out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target))); + if (target.getName().endsWith(".gz")) + { + out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target))); + } + else + { + out = new PrintWriter(new FileOutputStream(target)); + } for (Log log : logs) { @@ -203,61 +262,24 @@ public final class LogUtils * the target * @throws IOException */ - public static void sortAccessLog(final File file) throws IOException + public static void sortLogFile(final File file) throws IOException { - File work = new File(file.getParent(), file.getName() + ".tmp"); + File workFile = new File(file.getParent(), file.getName() + ".tmp"); - Logs logs = loadAccessLog(file); + Logs logs = loadLogFile(file); logs.sortByDatetime(); - saveLogs(work, logs); + saveLogs(workFile, logs); File backup = new File(file.getParentFile(), file.getName() + ".bak"); if (file.renameTo(backup)) { - if (!work.renameTo(file)) - { - backup.renameTo(file); - } - } - - try - { - String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum \""); - System.out.print(out); - out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + ".bak | sort | sha1sum \""); - System.out.println(out); - } - catch (CmdExecException exception) - { - exception.printStackTrace(); - } - } - - /** - * Sort error log. - * - * @param file - * the file - * @throws IOException - * Signals that an I/O exception has occurred. - */ - public static void sortErrorLog(final File file) throws IOException - { - File work = new File(file.getParent(), file.getName() + ".tmp"); - - Logs logs = loadErrorLog(file); - logs.sortByDatetime(); - saveLogs(work, logs); - - File backup = new File(file.getParentFile(), file.getName() + ".bak"); - if (file.renameTo(backup)) - { - if (!work.renameTo(file)) + if (!workFile.renameTo(file)) { backup.renameTo(file); } } + // Check. try { String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum \""); diff --git a/src/org/april/logar/cli/LogarCLI.java b/src/org/april/logar/cli/LogarCLI.java index 3428cea..e8d29c3 100644 --- a/src/org/april/logar/cli/LogarCLI.java +++ b/src/org/april/logar/cli/LogarCLI.java @@ -56,8 +56,8 @@ public final class LogarCLI message.appendln(" logar [ -v | -version | --version ]"); message.appendln(" logar anonymize fileordirectory [maptable] anonymize ip and login"); message.appendln(" logar archive source target archive previous month"); - message.appendln(" logar checksort fileordirectory check sort of an access log file"); message.appendln(" logar check fileordirectory census bad format line in log files"); + message.appendln(" logar checksort fileordirectory check sort of an access log file"); message.appendln(" logar sort fileordirectory sort log files by datetime"); message.appendln(" logar testarchive source test archive"); @@ -207,13 +207,13 @@ public final class LogarCLI { File source = new File(args[1]); - Logar.check(source); + Logar.checkLogFiles(source); } else if (isMatching(args, "checksort", "\\s*\\S+\\s*")) { File source = new File(args[1]); - Logar.checkSort(source); + Logar.checkSorts(source); } else if (isMatching(args, "sort", "\\s*\\S+\\s*")) { diff --git a/src/org/april/logar/util/Files.java b/src/org/april/logar/util/Files.java index 07c174e..dd713a4 100644 --- a/src/org/april/logar/util/Files.java +++ b/src/org/april/logar/util/Files.java @@ -48,6 +48,31 @@ public class Files extends ArrayList super(initialCapacity); } + /** + * Keep directories. + * + * @return the files + */ + public Files keepDirectoriesOnly() + { + Files result; + + Iterator iterator = iterator(); + while (iterator.hasNext()) + { + File file = iterator.next(); + if (!file.isDirectory()) + { + iterator.remove(); + } + } + + result = this; + + // + return result; + } + /** * Removes the containing. * @@ -75,6 +100,31 @@ public class Files extends ArrayList return result; } + /** + * Removes the file type. + * + * @return the files + */ + public Files removeFileType() + { + Files result; + + Iterator iterator = iterator(); + while (iterator.hasNext()) + { + File file = iterator.next(); + if (file.isFile()) + { + iterator.remove(); + } + } + + result = this; + + // + return result; + } + /** * Removes the hidden. *