Chapril/logar
Chapril/logar
25
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

Refactored code. Removed concate use for anonymize.

Browse Source
This commit is contained in:
Christian P. MOMON 2021-04-25 02:21:38 +02:00
parent 1e46e07c4b
commit d4d1b6038c
4 changed files with 31 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/fr/devinsy/logar/app/Logar.java
View File

@ -93,7 +93,7 @@ public final class Logar
anonymizer.loadMapTable(mapFile); anonymizer.loadMapTable(mapFile);
System.out.println("Table size=" + anonymizer.getMapTable().size()); System.out.println("Table size=" + anonymizer.getMapTable().size());
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeContaining("-anon.log"); Files files = FilesUtils.searchEndingWith(source, ".log", ".log.gz").keepFileType().removeContaining("-anon.log");
logger.info("file count={}", files.size()); logger.info("file count={}", files.size());
for (File file : files) for (File file : files)
{ {
@ -338,15 +338,7 @@ public final class Logar
else else
{ {
System.out.println("== Check parse log for [" + file.getName() + "]"); System.out.println("== Check parse log for [" + file.getName() + "]");
boolean isAccessFile; boolean isAccessFile = file.getName().contains("access");
if (file.getName().contains("access"))
{
isAccessFile = true;
}
else
{
isAccessFile = false;
}
int lineCount = 0; int lineCount = 0;
int badLineCount = 0; int badLineCount = 0;
@ -403,7 +395,7 @@ public final class Logar
*/ */
public static void checkLogFiles(final File source) public static void checkLogFiles(final File source)
{ {
Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName(); Files files = FilesUtils.search(source, LOGFILE_PATTERN).sortByName();
for (File file : files) for (File file : files)
{ {
@ -420,11 +412,11 @@ public final class Logar
*/ */
public static void checkSort(final File source) throws IOException public static void checkSort(final File source) throws IOException
{ {
Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName(); Files files = FilesUtils.search(source, LOGFILE_PATTERN).sortByName();
for (File file : files) for (File file : files)
{ {
checkSortOfFile(file); checkSortFile(file);
} }
} }
@ -435,7 +427,7 @@ public final class Logar
* the source * the source
* @throws IOException * @throws IOException
*/ */
public static void checkSortOfFile(final File file) throws IOException public static void checkSortFile(final File file) throws IOException
{ {
if (file == null) if (file == null)
{ {
@ -447,17 +439,8 @@ public final class Logar
} }
else else
{ {
boolean isAccessFile;
if (file.getName().contains("access"))
{
isAccessFile = true;
}
else
{
isAccessFile = false;
}
System.out.println("== Check sort for [" + file.getName() + "]"); System.out.println("== Check sort for [" + file.getName() + "]");
boolean isAccessFile = file.getName().contains("access");
LocalDateTime currentDate = null; LocalDateTime currentDate = null;
int lineCount = 0; int lineCount = 0;
int badLineCount = 0; int badLineCount = 0;
@ -501,7 +484,7 @@ public final class Logar
*/ */
public static void sort(final File source) throws IOException public static void sort(final File source) throws IOException
{ {
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeHidden().sortByName(); Files files = FilesUtils.searchEndingWith(source, LOGFILE_PATTERN).removeHidden().sortByName();
for (File file : files) for (File file : files)
{ {
@ -518,11 +501,11 @@ public final class Logar
*/ */
public static void testConcate(final File source) public static void testConcate(final File source)
{ {
Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName(); Files files = FilesUtils.searchEndingWith(source, ".log", ".log.gz").keepFileType().removeContaining("-anon.log");
for (File file : files) for (File file : files)
{ {
testConcateOnFile(file); testConcateFile(file);
} }
} }
@ -532,7 +515,7 @@ public final class Logar
* @param source * @param source
* the source * the source
*/ */
public static void testConcateOnFile(final File file) public static void testConcateFile(final File file)
{ {
if (file == null) if (file == null)
{ {
@ -545,15 +528,7 @@ public final class Logar
else else
{ {
System.out.println("== Test concate log for [" + file.getName() + "]"); System.out.println("== Test concate log for [" + file.getName() + "]");
boolean isAccessFile; boolean isAccessFile = file.getName().contains("access");
if (file.getName().contains("access"))
{
isAccessFile = true;
}
else
{
isAccessFile = false;
}
int lineCount = 0; int lineCount = 0;
int badLineCount = 0; int badLineCount = 0;

27
src/fr/devinsy/logar/app/anonymizer/Anonymizer.java
View File

@ -22,8 +22,6 @@ import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.format.DateTimeParseException; import java.time.format.DateTimeParseException;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -175,26 +173,19 @@ public final class Anonymizer
{ {
Log result; Log result;
result = new Log(log); //
String anonIp = this.map.anonymizeIp(log.getIp());
String line = log.getLine().replace(log.getIp(), anonIp);
result.setIp(this.map.anonymizeIp(log.getIp())); //
result.setUser(this.map.anonymizeUser(log.getUser())); if (!StringUtils.equals(log.getUser(), "-"))
// Anonymize ip.
result.setRequest(result.getRequest().replace(result.getIp(), result.getIp()));
result.setReferer(result.getReferer().replace(result.getIp(), result.getIp()));
// Anonymize user.
if (!log.getUser().equals("-"))
{ {
// URLEncode replaces ' ' with '+' so bad for us. String anonUser = this.map.anonymizeUser(log.getUser());
String userInUrl = URLEncoder.encode(log.getUser(), StandardCharsets.UTF_8).replace("+", "%20"); line.replaceFirst(" " + log.getUser(), anonUser);
result.setRequest(result.getRequest().replace(userInUrl, result.getUser()));
result.setReferer(result.getReferer().replace(userInUrl, result.getUser()));
} }
result.concateAccessLog(); //
result = new Log(line, log.getDatetime());
// //
return result; return result;

14
src/org/april/logar/cli/LogarCLI.java
View File

@ -55,13 +55,13 @@ public final class LogarCLI
message.appendln("Usage:"); message.appendln("Usage:");
message.appendln(" logar [ -h | -help | --help ]"); message.appendln(" logar [ -h | -help | --help ]");
message.appendln(" logar [ -v | -version | --version ]"); message.appendln(" logar [ -v | -version | --version ]");
message.appendln(" logar anonymize fileordirectory [maptable] anonymize ip and login"); message.appendln(" logar anonymize fileordirectory [mapfile] anonymize ip and user");
message.appendln(" logar archive source target archive previous month"); message.appendln(" logar archive source target archive previous month from /var/log/nginx/ tree");
message.appendln(" logar check fileordirectory check line format in log files"); message.appendln(" logar check fileordirectory check line format in log file");
message.appendln(" logar checksort fileordirectory check sort of an access log file"); message.appendln(" logar checksort fileordirectory check sort in log file");
message.appendln(" logar sort fileordirectory sort log files by datetime"); message.appendln(" logar sort fileordirectory sort log files by datetime");
message.appendln(" logar testarchive source test archive"); message.appendln(" logar testarchive source test archive without writing");
message.appendln(" logar testconcate fileordirectory test concate of log line"); message.appendln(" logar testconcate fileordirectory test line concate in log file");
logger.info(message.toString()); logger.info(message.toString());
} }

6
src/org/april/logar/util/FilesUtils.java
View File

@ -83,7 +83,7 @@ public class FilesUtils
* the extensions * the extensions
* @return the files * @return the files
*/ */
public static Files searchFileRecursively(final File source, final String... extensions) public static Files searchEndingWith(final File source, final String... extensions)
{ {
Files result; Files result;
@ -92,7 +92,7 @@ public class FilesUtils
Files full = listRecursively(source); Files full = listRecursively(source);
for (File file : full) for (File file : full)
{ {
if ((file.isFile()) && (StringUtils.endsWithAny(file.getName(), extensions))) if (StringUtils.endsWithAny(file.getName(), extensions))
{ {
result.add(file); result.add(file);
} }
@ -111,7 +111,7 @@ public class FilesUtils
* the regex * the regex
* @return the files * @return the files
*/ */
public static Files searchRecursively(final File source, final String regex) public static Files search(final File source, final String regex)
{ {
Files result; Files result;