Browse Source

Added anonymize feature.

master
Christian P. MOMON 6 months ago
parent
commit
2ff492fc83
  1. 1
      .classpath
  2. BIN
      lib/devinsy-cmdexec-0.11.1-sources.zip
  3. BIN
      lib/devinsy-cmdexec-0.11.1.jar
  4. 2
      resources/scripts/logar.sh
  5. 223
      src/anonymizer/AnonMap.java
  6. 157
      src/anonymizer/AnonMapFile.java
  7. 199
      src/anonymizer/Anonymizer.java
  8. 252
      src/anonymizer/Ipv4Generator.java
  9. 80
      src/anonymizer/Ipv6Generator.java
  10. 90
      src/anonymizer/UserGenerator.java
  11. 351
      src/fr/devinsy/logar/app/Logar.java
  12. 43
      src/fr/devinsy/logar/app/LogarUtils.java
  13. 102
      src/fr/devinsy/logar/app/log/Log.java
  14. 126
      src/fr/devinsy/logar/app/log/LogComparator.java
  15. 276
      src/fr/devinsy/logar/app/log/LogUtils.java
  16. 82
      src/fr/devinsy/logar/app/log/Logs.java
  17. 30
      src/org/april/logar/cli/LogarCLI.java
  18. 27
      src/org/april/logar/util/Files.java
  19. 103
      src/org/april/logar/util/FilesUtils.java
  20. 15
      test/org/april/logar/app/TestFoo.java

1
.classpath

@ -21,5 +21,6 @@
<classpathentry kind="lib" path="lib/commons-text-1.9.jar" sourcepath="lib/commons-text-1.9-sources.jar"/>
<classpathentry kind="lib" path="lib/devinsy-strings-0.11.3.jar" sourcepath="lib/devinsy-strings-0.11.3-sources.zip"/>
<classpathentry kind="lib" path="lib/threeten-extra-1.5.0.jar" sourcepath="lib/threeten-extra-1.5.0-sources.jar"/>
<classpathentry kind="lib" path="lib/devinsy-cmdexec-0.11.1.jar" sourcepath="lib/devinsy-cmdexec-0.11.1-sources.zip"/>
<classpathentry kind="output" path="bin"/>
</classpath>

BIN
lib/devinsy-cmdexec-0.11.1-sources.zip

Binary file not shown.

BIN
lib/devinsy-cmdexec-0.11.1.jar

Binary file not shown.

2
resources/scripts/logar.sh

@ -3,7 +3,7 @@
# Java check.
javaCheck=`which java`
if [[ "$javaCheck" =~ ^/.* ]]; then
echo "Java requirement............... OK"
#echo "Java requirement............... OK"
java -jar "$(dirname "$0")"/logar.jar $@
else
echo "Java requirement............... MISSING"

223
src/anonymizer/AnonMap.java

@ -0,0 +1,223 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package anonymizer;
import java.util.HashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.strings.StringSet;
/**
* The Class AnonMap.
*/
public final class AnonMap
{
private static Logger logger = LoggerFactory.getLogger(AnonMap.class);
private HashMap<String, String> map;
private HashMap<String, String> unmap;
/**
* Instantiates a new anon map.
*/
public AnonMap()
{
this.map = new HashMap<String, String>();
this.unmap = new HashMap<String, String>();
}
/**
* Adds the all.
*
* @param source
* the source
*/
public void addAll(final AnonMap source)
{
for (String key : source.getKeys())
{
String value = source.get(key);
this.map.put(key, value);
this.unmap.put(value, key);
}
}
/**
* Gets the ip.
*
* @param ip
* the ip
* @return the ip
*/
public String anonymizeIp(final String ip)
{
String result;
if (ip == null)
{
result = null;
}
else
{
result = this.map.get(ip);
if (result == null)
{
boolean ended = false;
while (!ended)
{
if (ip.contains(":"))
{
result = Ipv6Generator.random(ip);
}
else
{
result = Ipv4Generator.random(ip);
}
// Check it does not already exist.
if ((this.map.get(result) == null) && (this.unmap.get(result) == null))
{
this.map.put(ip, result);
this.unmap.put(ip, result);
ended = true;
}
}
}
}
//
return result;
}
/**
* Gets the user.
*
* @param user
* the user
* @return the user
*/
public String anonymizeUser(final String user)
{
String result;
if (user == null)
{
result = null;
}
else if (user.equals("-"))
{
result = user;
}
else
{
result = this.map.get(user);
if (result == null)
{
boolean ended = false;
while (!ended)
{
result = UserGenerator.random(user);
// Check it does not already exist.
if ((this.map.get(result) == null) && (this.unmap.get(result) == null))
{
this.map.put(user, result);
this.unmap.put(user, result);
ended = true;
}
}
}
}
//
return result;
}
/**
* Gets the.
*
* @param key
* the key
* @return the string
*/
public String get(final String key)
{
String result;
if (key == null)
{
result = null;
}
else
{
result = this.map.get(key);
}
//
return result;
}
/**
* Gets the keys.
*
* @return the keys
*/
public StringSet getKeys()
{
StringSet result;
result = new StringSet(this.map.keySet());
//
return result;
}
/**
* Put.
*
* @param key
* the key
* @param value
* the value
*/
public void put(final String key, final String value)
{
this.map.put(key, value);
}
/**
* Size.
*
* @return the int
*/
public int size()
{
int result;
result = this.map.size();
//
return result;
}
}

157
src/anonymizer/AnonMapFile.java

@ -0,0 +1,157 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package anonymizer;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class AnonMapFile.
*/
public final class AnonMapFile
{
private static Logger logger = LoggerFactory.getLogger(AnonMapFile.class);
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
/**
* Instantiates a new anon map file.
*/
private AnonMapFile()
{
}
/**
* Load.
*
* @param source
* the source
* @return the anon map
*/
public static AnonMap load(final File source)
{
AnonMap result;
result = new AnonMap();
if ((source != null) && (source.exists()))
{
BufferedReader in = null;
try
{
if (source.getName().endsWith(".gz"))
{
in = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(source))));
}
else
{
in = new BufferedReader(new InputStreamReader(new FileInputStream(source), DEFAULT_CHARSET_NAME));
}
boolean ended = false;
while (!ended)
{
String key = in.readLine();
String value = in.readLine();
if (key == null)
{
ended = true;
}
else
{
result.put(key, value);
}
}
}
catch (IOException exception)
{
exception.printStackTrace();
}
finally
{
IOUtils.closeQuietly(in);
}
}
//
return result;
}
/**
* Save.
*
* @param source
* the source
* @param map
* the map
*/
public static void save(final File target, final AnonMap map)
{
if (target == null)
{
throw new IllegalArgumentException("Null parameter source.");
}
else if (map == null)
{
throw new IllegalArgumentException("Null parameter map.");
}
else
{
PrintWriter out = null;
try
{
if (target.getName().endsWith(".gz"))
{
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
}
else
{
out = new PrintWriter(new FileOutputStream(target));
}
for (String key : map.getKeys())
{
out.println(key);
out.println(map.get(key));
}
}
catch (IOException exception)
{
System.err.println("Error with file [" + target.getAbsolutePath() + "]");
exception.printStackTrace();
}
finally
{
IOUtils.closeQuietly(out);
}
}
}
}

199
src/anonymizer/Anonymizer.java

@ -0,0 +1,199 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package anonymizer;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.time.format.DateTimeParseException;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.io.IOUtils;
import org.april.logar.util.LineIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogUtils;
/**
* The Class Anonymizer.
*/
public final class Anonymizer
{
private static Logger logger = LoggerFactory.getLogger(Anonymizer.class);
private AnonMap map;
/**
* Instantiates a new anonymizer.
*/
public Anonymizer()
{
this.map = new AnonMap();
}
/**
* Anonymize.
*
* @param source
* the source
* @param target
* the target
*/
public void anonymize(final File source)
{
if (source == null)
{
throw new IllegalArgumentException("Null parameter.");
}
else if (!source.isFile())
{
throw new IllegalArgumentException("Parameter is not a file.");
}
else
{
System.out.println("== Anonymize log for [" + source.getName() + "]");
File target;
if (source.getName().endsWith(".log.gz"))
{
target = new File(source.getParentFile(), source.getName().replace(".log.gz", "-anon.log.gz"));
}
else
{
target = new File(source.getParentFile(), source.getName().replace(".log", "-anon.log"));
}
PrintWriter out = null;
try
{
LineIterator iterator = new LineIterator(source);
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
while (iterator.hasNext())
{
String line = iterator.next();
try
{
Log log = LogUtils.parseAccessLog(line);
// logger.info("line={}", line);
// logger.info("log =[{}][{}][{}]", log.getIp(),
// log.getUser(), log.getDatetime());
Log anon = anonymize(log);
// logger.info("anon=[{}][{}][{}]", anon.getIp(),
// anon.getUser(), anon.getDatetime());
// logger.info("anon={}", anon);
out.println(anon);
}
catch (IllegalArgumentException exception)
{
System.out.println("Bad format line: " + line);
exception.printStackTrace();
}
catch (DateTimeParseException exception)
{
System.out.println("Bad datetime format: " + line);
}
}
}
catch (IOException exception)
{
System.err.println("Error with file [" + source.getAbsolutePath() + "]");
exception.printStackTrace();
}
finally
{
IOUtils.closeQuietly(out);
}
}
}
/**
* Anonymize.
*
* @param log
* the log
* @return the log
*/
public Log anonymize(final Log log)
{
Log result;
String anonIp = this.map.anonymizeIp(log.getIp());
String anonUser = this.map.anonymizeUser(log.getUser());
String line = log.getLine().replace(log.getIp(), anonIp);
if (!log.getUser().equals("-"))
{
line = line.replace(log.getUser(), anonUser);
}
result = new Log(line, log.getDatetime(), anonIp, anonUser);
//
return result;
}
/**
* Gets the map table.
*
* @return the map table
*/
public AnonMap getMapTable()
{
AnonMap result;
result = this.map;
//
return result;
}
/**
* Inits the map.
*
* @param source
* the source
*/
public void loadMapTable(final File source)
{
if (source != null)
{
this.map.addAll(AnonMapFile.load(source));
}
}
/**
* Save map table.
*
* @param target
* the target
*/
public void SaveMapTable(final File target)
{
if (target != null)
{
AnonMapFile.save(target, this.map);
}
}
}

252
src/anonymizer/Ipv4Generator.java

@ -0,0 +1,252 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package anonymizer;
import org.apache.commons.lang3.RandomUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class RandomIpv4Generator.
*/
public final class Ipv4Generator
{
private static Logger logger = LoggerFactory.getLogger(Ipv4Generator.class);
/**
* Instantiates a new random ipv 4 generator.
*/
private Ipv4Generator()
{
}
/**
* Gets the ipv 4 max length part.
*
* @param length
* the length (1..12)
* @param column
* the column (4...1)
* @return the ipv 4 max length part
*/
public static int getIpv4MaxLengthPart(final int length, final int column)
{
int result;
if ((length < 1) || (length > 12))
{
throw new IllegalArgumentException("Bad length value: " + length);
}
else if ((column < 1) || (column > 12))
{
throw new IllegalArgumentException("Bad column value:" + column);
}
else
{
final int[] col4 = { 0, 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 3 };
final int[] col3 = { 0, 0, 1, 2, 3, 3, 3, 3, 3, 0, 0, 0 };
final int[] col2 = { 0, 1, 2, 3, 3, 3, 0, 0, 0, 0, 0, 0 };
final int[] col1 = { 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
final int[][] table = { col1, col2, col3, col4 };
result = table[column - 1][length - 1];
if (result == 0)
{
throw new IllegalArgumentException(String.format("Zero detected (%d, %d).", length, column));
}
}
//
return result;
}
/**
* Gets the ipv 4 max value part.
*
* @param length
* the length
* @param column
* the column
* @return the ipv 4 max value part
*/
public static int getIpv4MaxValuePart(final int length, final int column)
{
int result;
int max = getIpv4MaxLengthPart(length, column);
switch (max)
{
case 1:
result = 9;
break;
case 2:
result = 99;
break;
case 3:
result = 255;
break;
default:
throw new IllegalArgumentException("Bad value: " + max);
}
//
return result;
}
/**
* Gets the ipv 4 min length part.
*
* @param length
* the length (1..12)
* @param column
* the column (4...1)
* @return the ipv 4 min length part
*/
public static int getIpv4MinLengthPart(final int length, final int column)
{
int result;
if ((length < 1) || (length > 12))
{
throw new IllegalArgumentException("Bad length value: " + length);
}
else if ((column < 1) || (column > 12))
{
throw new IllegalArgumentException("Bad column value:" + column);
}
else
{
final int[] col4 = { 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 3 };
final int[] col3 = { 0, 0, 1, 1, 1, 1, 1, 2, 3, 0, 0, 0 };
final int[] col2 = { 0, 1, 1, 1, 2, 3, 0, 0, 0, 0, 0, 0 };
final int[] col1 = { 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
final int[][] table = { col1, col2, col3, col4 };
result = table[column - 1][length - 1];
if (result == 0)
{
throw new IllegalArgumentException(String.format("Zero detected (%d, %d).", length, column));
}
}
//
return result;
}
public static int getIpv4MinValuePart(final int length, final int column)
{
int result;
int max = getIpv4MinLengthPart(length, column);
switch (max)
{
case 1:
result = 0;
break;
case 2:
result = 10;
break;
case 3:
result = 100;
break;
default:
throw new IllegalArgumentException("Bad value: " + max);
}
//
return result;
}
/**
* Generates a ipv4 of a fixed length.
*
* @param length
* the length
* @return the string
*/
public static String random(final int length)
{
String result;
if ((length < 7) || (length > 15))
{
throw new IllegalArgumentException("Bad parameter: " + length);
}
else
{
int size = length - 3;
int a = (int) RandomUtils.nextLong(getIpv4MinValuePart(size, 4), getIpv4MaxValuePart(size, 4) + 1);
size -= String.valueOf(a).length();
int b = (int) RandomUtils.nextLong(getIpv4MinValuePart(size, 3), getIpv4MaxValuePart(size, 3) + 1);
size -= String.valueOf(b).length();
int c = (int) RandomUtils.nextLong(getIpv4MinValuePart(size, 2), getIpv4MaxValuePart(size, 2) + 1);
size -= String.valueOf(c).length();
int d = (int) RandomUtils.nextLong(getIpv4MinValuePart(size, 1), getIpv4MaxValuePart(size, 1) + 1);
size -= String.valueOf(d).length();
result = String.format("%d.%d.%d.%d", a, b, c, d);
}
//
return result;
}
/**
* Generates a ipv4 with the same length than the parameter.
*
* @param ip
* the source
* @return the string
*/
public static String random(final String ip)
{
String result;
if (ip == null)
{
result = null;
}
else
{
result = random(ip.length());
if (result.equals(ip))
{
random(ip);
}
}
//
return result;
}
}

80
src/anonymizer/Ipv6Generator.java

@ -0,0 +1,80 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package anonymizer;
import org.apache.commons.lang3.RandomStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class RandomIpv6Generator.
*/
public final class Ipv6Generator
{
private static Logger logger = LoggerFactory.getLogger(Ipv6Generator.class);
/**
* Instantiates a new random ipv 6 generator.
*/
private Ipv6Generator()
{
}
/**
* Random.
*
* @param ip
* the source
* @return the string
*/
public static String random(final String ip)
{
String result;
if (ip == null)
{
result = null;
}
else
{
StringBuffer buffer = new StringBuffer(ip.length());
for (int index = 0; index < ip.length(); index++)
{
char c = ip.charAt(index);
if (c == ':')
{
buffer.append(c);
}
else
{
buffer.append(RandomStringUtils.random(1, "0123456789abcdef"));
}
}
result = buffer.toString();
if (result.equals(ip))
{
random(ip);
}
}
//
return result;
}
}

90
src/anonymizer/UserGenerator.java

@ -0,0 +1,90 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package anonymizer;
import org.apache.commons.lang3.RandomStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class RandomUserGenerator.
*/
public final class UserGenerator
{
private static Logger logger = LoggerFactory.getLogger(UserGenerator.class);
/**
* Instantiates a new random user generator.
*/
private UserGenerator()
{
}
/**
* Random.
*
* @param length
* the length
* @return the string
*/
public static String random(final int length)
{
String result;
if (length < 0)
{
throw new IllegalArgumentException("Bad parameter: " + length);
}
else
{
result = RandomStringUtils.random(length);
}
//
return result;
}
/**
* Random.
*
* @param source
* the source
* @return the string
*/
public static String random(final String source)
{
String result;
if (source == null)
{
result = null;
}
else if (source.equals("-"))
{
result = source;
}
else
{
result = RandomStringUtils.random(source.length(), "abcdefghijklmnopqrstuvwxyz");
}
//
return result;
}
}

351
src/fr/devinsy/logar/app/Logar.java

@ -34,10 +34,14 @@ import java.util.zip.GZIPOutputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.april.logar.util.Files;
import org.april.logar.util.FilesUtils;
import org.april.logar.util.LineIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import anonymizer.Anonymizer;
import fr.devinsy.logar.app.log.LogUtils;
/**
* The Class Logar.
*/
@ -45,9 +49,6 @@ public final class Logar
{
private static Logger logger = LoggerFactory.getLogger(Logar.class);
public static Pattern nginxAccessLogLinePatternFull = Pattern.compile(
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>\\S+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");
public static Pattern nginxAccessLogLinePattern = Pattern.compile("^\\S+ - [^\\[]+ \\[(?<time>[^\\]]+)\\] .*$");
public static Pattern nginxErrorLogLinePattern = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
@ -66,9 +67,55 @@ public final class Logar
* @param target
* the target
*/
public static void anonymize(final File source, final File target)
public static void anonymize(final File source)
{
anonymize(source, null);
}
/**
* Anonymize.
*
* @param source
* the source
* @param target
* the target
*/
public static void anonymize(final File source, final File mapFile)
{
System.out.println("En cours de développement…");
if (source == null)
{
throw new IllegalArgumentException("Null source file.");
}
else if (!source.exists())
{
throw new IllegalArgumentException("Source file does not exist.");
}
else
{
Anonymizer anonymizer = new Anonymizer();
anonymizer.loadMapTable(mapFile);
System.out.println("Table size=" + anonymizer.getMapTable().size());
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeContaining("-anon.log");
logger.info("file count={}", files.size());
for (File file : files)
{
if (file.getName().contains("access"))
{
anonymizer.anonymize(file);
}
else if (file.getName().contains("error"))
{
//
}
}
if (mapFile != null)
{
System.out.println("Table size=" + anonymizer.getMapTable().size());
anonymizer.SaveMapTable(mapFile);
}
}
}
/**
@ -295,19 +342,25 @@ public final class Logar
}
/**
* Check sort.
* Check.
*
* @param source
* the source
* @throws IOException
*/
public static void checkSort(final File source) throws IOException
public static void check(final File source)
{
if (source != null)
{
if (source.isFile())
{
checkSortForAccessFiles(source);
if (source.getName().contains("access"))
{
checkAccessFiles(source);
}
else if (source.getName().contains("error"))
{
checkErrorFiles(source);
}
}
else
{
@ -315,18 +368,165 @@ public final class Logar
{
if (file.isDirectory())
{
checkSort(file);
check(file);
}
else if (file.getName().contains("access"))
else if (StringUtils.endsWithAny(file.getName(), ".log", ".gz", ".1", ".2"))
{
checkSortForAccessFiles(file);
check(file);
}
else if (file.getName().contains("error"))
}
}
}
}
/**
* Check access files.
*
* @param file
* the source
*/
public static void checkAccessFiles(final File file)
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter.");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Parameter is not a file.");
}
else
{
System.out.println("== Check access log for [" + file.getName() + "]");
int lineCount = 0;
int badLineCount = 0;
try
{
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
lineCount += 1;
try
{
LogUtils.parseAccessLog(line).getDatetime();
}
catch (IllegalArgumentException exception)
{
checkSortForErrorFiles(file);
System.out.println("Bad format line: " + line);
badLineCount += 1;
exception.printStackTrace();
}
catch (DateTimeParseException exception)
{
System.out.println("Bad datetime format: " + line);
badLineCount += 1;
}
}
}
catch (IOException exception)
{
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
exception.printStackTrace();
}
if (badLineCount > 0)
{
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
}
}
}
/**
* Check error files.
*
* @param file
* the file
*/
public static void checkErrorFiles(final File file)
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter.");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Parameter is not a file.");
}
else
{
System.out.println("== Check error log for [" + file.getName() + "]");
int lineCount = 0;
int badLineCount = 0;
try
{
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
lineCount += 1;
try
{
LogUtils.parseErrorLog(line).getDatetime();
}
catch (IllegalArgumentException exception)
{
System.out.println("Bad format line: " + line);
badLineCount += 1;
}
catch (DateTimeParseException exception)
{
System.out.println("Bad datetime format: " + line);
badLineCount += 1;
}
}
}
catch (IOException exception)
{
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
exception.printStackTrace();
}
if (badLineCount > 0)
{
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
}
}
}
/**
* Check sort.
*
* @param source
* the source
* @throws IOException
*/
public static void checkSort(final File source) throws IOException
{
if (source != null)
{
if (source.isFile())
{
if (source.getName().contains("access"))
{
checkSortForAccessFiles(source);
}
else if (source.getName().contains("error"))
{
checkSortForErrorFiles(source);
}
}
else
{
for (File file : Files.of(source).removeHidden().sortByName())
{
checkSort(file);
}
}
}
}
@ -358,21 +558,18 @@ public final class Logar
{
String line = iterator.next();
lineCount += 1;
LocalDateTime date = getLogDate(line);
if (currentDate == null)
LocalDateTime date = LogUtils.parseAccessLog(line).getDatetime();
if ((currentDate != null) && (date.isBefore(currentDate)))
{
currentDate = date;
}
else if (date.isBefore(currentDate))
{
System.out.println(String.format("detected line: %d %s", lineCount, currentDate.toString()));
System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString()));
badLineCount += 1;
}
currentDate = date;
}
if (badLineCount > 0)
{
System.out.println("Bad line count: " + badLineCount);
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
}
}
}
@ -382,8 +579,9 @@ public final class Logar
*
* @param source
* the source
* @throws IOException
*/
public static void checkSortForErrorFiles(final File file)
public static void checkSortForErrorFiles(final File file) throws IOException
{
if (file == null)
{
@ -396,40 +594,110 @@ public final class Logar
else
{
System.out.println("== Check sort for [" + file.getName() + "]");
LocalDateTime currentDate = null;
int lineCount = 0;
int badLineCount = 0;
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
lineCount += 1;
LocalDateTime date = LogUtils.parseErrorLog(line).getDatetime();
if ((currentDate != null) && (date.isBefore(currentDate)))
{
System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString()));
badLineCount += 1;
}
currentDate = date;
}
if (badLineCount > 0)
{
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
}
}
}
/**
* Gets the log date.
* Sort.
*
* @param line
* the line
* @return the log date
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
private static LocalDateTime getLogDate(final String line)
public static void sort(final File source) throws IOException
{
LocalDateTime result;
Matcher matcher = nginxAccessLogLinePattern.matcher(line);
if (matcher.matches())
if (source != null)
{
String value = matcher.group("time");
try
if (source.isFile())
{
result = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
if (source.getName().contains("access"))
{
sortAccessFiles(source);
}
else if (source.getName().contains("error"))
{
sortErrorFiles(source);
}
}
catch (DateTimeParseException exception)
else
{
throw new IllegalArgumentException("Bad format time [" + value + "]");
for (File file : Files.of(source).removeHidden().sortByName())
{
sort(file);
}
}
}
}
/**
* Sort access files.
*
* @param file
* the file
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void sortAccessFiles(final File file) throws IOException
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
{
throw new IllegalArgumentException("Bad format line [" + line + "]");
System.out.println("== Sort for [" + file.getName() + "]");
LogUtils.sortAccessLog(file);
}
}
//
return result;
/**
* Sort error files.
*
* @param file
* the file
* @throws IOException
*/
public static void sortErrorFiles(final File file) throws IOException
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
{
System.out.println("== Sort for [" + file.getName() + "]");
LogUtils.sortErrorLog(file);
}
}
/**
@ -462,6 +730,7 @@ public final class Logar
}
else
{
System.out.println("== Test access log for [" + source.getName() + "]");
Stats counter = new Stats();
for (File directory : source.listFiles())
@ -523,7 +792,6 @@ public final class Logar
}
}
System.out.println("=====================================================");
counter.stop();
System.out.println(counter.toString());
}
@ -547,6 +815,7 @@ public final class Logar
}
else
{
System.out.println("== Test error log for [" + source.getName() + "]");
Stats counter = new Stats();
for (File directory : source.listFiles())

43
src/fr/devinsy/logar/app/LogarUtils.java

@ -18,6 +18,15 @@
*/
package fr.devinsy.logar.app;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -28,6 +37,8 @@ public final class LogarUtils
{
private static Logger logger = LoggerFactory.getLogger(LogarUtils.class);
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
/**
* Instantiates a new log tool utils.
*/
@ -35,6 +46,38 @@ public final class LogarUtils
{
}
/**
* Sha 1 sum.
*
* @param file
* the file
* @return the string
* @throws IOException
*/
public static String sha1sum(final File file) throws IOException
{
String result;
DigestInputStream digestStream = null;
try
{
digestStream = new DigestInputStream(new FileInputStream(file), MessageDigest.getInstance("SHA-1"));
digestStream.readAllBytes();
result = DigestUtils.sha1Hex(digestStream.getMessageDigest().digest());
}
catch (NoSuchAlgorithmException e1)
{
throw new IOException("Impossible to get SHA-1 digester", e1);
}
finally
{
IOUtils.closeQuietly(digestStream);
}
//
return result;
}
/**
* To human.
*

102
src/fr/devinsy/logar/app/log/Log.java

@ -0,0 +1,102 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.app.log;
import java.time.LocalDateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;