diff --git a/.classpath b/.classpath index 3287617..fc80bae 100644 --- a/.classpath +++ b/.classpath @@ -21,5 +21,6 @@ + diff --git a/lib/devinsy-cmdexec-0.11.1-sources.zip b/lib/devinsy-cmdexec-0.11.1-sources.zip new file mode 100644 index 0000000..dfffe60 Binary files /dev/null and b/lib/devinsy-cmdexec-0.11.1-sources.zip differ diff --git a/lib/devinsy-cmdexec-0.11.1.jar b/lib/devinsy-cmdexec-0.11.1.jar new file mode 100644 index 0000000..59afe89 Binary files /dev/null and b/lib/devinsy-cmdexec-0.11.1.jar differ diff --git a/resources/scripts/logar.sh b/resources/scripts/logar.sh index 340a338..4cd0db5 100755 --- a/resources/scripts/logar.sh +++ b/resources/scripts/logar.sh @@ -3,7 +3,7 @@ # Java check. javaCheck=`which java` if [[ "$javaCheck" =~ ^/.* ]]; then - echo "Java requirement............... OK" + #echo "Java requirement............... OK" java -jar "$(dirname "$0")"/logar.jar $@ else echo "Java requirement............... MISSING" diff --git a/src/anonymizer/AnonMap.java b/src/anonymizer/AnonMap.java new file mode 100644 index 0000000..28b0f62 --- /dev/null +++ b/src/anonymizer/AnonMap.java @@ -0,0 +1,223 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of Logar, simple tool to manage http log files. + * + * Logar is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * Logar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Logar. If not, see . + */ +package anonymizer; + +import java.util.HashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.strings.StringSet; + +/** + * The Class AnonMap. + */ +public final class AnonMap +{ + private static Logger logger = LoggerFactory.getLogger(AnonMap.class); + + private HashMap map; + private HashMap unmap; + + /** + * Instantiates a new anon map. + */ + public AnonMap() + { + this.map = new HashMap(); + this.unmap = new HashMap(); + } + + /** + * Adds the all. + * + * @param source + * the source + */ + public void addAll(final AnonMap source) + { + for (String key : source.getKeys()) + { + String value = source.get(key); + + this.map.put(key, value); + this.unmap.put(value, key); + } + } + + /** + * Gets the ip. + * + * @param ip + * the ip + * @return the ip + */ + public String anonymizeIp(final String ip) + { + String result; + + if (ip == null) + { + result = null; + } + else + { + result = this.map.get(ip); + + if (result == null) + { + boolean ended = false; + while (!ended) + { + if (ip.contains(":")) + { + result = Ipv6Generator.random(ip); + } + else + { + result = Ipv4Generator.random(ip); + } + + // Check it does not already exist. + if ((this.map.get(result) == null) && (this.unmap.get(result) == null)) + { + this.map.put(ip, result); + this.unmap.put(ip, result); + ended = true; + } + } + } + } + + // + return result; + } + + /** + * Gets the user. + * + * @param user + * the user + * @return the user + */ + public String anonymizeUser(final String user) + { + String result; + + if (user == null) + { + result = null; + } + else if (user.equals("-")) + { + result = user; + } + else + { + result = this.map.get(user); + + if (result == null) + { + boolean ended = false; + while (!ended) + { + result = UserGenerator.random(user); + + // Check it does not already exist. + if ((this.map.get(result) == null) && (this.unmap.get(result) == null)) + { + this.map.put(user, result); + this.unmap.put(user, result); + ended = true; + } + } + } + } + + // + return result; + } + + /** + * Gets the. + * + * @param key + * the key + * @return the string + */ + public String get(final String key) + { + String result; + + if (key == null) + { + result = null; + } + else + { + result = this.map.get(key); + } + + // + return result; + } + + /** + * Gets the keys. + * + * @return the keys + */ + public StringSet getKeys() + { + StringSet result; + + result = new StringSet(this.map.keySet()); + + // + return result; + } + + /** + * Put. + * + * @param key + * the key + * @param value + * the value + */ + public void put(final String key, final String value) + { + this.map.put(key, value); + } + + /** + * Size. + * + * @return the int + */ + public int size() + { + int result; + + result = this.map.size(); + + // + return result; + } +} diff --git a/src/anonymizer/AnonMapFile.java b/src/anonymizer/AnonMapFile.java new file mode 100644 index 0000000..6e5c650 --- /dev/null +++ b/src/anonymizer/AnonMapFile.java @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of Logar, simple tool to manage http log files. + * + * Logar is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * Logar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Logar. If not, see . + */ +package anonymizer; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class AnonMapFile. + */ +public final class AnonMapFile +{ + private static Logger logger = LoggerFactory.getLogger(AnonMapFile.class); + + public static final String DEFAULT_CHARSET_NAME = "UTF-8"; + + /** + * Instantiates a new anon map file. + */ + private AnonMapFile() + { + } + + /** + * Load. + * + * @param source + * the source + * @return the anon map + */ + public static AnonMap load(final File source) + { + AnonMap result; + + result = new AnonMap(); + if ((source != null) && (source.exists())) + { + BufferedReader in = null; + try + { + if (source.getName().endsWith(".gz")) + { + in = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(source)))); + } + else + { + in = new BufferedReader(new InputStreamReader(new FileInputStream(source), DEFAULT_CHARSET_NAME)); + } + + boolean ended = false; + while (!ended) + { + String key = in.readLine(); + String value = in.readLine(); + + if (key == null) + { + ended = true; + } + else + { + result.put(key, value); + } + } + } + catch (IOException exception) + { + exception.printStackTrace(); + } + finally + { + IOUtils.closeQuietly(in); + } + + } + + // + return result; + } + + /** + * Save. + * + * @param source + * the source + * @param map + * the map + */ + public static void save(final File target, final AnonMap map) + { + if (target == null) + { + throw new IllegalArgumentException("Null parameter source."); + } + else if (map == null) + { + throw new IllegalArgumentException("Null parameter map."); + } + else + { + PrintWriter out = null; + try + { + if (target.getName().endsWith(".gz")) + { + out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target))); + } + else + { + out = new PrintWriter(new FileOutputStream(target)); + } + + for (String key : map.getKeys()) + { + out.println(key); + out.println(map.get(key)); + } + } + catch (IOException exception) + { + System.err.println("Error with file [" + target.getAbsolutePath() + "]"); + exception.printStackTrace(); + } + finally + { + IOUtils.closeQuietly(out); + } + } + } +} diff --git a/src/anonymizer/Anonymizer.java b/src/anonymizer/Anonymizer.java new file mode 100644 index 0000000..87a8558 --- /dev/null +++ b/src/anonymizer/Anonymizer.java @@ -0,0 +1,199 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of Logar, simple tool to manage http log files. + * + * Logar is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * Logar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Logar. If not, see . + */ +package anonymizer; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.time.format.DateTimeParseException; +import java.util.zip.GZIPOutputStream; + +import org.apache.commons.io.IOUtils; +import org.april.logar.util.LineIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.logar.app.log.Log; +import fr.devinsy.logar.app.log.LogUtils; + +/** + * The Class Anonymizer. + */ +public final class Anonymizer +{ + private static Logger logger = LoggerFactory.getLogger(Anonymizer.class); + + private AnonMap map; + + /** + * Instantiates a new anonymizer. + */ + public Anonymizer() + { + this.map = new AnonMap(); + } + + /** + * Anonymize. + * + * @param source + * the source + * @param target + * the target + */ + public void anonymize(final File source) + { + if (source == null) + { + throw new IllegalArgumentException("Null parameter."); + } + else if (!source.isFile()) + { + throw new IllegalArgumentException("Parameter is not a file."); + } + else + { + System.out.println("== Anonymize log for [" + source.getName() + "]"); + + File target; + if (source.getName().endsWith(".log.gz")) + { + target = new File(source.getParentFile(), source.getName().replace(".log.gz", "-anon.log.gz")); + } + else + { + target = new File(source.getParentFile(), source.getName().replace(".log", "-anon.log")); + } + + PrintWriter out = null; + try + { + LineIterator iterator = new LineIterator(source); + out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target))); + while (iterator.hasNext()) + { + String line = iterator.next(); + + try + { + Log log = LogUtils.parseAccessLog(line); + // logger.info("line={}", line); + // logger.info("log =[{}][{}][{}]", log.getIp(), + // log.getUser(), log.getDatetime()); + + Log anon = anonymize(log); + // logger.info("anon=[{}][{}][{}]", anon.getIp(), + // anon.getUser(), anon.getDatetime()); + // logger.info("anon={}", anon); + + out.println(anon); + } + catch (IllegalArgumentException exception) + { + System.out.println("Bad format line: " + line); + exception.printStackTrace(); + } + catch (DateTimeParseException exception) + { + System.out.println("Bad datetime format: " + line); + } + } + } + catch (IOException exception) + { + System.err.println("Error with file [" + source.getAbsolutePath() + "]"); + exception.printStackTrace(); + } + finally + { + IOUtils.closeQuietly(out); + } + } + } + + /** + * Anonymize. + * + * @param log + * the log + * @return the log + */ + public Log anonymize(final Log log) + { + Log result; + + String anonIp = this.map.anonymizeIp(log.getIp()); + String anonUser = this.map.anonymizeUser(log.getUser()); + + String line = log.getLine().replace(log.getIp(), anonIp); + if (!log.getUser().equals("-")) + { + line = line.replace(log.getUser(), anonUser); + } + + result = new Log(line, log.getDatetime(), anonIp, anonUser); + + // + return result; + } + + /** + * Gets the map table. + * + * @return the map table + */ + public AnonMap getMapTable() + { + AnonMap result; + + result = this.map; + + // + return result; + } + + /** + * Inits the map. + * + * @param source + * the source + */ + public void loadMapTable(final File source) + { + if (source != null) + { + this.map.addAll(AnonMapFile.load(source)); + } + } + + /** + * Save map table. + * + * @param target + * the target + */ + public void SaveMapTable(final File target) + { + if (target != null) + { + AnonMapFile.save(target, this.map); + } + } +} diff --git a/src/anonymizer/Ipv4Generator.java b/src/anonymizer/Ipv4Generator.java new file mode 100644 index 0000000..e81106e --- /dev/null +++ b/src/anonymizer/Ipv4Generator.java @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of Logar, simple tool to manage http log files. + * + * Logar is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * Logar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Logar. If not, see . + */ +package anonymizer; + +import org.apache.commons.lang3.RandomUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class RandomIpv4Generator. + */ +public final class Ipv4Generator +{ + private static Logger logger = LoggerFactory.getLogger(Ipv4Generator.class); + + /** + * Instantiates a new random ipv 4 generator. + */ + private Ipv4Generator() + { + } + + /** + * Gets the ipv 4 max length part. + * + * @param length + * the length (1..12) + * @param column + * the column (4...1) + * @return the ipv 4 max length part + */ + public static int getIpv4MaxLengthPart(final int length, final int column) + { + int result; + + if ((length < 1) || (length > 12)) + { + throw new IllegalArgumentException("Bad length value: " + length); + } + else if ((column < 1) || (column > 12)) + { + throw new IllegalArgumentException("Bad column value:" + column); + } + else + { + final int[] col4 = { 0, 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 3 }; + final int[] col3 = { 0, 0, 1, 2, 3, 3, 3, 3, 3, 0, 0, 0 }; + final int[] col2 = { 0, 1, 2, 3, 3, 3, 0, 0, 0, 0, 0, 0 }; + final int[] col1 = { 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + final int[][] table = { col1, col2, col3, col4 }; + + result = table[column - 1][length - 1]; + + if (result == 0) + { + throw new IllegalArgumentException(String.format("Zero detected (%d, %d).", length, column)); + } + } + + // + return result; + } + + /** + * Gets the ipv 4 max value part. + * + * @param length + * the length + * @param column + * the column + * @return the ipv 4 max value part + */ + public static int getIpv4MaxValuePart(final int length, final int column) + { + int result; + + int max = getIpv4MaxLengthPart(length, column); + + switch (max) + { + case 1: + result = 9; + break; + + case 2: + result = 99; + break; + + case 3: + result = 255; + break; + + default: + throw new IllegalArgumentException("Bad value: " + max); + } + + // + return result; + } + + /** + * Gets the ipv 4 min length part. + * + * @param length + * the length (1..12) + * @param column + * the column (4...1) + * @return the ipv 4 min length part + */ + public static int getIpv4MinLengthPart(final int length, final int column) + { + int result; + + if ((length < 1) || (length > 12)) + { + throw new IllegalArgumentException("Bad length value: " + length); + } + else if ((column < 1) || (column > 12)) + { + throw new IllegalArgumentException("Bad column value:" + column); + } + else + { + final int[] col4 = { 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 3 }; + final int[] col3 = { 0, 0, 1, 1, 1, 1, 1, 2, 3, 0, 0, 0 }; + final int[] col2 = { 0, 1, 1, 1, 2, 3, 0, 0, 0, 0, 0, 0 }; + final int[] col1 = { 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + final int[][] table = { col1, col2, col3, col4 }; + + result = table[column - 1][length - 1]; + + if (result == 0) + { + throw new IllegalArgumentException(String.format("Zero detected (%d, %d).", length, column)); + } + } + + // + return result; + } + + public static int getIpv4MinValuePart(final int length, final int column) + { + int result; + + int max = getIpv4MinLengthPart(length, column); + + switch (max) + { + case 1: + result = 0; + break; + + case 2: + result = 10; + break; + + case 3: + result = 100; + break; + + default: + throw new IllegalArgumentException("Bad value: " + max); + } + + // + return result; + } + + /** + * Generates a ipv4 of a fixed length. + * + * @param length + * the length + * @return the string + */ + public static String random(final int length) + { + String result; + + if ((length < 7) || (length > 15)) + { + throw new IllegalArgumentException("Bad parameter: " + length); + } + else + { + int size = length - 3; + + int a = (int) RandomUtils.nextLong(getIpv4MinValuePart(size, 4), getIpv4MaxValuePart(size, 4) + 1); + size -= String.valueOf(a).length(); + + int b = (int) RandomUtils.nextLong(getIpv4MinValuePart(size, 3), getIpv4MaxValuePart(size, 3) + 1); + size -= String.valueOf(b).length(); + + int c = (int) RandomUtils.nextLong(getIpv4MinValuePart(size, 2), getIpv4MaxValuePart(size, 2) + 1); + size -= String.valueOf(c).length(); + + int d = (int) RandomUtils.nextLong(getIpv4MinValuePart(size, 1), getIpv4MaxValuePart(size, 1) + 1); + size -= String.valueOf(d).length(); + + result = String.format("%d.%d.%d.%d", a, b, c, d); + } + + // + return result; + } + + /** + * Generates a ipv4 with the same length than the parameter. + * + * @param ip + * the source + * @return the string + */ + public static String random(final String ip) + { + String result; + + if (ip == null) + { + result = null; + } + else + { + result = random(ip.length()); + + if (result.equals(ip)) + { + random(ip); + } + } + + // + return result; + } +} diff --git a/src/anonymizer/Ipv6Generator.java b/src/anonymizer/Ipv6Generator.java new file mode 100644 index 0000000..3d5e94e --- /dev/null +++ b/src/anonymizer/Ipv6Generator.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of Logar, simple tool to manage http log files. + * + * Logar is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * Logar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Logar. If not, see . + */ +package anonymizer; + +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class RandomIpv6Generator. + */ +public final class Ipv6Generator +{ + private static Logger logger = LoggerFactory.getLogger(Ipv6Generator.class); + + /** + * Instantiates a new random ipv 6 generator. + */ + private Ipv6Generator() + { + } + + /** + * Random. + * + * @param ip + * the source + * @return the string + */ + public static String random(final String ip) + { + String result; + + if (ip == null) + { + result = null; + } + else + { + StringBuffer buffer = new StringBuffer(ip.length()); + for (int index = 0; index < ip.length(); index++) + { + char c = ip.charAt(index); + if (c == ':') + { + buffer.append(c); + } + else + { + buffer.append(RandomStringUtils.random(1, "0123456789abcdef")); + } + } + result = buffer.toString(); + + if (result.equals(ip)) + { + random(ip); + } + } + + // + return result; + } +} diff --git a/src/anonymizer/UserGenerator.java b/src/anonymizer/UserGenerator.java new file mode 100644 index 0000000..2102026 --- /dev/null +++ b/src/anonymizer/UserGenerator.java @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of Logar, simple tool to manage http log files. + * + * Logar is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * Logar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Logar. If not, see . + */ +package anonymizer; + +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class RandomUserGenerator. + */ +public final class UserGenerator +{ + private static Logger logger = LoggerFactory.getLogger(UserGenerator.class); + + /** + * Instantiates a new random user generator. + */ + private UserGenerator() + { + } + + /** + * Random. + * + * @param length + * the length + * @return the string + */ + public static String random(final int length) + { + String result; + + if (length < 0) + { + throw new IllegalArgumentException("Bad parameter: " + length); + } + else + { + result = RandomStringUtils.random(length); + } + + // + return result; + } + + /** + * Random. + * + * @param source + * the source + * @return the string + */ + public static String random(final String source) + { + String result; + + if (source == null) + { + result = null; + } + else if (source.equals("-")) + { + result = source; + } + else + { + result = RandomStringUtils.random(source.length(), "abcdefghijklmnopqrstuvwxyz"); + } + + // + return result; + } +} diff --git a/src/fr/devinsy/logar/app/Logar.java b/src/fr/devinsy/logar/app/Logar.java index 620388e..aacb1c3 100644 --- a/src/fr/devinsy/logar/app/Logar.java +++ b/src/fr/devinsy/logar/app/Logar.java @@ -34,10 +34,14 @@ import java.util.zip.GZIPOutputStream; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.april.logar.util.Files; +import org.april.logar.util.FilesUtils; import org.april.logar.util.LineIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import anonymizer.Anonymizer; +import fr.devinsy.logar.app.log.LogUtils; + /** * The Class Logar. */ @@ -45,9 +49,6 @@ public final class Logar { private static Logger logger = LoggerFactory.getLogger(Logar.class); - public static Pattern nginxAccessLogLinePatternFull = Pattern.compile( - "^(?[a-zA-F0-9\\\\:\\\\.]+) - (?\\S+) \\[(?