logar/src/fr/devinsy/logar/app/log/LogUtils.java

/*
 * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
 * 
 * This file is part of Logar, simple tool to manage http log files.
 * 
 * Logar is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * Logar is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with Logar.  If not, see <http://www.gnu.org/licenses/>.
 */
package fr.devinsy.logar.app.log;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPOutputStream;

import org.apache.commons.io.IOUtils;
import org.april.logar.util.LineIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import fr.devinsy.cmdexec.CmdExecException;
import fr.devinsy.cmdexec.CmdExecUtils;

/**
 * The Class NginxAccessLogParser.
 */
public final class LogUtils
{
    private static Logger logger = LoggerFactory.getLogger(LogUtils.class);

    public static Pattern nginxAccessLogLinePatternFull = Pattern.compile(
            "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>\\S+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");

    public static Pattern NGINX_ACCESSLOG_LINE_PATTERN = Pattern.compile("^(?<remoteAddress>[a-fA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] .*$");
    public static Pattern NGINX_ERRORLOG_LINE_PATTERN = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");

    /**
     * Instantiates a new nginx access log parser.
     */
    private LogUtils()
    {
    }

    /**
     * Load.
     *
     * @param file
     *            the file
     * @return the logs
     * @throws IOException
     */
    public static Logs loadAccessLog(final File file) throws IOException
    {
        Logs result;

        result = new Logs();

        LineIterator iterator = new LineIterator(file);
        while (iterator.hasNext())
        {
            String line = iterator.next();
            Log log = parseAccessLog(line);
            result.add(log);
        }

        //
        return result;
    }

    /**
     * Load error log.
     *
     * @param file
     *            the file
     * @return the logs
     * @throws IOException
     *             Signals that an I/O exception has occurred.
     */
    public static Logs loadErrorLog(final File file) throws IOException
    {
        Logs result;

        result = new Logs();

        LineIterator iterator = new LineIterator(file);
        while (iterator.hasNext())
        {
            String line = iterator.next();
            Log log = parseErrorLog(line);
            result.add(log);
        }

        //
        return result;
    }

    /**
     * From access log.
     *
     * @param line
     *            the line
     * @return the log
     */
    public static Log parseAccessLog(final String line)
    {
        Log result;

        Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line);
        if (matcher.matches())
        {
            String value = matcher.group("time");
            LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));

            String ip = matcher.group("remoteAddress").trim();
            String user = matcher.group("remoteUser").trim();

            result = new Log(line, date, ip, user);
        }
        else
        {
            throw new IllegalArgumentException("Bad line format: " + line);
        }

        //
        return result;
    }

    /**
     * From error log.
     *
     * @param line
     *            the line
     * @return the log
     */
    public static Log parseErrorLog(final String line)
    {
        Log result;

        Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line);
        if (matcher.matches())
        {
            String value = matcher.group("time");
            LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));

            result = new Log(line, date);
        }
        else
        {
            throw new IllegalArgumentException("Bad line format: " + line);
        }

        //
        return result;
    }

    /**
     * Save access log.
     *
     * @param source
     *            the source
     * @throws IOException
     * @throws FileNotFoundException
     */
    public static void saveLogs(final File target, final Logs logs) throws FileNotFoundException, IOException
    {
        PrintWriter out = null;
        try
        {
            out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));

            for (Log log : logs)
            {
                out.println(log.getLine());
            }
        }
        finally
        {
            IOUtils.closeQuietly(out);
        }
    }

    /**
     * Sort access log.
     *
     * @param target
     *            the target
     * @throws IOException
     */
    public static void sortAccessLog(final File file) throws IOException
    {
        File work = new File(file.getParent(), file.getName() + ".tmp");

        Logs logs = loadAccessLog(file);
        logs.sortByDatetime();
        saveLogs(work, logs);

        File backup = new File(file.getParentFile(), file.getName() + ".bak");
        if (file.renameTo(backup))
        {
            if (!work.renameTo(file))
            {
                backup.renameTo(file);
            }
        }

        try
        {
            String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum  \"");
            System.out.print(out);
            out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + ".bak | sort | sha1sum  \"");
            System.out.println(out);
        }
        catch (CmdExecException exception)
        {
            exception.printStackTrace();
        }
    }

    /**
     * Sort error log.
     *
     * @param file
     *            the file
     * @throws IOException
     *             Signals that an I/O exception has occurred.
     */
    public static void sortErrorLog(final File file) throws IOException
    {
        File work = new File(file.getParent(), file.getName() + ".tmp");

        Logs logs = loadErrorLog(file);
        logs.sortByDatetime();
        saveLogs(work, logs);

        File backup = new File(file.getParentFile(), file.getName() + ".bak");
        if (file.renameTo(backup))
        {
            if (!work.renameTo(file))
            {
                backup.renameTo(file);
            }
        }

        try
        {
            String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum  \"");
            System.out.print(out);
            out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + ".bak | sort | sha1sum  \"");
            System.out.println(out);
        }
        catch (CmdExecException exception)
        {
            exception.printStackTrace();
        }
    }
}