#! /usr/bin/python3

"""File activity monitor, with quasi-load printout"""

import os
import math
import optparse
import glob
import time
import datetime
import sys
import re

fds={}
load={}

progress=""".234567890abcdefghijklmnopqrstuvwxyz_"""

class fileloader:
    re_syslogtime = re.compile(r"^([A-Z][a-z][a-z] )?([A-Z][a-z][a-z]  ?[0-9][0-9]? \d\d:\d\d:\d\d)");
    re_exim__time = re.compile(r"^(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)");
    #  [22/Jul/2010:12:12:27 +0200]
    re_apachetime      = re.compile(r"\[(\d\d/[A-Z][a-z][a-z]/\d\d\d\d:\d\d:\d\d:\d\d) ");
    re_apachetime_zone = re.compile(r"\[(\d\d/[A-Z][a-z][a-z]/\d\d\d\d:\d\d:\d\d:\d\d) ([\+-]\d\d)\d\d");
    # [18/Jun/2014:10:05:42.774]
    re_haproxytime    = re.compile(r"\[(\d\d/[A-Z][a-z][a-z]/\d\d\d\d:\d\d:\d\d:\d\d)(\.\d+)\]");
    
    # 1285664956.555     18 41.72.108.82 TCP_CLIENT_REFRESH_MISS/200
    re_squidtime = re.compile(r"^([0-9]{9,11})\.[0-9]+ +[0-9]+ [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ ");
    re_unixtime = re.compile(r'time\((\d+)\)');
    year = str(time.localtime().tm_year)
    verbose=0

    def __init__(self,filename,regex='',verbose=0,quiet=0):
        self.tail = []
        self.taillength = 10
        self.quiet=quiet
        self.verbose=verbose
        self.filename=filename
        self.regex=regex
        self.fd=open(self.filename,'r')
        self.fd.seek(0,2)
        self.lastlen = self.fd.tell()
        self.load1=0.0
        self.load5=0.0
        self.load15=0.0
        self.lines=0
        self.minutes = 1  # number of minutes to do the load over
        self.start_time = time.time()
        self.now = self.start_time
        self.daylight = time.daylight  # different files may have different ideas of daylight saving ...
        self.time_lines = 0
        self.timefunctions = [
            self.find_re_syslogtime,
            self.find_re_squidtime,
            self.find_re_apachetime,
            self.find_re_haproxytime,
            self.find_re_exim__time,
            self.find_re_unixtime, ]

    def gettimestamp(self,line):
        for function in self.timefunctions:
            r = function(line)
            if r:
                if self.time_lines<20: # re-order the list so we use this function preferentially
                    self.timefunctions.remove(function)
                    self.timefunctions.insert(0,function)
                    self.time_lines+=1
                return r

    def find_re_syslogtime(self,line):
        m = self.re_syslogtime.search(line)
        if m:
            fmt = "%Y %b %d %H:%M:%S"
            timestring = self.year+' '+m.group(2).replace("  "," ")
            timevals = time.strptime(timestring,fmt)
            timeunix = time.mktime(timevals)
            if self.verbose:
                print(timestring)
            if timeunix>time.time()+86400*7:
                timestring = str(int(self.year)-1)+' '+m.group(2).replace("  "," ")
                timevals = time.strptime(timestring,fmt)
                timeunix = time.mktime(timevals)
            return timeunix

    def find_re_squidtime(self,line):
        m = self.re_squidtime.search(line)
        if m:
            timestring = m.group(1)
            return int(timestring)
        m = self.re_apachetime_zone.search(line)
        if m:
            fmt = "%d/%b/%Y:%H:%M:%S"
            timestring = m.group(1)
            zonehours = int(m.group(2))
            t0 = time.strptime(timestring,fmt)
            # >>> t0 = time.mktime(time.strptime(timestring,fmt)) - 3600 * (time.timezone/3600 +2 - time.daylight)
            r = time.mktime(time.strptime(timestring,fmt)) - time.timezone + 3600 * (self.daylight - zonehours )  # randomly undo stuff
            # ugly hack: if time goes forwards, undo daylight saving, since the app doesn't know about it...
            if r > self.start_time+15:
                self.daylight=0
            return r

    def find_re_apachetime(self,line):
        m = self.re_apachetime.search(line)
        if m:
            fmt = "%d/%b/%Y:%H:%M:%S"
            timestring = m.group(1)
            timevals = time.strptime(timestring,fmt)
            return time.mktime(timevals)

    def find_re_haproxytime(self,line):
        m = self.re_haproxytime.search(line)
        if m:
            fmt = "%d/%b/%Y:%H:%M:%S"
            timestring = m.group(1)
            timevals = time.strptime(timestring,fmt)
            return time.mktime(timevals)

    def find_re_exim__time(self,line):
        m = self.re_exim__time.search(line)
        if m:
            fmt = "%Y-%m-%d %H:%M:%S"
            timestring = m.group(1)
            timevals = time.strptime(timestring,fmt)
            return time.mktime(timevals)

    def find_re_unixtime(self,line):
        m= self.re_unixtime.search(line)
        if m:
            timestring = m.group(1)
            return int(timestring)

    def analyse(self,minutes):
        'Read simple averages from files'
        targettime = self.start_time - minutes*60;
        position = self.fd.tell()
        endpoint = position
        stop = position
        jump = 63336; # 64kb at a time
        # analyse = 1024*1024; # 1Mb max .. oh wait, unlimited!
        seek_to  = endpoint - (endpoint % 2048 ) - 2048;
        attempts = 0
        iteration=0
        self.linecount = 0
        if not self.quiet:
            sys.stderr.write(".")
            sys.stderr.flush()
        self.tail = []
        while True:
            if seek_to < 0 : seek_to =0
            try:
                self.fd.seek(seek_to ,0)
                position=seek_to
            except IOError:
                break # give up
            if iteration>0:
                c=progress[iteration:iteration+1]
                if not c: c=progress[-1]
                if not self.quiet:
                    sys.stderr.write("\b"+c)
                    sys.stderr.flush()
            iteration+=1

            line1 = self.fd.readline() # read partial line
            line = self.fd.readline() # read full line
            if len(self.tail)==0:
                self.tail = [line.strip('\n'), ]
                self.tail.extend( self.fd.read().split('\n') )
                self.tail=self.tail[-self.taillength:]
            position += len(line)+len(line1)
            timestamp=self.gettimestamp(line)
            if not timestamp:
                attempts+=1
                if attempts == 5: break # 5 bad lines in a row means blah
            if timestamp and timestamp<targettime:
                if self.verbose > 2:
                    sys.stderr.write("%s\n" % str( dict(timestamp=timestamp, targettime=targettime, attempts=attempts,iteration=iteration)))
                break
            if seek_to ==0: break;
            seek_to -= jump;


        for line in self.fd:
            self.lines+=1
            position+=len(line)
            if self.matches_regex(line):
                timestamp=self.gettimestamp(line)
                if timestamp:
                    self.linecount += self.perturb(self.start_time-timestamp)
            if position >= stop: break

        if self.verbose > 2:
            print(dict(seek_to=seek_to, iteration=iteration, lines=self.lines, stop=stop,position=position))

        return self.linecount

    def matches_regex(self,line):
        if not self.regex:
            return True
        return self.regex.search(line)


    def __repr__(self):
        return "%9.3f" % self.load1

    def perturb(self,seconds):
        """Record a tick of load happening back when as a straightforward
        strict average.  An exponential thing would be nice, but it's hard
        to get right.  Returns the number of lines counted."""
        rc = 0
        if seconds<60: 
            self.load1  += 1.0/60
            rc = 1
        if seconds<300:
            self.load5  += 1.0/300
            rc = 1
        if seconds<900:
            self.load15 += 1.0/900
            rc = 1
        return rc

    def adjustload(self,load,seconds):
        'seconds should really be a float, please'
        E = math.exp(-seconds/(self.minutes*60)) # decay over n minutes
        self.load1 = self.load1*E + load*(1.0-E)

        E = math.exp(-seconds/(300)) # decay over 5 minutes
        self.load5 = self.load5*E + load*(1.0-E)

        E = math.exp(-seconds/(900)) # decay over 15 minutes
        self.load15 = self.load15*E + load*(1.0-E)

    def updateusage(self):
        self.fd.seek(0,2)
        newlen = self.fd.tell()
        bytes = newlen - self.lastlen
        self.fd.seek(self.lastlen,0)
        p = self.fd.read(bytes)
        if not self.regex:
            self.tail=p.split('\n')
            lines = len(self.tail); # p.count('\n')
        else:
            self.tail=[]
            lines=0
            for line in p.split('\n'):  # this is inexact, since we are not retaining data over buffer splits ...
                if self.regex.search(line):
                    self.tail.append(line.strip('\n'))
                    lines+=1
        self.lines+=lines
        self.lastlen=newlen
        now = time.time()
        dtime=now-self.now
        self.now=now
        if dtime>self.minutes*60: dtime=self.minutes*60;
        self.adjustload(lines/dtime,dtime)
        self.tail=self.tail[-self.taillength:]

files = []

def fcmp(f1,f2):
    if f1.load5<=f2.load5: return -1
    if f1.load5>=f2.load5: return 1
    return 0

def fkey(f1):
    return f1.load5

if len(sys.argv)>1:
    globs = sys.argv[1:]
else:
    globs ="/usr/local/apache/domlogs/*.*"

def main(filelist, regex='', mmin=10, sleep=1,display=10, seconds=0, zero=False,quiet=False, verbose=0,times=-1,follow=0):
    if regex:
        regex=re.compile(regex)
    discards=0
    for globspec in filelist:
        for filename in glob.glob(globspec):
            if filename.find("-bytes_log") >=0 : continue # ignore the byteslogs files .. just ignore them!
            if filename.find("-smtpbytes_log") >=0 : continue # ignore the smtp byteslogs files .. really.
            if time.time()-os.stat(filename).st_mtime > mmin*60:
                discards += 1
                continue
            try:
                f=fileloader(filename,regex=regex,verbose=verbose, quiet=quiet)
            except IOError:
                continue
            files.append(f)

    if not quiet:
        print("%d of %d files opened" % (len(files),len(files)+discards))
    # if discards: print "%d files ignored (older than %d minutes)" % (discards,mmin)

    timescount = 0
    if zero:
        time.sleep(2)
        for f in files:
            f.updateusage()
    else:
        if not quiet:
            sys.stderr.write("Reading simple averages ")
            sys.stderr.flush()
        # Do historical analysis
        linecount = 0
        for f in files:
            linecount += f.analyse(20)  # go back 20 minutes, because some log files have mixed timestamps ...
        if not quiet:
            sys.stderr.write(" (%d lines)\n" % linecount)

    if not quiet:
        sys.stderr.write( "Hits per minute: moving average over 1, 5 and 15 minutes:\n")
    iterationcount = 0
    while True:
        activefiles = []
        files.sort(key=fkey)
        for f in files[-display:]: 
            print("%9.3f %9.3f %9.3f %s" % (
                f.load1*60, 
                f.load5*60, 
                f.load15*60, 
                f.filename))
            if verbose>0:
                print("  lines=%d" % f.lines)
        if not quiet:
            os.system("uptime")
            print()
        if follow:
            for f in files[-follow:]:
                print( '\n'.join(f.tail) )
        if iterationcount == times:
            break
        iterationcount += 1
        time.sleep(sleep)
        for f in files:
            f.updateusage()
    return files
        

if __name__=="__main__":
    usage="Usage: %prog [<options>]\n"+__doc__+"\n"
    parser=optparse.OptionParser(usage)
    parser.add_option("-m","--mmin",dest="mmin", action="store",type='int', default=10, help="ignore files older than m minutes (10)")
    parser.add_option("-n","--display",dest="display", action="store",type='int', default=10, help="display this many files")
    parser.add_option("-s","--sleep",dest="sleep", action="store", type='int', default=10, help="Sleep interval between checks (10 seconds)")
    parser.add_option("-c","--count",dest="count", action="store", type='int', default=10, help="Number of minutes to run for")
    parser.add_option("-z","--zero",dest="zero", action="store_true", default=False, help="Start with zero counters")
    parser.add_option("-r","--re",dest="regex", action="append", default=[], help="Regexes to allow (default is any)")
    parser.add_option("-q","--quiet",dest="quiet", action="store_true", default=None, help="Don't display extra information (uptime)")
    parser.add_option("-x","--times",dest="times", action="store", type='int', default=-1, help="Number of times to run (-1 by default, indefinitely)")
    parser.add_option("-v","--verbose",dest="verbose", action="count", default=0, help="Verbosityness")
    parser.add_option("-f","--follow",dest="follow", action="count", default=0, help="Follow most active file")
    (options,args) = parser.parse_args()
    if len(args)==0: args=['/usr/local/apache/domlogs/*.*'];
    try:
        main(args,
            regex='|'.join(options.regex),
            mmin=options.mmin,
            display=options.display,
            sleep=options.sleep,
            zero=options.zero,
            quiet=options.quiet,
            follow=options.follow,
            verbose=options.verbose,
            times=options.times,
            seconds=options.count*60)
    except KeyboardInterrupt:
        pass