dirty-plot

#!/usr/bin/env python3
import time
import dateutil.parser
import fileinput
import csv

import logging
dblevel=logging.INFO
logging.basicConfig(level=dblevel)
logger = logging.getLogger(__name__)
logger.setLevel(dblevel)

def try_pandas(s,fn):
    want_pandas = False
    if not want_pandas:
        return

    try:
        import pandas as pd
    except:
        return

    df=pd.DataFrame(s)
    df.to_json(("%s.json" % fn))
    logger.debug("pandas saved to '%s.json'",fn)

    combined={}
    for section, metrics in df.items():
        for metric,rows in metrics.items():
            list_ts=[]
            list_val=[]
            list_metric=[]

            for row in rows:
                for ts in row.keys():
                    val=row[ts]
                    list_ts.append(ts)
                    list_val.append(val)
                    list_metric.append(metric)

            ts_series=pd.to_datetime(list_ts)
            val_series=pd.to_numeric(list_val)
            metric_series=pd.Series(list_metric)

            ts_df=pd.DataFrame(ts_series,columns=["Time"])
            val_df=pd.DataFrame(val_series,columns=[metric])
            metric_df=pd.DataFrame(metric_series,columns=["Metric"])

            all_df=pd.concat([ts_df,val_df,metric_df],axis=1)
            all_df.set_index('Time',inplace=True)
            if (False):  # Disabled right now
                all_df.to_csv("%s.csv" % fn)
                # TODO This is saving one metric worth of data to a CSV.
                # Needs to run against a list of all the metrics
                # Maybe build a combined frame here instead of the
                # dictionary of metrics I have right now?
                logger.info("metrics saved to '%s.csv'",fn)

            combined[metric]=all_df

        # This part will return after processing only one section; all there is so far.
        return combined

def main():
    section="meminfo"
    sets={section:{}}
    rows=[]
    fields = ['collected','value','metric']

    for l in fileinput.input():
        l=l.strip()
        (ts,key,val)=str.split(l,"\t",2)
        t=dateutil.parser.parse(ts)
        epoch=time.mktime(t.timetuple())
        formatted="%s %s" % (epoch,val)

        # Backward compatible output
        if key=="Dirty":
            print (formatted)
        value=float(val)

        metric_row=[ts,val,key]
        rows.append(metric_row)

        if key not in sets[section]:
            sets[section][key] = []
        sets[section][key].append({t:value})

    csvfn="%s.csv" % section
    with open(csvfn, 'w') as csvfile:
        csvwriter = csv.writer(csvfile)
        csvwriter.writerow(fields)
        csvwriter.writerows(rows)
        logger.info("metrics saved to '%s.csv'",section)

    # TODO Switch these to logger
    if (False):  print(rows)

    return try_pandas(sets,section)

if __name__ == '__main__':
    df=main()