collectPandas

#!/usr/bin/env python3

import pandas as pd
import matplotlib.pyplot as plt
import os

import logging
dblevel=logging.INFO
logging.basicConfig(level=dblevel)
logger = logging.getLogger(__name__)
logger.setLevel(dblevel)

def json_pandas(fn):
        df=pd.read_json(fn)
        logger.debug("pandas loaded from '%s'",fn)
        return df

def csv_in(fn,index_col):
    df = pd.read_csv(fn,index_col=index_col,parse_dates=True)
    logger.info("CSV loaded from '%s'",fn)
    return df
 
def main():
    base="regress/480/"
    if False:
        io=json_pandas(os.path.join(base,"iostat.json"))
        vm=json_pandas(os.path.join(base,"vmstat.json"))
        me=json_pandas(os.path.join(base,"meminfo.json"))

    tps_stats=csv_in(os.path.join(base,"latency_metric.csv"),'collected')
    iostat_stats=csv_in(os.path.join(base,"iostat.csv"),'collected')
    vmstat_stats=csv_in(os.path.join(base,"vmstat.csv"),'collected')
    meminfo_stats=csv_in(os.path.join(base,"meminfo.csv"),'collected')

    t=tps_stats
    t=t.append(iostat_stats)
    t=t.append(vmstat_stats)
    t=t.append(meminfo_stats)

    fn=os.path.join(base,"combined.csv")
    t.to_csv(fn)
    logger.info("csv saved to '%s'",fn)

    # Round here to make the Pandas merge give less NaN values
    t.index=t.index.round('S')

    combined=pd.DataFrame()
    grouped = t.groupby('metric')
    for metric,group in grouped:
        logger.info(metric)
        metric_data=group['value']
        logger.debug(metric_data)

        # Rename the new column to have the metric name
        renamed=pd.DataFrame(metric_data)
        renamed.rename(columns={'value':metric},inplace=True)

        combined=combined.merge(renamed,how="outer",left_index=True,right_index=True)

    fn=os.path.join(base,"combined-wide.csv")
    combined.to_csv(fn)
    logger.info("csv saved to '%s'",fn)

    # Redundant and broken by CSV changes, "DataFrame index must be unique for orient='columns'."
    if (False):
        fn=os.path.join(base,"combined.json")
        t.to_json()
        logger.warning("json saved to '%s'",fn)

    return combined

if __name__ == '__main__':
    df=main()

    base="regress/480/metrics"
    for c in df.columns:
        print("Processing",c)
        ax=df[c].plot(grid=True,title=c,figsize=(8,6),legend=True,color='m')
        ax.xaxis.grid(True, which="minor")

        unslashed=c.replace("/","-")
        fn=os.path.join(base,unslashed)

        plt.savefig(fn,dpi=80)  # gives 640x480 figures
        plt.close()
        logger.info("saved to '%s.png'",fn)