From 09af817c6a9ae75ada44295abb60477a465c49e8 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 16 Aug 2021 01:14:36 +0800 Subject: [PATCH] testing for now - probably has bugs --- .gitignore | 4 ++ csvfile.py | 124 +++++++++++++++++++++++++++++++++ disk-usage.py | 79 +++++++++++++++++++++ drive.py | 52 ++++++++++++++ utils.py | 41 +++++++++++ viewer.py | 187 ++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 487 insertions(+) create mode 100644 .gitignore create mode 100755 csvfile.py create mode 100755 disk-usage.py create mode 100755 drive.py create mode 100755 utils.py create mode 100755 viewer.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5a06c93 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +test* +*.csv +.venv/ +__pycache__/ diff --git a/csvfile.py b/csvfile.py new file mode 100755 index 0000000..cdae917 --- /dev/null +++ b/csvfile.py @@ -0,0 +1,124 @@ +import csv +from pathlib import Path + +class CSVFile: + def __init__(self, filePath, delimiter=",") -> None: + self.__rewrite = False + self.__delimiter = delimiter + self.__filePath = Path(filePath) + self.__addBuffer = [] + self.__newFile = self.fileInit() + self.length = 0 + with open(filePath, "r") as file: + data = csv.DictReader(file,delimiter=delimiter) + if data.fieldnames == None: + self.__fields = list() + self.__newFile = True + else: + self.__fields = list(data.fieldnames) + self.__data = list(data) + self.length = len(self.__data) + pass + + def fileInit(self): + if not self.__filePath.exists(): + open(self.__filePath,'a').close() + return True + return False + + def addRow(self,row): + print("ROW", row) + print("ROW", row.keys()) + fields = row.keys() + for field in fields: + if not field in self.__fields: + self.__fields.append(field) + self.__data.append(row) + self.__addBuffer.append(row) + + def getRow(self,rowIdx): + return self.__data[rowIdx] + + def validate(self): + if self.__rewrite: + return False + alreadyValid = True + expectedFieldLen = len(self.__fields) + addIdx = len(self.__data)-len(self.__addBuffer) + idx = 0 + for row in self.__data: + fields = row.keys() + fieldsLen = len(fields) + if fieldsLen < expectedFieldLen: + for expectedField in self.__fields: + if expectedField not in fields: + row.update({expectedField: ""}) + if idx < addIdx: + alreadyValid = False + elif fieldsLen > expectedFieldLen: + # Omit invalid data. + firstField = self.__fields[0] + commented = "#"+row[firstField] + row.update({firstField : commented}) + print("Error on line "+str(idx)+" - Omitted from file") + alreadyValid = False + idx += 1 + return alreadyValid + + def writeAppend(self,filepath): + print("Append - no header change") + with open(filepath, "a", newline="") as file: + w = csv.DictWriter(file, fieldnames=self.__fields) + w.writerows(self.__addBuffer) + def writeFile(self,filepath): + print("Rewrite - header changed") + with open(filepath, "w") as file: + w = csv.DictWriter(file, self.__fields) + w.writeheader() + w.writerows(self.__data) + + def efficientWriteFile(self): + filepath = self.__filePath + if self.validate() and not self.__newFile: + self.writeAppend(filepath) + else: + self.writeFile(filepath) + + def search(self,k,v,adjacent=False,backwards=False,start=0): + results = [] + data = self.__data + r = range(start,len(data)) + if backwards: + r = range(start,-len(data)-1,-1) + for i in r: + row = self.__data[i] + if k in row.keys() and row[k] == v: + results.append(row) + elif adjacent: + break + return results + + # matches=-1 ==> infinite replacements + def replaceByKV(self,k,v,newRow,backwards=False,start=0,matches=1): + infinite = matches < 0 + data = self.__data + count = 0 + r = range(start,len(data)) + if backwards: + r = range(start,-len(data)-1,-1) + for i in r: + row = self.__data[i] + if k in row.keys() and row[k] == v: + self.__data[i] = newRow + self.__rewrite = True + count += 1 + if not infinite and count >= matches: + return True + if infinite: + return count > 0 + return False + + def fill(self, v=0): + if len(self.__data) == 0: + return False + return { k:v for k in self.__data[0].keys() } \ No newline at end of file diff --git a/disk-usage.py b/disk-usage.py new file mode 100755 index 0000000..9c587b4 --- /dev/null +++ b/disk-usage.py @@ -0,0 +1,79 @@ +import sys +import os +import shutil +import csv +import datetime +from drive import Drive +from csvfile import CSVFile +from pathlib import Path +from utils import subtract, nextWeek, sumKV + +SEARCH_PATH = "/mnt/" + +class DiskUsage: + # I probably should be using a database instead of CSV. + def __init__(self, basepath) -> None: + basepath = Path(basepath) + self.SIZE_FILE = basepath.joinpath("size_daily.csv") + self.DIFF_FILE = basepath.joinpath("diff_daily.csv") + self.WEEKLY_SIZE_FILE = basepath.joinpath("size_weekly.csv") + self.WEEKLY_DIFF_FILE = basepath.joinpath("diff_weekly.csv") + pass + + @staticmethod + def __daily2weeklyRow(row_): + row = row_.copy() + row["time"] = row["next_week"] + row.pop("next_week") + return row + + + def update(self): + nextWeek_ = nextWeek().isoformat() + + sizeDaily = CSVFile(self.SIZE_FILE) + currentlyUsed = Drive.getUsed(SEARCH_PATH) + sizeDaily.addRow(currentlyUsed) + sizeDaily.efficientWriteFile() + if sizeDaily.length < 2: + return 0 # Break for new files + + sizeWeekly = CSVFile(self.WEEKLY_SIZE_FILE) + sizeWeeklyRow = self.__daily2weeklyRow(currentlyUsed) + previousNextWeek = sizeDaily.getRow(-2).get("next_week") + if sizeWeekly.length < 1: + sizeWeekly.addRow(sizeWeeklyRow) + elif previousNextWeek == nextWeek_: + sizeWeekly.replaceByKV("time", nextWeek_, sizeWeeklyRow, backwards=True, start=-1) + else: + sizeWeekly.addRow(sizeWeeklyRow) + previousWeeklyRow = sizeWeeklyRow.copy() + previousWeeklyRow.update({ "time" : previousNextWeek }) + sizeWeekly.replaceByKV("time", previousNextWeek, previousWeeklyRow, backwards=True, start=-1) + sizeWeekly.efficientWriteFile() + + delta = subtract(sizeDaily.getRow(-1),sizeDaily.getRow(-2)) + diffDaily = CSVFile(self.DIFF_FILE) + diffDaily.addRow(delta) + diffDaily.efficientWriteFile() + + + if sizeWeekly.length < 2: + return 0 # Break for new files + diffWeekly = CSVFile(self.WEEKLY_DIFF_FILE) + previousNextWeek = diffDaily.getRow(-2).get("next_week") + delta = subtract(sizeWeekly.getRow(-1), sizeWeekly.getRow(-2)) + print(previousNextWeek == nextWeek_) + print(delta) + if previousNextWeek == nextWeek_: + diffWeekly.replaceByKV("time", previousNextWeek, delta, backwards=True, start=-1) + else: + delta.update({ "time" : previousNextWeek }) + diffWeekly.replaceByKV("time", previousNextWeek, delta, backwards=True, start=-1) + empty = diffDaily.fill() + empty.update({"time" : nextWeek_}) + diffWeekly.addRow(empty) + diffWeekly.efficientWriteFile() + +if __name__ == '__main__': + DiskUsage("/home/peter/scripts/datalogging/disk-usage/testfiles").update() \ No newline at end of file diff --git a/drive.py b/drive.py new file mode 100755 index 0000000..e4a92d3 --- /dev/null +++ b/drive.py @@ -0,0 +1,52 @@ +import shutil +from datetime import datetime, date, timedelta +from utils import nextWeek +import os + +class Drive: + def __init__(self, drivePath) -> None: + self.__drivePath = drivePath + try: + self.total, self.used, self.free = shutil.disk_usage(self.__drivePath) + except: + self.total = self.used = self.free = "" + pass + + def toKV_used(self): + return {self.__drivePath : self.used} + def toKV_total(self): + return {self.__drivePath : self.total} + def toKV_free(self): + return {self.__drivePath : self.free} + + @staticmethod + def getDrives(mountPath): + drives = [] + for drive in os.listdir(mountPath): + path = Drive.__normalizeDirPath(mountPath+drive) + drives.append(Drive(path)) + return drives + + @staticmethod + def getUsed(mountPath): + __now = datetime.now().isoformat(sep="_") + row = { + "time" : __now, + "next_week" : nextWeek().isoformat() + } + for drive in Drive.getDrives(mountPath): + row.update(drive.toKV_used()) + return row + + @staticmethod + def getMaxTotal(mountPath): + drives = Drive.getDrives(mountPath) + return max(drives, key = lambda d : d.total).total + + + @staticmethod + def __normalizeDirPath(dirPath): + if dirPath[-1] == '/': + return dirPath + return dirPath+'/' + diff --git a/utils.py b/utils.py new file mode 100755 index 0000000..5c6330f --- /dev/null +++ b/utils.py @@ -0,0 +1,41 @@ +from datetime import date, timedelta + +def nextWeek(): + __today = date.today() + return __today + timedelta(days=-__today.weekday(), weeks=1) + +def subtract(row1,row2,castmode=int): + if (type(row1) == list and type(row2) == list): + if (not len(row1) == len(row2)): + True + #throw or something + for i in range(0,len(row1)): + try: + row1[i] = castmode(row1[i]) - castmode(row2[i]) + except: + True # keep first v if not a number type. + return row1 + elif (type(row1) == dict and type(row2) == dict): + if (row1.keys() == row2.keys()): + for k in row1.keys(): + try: + row1.update({k : castmode(row1[k]) - castmode(row2[k])}) + except: + True # keep first v if not a number type. + return row1 + +def sumKV(rows,castmode=int): + result = {} + for row in rows: + for k in row.keys(): + if not k in result.keys(): + try: + result.update({ k : castmode(row[k]) }) + except: + result.update({ k : row[k] }) + else: + try: + result[k] += castmode(row[k]) + except: + True + return result \ No newline at end of file diff --git a/viewer.py b/viewer.py new file mode 100755 index 0000000..23ace1f --- /dev/null +++ b/viewer.py @@ -0,0 +1,187 @@ + +from drive import Drive +from pygnuplot import gnuplot +from utils import nextWeek +from pathlib import Path +from datetime import datetime, timedelta + +BASE = Path("/home/peter/scripts/datalogging/disk-usage/testfiles/") +SIZE_FILE = BASE.joinpath("size_daily.csv") +DIFF_FILE = BASE.joinpath("diff_daily.csv") +WEEKLY_SIZE_FILE = BASE.joinpath("size_weekly.csv") +WEEKLY_DIFF_FILE = BASE.joinpath("diff_weekly.csv") + +NOW = datetime.now().isoformat(sep="_") + +__MAX = nextWeek() # Next monday +T_MAX = __MAX.isoformat() +T_MIN = (__MAX - timedelta(days=28)).isoformat() +TY_MIN = (__MAX - timedelta(days=364)).isoformat() + +Y_MAX = Drive.getMaxTotal("/mnt/") + +def fmtRange(min, max): + return str.format("['{0}':'{1}']", min, max) + +array = "" +with open(SIZE_FILE) as f: + array = f.readline() +array = array.replace(","," ").strip() +HEADER_LENGTH = len(array.split(' ')) + + +PLOT_INIT=""" +set terminal sixelgd background rgb 'black' size 1920, 960; set multiplot layout 2,2; set encoding utf8; +set key top left; set key tc rgb 'white'; set border lc rgb 'white'; set key tc rgb 'white'; set datafile separator ','; + +set grid ytics; set ytics scale 2,1; + +set xdata time; set x2data time; set timefmt '%Y-%m-%d_%H:%M:%S'; set x2label 'Date' textcolor 'white'; + +set border lc rgb 'white'; +""" + +PLOT_28D=""" +set format x '%Y-%m-%d'; +set xtics font ', 10' rotate by 90 right; +set x2tics 4*24*60*60, 7*24*60*60 textcolor 'white'; +set x2tics format '%Y-%m-%d'; +set x2tics scale 37,1; +set y2tics scale 2,1; +set tics nomirror; + +set xlabel 'Day' textcolor 'white'; +set xtics 12*60*60, 24*60*60; +set xtics format '%a'; +set mxtics 2; +set xtics scale 0,1; +set grid mxtics; +""" + +# Some bug with scaling meant that I needed to use 59.999999999999 as opposed to +# 60 for the offsets of the weeks from the months IIRC. Haven't checked if this +# bug is still here. +PLOT_364D=""" +set x2tics font ', 10'; +set x2tics 11*24*60*60, 3*7*24*60*60+7*24*60*59.999999999999 textcolor 'white'; +set x2tics format '%m-%d'; +set x2tics scale 2,1; +set mx2tics 4; + +set xlabel 'Month' textcolor 'white'; +set xtics 4*7*24*60*60; +set xtics format '%b'; +set xtics font ', 10' rotate by 0 center; +set mxtics 1; +set xtics scale 1,1; +set grid xtics; +""" + +PLOT_DIFF = """ +set autoscale y +set ylabel '{/Symbol D}Used [ISO definition]' textcolor 'white'; +set format y '%+08.3s %cB'; + +set y2label '{/Symbol D}Used (GB)' textcolor 'white'; +set link y2 via y/(1024**3) inverse y*(1024**3); +set format y2 '%+08.3s %c'; +""" + +PLOT_0="""set title 'Used capacity of disks (last 28 days)' textcolor 'white'; + +set format y '%.2s %cB'; +set ylabel 'Used [ISO definition]' textcolor 'white'; +set ytics 0.25*10**12; +set mytics 2; + +set link y2 via y/1024**4 inverse y*1024**4; +set format y2 '%05.3f'; +interval = (2.5*10**11)/(1024**4); set y2tics interval; +set y2label 'Used (TB)' textcolor 'white'; +set my2tics 1; +""" + +# +# plot for [col=3:$columns] '$(sed 's/\///1;s/\//:\//1' <<< "$FILE")' using 1:col with linespoints title word(array, col) pointtype 5""" + +PLOT_1=""" +set title 'Change in used capacity (last 28 days)' textcolor 'white'; + +set ytics 5*10**9; +set mytics 2; + +interval = (5.0*10**9)/(1024**3); set y2tics interval; +set my2tics 2; + +set offset graph 0, graph 0, graph 0.02, graph 0.02; +"""+PLOT_DIFF + +PLOT_2=""" +set title 'Used capacity of disks (last 364 days)' textcolor 'white'; + +set ylabel 'Used [ISO definition]' textcolor 'white'; +set format y '%.2s %cB'; +set ytics 0.25*10**12; +set mytics 2; + +set y2label 'Used (TB)' textcolor 'white'; +set link y2 via y/1024**4 inverse y*1024**4; +set format y2 '%05.3f'; +interval = (2.5*10**11)/(1024**4); set y2tics interval; +set my2tics 1; + +set offset graph 0, graph 0, graph 0, graph 0; +""" + + +PLOT_3=""" +set title 'Change in used capacity (last 364 days)' textcolor 'white'; + +set ytics 10*10**9; +set mytics 2; + +interval = (10.0*10**9)/(1024**3); set y2tics interval; +set my2tics 2; + +set offset graph 0, graph 0, graph 0.02, graph 0.02; +"""+PLOT_DIFF + +g = gnuplot.Gnuplot(out = '"test.six"') +g.cmd(PLOT_INIT) + +# Reset line colors cycle +columns = 3 +g.set(linetype = "cycle "+str(columns-2)) +for i in range(columns,9): + g.unset("linetype "+str(i)) + +# set up ranges 28d +g.set( + yrange = fmtRange(0,Y_MAX), + xrange = fmtRange(T_MIN,T_MAX), + x2range = fmtRange(T_MIN,T_MAX), +) +g.cmd(PLOT_28D) +g.set( + arrow = "1 from first '"+NOW+"',graph 0 to first '"+NOW+"',graph 1 nohead lw 1.5 dt 2 lc rgb '#00000099'", + object = "1 rect from '"+NOW+"',graph 0 to graph 1, graph 1 fc rgb '#00444444' fillstyle pattern 4 noborder transparent" +) # PLOT_CROSS + +PLOT_RANGE = "for [col=3:"+str(HEADER_LENGTH)+"]" + +g.cmd(PLOT_0) +g.cmd("plot "+PLOT_RANGE+" '"+SIZE_FILE.as_posix()+"' using 1:col with linespoints title word('"+array+"', col) pointtype 5 pointsize 0.5") +g.cmd(PLOT_1) +g.cmd("plot "+PLOT_RANGE+" '"+WEEKLY_DIFF_FILE.as_posix()+"' using 1:col-1 with linespoints title word('"+array+"', col) pointtype 5, \ + "+PLOT_RANGE+" '"+DIFF_FILE.as_posix()+"' using 1:col with points notitle pointtype 12 pointsize 2") +# set up ranges 365d +g.set( + yrange = fmtRange(0,Y_MAX), + xrange = fmtRange(TY_MIN,T_MAX), + x2range = fmtRange(TY_MIN,T_MAX), +) +g.cmd(PLOT_364D) +g.cmd(PLOT_2) +g.cmd("plot "+PLOT_RANGE+" '"+WEEKLY_SIZE_FILE.as_posix()+"' using 1:col-1 with linespoints title word('"+array+"', col) pointtype 5") +g.cmd(PLOT_3) +g.cmd("plot "+PLOT_RANGE+" '"+WEEKLY_DIFF_FILE.as_posix()+"' using 1:col-1 with linespoints title word('"+array+"', col) pointtype 5")