testing for now - probably has bugs

.gitignore

@@ -0,0 +1,4 @@
+test*
+*.csv
+.venv/
+__pycache__/

csvfile.py

@@ -0,0 +1,124 @@
+import csv
+from pathlib import Path
+
+class CSVFile:
+    def __init__(self, filePath, delimiter=",") -> None:
+        self.__rewrite = False
+        self.__delimiter = delimiter
+        self.__filePath = Path(filePath)
+        self.__addBuffer = []
+        self.__newFile = self.fileInit()
+        self.length = 0
+        with open(filePath, "r") as file:
+            data = csv.DictReader(file,delimiter=delimiter)
+            if data.fieldnames == None:
+                self.__fields = list()
+                self.__newFile = True
+            else:
+                self.__fields = list(data.fieldnames)
+            self.__data = list(data)
+            self.length = len(self.__data)
+        pass
+
+    def fileInit(self):
+        if not self.__filePath.exists():
+            open(self.__filePath,'a').close()
+            return True
+        return False
+
+    def addRow(self,row):
+        print("ROW", row)
+        print("ROW", row.keys())
+        fields = row.keys()
+        for field in fields:
+            if not field in self.__fields:
+                self.__fields.append(field)
+        self.__data.append(row)
+        self.__addBuffer.append(row)
+
+    def getRow(self,rowIdx):
+        return self.__data[rowIdx]
+
+    def validate(self):
+        if self.__rewrite:
+            return False
+        alreadyValid = True
+        expectedFieldLen = len(self.__fields)
+        addIdx = len(self.__data)-len(self.__addBuffer)
+        idx = 0
+        for row in self.__data:
+            fields = row.keys()
+            fieldsLen = len(fields)
+            if fieldsLen < expectedFieldLen:
+                for expectedField in self.__fields:
+                    if expectedField not in fields:
+                        row.update({expectedField: ""})
+                        if idx < addIdx:
+                            alreadyValid = False
+            elif fieldsLen > expectedFieldLen:
+                # Omit invalid data.
+                firstField = self.__fields[0]
+                commented = "#"+row[firstField]
+                row.update({firstField : commented})
+                print("Error on line "+str(idx)+" - Omitted from file")
+                alreadyValid = False
+            idx += 1
+        return alreadyValid
+
+    def writeAppend(self,filepath):
+        print("Append - no header change")
+        with open(filepath, "a", newline="") as file:
+            w = csv.DictWriter(file, fieldnames=self.__fields)
+            w.writerows(self.__addBuffer)
+    def writeFile(self,filepath):
+        print("Rewrite - header changed")
+        with open(filepath, "w") as file:
+            w = csv.DictWriter(file, self.__fields)
+            w.writeheader()
+            w.writerows(self.__data)
+    
+    def efficientWriteFile(self):
+        filepath = self.__filePath
+        if self.validate() and not self.__newFile:
+            self.writeAppend(filepath)
+        else:
+            self.writeFile(filepath)
+    
+    def search(self,k,v,adjacent=False,backwards=False,start=0):
+        results = []
+        data = self.__data
+        r = range(start,len(data))
+        if backwards:
+            r = range(start,-len(data)-1,-1)
+        for i in r:
+            row = self.__data[i]
+            if k in row.keys() and row[k] == v:
+                results.append(row)
+            elif adjacent:
+                break
+        return results
+    
+    # matches=-1 ==> infinite replacements
+    def replaceByKV(self,k,v,newRow,backwards=False,start=0,matches=1):
+        infinite = matches < 0
+        data = self.__data
+        count = 0
+        r = range(start,len(data))
+        if backwards:
+            r = range(start,-len(data)-1,-1)
+        for i in r:
+            row = self.__data[i]
+            if k in row.keys() and row[k] == v:
+                self.__data[i] = newRow
+                self.__rewrite = True
+                count += 1
+                if not infinite and count >= matches:
+                    return True
+        if infinite:
+            return count > 0
+        return False
+    
+    def fill(self, v=0):
+        if len(self.__data) == 0:
+            return False
+        return { k:v for k in self.__data[0].keys() }

disk-usage.py

@@ -0,0 +1,79 @@
+import sys
+import os
+import shutil
+import csv
+import datetime
+from drive import Drive
+from csvfile import CSVFile
+from pathlib import Path
+from utils import subtract, nextWeek, sumKV
+
+SEARCH_PATH = "/mnt/"
+
+class DiskUsage:
+    # I probably should be using a database instead of CSV.
+    def __init__(self, basepath) -> None:
+        basepath = Path(basepath)
+        self.SIZE_FILE = basepath.joinpath("size_daily.csv")
+        self.DIFF_FILE = basepath.joinpath("diff_daily.csv")
+        self.WEEKLY_SIZE_FILE = basepath.joinpath("size_weekly.csv")
+        self.WEEKLY_DIFF_FILE = basepath.joinpath("diff_weekly.csv")
+        pass
+
+    @staticmethod
+    def __daily2weeklyRow(row_):
+        row = row_.copy()
+        row["time"] = row["next_week"]
+        row.pop("next_week")
+        return row
+
+
+    def update(self):
+        nextWeek_ = nextWeek().isoformat()
+
+        sizeDaily = CSVFile(self.SIZE_FILE)
+        currentlyUsed = Drive.getUsed(SEARCH_PATH)
+        sizeDaily.addRow(currentlyUsed)
+        sizeDaily.efficientWriteFile()
+        if sizeDaily.length < 2:
+            return 0 # Break for new files
+        
+        sizeWeekly = CSVFile(self.WEEKLY_SIZE_FILE)
+        sizeWeeklyRow = self.__daily2weeklyRow(currentlyUsed)
+        previousNextWeek = sizeDaily.getRow(-2).get("next_week")
+        if sizeWeekly.length < 1:
+            sizeWeekly.addRow(sizeWeeklyRow)
+        elif previousNextWeek == nextWeek_:
+            sizeWeekly.replaceByKV("time", nextWeek_, sizeWeeklyRow, backwards=True, start=-1)
+        else:
+            sizeWeekly.addRow(sizeWeeklyRow)
+            previousWeeklyRow = sizeWeeklyRow.copy()
+            previousWeeklyRow.update({ "time" : previousNextWeek })
+            sizeWeekly.replaceByKV("time", previousNextWeek, previousWeeklyRow, backwards=True, start=-1)
+        sizeWeekly.efficientWriteFile()
+
+        delta = subtract(sizeDaily.getRow(-1),sizeDaily.getRow(-2))
+        diffDaily = CSVFile(self.DIFF_FILE)
+        diffDaily.addRow(delta)
+        diffDaily.efficientWriteFile()
+        
+        
+        if sizeWeekly.length < 2:
+            return 0 # Break for new files
+        diffWeekly = CSVFile(self.WEEKLY_DIFF_FILE)
+        previousNextWeek = diffDaily.getRow(-2).get("next_week")
+        delta = subtract(sizeWeekly.getRow(-1), sizeWeekly.getRow(-2))
+        print(previousNextWeek == nextWeek_)
+        print(delta)
+        if previousNextWeek == nextWeek_:
+            diffWeekly.replaceByKV("time", previousNextWeek, delta, backwards=True, start=-1)
+        else:
+            delta.update({ "time" : previousNextWeek })
+            diffWeekly.replaceByKV("time", previousNextWeek, delta, backwards=True, start=-1)
+            empty = diffDaily.fill()
+            empty.update({"time" : nextWeek_})
+            diffWeekly.addRow(empty)
+        diffWeekly.efficientWriteFile()
+
+if __name__ == '__main__':
+    DiskUsage("/home/peter/scripts/datalogging/disk-usage/testfiles").update()

drive.py

@@ -0,0 +1,52 @@
+import shutil
+from datetime import datetime, date, timedelta
+from utils import nextWeek
+import os
+
+class Drive:
+    def __init__(self, drivePath) -> None:
+        self.__drivePath = drivePath
+        try:
+            self.total, self.used, self.free = shutil.disk_usage(self.__drivePath)
+        except:
+            self.total = self.used = self.free = ""
+        pass
+
+    def toKV_used(self):
+        return {self.__drivePath : self.used}
+    def toKV_total(self):
+        return {self.__drivePath : self.total}
+    def toKV_free(self):
+        return {self.__drivePath : self.free}
+
+    @staticmethod
+    def getDrives(mountPath):
+        drives = []
+        for drive in os.listdir(mountPath):
+            path = Drive.__normalizeDirPath(mountPath+drive)
+            drives.append(Drive(path))
+        return drives
+
+    @staticmethod
+    def getUsed(mountPath):
+        __now = datetime.now().isoformat(sep="_")
+        row = {
+            "time" : __now,
+            "next_week" : nextWeek().isoformat()
+        }
+        for drive in Drive.getDrives(mountPath):
+            row.update(drive.toKV_used())
+        return row
+
+    @staticmethod
+    def getMaxTotal(mountPath):
+        drives = Drive.getDrives(mountPath)
+        return max(drives, key = lambda d : d.total).total
+        
+
+    @staticmethod    
+    def __normalizeDirPath(dirPath):
+        if dirPath[-1] == '/':
+            return dirPath
+        return dirPath+'/'
+

utils.py

@@ -0,0 +1,41 @@
+from datetime import date, timedelta
+
+def nextWeek():
+    __today = date.today()
+    return __today + timedelta(days=-__today.weekday(), weeks=1)
+
+def subtract(row1,row2,castmode=int):
+    if (type(row1) == list and type(row2) == list):
+        if (not len(row1) == len(row2)):
+            True
+            #throw or something
+        for i in range(0,len(row1)):
+            try:
+                row1[i] = castmode(row1[i]) - castmode(row2[i])
+            except:
+                True    # keep first v if not a number type.
+        return row1
+    elif (type(row1) == dict and type(row2) == dict):
+        if (row1.keys() == row2.keys()):
+            for k in row1.keys():
+                try:
+                    row1.update({k : castmode(row1[k]) - castmode(row2[k])})
+                except:
+                    True    # keep first v if not a number type.
+            return row1
+
+def sumKV(rows,castmode=int):
+    result = {}
+    for row in rows:
+        for k in row.keys():
+            if not k in result.keys():
+                try:
+                    result.update({ k : castmode(row[k]) })
+                except:
+                    result.update({ k : row[k] })
+            else:
+                try:
+                    result[k] += castmode(row[k])
+                except:
+                    True
+    return result

viewer.py

@@ -0,0 +1,187 @@
+
+from drive import Drive
+from pygnuplot import gnuplot
+from utils import nextWeek
+from pathlib import Path
+from datetime import datetime, timedelta
+
+BASE = Path("/home/peter/scripts/datalogging/disk-usage/testfiles/")
+SIZE_FILE = BASE.joinpath("size_daily.csv")
+DIFF_FILE = BASE.joinpath("diff_daily.csv")
+WEEKLY_SIZE_FILE = BASE.joinpath("size_weekly.csv")
+WEEKLY_DIFF_FILE = BASE.joinpath("diff_weekly.csv")
+
+NOW = datetime.now().isoformat(sep="_")
+
+__MAX = nextWeek()   # Next monday
+T_MAX = __MAX.isoformat()
+T_MIN = (__MAX - timedelta(days=28)).isoformat()
+TY_MIN = (__MAX - timedelta(days=364)).isoformat()
+
+Y_MAX = Drive.getMaxTotal("/mnt/")
+
+def fmtRange(min, max):
+    return str.format("['{0}':'{1}']", min, max)
+
+array = ""
+with open(SIZE_FILE) as f:
+    array = f.readline()
+array = array.replace(","," ").strip()
+HEADER_LENGTH = len(array.split(' '))
+
+
+PLOT_INIT="""
+set terminal sixelgd background rgb 'black' size 1920, 960; set multiplot layout 2,2; set encoding utf8;
+set key top left; set key tc rgb 'white'; set border lc rgb 'white'; set key tc rgb 'white'; set datafile separator ',';
+
+set grid ytics; set ytics scale 2,1;
+
+set xdata time; set x2data time; set timefmt '%Y-%m-%d_%H:%M:%S'; set x2label 'Date' textcolor 'white';
+
+set border lc rgb 'white';
+"""
+
+PLOT_28D="""
+set format x '%Y-%m-%d';
+set xtics font ', 10' rotate by 90 right;
+set x2tics 4*24*60*60, 7*24*60*60 textcolor 'white';
+set x2tics format '%Y-%m-%d';
+set x2tics scale 37,1;
+set y2tics scale 2,1;
+set tics nomirror;
+
+set xlabel 'Day' textcolor 'white';
+set xtics 12*60*60, 24*60*60;
+set xtics format '%a';
+set mxtics 2;
+set xtics scale 0,1;
+set grid mxtics;
+"""
+
+# Some bug with scaling meant that I needed to use 59.999999999999 as opposed to
+# 60 for the offsets of the weeks from the months IIRC. Haven't checked if this
+# bug is still here.
+PLOT_364D="""
+set x2tics font ', 10';
+set x2tics 11*24*60*60, 3*7*24*60*60+7*24*60*59.999999999999 textcolor 'white';
+set x2tics format '%m-%d';
+set x2tics scale 2,1;
+set mx2tics 4;
+
+set xlabel 'Month' textcolor 'white';
+set xtics 4*7*24*60*60;
+set xtics format '%b';
+set xtics font ', 10' rotate by 0 center;
+set mxtics 1;
+set xtics scale 1,1;
+set grid xtics;
+"""
+
+PLOT_DIFF = """
+set autoscale y
+set ylabel '{/Symbol D}Used [ISO definition]' textcolor 'white';
+set format y '%+08.3s %cB';
+
+set y2label '{/Symbol D}Used (GB)' textcolor 'white';
+set link y2 via y/(1024**3) inverse y*(1024**3);
+set format y2 '%+08.3s %c';
+"""
+	
+PLOT_0="""set title 'Used capacity of disks (last 28 days)' textcolor 'white';
+
+set format y '%.2s %cB';
+set ylabel 'Used [ISO definition]' textcolor 'white';
+set ytics 0.25*10**12;
+set mytics 2;
+
+set link y2 via y/1024**4 inverse y*1024**4;
+set format y2 '%05.3f';
+interval = (2.5*10**11)/(1024**4); set y2tics interval;
+set y2label 'Used (TB)' textcolor 'white';
+set my2tics 1;
+"""
+
+# 
+# plot for [col=3:$columns] '$(sed 's/\///1;s/\//:\//1' <<< "$FILE")' using 1:col with linespoints title word(array, col) pointtype 5"""
+
+PLOT_1="""
+set title 'Change in used capacity (last 28 days)' textcolor 'white';
+
+set ytics 5*10**9;
+set mytics 2;
+
+interval = (5.0*10**9)/(1024**3); set y2tics interval;
+set my2tics 2;
+
+set offset graph 0, graph 0, graph 0.02, graph 0.02;
+"""+PLOT_DIFF
+
+PLOT_2="""
+set title 'Used capacity of disks (last 364 days)' textcolor 'white';
+
+set ylabel 'Used [ISO definition]' textcolor 'white';
+set format y '%.2s %cB';
+set ytics 0.25*10**12;
+set mytics 2;
+
+set y2label 'Used (TB)' textcolor 'white';
+set link y2 via y/1024**4 inverse y*1024**4;
+set format y2 '%05.3f';
+interval = (2.5*10**11)/(1024**4); set y2tics interval;
+set my2tics 1;
+
+set offset graph 0, graph 0, graph 0, graph 0;
+"""
+
+
+PLOT_3="""
+set title 'Change in used capacity (last 364 days)' textcolor 'white';
+
+set ytics 10*10**9;
+set mytics 2;
+
+interval = (10.0*10**9)/(1024**3); set y2tics interval;
+set my2tics 2;
+
+set offset graph 0, graph 0, graph 0.02, graph 0.02;
+"""+PLOT_DIFF
+
+g = gnuplot.Gnuplot(out = '"test.six"')
+g.cmd(PLOT_INIT)
+
+# Reset line colors cycle
+columns = 3
+g.set(linetype = "cycle "+str(columns-2))
+for i in range(columns,9):
+    g.unset("linetype "+str(i))
+
+# set up ranges 28d
+g.set(
+    yrange =  fmtRange(0,Y_MAX),
+    xrange =  fmtRange(T_MIN,T_MAX),
+    x2range = fmtRange(T_MIN,T_MAX),
+)
+g.cmd(PLOT_28D)
+g.set(
+    arrow  = "1 from first '"+NOW+"',graph 0 to first '"+NOW+"',graph 1 nohead lw 1.5 dt 2 lc rgb '#00000099'",
+    object = "1 rect from '"+NOW+"',graph 0 to graph 1, graph 1 fc rgb '#00444444' fillstyle pattern 4 noborder transparent"
+) # PLOT_CROSS
+
+PLOT_RANGE = "for [col=3:"+str(HEADER_LENGTH)+"]"
+
+g.cmd(PLOT_0)
+g.cmd("plot "+PLOT_RANGE+" '"+SIZE_FILE.as_posix()+"' using 1:col with linespoints title word('"+array+"', col) pointtype 5 pointsize 0.5")
+g.cmd(PLOT_1)
+g.cmd("plot "+PLOT_RANGE+" '"+WEEKLY_DIFF_FILE.as_posix()+"' using 1:col-1 with linespoints title word('"+array+"', col) pointtype 5, \
+	        "+PLOT_RANGE+" '"+DIFF_FILE.as_posix()+"' using 1:col with points notitle pointtype 12 pointsize 2")
+# set up ranges 365d
+g.set(
+    yrange =  fmtRange(0,Y_MAX),
+    xrange =  fmtRange(TY_MIN,T_MAX),
+    x2range = fmtRange(TY_MIN,T_MAX),
+)
+g.cmd(PLOT_364D)
+g.cmd(PLOT_2)
+g.cmd("plot "+PLOT_RANGE+" '"+WEEKLY_SIZE_FILE.as_posix()+"' using 1:col-1 with linespoints title word('"+array+"', col) pointtype 5")
+g.cmd(PLOT_3)
+g.cmd("plot "+PLOT_RANGE+" '"+WEEKLY_DIFF_FILE.as_posix()+"' using 1:col-1 with linespoints title word('"+array+"', col) pointtype 5")