Update Scripts/PythonScripts/compare_key_tls.py,...

Update Scripts/PythonScripts/compare_key_tls.py, Scripts/PythonScripts/compare_key_tls_periodicity.py, Scripts/PythonScripts/compare_key_tls_time_K.py, Scripts/PythonScripts/compare_key_tls_timing.py, Scripts/PythonScripts/compare_key_tls_timing_pretty.py, Scripts/PythonScripts/frame_length_of_messages.py, Scripts/PythonScripts/GAR.py, Scripts/PythonScripts/periodogram copy.py, Scripts/PythonScripts/periodogram.py, Scripts/PythonScripts/periodogram_compare.py, Scripts/PythonScripts/plot_psd.py files

Scripts/PythonScripts/GAR.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+files = [
+    r"C:\Users\aharn\OneDrive - Naval Postgraduate School\Stuff\Capstone\F-20200226\FA-20200226-keystroke.csv",
+    r"C:\Users\aharn\OneDrive - Naval Postgraduate School\Stuff\Capstone\F-20200226\FA-20200226-tls_export_unix.csv",
+    ]
+
+fig, plots = plt.subplots(2, sharex=True)
+# plots.add_subplot(yticks="")
+fig.suptitle('Comparing Keystoke and TLS traffic Times')
+# cols = []
+
+keysdata = pd.read_csv(files[0])
+keystimes = keysdata["press_time"]
+keysnames = keysdata["key_name"]
+keystimes /= 1000
+
+print(keystimes)
+print(keysnames)
+
+# keystimes_slice= keystimes.loc[keystimes > 1582752067]
+# print(keystimes_slice)
+# keystimes_slice = keystimes.loc[keystimes < 1582752081]
+# print(keystimes_slice)
+
+keystimes_slice = keystimes#.loc[130:230]#155:208
+keysnames_slice = keysnames#.loc[130:230]
+
+# print(keystimes_slice)
+
+# keystimes_slice = keystimes.loc[keystimes < 1582752081000]
+# print(keystimes_slice)
+
+keysnames.replace(
+    {
+        "backspace" : "<", 
+        "space"     : "*", 
+        # "enter"     : "E", 
+        # "shift"     : "S", 
+        "comma"     : ",",
+        # "quote"     : "\"",
+        "dead_acute": " ",
+        }, inplace=True)
+
+tlsdata = pd.read_csv(files[1])
+tlstimes = tlsdata["Time"]
+tlslengths = tlsdata["Length"]
+
+print(tlstimes)
+print(tlslengths)
+
+# tlstimes_slice = tlstimes.loc[tlstimes > 1582752067]
+# print(tlstimes_slice)
+# tlstimes_slice = tlstimes.loc[tlstimes < 1582752081]
+# print(tlstimes_slice)
+tlstimes_slice = tlstimes#.loc[100:175]#124:149
+tlslengths_slice = tlslengths#.loc[100:175]
+
+# tlslengths = [1 for x in tlstimes_slice]
+
+plots[0].set_title("A Keypresses")
+plots[0].set(xlabel='Time')
+plots[0].set(ylabel="Key Pressed")
+plots[0].plot(keystimes_slice, keysnames_slice, ".b")
+for x,y in zip(keystimes_slice, keysnames_slice):
+    # label = y
+    plots[0].annotate(y, # this is the text
+                (x,y), # this is the point to label
+                textcoords="offset points", # how to position the text
+                xytext=(0,2), # distance from text to points (x,y)
+                ha='center') # horizontal alignment can be left, right or center
+
+plots[1].set_title("Upstream TLS traffic")
+plots[1].set(ylabel='Frame Size')
+# plots[1].label_outer()
+plots[1].plot(tlstimes_slice, tlslengths_slice, ".r")
+
+labels = set(keysnames)
+# plots[0].yticks(np.arange(len(labels)), sorted(labels))
+
+plt.xlim(1582751210, 1582751230)
+
+plt.show()
+
+# plt.savefig(r"C:\Users\aharn\OneDrive - Naval Postgraduate School\Stuff\Capstone\G-20200226\GA-20200226-keystroke_time_traffic_time.pdf", format="pdf")
\ No newline at end of file

Scripts/PythonScripts/compare_key_tls.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+def get_file(title="Choose a file", type="get", 
+            filetypes="", ifile="", ext="", keyword=None):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # No Tk GUI
+    filetypes = [
+        ("csv files","*.csv"),
+        ("excel files","*.xls*"),
+        ("all files","*.*"),
+        ]
+    if keyword:
+        filetypes.insert(0, ("keyword", "*"+keyword+"*"))
+    
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = filetypes,
+                initialfile = ifile,
+                )
+    
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+
+    if not len(file):
+        from sys import exit
+        exit()
+        
+    app.destroy()
+    return Path(file)
+
+def get_data(file, columns):
+    if file.suffix == ".csv":
+        data = pd.read_csv(file)
+    elif file.suffix[:4] == '.xls':
+        data = pd.read_excel(file)
+    else:
+        return []
+    if isinstance(columns, (list, tuple)):
+        cdata = []
+        for column in columns:
+            cdata.append(data[column])
+    else:
+        cdata = data[columns]
+    return cdata
+
+def timing(files):
+    keystimes, keysnames = get_data(files[0], ["press_time", "key_name"])
+    keystimes /= 1000
+    keysnames.replace(
+        {
+            "backspace" : "<", 
+            "space"     : "*", 
+            # "enter"     : "E", 
+            # "shift"     : "S", 
+            "comma"     : ",",
+            # "quote"     : "\"",
+            "dead_acute": " ",
+            }, inplace=True)
+
+    tlstimes, tlslengths = get_data(files[1], ["Time", "Length"])
+
+    fig, plots = plt.subplots(2, sharex=True)
+    plt.xlabel("Time (Unix seconds)")
+    fig.suptitle('Comparing Keystoke and TLS Traffic Times')
+
+    plots[0].set_title("A Keypresses")
+    plots[0].set(ylabel="Key Pressed")
+    plots[0].plot(keystimes, keysnames, ".")
+    for x,y in zip(keystimes, keysnames):
+        plots[0].annotate(y, # this is the text
+                    (x,y), # this is the point to label
+                    textcoords="offset points", # how to position the text
+                    xytext=(0,2), # distance from text to points (x,y)
+                    ha='center') # horizontal alignment can be left, right or center
+
+    plots[1].set_title("Upstream TLS traffic")
+    plots[1].set(ylabel='Frame Size')
+    plots[1].plot(tlstimes, tlslengths, ".")
+
+    plt.xlim(1582752125, 1582752140)
+
+    if show:
+        plt.show()
+    
+    return plt
+
+def periodicity(files):
+    keystimes = periodogram(get_data(files[0], "press_time")/1000)
+    tlsdata = periodogram(get_data(files[1], "Time"))
+
+    fig, plots = plt.subplots(2, sharex=True)
+    plt.xlabel("Frequency (Hz)")
+    fig.suptitle('Comparing Keystoke and TLS Periodicity')
+
+    plots[0].set_title("Client A Keypress Periodicity")
+    plots[0].set(ylabel='Count')
+    plots[0].plot(keystimes)
+
+    plots[1].set_title("Upstream TLS Periodicity")
+    plots[1].set(ylabel='Count')
+    plots[1].plot(tlsdata)
+
+    if show:
+        plt.show()
+    
+    return plt
+
+if __name__ == "__main__":
+    show = True
+    save = True
+
+    files = [
+        get_file("Choose Keystroke File", keyword="keystroke"),
+        get_file("Choose Traffic File", keyword="tls"),
+        ]
+
+    tplt = timing(files)
+    pplt = periodicity(files)
+    
+    if save:
+        out_name = files[0].stem[:12]
+        tfile_name = out_name + "compare_timing"
+        pfile_name = out_name + "compare_periods"
+        ext = 'pdf'
+
+        tfile = get_file("Save Timing Graph as...", type="save", ifile=tfile_name, ext=ext)
+        if tfile:
+            print(tfile)
+            # tplt.savefig(tfile, format=ext)
+
+        pfile = get_file("Save Periodicity Graph as...", type="save", ifile=pfile_name, ext=ext)
+        if pfile:
+            print(pfile)
+            # pplt.savefig(pfile, format=ext)
\ No newline at end of file

Scripts/PythonScripts/compare_key_tls_periodicity.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from glob import glob
+from pathlib import Path
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+def get_file(title="Choose a file", type="get", 
+            filetypes="", ifile="", ext="", keyword=None):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # No Tk GUI
+    filetypes = [
+        ("csv files","*.csv"),
+        ("excel files","*.xls*"),
+        ("all files","*.*"),
+        ]
+    if keyword:
+        filetypes.insert(0, ("keyword", "*"+keyword+"*"))
+    
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = filetypes,
+                initialfile = ifile,
+                )
+    
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+
+    if not len(file):
+        from sys import exit
+        exit()
+        
+    app.destroy()
+    return Path(file)
+
+def get_data(file, columns):
+    if file.suffix == ".csv":
+        data = pd.read_csv(file)
+    elif file.suffix[:4] == '.xls':
+        data = pd.read_excel(file)
+    else:
+        return []
+    if isinstance(columns, (list, tuple)):
+        cdata = []
+        for column in columns:
+            cdata.append(data[column])
+    else:
+        cdata = data[columns]
+    return cdata
+
+def main():
+    show = False
+    save = True
+
+    for client in ["A", "B"]:
+        files = [
+            Path(glob(r"*"+client+r"*keystroke*.csv")[0]),
+            Path(glob(r"*"+client+r"*tls*.csv")[0])
+            ]
+
+        if save:
+            ext = 'png'
+            outfile_name = files[0].stem[:12] + "compare_periods"
+            outfile = get_file("Save as...", type="save", ifile=outfile_name, ext=ext)
+
+        keystimes = periodogram(get_data(files[0], "press_time")/1000)
+        tlsdata = periodogram(get_data(files[1], "Time"))
+
+        fig = plt.figure(figsize=(10, 5))
+        plt.xlabel("Frequency (Hz)")
+        plt.ylabel("Count")
+        fig.suptitle('Comparing Keystoke and TLS Periodicity')
+
+        plt.plot(keystimes, label=f"Client {client} Keypress Periodicity")
+        plt.plot(tlsdata, label="Upstream TLS Periodicity")
+        plt.legend()
+
+        if save:
+            plt.savefig(outfile, format=ext)
+
+        if show:
+            plt.show()
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

Scripts/PythonScripts/compare_key_tls_time_K.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from glob import glob
+from pathlib import Path
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+def get_file(title="Choose a file", type="get", 
+            filetypes="", ifile="", ext="", keyword=None):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # No Tk GUI
+    filetypes = [
+        ("csv files","*.csv"),
+        ("excel files","*.xls*"),
+        ("all files","*.*"),
+        ]
+    if keyword:
+        filetypes.insert(0, ("keyword", "*"+keyword+"*"))
+    
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = filetypes,
+                initialfile = ifile,
+                )
+    
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+
+    if not len(file):
+        from sys import exit
+        exit()
+        
+    app.destroy()
+    return Path(file)
+
+def get_data(file, columns):
+    if file.suffix == ".csv":
+        data = pd.read_csv(file)
+    elif file.suffix[:4] == '.xls':
+        data = pd.read_excel(file)
+    else:
+        return []
+    if isinstance(columns, (list, tuple)):
+        cdata = []
+        for column in columns:
+            cdata.append(data[column])
+    else:
+        cdata = data[columns]
+    return cdata
+
+def main():
+    show = False
+    save = True
+    
+
+    files = [
+        # get_file("Choose Keystroke A File", keyword="A*keystroke"),
+        # get_file("Choose Keystroke B File", keyword="B*keystroke"),
+        # get_file("Choose Traffic File A1", keyword="A*do-a*csv"),
+        # get_file("Choose Traffic File A2", keyword="A*kiwi*csv"),
+        # get_file("Choose Traffic File B1", keyword="B*do-a*2*csv"),
+        # get_file("Choose Traffic File B2", keyword="B*kiwi*2*csv"),
+        Path(glob(r"*A*keystroke.csv")[0]),
+        Path(glob(r"*A*do-a*.csv")[0]),
+        Path(glob(r"*A*kiwi*.csv")[0]),
+        ]
+
+    if save:
+        ext = 'png'
+        outfile_name = files[0].stem[:12] + "compare_timing_both"
+        outfile = get_file("Save as...", type="save", ifile=outfile_name, ext=ext)
+    file = 0
+    keystimesA, keysAy = get_data(files[file], ["press_time", "key_code"])
+    keystimesA /= 1000
+    # keysAy = list(map(lambda y: y%30, keystimesA))
+    file += 1
+    tlstimesA1, tlsA1y = get_data(files[file], ["Time", "Length"])
+    file += 1
+    tlstimesA2, tlsA2y = get_data(files[file], ["Time", "Length"])
+    time_zero = min(keystimesA[0], tlstimesA1[0], tlstimesA2[0])
+    keystimesA -= time_zero
+    tlstimesA1 -= time_zero
+    tlstimesA2 -= time_zero
+
+    fig = plt.figure(figsize=[10, 5])
+    plt.plot(keystimesA, keysAy, ".r", label="Key Strokes", figure=fig)
+    plt.plot(tlstimesA1, tlsA1y, ".b", label="TLS Frames Sent to Server 1", figure=fig)
+    plt.plot(tlstimesA2, tlsA2y, ".g", label="TLS Frames Sent to Server 2", figure=fig)
+
+    plt.title("Which Kiwi Server Handles Message Data")
+    plt.xlabel("Time of Frame/Keystroke (seconds)")
+    plt.ylabel("Keystroke Code or Frame Length")
+    plt.legend()
+
+    if save:
+        plt.savefig(outfile, format=ext)
+
+    if show:
+        plt.show()
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

Scripts/PythonScripts/compare_key_tls_timing.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+def get_file(title="Choose a file", type="get", 
+            filetypes="", ifile="", ext="", keyword=None):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # No Tk GUI
+    filetypes = [
+        ("csv files","*.csv"),
+        ("excel files","*.xls*"),
+        ("all files","*.*"),
+        ]
+    if keyword:
+        filetypes.insert(0, ("keyword", "*"+keyword+"*"))
+    
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = filetypes,
+                initialfile = ifile,
+                )
+    
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+
+    if not len(file):
+        from sys import exit
+        exit()
+        
+    app.destroy()
+    return Path(file)
+
+def get_data(file, columns):
+    if file.suffix == ".csv":
+        data = pd.read_csv(file)
+    elif file.suffix[:4] == '.xls':
+        data = pd.read_excel(file)
+    else:
+        return []
+    if isinstance(columns, (list, tuple)):
+        cdata = []
+        for column in columns:
+            cdata.append(data[column])
+    else:
+        cdata = data[columns]
+    return cdata
+
+def main():
+    show = True
+    save = False#True
+
+    files = [
+        get_file("Choose Keystroke File", keyword="keystroke"),
+        get_file("Choose Traffic File", keyword="tls"),
+        ]
+
+    if save:
+        ext = 'pdf'
+        outfile_name = files[0].stem[:12] + "compare_timing"
+        outfile = get_file("Save as...", type="save", ifile=outfile_name, ext=ext)
+
+    keystimes, keysnames = get_data(files[0], ["press_time", "key_name"])
+    keystimes /= 1000
+    keysnames.replace(
+        {
+            "backspace" : "<", 
+            "space"     : "*", 
+            # "enter"     : "E", 
+            # "shift"     : "S", 
+            "comma"     : ",",
+            # "quote"     : "\"",
+            "dead_acute": " ",
+            }, inplace=True)
+
+    tlstimes, tlslengths = get_data(files[1], ["Time", "Length"])
+
+    fig, plots = plt.subplots(2, sharex=True)
+    plt.xlabel("Time (Unix seconds)")
+    fig.suptitle('Comparing Keystoke and TLS Traffic Times')
+
+    plots[0].set_title("A Keypresses")
+    plots[0].set(ylabel="Key Pressed")
+    plots[0].plot(keystimes, keysnames, ".", label="Key Strokes")
+    plots[0].legend()
+    for x,y in zip(keystimes, keysnames):
+        plots[0].annotate(y, # this is the text
+                    (x,y), # this is the point to label
+                    textcoords="offset points", # how to position the text
+                    xytext=(0,2), # distance from text to points (x,y)
+                    ha='center') # horizontal alignment can be left, right or center
+
+    plots[1].set_title("Upstream TLS traffic")
+    plots[1].set(ylabel='Frame Size')
+    plots[1].plot(tlstimes, tlslengths, ".", label="Frame Size (bytes)")
+    plots[1].legend()
+
+
+    plt.xlim(1582752125, 1582752140)
+
+    if save:
+        plt.savefig(outfile, format=ext)
+
+    if show:
+        plt.show()
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

Scripts/PythonScripts/compare_key_tls_timing_pretty.py

+import pandas as pd
+import matplotlib.pyplot as plt
+from glob import glob
+from pathlib import Path
+
+def get_file(title="Choose a file", type="get", 
+            filetypes="", ifile="", ext="", keyword=None):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # No Tk GUI
+    filetypes = [
+        ("csv files","*.csv"),
+        ("excel files","*.xls*"),
+        ("all files","*.*"),
+        ]
+    if keyword:
+        filetypes.insert(0, ("keyword", "*"+keyword+"*"))
+    
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = filetypes,
+                initialfile = ifile,
+                )
+
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+    if not len(file):
+        from sys import exit
+        exit()
+
+    app.destroy()
+    return Path(file)
+
+def get_data(file, columns):
+    if file.suffix == ".csv":
+        data = pd.read_csv(file)
+    elif file.suffix[:4] == '.xls':
+        data = pd.read_excel(file)
+    else:
+        return []
+    if isinstance(columns, (list, tuple)):
+        cdata = []
+        for column in columns:
+            cdata.append(data[column])
+        if len(cdata) == 1:
+            cdata = cdata[0]
+    else:
+        cdata = data[columns]
+    return cdata
+
+def main():
+    show = False
+    save = True
+    keystroke = Path(glob(r"*A*keystroke.csv")[0])
+    tls = Path(glob(r"*A*tls*.csv")[0])
+    if save:
+        ext = "png"
+        outfile = get_file("Save as...", type="save", ifile=keystroke.stem[:12]+"keystroke_tls_time_pretty", ext=ext)
+
+    key_time, key_name = get_data(keystroke, ["press_time", "key_name"])
+    tls_time, tls_length = get_data(tls, ["Time", "Length"])
+    key_time /= 1000
+    time_zero = min(tls_time[0], key_time[0])
+    key_time -= time_zero
+    tls_time -= time_zero
+    key_dot = key_time * 0
+
+    fig = plt.figure(figsize=[10, 5])
+    
+    plt.xlim(1582751282 - time_zero, 1582751291 - time_zero)
+    # plt.ylim(0, 550)
+
+    plt.bar(tls_time, tls_length, 0.02, label="Frame Size", figure=fig)
+    plt.plot(key_time, key_dot, ".r", label="Key Stroke", figure=fig)
+    plt.title("Comparing Keystroke Times and TLS Traffic Times")
+    plt.ylabel("Frame Size (Bytes)")
+    plt.xlabel("Time of Frame/Keystroke (seconds)")
+    plt.legend()
+    for x, y in zip(key_time, key_name):
+        plt.annotate(
+            y, 
+            (x, 10), 
+            rotation=45, #90 if len(y)>1 else 0, 
+            textcoords="offset points", 
+            xytext=(-5, 0), 
+            ha='left',
+            )
+
+    for x, y in zip(tls_time, tls_length):
+        plt.annotate(
+            y, 
+            (x, y), 
+            rotation=45, 
+            textcoords="offset points", 
+            xytext=(5, 0), 
+            ha='center',
+            )
+
+    if save:
+        plt.savefig(outfile, format=ext)
+
+    if show:
+        plt.show()
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

Scripts/PythonScripts/frame_length_of_messages.py

+import pandas as pd
+import matplotlib.pyplot as plt
+from glob import glob
+from pathlib import Path
+
+def get_file(title="Choose a file", type="get", 
+            filetypes="", ifile="", ext="", keyword=None):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # No Tk GUI
+    filetypes = [
+        ("csv files","*.csv"),
+        ("excel files","*.xls*"),
+        ("all files","*.*"),
+        ]
+    if keyword:
+        filetypes.insert(0, ("keyword", "*"+keyword+"*"))
+    
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = filetypes,
+                initialfile = ifile,
+                )
+
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+    if not len(file):
+        from sys import exit
+        exit()
+
+    app.destroy()
+    return Path(file)
+
+def get_data(file, columns):
+    if file.suffix == ".csv":
+        data = pd.read_csv(file)
+    elif file.suffix[:4] == '.xls':
+        data = pd.read_excel(file)
+    else:
+        return []
+    if isinstance(columns, (list, tuple)):
+        cdata = []
+        for column in columns:
+            cdata.append(data[column])
+    else:
+        cdata = data[columns]
+    return cdata
+
+def main():
+    show = True
+    save = True
+    headers_length = 59
+
+    # files = [
+    #     get_file("Choose A File", keyword="*A*tls"),
+    #     get_file("Choose B File", keyword="*B*tls"),
+    #     ]
+
+    files = glob(r"*do*tls*unix.csv")
+    files = list(map(lambda file: Path(file), files))
+    # print(files)
+    if save:
+        ext = "png"
+        outfile = get_file("Save as...", type="save", ifile=files[0].stem, ext=ext)
+
+    lower, upper = map(lambda length: length + headers_length, {
+        "F": (955, 1100),
+        "G": (330,  420),
+        "K": ( 74,  120),
+        }[files[0].stem[0]])
+
+    fig, plots = plt.subplots(ncols=len(files), sharey=True)
+    fig.set_size_inches(10, 5, forward=True)
+    fig.suptitle("Frames Containing Message Content")
+    # plt.ylabel('Frame Size (Bytes)')
+
+    plt.ylim(lower*0.8, upper*1.2)
+    for n, plot in enumerate(plots):
+        time, length = get_data(files[n], ["Time", "Length"])
+        r = int(max(time)-min(time))
+        time = list(map(lambda t: t%r, time))
+        plot.set_title("Client "+ files[n].stem[1])# xlabel('Time of Frame (UNIX time)')
+        plot.set(ylabel="Frame size (Bytes)")
+        plot.set(xlabel="Time of Frame (seconds)")
+        # plot.set(xlabel="Time of Client " + files[n].stem[1] + " Frame (UNIX Time)")
+        plot.plot(time, length, ".", label="Frame Length (Bytes)")
+        # i = 3
+        for x, y in zip(time, length):
+            if y in range(lower, upper):
+                plot.annotate(
+                    y,
+                    (x,y),
+                    rotation=45,
+                    textcoords="offset points", 
+                    xytext=(
+                        -5,
+                        0
+                        # [-15, 15][i%2], 
+                        # [20, -30, 30, -50, 20, -30][(i%4)]
+                        ),
+                    ha='left',
+                    # arrowprops={
+                        # "arrowstyle":"->", 
+                        # "color":["blue","red"][i%i]
+                        # },
+                    )
+                # i += 1 
+        
+        x1, y1 = (min(time),max(time)), (lower-1,lower-1)
+        x2, y2 = (min(time),max(time)), (upper+1,upper+1)
+        # x3, y3 = (min(t),max(t)), (154,154)
+        
+        plot.label_outer()
+        plot.plot(x1,y1, label="Lower Bound to Find Message Content")
+        plot.annotate(
+            lower, 
+            (min(time), lower),
+            textcoords="offset points",     
+            xytext=(0,-10),
+            ha='center',                    
+            # arrowprops={"arrowstyle":"-", "color":"orange"},
+            )
+
+        if not files[n].stem[0] == "K":
+            plot.plot(
+                x2,y2, label="Upper Bound to Find Message Content"
+            #  x3,y3,
+            )
+            plot.annotate(
+                upper, 
+                (min(time), upper), 
+                textcoords="offset points",    
+                xytext=(0, -10),
+                ha='center',
+                # arrowprops={"arrowstyle":"-", "color":"green"},
+                )
+        if n:
+            plot.legend(fancybox=True, framealpha=0.5)
+
+
+    if save:
+        plt.savefig(outfile, format=ext)
+
+    if show:
+        plt.show()
+
+if __name__ == "__main__":
+    # files = glob.glob(r"*do-a*tls*.csv")
+    # print(files)
+    main()
\ No newline at end of file

Scripts/PythonScripts/periodogram copy.py

+import numpy as np
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+def get_file(title="Choose a file", type="get", 
+            filetypes="", ifile="", ext="", keyword=None):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # No Tk GUI
+    filetypes = [
+        ("csv files","*.csv"),
+        ("excel files","*.xls*"),
+        ("all files","*.*"),
+        ]
+    if keyword:
+        filetypes.insert(0, ("keyword", "*"+keyword+"*"))
+    
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = filetypes,
+                initialfile = ifile,
+                )
+
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+    if not len(file):
+        from sys import exit
+        exit()
+
+    app.destroy()
+    return Path(file)
+
+def get_data(file, columns):
+    if file.suffix == ".csv":
+        data = pd.read_csv(file)
+    elif file.suffix[:4] == '.xls':
+        data = pd.read_excel(file)
+    else:
+        return []
+    if isinstance(columns, (list, tuple)):
+        cdata = []
+        for column in columns:
+            cdata.append(data[column])
+        if len(cdata) == 1:
+            cdata = cdata[0]
+    else:
+        cdata = data[columns]
+    return cdata
+
+def main():
+    import pandas as pd
+    import matplotlib.pyplot as plt
+    files = [
+        get_file("Choose Keystroke File...", keyword="*keystroke*.csv"),
+        get_file("Choose Traffic File...", keyword="*.csv"),
+        ]
+    keytimes = get_data(files[0], "key_press")/1000
+    frametimes = get_data(files[0], "Length")
+    # t = data[column]/1000
+    keyP = periodogram(keytimes)
+    frameP = periodogram(frametimes)
+
+    plt.plot(keyP, label="Key Stroke Time")
+    plt.plot(frameP)
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

Scripts/PythonScripts/periodogram.py

+import numpy as np
+
+def parseargs():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("infile", type=str, 
+        help="The name of the csv file from which to \
+            retrieve the times")
+    parser.add_argument("-o", "--outfile",  type=str, 
+        help="The name file to output the graph", default=False)
+    parser.add_argument("-n", "--noshow",  action="store_true", 
+        help="Display the graph in a new window", default=False)
+    parser.add_argument("-c", "--column",  type=str, 
+        help="Name of the column with times values", default="Time")
+    parser.add_argument("-f", "--format",  type=str, 
+        help="Format of the time data")
+    parser.add_argument("-p", "--pdf",  action="store_true", 
+        help="File format to output the graph")
+    args = parser.parse_args()
+    for arg in ["noshow", "pdf"]:
+        if (vars(args)[arg] and not args.outfile):
+            parser.error(
+                'Do not use -'+ arg[0] +' without -o. \
+                    Nothing will happen.')
+    return args
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+def checkfiles(infile, outfile=False):
+    from pathlib import Path
+    if not Path(infile).is_file():
+        raise Exception("Input file does not exist")
+
+    if outfile:
+        if Path(outfile).is_file():
+            raise Exception("Output file already exists")
+    return outfile
+
+def main(infile, outfile=False, 
+        pdf=True, column="Time", 
+        format=None, noshow=False):
+    import pandas as pd
+    import matplotlib.pyplot as plt
+    from brokenaxes import brokenaxes
+    data = pd.read_csv(infile)
+    if format:
+        times = pd.to_datetime(data[column], format=format)
+        t = np.asarray(
+            list(
+                map(
+                    lambda min, sec, mic: 
+                        min*60 + sec + mic/1000000, 
+                            times.dt.minute, 
+                            times.dt.second, 
+                            times.dt.microsecond,
+                )   
+            )   
+        )
+    else:
+        try:
+            t = data[column]/1000
+        except KeyError:
+            print(
+                "Cannot find time data. Please supply the name of the column using the -c option")
+            quit()
+    P = periodogram(t)
+    slice = False
+    if slice:
+        bax = brokenaxes(
+            ylims=(
+                (min(P), 8), 
+                (24, 26), 
+                (50, max(P)),
+                )   
+            )
+        bax.plot(P)
+    else:
+        plt.plot(P)
+
+    if outfile:
+        if pdf:
+            plt.savefig(outfile, format="pdf")
+        else:
+            plt.savefig(outfile)
+    if not noshow:
+        plt.show()
+
+if __name__ == "__main__":
+    args = parseargs()
+    args.outfile = checkfiles(args.infile, args.outfile)
+    main(**vars(args))
\ No newline at end of file

Scripts/PythonScripts/periodogram_compare.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+def get_file(title="Choose a file", type="get", filetypes="", ifile="", ext=""):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # we don't want a full GUI, so keep the root window from appearing
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = (
+                    ("csv files","*.csv"),
+                    ("all files","*.*")
+                    )
+            )
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+    app.destroy()
+    return Path(file)
+
+files = [
+    {
+        "file": get_file("Choose Keystroke File"),
+        "columns":[
+            "press_time",
+            "key_name",
+            ],
+        "replace": {
+            "backspace" : "<", 
+            "space"     : "_", 
+            "enter"     : "E", 
+            # "shift"     : "S", 
+            "comma"     : ",",
+            "quote"     : "\"",
+            "dead_acute": " ",
+            },
+        "title": "A Keypresses",
+        },
+    {
+        "file": get_file("Choose Traffic File"),
+        "columns": [
+            "Time",
+            "Length",
+            ],
+        "title": "Upstream TLS traffic",
+        },
+    ]
+
+ext = 'pdf'
+outfile_name = files[0]["file"].stem[:12] + "compare_periods"
+outfile = get_file("Save as...", type="save", ifile=outfile_name, ext=ext)
+
+fig, plots = plt.subplots(len(files), sharex=True)
+fig.suptitle('Comparing Keystoke and TLS traffic Times')
+colors = ["b", "r", "y"]
+for i, f in enumerate(files):
+    cols = []
+    if f["file"].suffix == ".csv":
+        data = pd.read_csv(f["file"])
+    elif f["file"].suffix[:4] == '.xls':
+        data = pd.read_excel(f["file"])
+    for j, column in enumerate(f["columns"]):
+        d = data[column]
+        if any([not type(val)==str for val in d]) and any([val > (2 * pow(10,9)) for val in d]):
+            d /= pow(10,3)
+        if j==1 and "replace" in f:
+            d.replace(f["replace"], inplace=True)
+        cols.append(list(d))
+
+    if len(cols) == 1:
+        cols.append([1 for x in cols[0]])
+
+    plots[i].set_title(f["title"])
+    plots[i].set(xlabel='Time', ylabel=f["columns"][1])
+    plots[i].label_outer()
+
+    plots[i].plot(*cols, "."+ colors[i])
+
+    # print(cols[1][0])
+    labels = set(cols[1])
+    # print(sorted(labels))
+    if type(cols[1][0]) == type(str):
+        plt.yticks(np.arange(len(labels)), sorted(labels))
+    if i == 0:
+        for x,y in zip(*cols):
+            plots[i].annotate(
+                    y, # this is the text
+                    # (x,y), # this is the point to label
+                    # textcoords="offset points", # how to position the text
+                    # xytext=(0, [15, -20, 21, -31][int(y)%4]), # distance from text to points (x,y)
+                    # ha=['right','center','left'][int(y)%3],# horizontal alignment can be left, right or center
+                    # arrowprops={"arrowstyle":"->"},
+                    # )
+                    (x,y), # this is the point to label
+                    textcoords="offset points", # how to position the text
+                    xytext=(0,2), # distance from text to points (x,y)
+                    ha='center',
+                ) # horizontal alignment can be left, right or center
+
+
+    plt.xlim(1582752067, 1582752081)
+
+plt.savefig(outfile)
+
+plt.show()
\ No newline at end of file

Scripts/PythonScripts/plot_psd.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def psd_x(t, fs=1000, n=1000):
+    t = t/1000
+    f = np.fft.rfftfreq(n=n, d=1/fs)
+    f = f[f>0]
+    Pxx = np.exp(2j * np.pi * f[:, np.newaxis] * t).sum(axis=1)
+    Pxx = (np.abs(Pxx)**2)/len(t)
+    return f, Pxx
+
+df = pd.read_excel('GA-20200226-keystroke_time_traffic_time.xlsx')
+
+plt.figure()
+t = df.loc[~df['key_name'].isnull(), 'Time'].values
+f, Pxx = psd_x(t)
+plt.plot(f[(f>=20)&(f<250)],Pxx[(f>=20)&(f<250)])
+plt.title('keystrokes')
+
+plt.figure()
+t = df.loc[df['code or key']=='sendchatmessage', 'Time'].values
+f, Pxx = psd_x(t)
+plt.plot(f[(f>=20)&(f<250)],Pxx[(f>=20)&(f<250)])
+plt.title('sendchatmessage')
+
+plt.figure()
+t = df.loc[df['code or key']=='setfocus', 'Time'].values
+f, Pxx = psd_x(t)
+plt.plot(f[(f>=20)&(f<250)],Pxx[(f>=20)&(f<250)])
+plt.title('setfocus')
+
+plt.figure()
+t = df.loc[df['code or key']=='settyping', 'Time'].values
+f, Pxx = psd_x(t)
+plt.plot(f[(f>=20)&(f<250)],Pxx[(f>=20)&(f<250)])
+plt.title('settyping')
+
+plt.show()