Update Scripts/PythonScripts/compare_key_tls.py,...

Update Scripts/PythonScripts/compare_key_tls.py, Scripts/PythonScripts/compare_key_tls_periodicity.py, Scripts/PythonScripts/compare_key_tls_time_K.py, Scripts/PythonScripts/compare_key_tls_timing.py, Scripts/PythonScripts/compare_key_tls_timing_pretty.py, Scripts/PythonScripts/frame_length_of_messages.py, Scripts/PythonScripts/GAR.py, Scripts/PythonScripts/periodogram copy.py, Scripts/PythonScripts/periodogram.py, Scripts/PythonScripts/periodogram_compare.py, Scripts/PythonScripts/plot_psd.py files

Scripts/PythonScripts/GAR.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+files = [
+    r"C:\Users\aharn\OneDrive - Naval Postgraduate School\Stuff\Capstone\F-20200226\FA-20200226-keystroke.csv",
+    r"C:\Users\aharn\OneDrive - Naval Postgraduate School\Stuff\Capstone\F-20200226\FA-20200226-tls_export_unix.csv",
+    ]
+
+fig, plots = plt.subplots(2, sharex=True)
+# plots.add_subplot(yticks="")
+fig.suptitle('Comparing Keystoke and TLS traffic Times')
+# cols = []
+
+keysdata = pd.read_csv(files[0])
+keystimes = keysdata["press_time"]
+keysnames = keysdata["key_name"]
+keystimes /= 1000
+
+print(keystimes)
+print(keysnames)
+
+# keystimes_slice= keystimes.loc[keystimes > 1582752067]
+# print(keystimes_slice)
+# keystimes_slice = keystimes.loc[keystimes < 1582752081]
+# print(keystimes_slice)
+
+keystimes_slice = keystimes#.loc[130:230]#155:208
+keysnames_slice = keysnames#.loc[130:230]
+
+# print(keystimes_slice)
+
+# keystimes_slice = keystimes.loc[keystimes < 1582752081000]
+# print(keystimes_slice)
+
+keysnames.replace(
+    {
+        "backspace" : "<", 
+        "space"     : "*", 
+        # "enter"     : "E", 
+        # "shift"     : "S", 
+        "comma"     : ",",
+        # "quote"     : "\"",
+        "dead_acute": " ",
+        }, inplace=True)
+
+tlsdata = pd.read_csv(files[1])
+tlstimes = tlsdata["Time"]
+tlslengths = tlsdata["Length"]
+
+print(tlstimes)
+print(tlslengths)
+
+# tlstimes_slice = tlstimes.loc[tlstimes > 1582752067]
+# print(tlstimes_slice)
+# tlstimes_slice = tlstimes.loc[tlstimes < 1582752081]
+# print(tlstimes_slice)
+tlstimes_slice = tlstimes#.loc[100:175]#124:149
+tlslengths_slice = tlslengths#.loc[100:175]
+
+# tlslengths = [1 for x in tlstimes_slice]
+
+plots[0].set_title("A Keypresses")
+plots[0].set(xlabel='Time')
+plots[0].set(ylabel="Key Pressed")
+plots[0].plot(keystimes_slice, keysnames_slice, ".b")
+for x,y in zip(keystimes_slice, keysnames_slice):
+    # label = y
+    plots[0].annotate(y, # this is the text
+                (x,y), # this is the point to label
+                textcoords="offset points", # how to position the text
+                xytext=(0,2), # distance from text to points (x,y)
+                ha='center') # horizontal alignment can be left, right or center
+
+plots[1].set_title("Upstream TLS traffic")
+plots[1].set(ylabel='Frame Size')
+# plots[1].label_outer()
+plots[1].plot(tlstimes_slice, tlslengths_slice, ".r")
+
+labels = set(keysnames)
+# plots[0].yticks(np.arange(len(labels)), sorted(labels))
+
+plt.xlim(1582751210, 1582751230)
+
+plt.show()
+
+# plt.savefig(r"C:\Users\aharn\OneDrive - Naval Postgraduate School\Stuff\Capstone\G-20200226\GA-20200226-keystroke_time_traffic_time.pdf", format="pdf")
\ No newline at end of file

Scripts/PythonScripts/compare_key_tls.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+def get_file(title="Choose a file", type="get", 
+            filetypes="", ifile="", ext="", keyword=None):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # No Tk GUI
+    filetypes = [
+        ("csv files","*.csv"),
+        ("excel files","*.xls*"),
+        ("all files","*.*"),
+        ]
+    if keyword:
+        filetypes.insert(0, ("keyword", "*"+keyword+"*"))
+    
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = filetypes,
+                initialfile = ifile,
+                )
+    
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+
+    if not len(file):
+        from sys import exit
+        exit()
+        
+    app.destroy()
+    return Path(file)
+
+def get_data(file, columns):
+    if file.suffix == ".csv":
+        data = pd.read_csv(file)
+    elif file.suffix[:4] == '.xls':
+        data = pd.read_excel(file)
+    else:
+        return []
+    if isinstance(columns, (list, tuple)):
+        cdata = []
+        for column in columns:
+            cdata.append(data[column])
+    else:
+        cdata = data[columns]
+    return cdata
+
+def timing(files):
+    keystimes, keysnames = get_data(files[0], ["press_time", "key_name"])
+    keystimes /= 1000
+    keysnames.replace(
+        {
+            "backspace" : "<", 
+            "space"     : "*", 
+            # "enter"     : "E", 
+            # "shift"     : "S", 
+            "comma"     : ",",
+            # "quote"     : "\"",
+            "dead_acute": " ",
+            }, inplace=True)
+
+    tlstimes, tlslengths = get_data(files[1], ["Time", "Length"])
+
+    fig, plots = plt.subplots(2, sharex=True)
+    plt.xlabel("Time (Unix seconds)")
+    fig.suptitle('Comparing Keystoke and TLS Traffic Times')
+
+    plots[0].set_title("A Keypresses")
+    plots[0].set(ylabel="Key Pressed")
+    plots[0].plot(keystimes, keysnames, ".")
+    for x,y in zip(keystimes, keysnames):
+        plots[0].annotate(y, # this is the text
+                    (x,y), # this is the point to label
+                    textcoords="offset points", # how to position the text
+                    xytext=(0,2), # distance from text to points (x,y)
+                    ha='center') # horizontal alignment can be left, right or center
+
+    plots[1].set_title("Upstream TLS traffic")
+    plots[1].set(ylabel='Frame Size')
+    plots[1].plot(tlstimes, tlslengths, ".")
+
+    plt.xlim(1582752125, 1582752140)
+
+    if show:
+        plt.show()
+    
+    return plt
+
+def periodicity(files):
+    keystimes = periodogram(get_data(files[0], "press_time")/1000)
+    tlsdata = periodogram(get_data(files[1], "Time"))
+
+    fig, plots = plt.subplots(2, sharex=True)
+    plt.xlabel("Frequency (Hz)")
+    fig.suptitle('Comparing Keystoke and TLS Periodicity')
+
+    plots[0].set_title("Client A Keypress Periodicity")
+    plots[0].set(ylabel='Count')
+    plots[0].plot(keystimes)
+
+    plots[1].set_title("Upstream TLS Periodicity")
+    plots[1].set(ylabel='Count')
+    plots[1].plot(tlsdata)
+
+    if show:
+        plt.show()
+    
+    return plt
+
+if __name__ == "__main__":
+    show = True
+    save = True
+
+    files = [
+        get_file("Choose Keystroke File", keyword="keystroke"),
+        get_file("Choose Traffic File", keyword="tls"),
+        ]
+
+    tplt = timing(files)
+    pplt = periodicity(files)
+    
+    if save:
+        out_name = files[0].stem[:12]
+        tfile_name = out_name + "compare_timing"
+        pfile_name = out_name + "compare_periods"
+        ext = 'pdf'
+
+        tfile = get_file("Save Timing Graph as...", type="save", ifile=tfile_name, ext=ext)
+        if tfile:
+            print(tfile)
+            # tplt.savefig(tfile, format=ext)
+
+        pfile = get_file("Save Periodicity Graph as...", type="save", ifile=pfile_name, ext=ext)
+        if pfile:
+            print(pfile)
+            # pplt.savefig(pfile, format=ext)
\ No newline at end of file

Scripts/PythonScripts/compare_key_tls_periodicity.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from glob import glob
+from pathlib import Path
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+def get_file(title="Choose a file", type="get", 
+            filetypes="", ifile="", ext="", keyword=None):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # No Tk GUI
+    filetypes = [
+        ("csv files","*.csv"),
+        ("excel files","*.xls*"),
+        ("all files","*.*"),
+        ]
+    if keyword:
+        filetypes.insert(0, ("keyword", "*"+keyword+"*"))
+    
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = filetypes,
+                initialfile = ifile,
+                )
+    
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+
+    if not len(file):
+        from sys import exit
+        exit()
+        
+    app.destroy()
+    return Path(file)
+
+def get_data(file, columns):
+    if file.suffix == ".csv":
+        data = pd.read_csv(file)
+    elif file.suffix[:4] == '.xls':
+        data = pd.read_excel(file)
+    else:
+        return []
+    if isinstance(columns, (list, tuple)):
+        cdata = []
+        for column in columns:
+            cdata.append(data[column])
+    else:
+        cdata = data[columns]
+    return cdata
+
+def main():
+    show = False
+    save = True
+
+    for client in ["A", "B"]:
+        files = [
+            Path(glob(r"*"+client+r"*keystroke*.csv")[0]),
+            Path(glob(r"*"+client+r"*tls*.csv")[0])
+            ]
+
+        if save:
+            ext = 'png'
+            outfile_name = files[0].stem[:12] + "compare_periods"
+            outfile = get_file("Save as...", type="save", ifile=outfile_name, ext=ext)
+
+        keystimes = periodogram(get_data(files[0], "press_time")/1000)
+        tlsdata = periodogram(get_data(files[1], "Time"))
+
+        fig = plt.figure(figsize=(10, 5))
+        plt.xlabel("Frequency (Hz)")
+        plt.ylabel("Count")
+        fig.suptitle('Comparing Keystoke and TLS Periodicity')
+
+        plt.plot(keystimes, label=f"Client {client} Keypress Periodicity")
+        plt.plot(tlsdata, label="Upstream TLS Periodicity")
+        plt.legend()
+
+        if save:
+            plt.savefig(outfile, format=ext)
+
+        if show:
+            plt.show()
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

Scripts/PythonScripts/compare_key_tls_time_K.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from glob import glob
+from pathlib import Path
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+def get_file(title="Choose a file", type="get", 
+            filetypes="", ifile="", ext="", keyword=None):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # No Tk GUI
+    filetypes = [
+        ("csv files","*.csv"),
+        ("excel files","*.xls*"),
+        ("all files","*.*"),
+        ]
+    if keyword:
+        filetypes.insert(0, ("keyword", "*"+keyword+"*"))
+    
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = filetypes,
+                initialfile = ifile,
+                )
+    
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+
+    if not len(file):
+        from sys import exit
+        exit()
+        
+    app.destroy()
+    return Path(file)
+
+def get_data(file, columns):
+    if file.suffix == ".csv":
+        data = pd.read_csv(file)
+    elif file.suffix[:4] == '.xls':
+        data = pd.read_excel(file)
+    else:
+        return []
+    if isinstance(columns, (list, tuple)):
+        cdata = []
+        for column in columns:
+            cdata.append(data[column])
+    else:
+        cdata = data[columns]
+    return cdata
+
+def main():
+    show = False
+    save = True
+    
+
+    files = [
+        # get_file("Choose Keystroke A File", keyword="A*keystroke"),
+        # get_file("Choose Keystroke B File", keyword="B*keystroke"),
+        # get_file("Choose Traffic File A1", keyword="A*do-a*csv"),
+        # get_file("Choose Traffic File A2", keyword="A*kiwi*csv"),
+        # get_file("Choose Traffic File B1", keyword="B*do-a*2*csv"),
+        # get_file("Choose Traffic File B2", keyword="B*kiwi*2*csv"),
+        Path(glob(r"*A*keystroke.csv")[0]),
+        Path(glob(r"*A*do-a*.csv")[0]),
+        Path(glob(r"*A*kiwi*.csv")[0]),
+        ]
+
+    if save:
+        ext = 'png'
+        outfile_name = files[0].stem[:12] + "compare_timing_both"
+        outfile = get_file("Save as...", type="save", ifile=outfile_name, ext=ext)
+    file = 0
+    keystimesA, keysAy = get_data(files[file], ["press_time", "key_code"])
+    keystimesA /= 1000
+    # keysAy = list(map(lambda y: y%30, keystimesA))
+    file += 1
+    tlstimesA1, tlsA1y = get_data(files[file], ["Time", "Length"])
+    file += 1
+    tlstimesA2, tlsA2y = get_data(files[file], ["Time", "Length"])
+    time_zero = min(keystimesA[0], tlstimesA1[0], tlstimesA2[0])
+    keystimesA -= time_zero
+    tlstimesA1 -= time_zero
+    tlstimesA2 -= time_zero
+
+    fig = plt.figure(figsize=[10, 5])
+    plt.plot(keystimesA, keysAy, ".r", label="Key Strokes", figure=fig)
+    plt.plot(tlstimesA1, tlsA1y, ".b", label="TLS Frames Sent to Server 1", figure=fig)
+    plt.plot(tlstimesA2, tlsA2y, ".g", label="TLS Frames Sent to Server 2", figure=fig)
+
+    plt.title("Which Kiwi Server Handles Message Data")
+    plt.xlabel("Time of Frame/Keystroke (seconds)")
+    plt.ylabel("Keystroke Code or Frame Length")
+    plt.legend()
+
+    if save:
+        plt.savefig(outfile, format=ext)
+
+    if show:
+        plt.show()
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

Scripts/PythonScripts/compare_key_tls_timing.py

+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def periodogram(t):
+    f = np.linspace(0, 500, 1001)
+    f = f[f>=1]
+    P = np.zeros(len(f), dtype=np.complex)
+    for k in range(len(f)):
+        P[k] = np.exp(2j * np.pi * f[k] * t).sum()
+    P = (np.abs(P)**2)/len(t)
+    return P
+
+def get_file(title="Choose a file", type="get", 
+            filetypes="", ifile="", ext="", keyword=None):
+    from tkinter import Tk
+    from tkinter.filedialog import askopenfilename, asksaveasfilename
+    from pathlib import Path
+    app = Tk()
+    app.withdraw() # No Tk GUI
+    filetypes = [
+        ("csv files","*.csv"),
+        ("excel files","*.xls*"),
+        ("all files","*.*"),
+        ]
+    if keyword:
+        filetypes.insert(0, ("keyword", "*"+keyword+"*"))
+    
+    if type == "get":
+        file = askopenfilename(
+                title = title,
+                filetypes = filetypes,
+                initialfile = ifile,
+                )
+    
+    elif type == "save":
+        file = asksaveasfilename(
+                title = title,
+                filetypes = (
+                    ("all files","*.*"),
+                    ("pdf files","*.pdf"),
+                    ),
+                initialfile = ifile,
+                defaultextension = ext,
+                )
+
+    if not len(file):
+        from sys import exit
+        exit()
+        
+    app.destroy()
+    return Path(file)
+
+def get_data(file, columns):
+    if file.suffix == ".csv":
+        data = pd.read_csv(file)
+    elif file.suffix[:4] == '.xls':
+        data = pd.read_excel(file)
+    else:
+        return []
+    if isinstance(columns, (list, tuple)):
+        cdata = []
+        for column in columns:
+            cdata.append(data[column])
+    else:
+        cdata = data[columns]
+    return cdata
+
+def main():
+    show = True
+    save = False#True
+
+    files = [
+        get_file("Choose Keystroke File", keyword="keystroke"),
+        get_file("Choose Traffic File", keyword="tls"),
+        ]
+
+    if save:
+        ext = 'pdf'
+        outfile_name = files[0].stem[:12] + "compare_timing"
+        outfile = get_file("Save as...", type="save", ifile=outfile_name, ext=ext)
+
+    keystimes, keysnames = get_data(files[0], ["press_time", "key_name"])
+    keystimes /= 1000
+    keysnames.replace(
+        {
+            "backspace" : "<", 
+            "space"     : "*", 
+            # "enter"     : "E", 
+            # "shift"     : "S", 
+            "comma"     : ",",
+            # "quote"     : "\"",
+            "dead_acute": " ",
+            }, inplace=True)
+
+    tlstimes, tlslengths = get_data(files[1], ["Time", "Length"])
+
+    fig, plots = plt.subplots(2, sharex=True)
+    plt.xlabel("Time (Unix seconds)")
+    fig.suptitle('Comparing Keystoke and TLS Traffic Times')
+
+    plots[0].set_title("A Keypresses")
+    plots[0].set(ylabel="Key Pressed")
+    plots[0].plot(keystimes, keysnames, ".", label="Key Strokes")
+    plots[0].legend()
+    for x,y in zip(keystimes, keysnames):
+        plots[0].annotate(y, # this is the text
+                    (x,y), # this is the point to label
+                    textcoords="offset points", # how to position the text
+                    xytext=(0,2), # distance from text to points (x,y)
+                    ha='center') # horizontal alignment can be left, right or center
+
+    plots[1].set_title("Upstream TLS traffic")
+    plots[1].set(ylabel='Frame Size')
+    plots[1].plot(tlstimes, tlslengths, ".", label="Frame Size (bytes)")
+    plots[1].legend()
+
+
+    plt.xlim(1582752125, 1582752140)
+
+    if save:
+        plt.savefig(outfile, format=ext)
+
+    if show:
+        plt.show()
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

Scripts/PythonScripts/compare_key_tls_timing_pretty.py

+import pandas as pd
+import matplotlib.pyplot as plt
+from glob import glob
+from pathlib import Path