identify sortable SAY columns directly from the MP code

python/export_trace_manager.py

@@ -8,7 +8,7 @@ from message_popup import message_popup
 from preferences import preferences
 from settings import settings
 from mp_file_dialog import get_save_filename, get_existing_directory
-from mp_code_event_dict import mp_code_schema
+from mp_code_parser import mp_code_schema
 from export_trace import export_trace
 
 def _image_format():

python/filter_event_item_delegates.py

 from PySide6.QtCore import Qt, Signal, Slot
 from PySide6.QtWidgets import QStyledItemDelegate, QComboBox, QSpinBox
-from mp_code_event_dict import mp_code_event_dict, mp_code_event_list
+from mp_code_parser import mp_code_event_dict, mp_code_event_list
 
 """
 Provides widgets for the cells for relationship and recurrence event filters.

python/graph_list_table_model.py

 from PySide6.QtCore import Qt, QAbstractTableModel, QModelIndex
-from say_info import say_column_names, say_column_map, say_info
+from mp_code_parser import mp_code_say_headers, say_info
 
 # graph list model column constants
 GRAPH_COLUMN = 0 # the graph item
@@ -35,13 +35,18 @@ class GraphListTableModel(QAbstractTableModel):
 
         self.graphs = list()
         self.headers = list()
-        self.say_map = say_column_map(self.graphs, len(_HARDCODED_HEADERS))
 
-    def set_graph_list(self, graphs):
-        self.beginResetModel() # because self.graphs change
+    def set_graph_list(self, graphs, mp_code):
+        self.beginResetModel()
+
+        # set the data
         self.graphs = graphs
+
+        # set the headers
+        say_headers = mp_code_say_headers(mp_code)
         self.headers = _HARDCODED_HEADERS.copy()
-        self.headers.extend(say_column_names(graphs))
+        self.headers.extend(say_headers)
+
         self.endResetModel()
 
     def headerData(self, section, orientation,
@@ -84,8 +89,12 @@ class GraphListTableModel(QAbstractTableModel):
                 gry_trace = gry_graph["trace"]
                 return len(gry_trace["nodes"]) + len(gry_trace["edges"])
             if column >= len(self.headers):
+                # column number out of range
                 raise RuntimeError("bad %d"%column)
 
+            # get the column value based on SAY text in the gry_graph
+            return say_info(gry_graph, self.headers[column])
+
             # provide values for columns dynamically created from SAY nodes
             header = self.headers[column]
             says = say_info(gry_graph)

python/graph_metadata_label.py

 from PySide6.QtCore import QObject # for signal/slot support
 from PySide6.QtCore import Slot
 from PySide6.QtWidgets import QLabel
-from mp_code_event_dict import mp_code_schema
+from mp_code_parser import mp_code_schema
 
 class GraphMetadataLabel(QObject):
     """Provides the active scope and MP Code schema name in a QLabel."""

python/graphs_manager.py

@@ -52,7 +52,7 @@ class GraphsManager(QObject):
         # clear graphs
         self.graphs = list()
         self.scope = scope
-        self.graph_list_table_model.set_graph_list(list())
+        self.graph_list_table_model.set_graph_list(list(), "")
 
         # repopulate graphs
         log_to_statusbar("Creating %d graphs..."%len(gry_graphs))
@@ -61,7 +61,7 @@ class GraphsManager(QObject):
         log_to_statusbar("Done creating %d graphs"%len(gry_graphs))
 
         # set graphs
-        self.graph_list_table_model.set_graph_list(self.graphs)
+        self.graph_list_table_model.set_graph_list(self.graphs, self.mp_code)
 
         # remember spacing for reference in case spacing changes
         self._remember_spacing()
@@ -73,7 +73,7 @@ class GraphsManager(QObject):
         self.mp_code = ""
         self.graphs = list()
         self.scope = 1
-        self.graph_list_table_model.set_graph_list(list())
+        self.graph_list_table_model.set_graph_list(list(), "")
 
         # signal
         self.signal_graphs_loaded.emit()
@@ -84,7 +84,7 @@ class GraphsManager(QObject):
         self.mp_code = existing_mp_code
         self.graphs = existing_graphs
         self.scope = existing_scope
-        self.graph_list_table_model.set_graph_list(self.graphs)
+        self.graph_list_table_model.set_graph_list(self.graphs, self.mp_code)
 
         # signal
         self.signal_graphs_loaded.emit()

python/gry_manager.py

@@ -7,7 +7,7 @@ from settings import settings
 import mp_json_io_manager
 from mp_file_dialog import get_open_filename, get_save_filename
 from mp_logger import log, begin_timestamps, timestamp, show_timestamps
-from mp_code_event_dict import mp_code_schema
+from mp_code_parser import mp_code_schema
 from version_file import VERSION
 from message_popup import message_popup
 

python/gui_manager.py

@@ -77,7 +77,7 @@ from report_an_issue import report_an_issue
 from startup_options import parse_startup_options
 from mp_file_dialog import get_open_filename, get_save_filename, \
                            get_existing_directory
-from mp_code_event_dict import mp_code_schema
+from mp_code_parser import mp_code_schema
 from mp_code_filename import active_mp_code_filename, \
                              set_active_mp_code_filename
 from verbose import verbose

python/model_statistics.py

 from version_file import VERSION
 from collections import defaultdict
 from hashlib import blake2b # supposed to be fast
-from mp_code_event_dict import mp_code_event_dict, mp_code_schema
+from mp_code_parser import mp_code_event_dict, mp_code_schema
 from verbose import verbose
 
 _EVENT_TYPES = ["root", "composite", "atomic"]

python/mp_code_manager.py

@@ -5,7 +5,7 @@ from PySide6.QtGui import QTextDocument
 from PySide6.QtWidgets import QPlainTextDocumentLayout
 from mp_code_syntax_highlighter import MPCodeSyntaxHighlighter
 from mp_code_syntax_checker import mp_check_syntax
-from mp_code_event_dict import mp_code_schema
+from mp_code_parser import mp_code_schema
 from mp_code_filename import set_active_mp_code_filename
 
 # use this to signal completion to GUI.

python/mp_code_event_dict.py → python/mp_code_parser.py

+import re
 from copy import deepcopy
+from collections import defaultdict
 from PySide6.QtCore import QRegularExpression
 from mp_code_expressions import MP_KEYWORDS, MP_META_SYMBOL_EXPRESSION, \
                   COMMENT_START_EXPRESSION, \
@@ -17,6 +19,15 @@ _STARTING_EVENT_COLON_EXPRESSION \
 _EVENT_WORD_EXPRESSION \
                   = QRegularExpression(r"(\$\$[a-zA-Z_]\w*|[a-zA-Z_]\w*)")
 
+# SAY keyword
+_SAY_KEYWORD_EXPRESSION = QRegularExpression(r"\bSAY\s*\(")
+
+# SAY content
+_SAY_CONTENT = QRegularExpression(r"\bSAY\s*\((.*)\)\s*$")
+
+# increment_report_command "=>" (97) or add_tuple_command "<|" (106)
+_SAY_EXEMPTION_EXPRESSION = QRegularExpression(r"=\s*>|<\s*\|")
+
 # MP keywords
 _STARTING_KEYWORD_EXPRESSION = QRegularExpression(r"^\s*(%s)"%MP_KEYWORDS)
 
@@ -29,20 +40,16 @@ _mp_code_text_cache = ""
 _mp_lines_cache = ""
 _event_dict = deepcopy(_EMPTY_EVENT_DICT)
 
-# get list of lines of text with comments and quoted text removed
-# and reorganized to delimit lines by ; instead of \n.
-def _parsable_mp_lines(mp_code_text):
+# return non-commented lines
+def _non_comment_lines(mp_code_text):
     in_comment = False
-    in_custom_view_declaration = False
-    text_start_index = 0
     stripped_lines = list()
-    parts = list()
     lines = mp_code_text.split("\n")
     for line in lines:
 
         # remove multi-line comments
         text_start_index = 0
-        parts.clear()
+        parts = list()
         while text_start_index != -1:
             if in_comment == False:
                 # in non-comment
@@ -80,26 +87,30 @@ def _parsable_mp_lines(mp_code_text):
         if single_line_comment_index != -1:
             stripped_line = stripped_line[0:single_line_comment_index]
 
-        # remove quoted text
-        while True:
-            match = QUOTED_TEXT_EXPRESSION.match(stripped_line)
-            quoted_text_index = match.capturedStart()
-            if quoted_text_index == -1:
-                # no quoted text
-                break
-            # remove quoted text including quotation marks
-            stripped_line = stripped_line[:quoted_text_index] \
-                           + stripped_line[match.capturedEnd():]
-
-        # keep non-empty lines
-        if stripped_line != "\n":
-            stripped_lines.append(stripped_line)
-
-    # convert CR-delimited lines into semicolon-delimited lines
+        stripped_lines.append(stripped_line)
+
+    # convert list of stripped lines into a list made from semicolon boundaries
     parsable_mp_lines = (" ".join(stripped_lines)).split(";")
 
     return parsable_mp_lines
 
+def _non_quote_line(non_comment_line):
+    # remove quoted text
+    while True:
+        match = QUOTED_TEXT_EXPRESSION.match(non_comment_line)
+        quoted_text_index = match.capturedStart()
+        if quoted_text_index == -1:
+            # no quoted text
+            break
+        # remove quoted text including quotation marks
+        non_comment_line = non_comment_line[:quoted_text_index] \
+                           + non_comment_line[match.capturedEnd():]
+    return non_comment_line
+
+def _non_quote_lines(non_comment_lines):
+    non_quote_lines = [_non_quote_line(line) for line in non_comment_lines]
+    return non_quote_lines
+
 # return tuple schema, remainder
 def _parse_schema(mp_line):
     match = _STARTING_SCHEMA_EXPRESSION.match(mp_line)
@@ -151,9 +162,9 @@ def mp_code_event_dict(mp_code_text):
         return deepcopy(_event_dict)
     _mp_code_text_cache = mp_code_text
 
-    # convert text into lines without comments or quoted text and delineated
-    # with ; instead of \n.
-    mp_lines = _parsable_mp_lines(mp_code_text)
+    # convert text into lines without comments or quoted text
+    non_comment_lines = _non_comment_lines(mp_code_text)
+    mp_lines = _non_quote_lines(non_comment_lines)
 
     # optimization
     if mp_lines == _mp_lines_cache:
@@ -205,3 +216,91 @@ def mp_code_event_list(event_dict):
 def mp_code_schema(mp_code_text):
     return mp_code_event_dict(mp_code_text)["schema"]
 
+def _say_text_and_number_count(say_line):
+    say_content = _SAY_CONTENT.match(say_line).captured(1)
+
+    quoted_text = ""
+    between_text = list()
+    number_count = 0
+
+    global_match = QUOTED_TEXT_EXPRESSION.globalMatch(say_content)
+
+    # text and count
+    start = 0
+    while global_match.hasNext():
+        match = global_match.next()
+        quoted_text += match.captured()[1:-1] # text with quotes stripped
+        captured_start = match.capturedStart()
+        between_text = say_content[start:match.capturedStart()]
+        if between_text.strip():
+            number_count += 1
+        start = match.capturedEnd()
+
+    # any final count
+    final_text = say_content[start:]
+    if final_text.strip():
+        number_count += 1
+
+    return quoted_text, number_count
+
+# derive sorted say headers from mp_code_text
+# We do not cache these results.  This mp_code_text is from trace generation
+# and is called by graph_list_table_model when the graph is loaded.
+def mp_code_say_headers(mp_code_text):
+
+    # convert text into lines without comments
+    non_comment_lines = _non_comment_lines(mp_code_text)
+
+    header_list = list()
+    for line in non_comment_lines:
+        non_quote_line = _non_quote_line(line)
+
+        # SAY keyword is required
+        if not _SAY_KEYWORD_EXPRESSION.globalMatch(non_quote_line).hasNext():
+            continue
+
+        # SAY directives increment_report_command "=>" (97) and
+        # add_tuple_command "<|" (106) do not generate SAY event blocks
+        if _SAY_EXEMPTION_EXPRESSION.globalMatch(non_quote_line).hasNext():
+            continue
+
+        say_text, number_count = _say_text_and_number_count(line)
+
+        # add to texts and columns
+        if number_count == 0:
+            header_list.append(say_text)
+        else:
+            for i in range(number_count):
+                header_list.append("%s (%d)"%(say_text, i+1))
+
+    header_list = sorted(header_list, key=str.casefold)
+    return header_list
+
+# Return text for the matching column else X or checkmark
+# Say nodes are type="T"."""
+_number_pattern = re.compile(r"[0-9.]+")
+def say_info(gry_graph, column_title):
+    if not "trace" in gry_graph:
+        raise RuntimeError("bad")
+
+    for gry_node in gry_graph["trace"]["nodes"]:
+        if gry_node["type"] == "T":
+
+            # try exact column match
+            label = gry_node["label"]
+            if label == column_title:
+                return "\u2713" # checkmark
+
+            # match the label with numbers extracted with the column title
+            prefix = _number_pattern.sub("", label)
+            numbers = _number_pattern.findall(label)
+            for i, number in enumerate(numbers):
+                # key is column without numbers and with index appended
+                key = "%s (%i)"%(prefix, i+1)
+                if key == column_title:
+                    # column matches for this index so return its number
+                    return float(numbers[i])
+
+    # no column match found
+    return "X"
+

python/mp_code_syntax_highlighter.py

@@ -9,7 +9,7 @@ from mp_code_expressions import ATOMIC_NAME_EXPRESSION, \
                MP_KEYWORD_EXPRESSION, MP_META_SYMBOL_EXPRESSION, \
                OPERATOR_EXPRESSION, NUMBER_EXPRESSION, \
                VARIABLE_EXPRESSION, QUOTED_TEXT_EXPRESSION
-from mp_code_event_dict import mp_code_event_dict
+from mp_code_parser import mp_code_event_dict
 from spellcheck import unknown_words_regex
 
 """

python/say_info.py

-import re
-from collections import defaultdict
-"""Extract SAY titles and numbers from SAY trace nodes."""
-
-_pattern = re.compile(r"[0-9.]+")
-
-# Return {say text, [numbers]} where:
-#   * say_text is say text with index prepended and numbers removed
-#   * and numbers is a list of zero or more numbers.
-# Say nodes are type="T"."""
-def say_info(gry_graph):
-    if not "trace" in gry_graph:
-        raise RuntimeError("bad")
-
-    says = defaultdict(list)
-    for gry_node in gry_graph["trace"]["nodes"]:
-        if gry_node["type"] == "T":
-            label = gry_node["label"]
-            key = _pattern.sub(" ", label)
-            numbers = _pattern.findall(label)
-            for i, number in enumerate(numbers, 1):
-                try:
-                    says["%i: %s"%(i, key)] = float(number)
-                except Exception as e:
-                    # multiple decimal places can thwart _pattern so manage
-                    # invalid sortable numbers as text
-                    print("say_info.say_info: invalid number: %s"%number)
-                    says["%i: %s"%(i, key)] = number
-    return says
-
-# Return a list of "say text" column names.
-def say_column_names(graphs):
-    if len(graphs) >= 2:
-        says = say_info(graphs[1].gry_graph)
-        return sorted(says.keys())
-    else:
-        return list()
-
-# Return a dict of {key="say text", value=column index}
-def say_column_map(graphs, start_index):
-    column_names = say_column_names(graphs)
-    column_map = dict()
-    for i, column_name in enumerate(column_names, start_index):
-         column_map[column_name] = i
-    return column_map
-

python/session_snapshots.py

@@ -4,7 +4,7 @@ from time import strftime
 from PySide6.QtGui import QAction
 from paths_gryphon import SNAPSHOTS_PATH
 from session_snapshots_dialog_wrapper import SessionSnapshotsDialogWrapper
-from mp_code_event_dict import mp_code_schema
+from mp_code_parser import mp_code_schema
 
 def schema_filename(schema_name):
     # filename = <schema name>_yy_mm_dd-hh_mm_ss.gry

python/test.py

@@ -19,7 +19,7 @@ from PySide6.QtWidgets import QApplication
 from version_file import VERSION
 import resources_rc
 from settings import settings
-from mp_code_event_dict import mp_code_event_dict
+from mp_code_parser import mp_code_event_dict
 from export_trace import export_trace
 from trace_generator_manager import TraceGeneratorManager
 from mp_json_io_manager import read_mp_code_file