mps-cli-py: Implemented NumPy flat storage, JAR-direct parsing and fixed caching logic

.github/workflows/mps_cli_py_build.yaml

@@ -25,8 +25,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        cd mps-cli-py
+        pip install -e .
         pip install parameterized
-      # pip install -r requirements.txt
     - name: Test with unittest
       run: |
         cd mps-cli-py

mps-cli-py/Readme.md → mps-cli-py/README.md

@@ -2,12 +2,18 @@
 
 This project provides a Python library which parses MPS files and builds the object model.
 
+### Requirements
+
+- numpy >= 1.20
+
 ### Features
 The following features are available:
 - load MPS files (*.mpsr, *.mps, *.mpb, *.jar) and expose their content as Python object model 
   - solutions, models, root nodes, nodes, children, references, properties
 - extract the meta-information and expose it as Python object model
   - list of languages, their concepts with information about properties, references, children
+- JAR files are read directly without extracting to disk
+- Parsed models are cached in '~/.mps_cli_cache' directory and invalidated automatically when JARs change on disk
 
 The core of the Python object model is given by the following classes:
 - `SNode` - represents a node

mps-cli-py/pyproject.toml

@@ -18,6 +18,12 @@ classifiers = [
     "License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)",
     "Operating System :: OS Independent",
 ]
+dependencies = [
+    "numpy>=1.20",
+]
+
+[project.optional-dependencies]
+fast = ["lxml>=4.0"]
 
 [project.urls]
 "Homepage" = "https://github.com/mbeddr/mps-cli"

mps-cli-py/src/mpscli/demo.py

@@ -9,6 +9,7 @@
     MODE = "test_project": parse a single mps_test_projects directory (original behaviour)
 """
 
+import logging
 import sys
 import time
 from datetime import datetime
@@ -283,6 +284,7 @@ def main() -> None:
         for c in lan.concepts:
             c.print_concept_details()
 
+
 if __name__ == "__main__":
     import multiprocessing
 
@@ -293,4 +295,13 @@ def main() -> None:
     sys.stdout = open(log_file, "w", encoding="utf-8", buffering=1)
     print(f"Logging to: {log_file}")
 
+    # route library log output (each parse phasee timings and cache progress) to stderr so they appear on
+    # console alongside the summary lines...
+    # format leaves out timestamps and level names to keep the output clean
+    logging.basicConfig(
+        stream=sys.stderr,
+        level=logging.INFO,
+        format="%(message)s",
+    )
+
     main()

mps-cli-py/src/mpscli/demo_language_extraction.py

@@ -5,13 +5,14 @@
 """
 
 import sys
+import time
 from datetime import datetime
 from pathlib import Path
 
 sys.path.insert(1, "..")
 
 MODE = "plugins"
-PLUGINS_PATH = r"C:\Temp\plugins"
+PLUGINS_PATH = r"C:\Users\emb-venkpri\vemb\arch"
 TEST_PROJECT = r"..\..\mps_test_projects\mps_cli_binary_persistency_language"
 OUTPUT_FILE = Path(f"language_concepts_{datetime.now().strftime('%H%M%S')}.md")
 
@@ -224,16 +225,27 @@ def build_markdown(languages_with_structure: list) -> str:
 def main():
     path = PLUGINS_PATH if MODE == "plugins" else TEST_PROJECT
 
-    print(f"Parsing ({MODE}): {path}", flush=True)
+    msg = f"Parsing ({MODE}): {path}"
+    print(msg, flush=True)
+    sys.stderr.write(msg + "\n")
+    sys.stderr.flush()
+
+    t0 = time.perf_counter()
     builder = SSolutionsRepositoryBuilder()
-    builder.USE_CACHE = False
     builder.build(path)
+    elapsed = time.perf_counter() - t0
+
+    timing_msg = f"Parsing complete in {elapsed:.1f}s"
+    print(f"\n{timing_msg}")
+    sys.stderr.write(timing_msg + "\n")
+    sys.stderr.flush()
 
     languages_with_structure = collect_languages_with_structure()
 
-    print(
-        f"Languages with structure aspect: {len(languages_with_structure)}", flush=True
-    )
+    lang_msg = f"Languages with structure aspect: {len(languages_with_structure)}"
+    print(lang_msg, flush=True)
+    sys.stderr.write(lang_msg + "\n")
+    sys.stderr.flush()
 
     for lang, structure in languages_with_structure:
         print(f"\n{'=' * 60}")
@@ -258,8 +270,22 @@ def main():
 
     md = build_markdown(languages_with_structure)
     OUTPUT_FILE.write_text(md, encoding="utf-8")
-    print(f"\nmarkdown file written: {OUTPUT_FILE}", flush=True)
+    done_msg = f"markdown file written: {OUTPUT_FILE}"
+    print(f"\n{done_msg}", flush=True)
+    sys.stderr.write(done_msg + "\n")
+    sys.stderr.flush()
 
 
 if __name__ == "__main__":
+    import multiprocessing
+
+    multiprocessing.freeze_support()
+
+    # redirect stdout to a log file so all output goess to disk..
+    log_file = f"lang_extraction_{datetime.now().strftime('%H%M%S')}.log"
+    sys.stdout = open(log_file, "w", encoding="utf-8", buffering=1)
+    print(f"Logging to: {log_file}")
+    sys.stderr.write(f"Logging to: {log_file}\n")
+    sys.stderr.flush()
+
     main()

mps-cli-py/src/mpscli/model/SConcept.py

@@ -1,3 +1,8 @@
+import logging
+
+_log = logging.getLogger(__name__)
+
+
 class SConcept:
 
     def __init__(self, name, uuid):
@@ -8,14 +13,14 @@ def __init__(self, name, uuid):
         self.references = []
 
     def print_concept_details(self):
-        print("concept: " + self.name)
-        print("\tproperties: ")
+        _log.debug("concept: %s", self.name)
+        _log.debug("\tproperties: ")
         for property in self.properties:
-            print("\t\t" + property)
-        print("\tchildren: ")
+            _log.debug("\t\t%s", property)
+        _log.debug("\tchildren: ")
         for child in self.children:
-            print("\t\t" + child)
-        print("\treferences: ")
+            _log.debug("\t\t%s", child)
+        _log.debug("\treferences: ")
         for reference in self.references:
-            print("\t\t" + reference)
-        print("<<<")
+            _log.debug("\t\t%s", reference)
+        _log.debug("<<<")

mps-cli-py/src/mpscli/model/SLanguage.py

@@ -5,6 +5,9 @@ def __init__(self, name, uuid):
         self.uuid = uuid
         self.concepts = []
 
+        # internal dict for concept lookup by name
+        self._concepts_by_name = {}
+
         # version number from the languageVersion attribute in the .mpl file..
         # stays 0 if this language was only seen via registry (never had its .mpl read)
         self.language_version = 0
@@ -15,10 +18,8 @@ def __init__(self, name, uuid):
         self.models = []
 
     def find_concept_by_name(self, name):
-        for c in self.concepts:
-            if c.name == name:
-                return c
-        return None
+        # lookupp via internal dict
+        return self._concepts_by_name.get(name)
 
     def find_model_by_name(self, suffix):
         # find a specific aspect model by the last segment of its name.