[core] Graph: initial refactoring of graph loading API and logic

* API
Instead of having a single `load` function that exposes in its API
some elements only applicable to initializing a graph from a templates,
split it into 2 distinct functions: `load` and `initFromTemplate`.
Apply those changes to users of the API (UI, CLI), and simplify Graph
wrapper classes to better align with those concepts.

* Deserialization
Reduce the cognitive complexity of the deserizalization process
by splitting it into more atomic functions, while maintaining the
current behavior.

bin/meshroom_batch

@@ -154,10 +154,10 @@ with meshroom.core.graph.GraphModification(graph):
     # initialize template pipeline
     loweredPipelineTemplates = dict((k.lower(), v) for k, v in meshroom.core.pipelineTemplates.items())
     if args.pipeline.lower() in loweredPipelineTemplates:
-        graph.load(loweredPipelineTemplates[args.pipeline.lower()], setupProjectFile=False, publishOutputs=True if args.output else False)
+        graph.initFromTemplate(loweredPipelineTemplates[args.pipeline.lower()], publishOutputs=True if args.output else False)
     else:
         # custom pipeline
-        graph.load(args.pipeline, setupProjectFile=False, publishOutputs=True if args.output else False)
+        graph.initFromTemplate(args.pipeline, publishOutputs=True if args.output else False)
     
     def parseInputs(inputs, uniqueInitNode):
         """Utility method for parsing the input and inputRecursive arguments."""

meshroom/core/graph.py

@@ -4,6 +4,7 @@ import json
 import logging
 import os
 import re
+from typing import Optional
 import weakref
 from collections import defaultdict, OrderedDict
 from contextlib import contextmanager
@@ -18,6 +19,7 @@ from meshroom.core.attribute import Attribute, ListAttribute, GroupAttribute
 from meshroom.core.exception import GraphCompatibilityError, StopGraphVisit, StopBranchVisit
 from meshroom.core.node import Status, Node, CompatibilityNode
 from meshroom.core.nodeFactory import nodeFactory
+from meshroom.core.typing import PathLike
 
 # Replace default encoder to support Enums
 
@@ -149,6 +151,21 @@ def changeTopology(func):
     return decorator
 
 
+def blockNodeCallbacks(func):
+    """
+    Graph methods loading serialized graph content must be decorated with 'blockNodeCallbacks',
+    to avoid attribute changed callbacks defined on node descriptions to be triggered during
+    this process.
+    """
+    def inner(self, *args, **kwargs):
+        self._loading = True
+        try:
+            return func(self, *args, **kwargs)
+        finally:
+            self._loading = False
+    return inner
+
+
 class Graph(BaseObject):
     """
     _________________      _________________      _________________
@@ -260,37 +277,88 @@ class Graph(BaseObject):
         return self._saving
 
     @Slot(str)
-    def load(self, filepath, setupProjectFile=True, importProject=False, publishOutputs=False):
+    def load(self, filepath: PathLike):
         """
-        Load a Meshroom graph ".mg" file.
+        Load a Meshroom Graph ".mg" file in place.
 
         Args:
-            filepath: project filepath to load
+            filepath: The path to the Meshroom Graph file to load.
-            setupProjectFile: Store the reference to the project file and setup the cache directory.
-                              If false, it only loads the graph of the project file as a template.
-            importProject: True if the project that is loaded will be imported in the current graph, instead
-                           of opened.
-            publishOutputs: True if "Publish" nodes from templates should not be ignored.
         """
-        self._loading = True
+        self._deserialize(Graph._loadGraphData(filepath))
-        try:
+        self._setFilepath(filepath)
-            return self._load(filepath, setupProjectFile, importProject, publishOutputs)
+        self._fileDateVersion = os.path.getmtime(filepath)
-        finally:
-            self._loading = False
 
-    def _load(self, filepath, setupProjectFile, importProject, publishOutputs):
+    def initFromTemplate(self, filepath: PathLike, publishOutputs: bool = False):
-        if not importProject:
+        """
-            self.clear()
+        Deserialize a template Meshroom Graph ".mg" file in place.
-        with open(filepath) as jsonFile:
-            fileData = json.load(jsonFile)
 
-        self.header = fileData.get(Graph.IO.Keys.Header, {})
+        When initializing from a template, the internal filepath of the graph instance is not set.
+        Saving the file on disk will require to specify a filepath.
 
-        fileVersion = self.header.get(Graph.IO.Keys.FileVersion, "0.0")
+        Args:
-        # Retro-compatibility for all project files with the previous UID format
+            filepath: The path to the Meshroom Graph file to load.
-        if Version(fileVersion) < Version("2.0"):
+            publishOutputs: (optional) Whether to keep 'Publish' nodes.
+        """
+        self._deserialize(Graph._loadGraphData(filepath))
+
+        if not publishOutputs:
+            for node in [node for node in self.nodes if node.nodeType == "Publish"]:
+                self.removeNode(node.name)
+
+    @staticmethod
+    def _loadGraphData(filepath: PathLike) -> dict:
+        """Deserialize the content of the Meshroom Graph file at `filepath` to a dictionnary."""
+        with open(filepath) as file:
+            graphData = json.load(file)
+        return graphData
+
+    @blockNodeCallbacks
+    def _deserialize(self, graphData: dict):
+        """Deserialize `graphData` in the current Graph instance.
+
+        Args:
+            graphData: The serialized Graph.
+        """
+        self.clear()
+        self.header = graphData.get(Graph.IO.Keys.Header, {})
+        fileVersion = Version(self.header.get(Graph.IO.Keys.FileVersion, "0.0"))
+        graphContent = self._normalizeGraphContent(graphData, fileVersion)
+        isTemplate = self.header.get("template", False)
+
+        with GraphModification(self):
+            # iterate over nodes sorted by suffix index in their names
+            for nodeName, nodeData in sorted(
+                graphContent.items(), key=lambda x: self.getNodeIndexFromName(x[0])
+            ):
+                self._deserializeNode(nodeData, nodeName)
+
+            # Create graph edges by resolving attributes expressions
+            self._applyExpr()
+            
+            # Templates are specific: they contain only the minimal amount of 
+            # serialized data to describe the graph structure.
+            # They are not meant to be computed: therefore, we can early return here,
+            # as uid conflict evaluation is only meaningful for nodes with computed data.
+            if isTemplate:
+                return
+
+            # By this point, the graph has been fully loaded and an updateInternals has been triggered, so all the
+            # nodes' links have been resolved and their UID computations are all complete.
+            # It is now possible to check whether the UIDs stored in the graph file for each node correspond to the ones
+            # that were computed.
+            self.updateInternals()
+            self._evaluateUidConflicts(graphContent)
+            try:
+                self._applyExpr()
+            except Exception as e:
+                logging.warning(e)
+
+    def _normalizeGraphContent(self, graphData: dict, fileVersion: Version) -> dict:
+        graphContent = graphData.get(Graph.IO.Keys.Graph, graphData)
+
+        if fileVersion < Version("2.0"):
             # For internal folders, all "{uid0}" keys should be replaced with "{uid}"
-            updatedFileData = json.dumps(fileData).replace("{uid0}", "{uid}")
+            updatedFileData = json.dumps(graphContent).replace("{uid0}", "{uid}")
 
             # For fileVersion < 2.0, the nodes' UID is stored as:
             # "uids": {"0": "hashvalue"}
@@ -302,74 +370,25 @@ class Graph(BaseObject):
                 uid = occ.split("\"")[-2]  # UID is second to last element
                 newUidStr = r'"uid": "{}"'.format(uid)
                 updatedFileData = updatedFileData.replace(occ, newUidStr)
-            fileData = json.loads(updatedFileData)
+            graphContent = json.loads(updatedFileData)
 
-        # Older versions of Meshroom files only contained the serialized nodes
+        return graphContent
-        graphData = fileData.get(Graph.IO.Keys.Graph, fileData)
 
-        if importProject:
+    def _deserializeNode(self, nodeData: dict, nodeName: str):
-            self._importedNodes.clear()
+        # Retrieve version from
-            graphData = self.updateImportedProject(graphData)
+        #   1. nodeData: node saved from a CompatibilityNode
+        #   2. nodesVersion in file header: node saved from a Node
+        #   3. fallback behavior: default to "0.0"
+        if "version" not in nodeData:
+            nodeData["version"] = self._getNodeTypeVersionFromHeader(nodeData["nodeType"], "0.0")
+        inTemplate = self.header.get("template", False)
+        node = nodeFactory(nodeData, nodeName, inTemplate=inTemplate)
+        self._addNode(node, nodeName)
+        return node
 
-        if not isinstance(graphData, dict):
+    def _getNodeTypeVersionFromHeader(self, nodeType: str, default: Optional[str] = None) -> Optional[str]:
-            raise RuntimeError('loadGraph error: Graph is not a dict. File: {}'.format(filepath))
+        nodeVersions = self.header.get(Graph.IO.Keys.NodesVersions, {})
-
+        return nodeVersions.get(nodeType, default)
-        nodesVersions = self.header.get(Graph.IO.Keys.NodesVersions, {})
-
-        self._fileDateVersion = os.path.getmtime(filepath)
-
-        # Check whether the file was saved as a template in minimal mode
-        isTemplate = self.header.get("template", False)
-
-        with GraphModification(self):
-            # iterate over nodes sorted by suffix index in their names
-            for nodeName, nodeData in sorted(graphData.items(), key=lambda x: self.getNodeIndexFromName(x[0])):
-                if not isinstance(nodeData, dict):
-                    raise RuntimeError('loadGraph error: Node is not a dict. File: {}'.format(filepath))
-
-                # retrieve version from
-                #   1. nodeData: node saved from a CompatibilityNode
-                #   2. nodesVersion in file header: node saved from a Node
-                #   3. fallback to no version "0.0": retro-compatibility
-                if "version" not in nodeData:
-                    nodeData["version"] = nodesVersions.get(nodeData["nodeType"], "0.0")
-
-                # if the node is a "Publish" node and comes from a template file, it should be ignored
-                # unless publishOutputs is True
-                if isTemplate and not publishOutputs and nodeData["nodeType"] == "Publish":
-                    continue
-
-                n = nodeFactory(nodeData, nodeName, inTemplate=isTemplate)
-
-                # Add node to the graph with raw attributes values
-                self._addNode(n, nodeName)
-
-                if importProject:
-                    self._importedNodes.add(n)
-
-            # Create graph edges by resolving attributes expressions
-            self._applyExpr()
-
-            if setupProjectFile:
-                # Update filepath related members
-                # Note: needs to be done at the end as it will trigger an updateInternals.
-                self._setFilepath(filepath)
-            elif not isTemplate:
-                # If no filepath is being set but the graph is not a template, trigger an updateInternals either way.
-                self.updateInternals()
-
-            # By this point, the graph has been fully loaded and an updateInternals has been triggered, so all the
-            # nodes' links have been resolved and their UID computations are all complete.
-            # It is now possible to check whether the UIDs stored in the graph file for each node correspond to the ones
-            # that were computed.
-            if not isTemplate:  # UIDs are not stored in templates
-                self._evaluateUidConflicts(graphData)
-                try:
-                    self._applyExpr()
-                except Exception as e:
-                    logging.warning(e)
-
-        return True
 
     def _evaluateUidConflicts(self, data):
         """

meshroom/core/typing.py

@@ -0,0 +1,8 @@
+"""
+Common typing aliases used in Meshroom.
+"""
+
+from pathlib import Path
+from typing import Union
+
+PathLike = Union[Path, str]

meshroom/ui/graph.py

@@ -451,17 +451,21 @@ class UIGraph(QObject):
         self.stopExecution()
         self._chunksMonitor.stop()
 
-    @Slot(str, result=bool)
+    @Slot(str)
-    def loadGraph(self, filepath, setupProjectFile=True, publishOutputs=False):
+    def loadGraph(self, filepath):
-        g = Graph('')
+        g = Graph("")
-        status = True
         if filepath:
-            status = g.load(filepath, setupProjectFile, importProject=False, publishOutputs=publishOutputs)
+            g.load(filepath)
             if not os.path.exists(g.cacheDir):
                 os.mkdir(g.cacheDir)
-            g.fileDateVersion = os.path.getmtime(filepath)
         self.setGraph(g)
-        return status
+
+    @Slot(str, bool, result=bool)
+    def initFromTemplate(self, filepath, publishOutputs=False):
+        graph = Graph("")
+        if filepath:
+            graph.initFromTemplate(filepath, publishOutputs=publishOutputs)
+        self.setGraph(graph)
 
     @Slot(QUrl, result="QVariantList")
     @Slot(QUrl, QPoint, result="QVariantList")

meshroom/ui/qml/Application.qml

@@ -185,7 +185,7 @@ Page {
         nameFilters: ["Meshroom Graphs (*.mg)"]
         onAccepted: {
             // Open the template as a regular file
-            if (_reconstruction.loadUrl(currentFile, true, true)) {
+            if (_reconstruction.load(currentFile)) {
                 MeshroomApp.addRecentProjectFile(currentFile.toString())
             }
         }
@@ -400,7 +400,7 @@ Page {
                     text: "Reload File"
 
                     onClicked: {
-                        _reconstruction.loadUrl(_reconstruction.graph.filepath)
+                        _reconstruction.load(_reconstruction.graph.filepath)
                         fileModifiedDialog.close()
                     }
                 }
@@ -705,7 +705,7 @@ Page {
                         MenuItem {
                             onTriggered: ensureSaved(function() {
                                 openRecentMenu.dismiss()
-                                if (_reconstruction.loadUrl(modelData["path"])) {
+                                if (_reconstruction.load(modelData["path"])) {
                                     MeshroomApp.addRecentProjectFile(modelData["path"])
                                 } else {
                                     MeshroomApp.removeRecentProjectFile(modelData["path"])

meshroom/ui/qml/Homepage.qml

@@ -389,7 +389,7 @@ Page {
                                     } else {
                                         // Open project
                                         mainStack.push("Application.qml")
-                                        if (_reconstruction.loadUrl(modelData["path"])) {
+                                        if (_reconstruction.load(modelData["path"])) {
                                             MeshroomApp.addRecentProjectFile(modelData["path"])
                                         } else {
                                             MeshroomApp.removeRecentProjectFile(modelData["path"])

meshroom/ui/qml/main.qml

@@ -128,7 +128,7 @@ ApplicationWindow {
             if (mainStack.currentItem instanceof Homepage) {
                 mainStack.push("Application.qml")
             }
-            if (_reconstruction.loadUrl(currentFile)) {
+            if (_reconstruction.load(currentFile)) {
                 MeshroomApp.addRecentProjectFile(currentFile.toString())
             }
         }

meshroom/ui/reconstruction.py

@@ -5,6 +5,7 @@ import os
 from collections.abc import Iterable
 from multiprocessing.pool import ThreadPool
 from threading import Thread
+from typing import Callable
 
 from PySide6.QtCore import QObject, Slot, Property, Signal, QUrl, QSizeF, QPoint
 from PySide6.QtGui import QMatrix4x4, QMatrix3x3, QQuaternion, QVector3D, QVector2D
@@ -534,17 +535,24 @@ class Reconstruction(UIGraph):
         # - correct pipeline name but the case does not match (e.g. panoramaHDR instead of panoramaHdr)
         # - lowercase pipeline name given through the "New Pipeline" menu
         loweredPipelineTemplates = dict((k.lower(), v) for k, v in meshroom.core.pipelineTemplates.items())
-        if p.lower() in loweredPipelineTemplates:
+        filepath = loweredPipelineTemplates.get(p.lower(), p)
-            self.load(loweredPipelineTemplates[p.lower()], setupProjectFile=False)
+        return self._loadWithErrorReport(self.initFromTemplate, filepath)
-        else:
-            # use the user-provided default project file
-            self.load(p, setupProjectFile=False)
 
     @Slot(str, result=bool)
-    def load(self, filepath, setupProjectFile=True, publishOutputs=False):
+    @Slot(QUrl, result=bool)
+    def load(self, url):
+        if isinstance(url, QUrl):
+            # depending how the QUrl has been initialized,
+            # toLocalFile() may return the local path or an empty string
+            localFile = url.toLocalFile() or url.toString()
+        else:
+            localFile = url
+        return self._loadWithErrorReport(self.loadGraph, localFile)
+
+    def _loadWithErrorReport(self, loadFunction: Callable[[str], None], filepath: str):
         logging.info(f"Load project file: '{filepath}'")
         try:
-            status = super(Reconstruction, self).loadGraph(filepath, setupProjectFile, publishOutputs)
+            loadFunction(filepath)
             # warn about pre-release projects being automatically upgraded
             if Version(self._graph.fileReleaseVersion).major == "0":
                 self.warning.emit(Message(
@@ -554,8 +562,8 @@ class Reconstruction(UIGraph):
                     "Open it with the corresponding version of Meshroom to recover your data."
                 ))
             self.setActive(True)
-            return status
+            return True
-        except FileNotFoundError as e:
+        except FileNotFoundError:
             self.error.emit(
                 Message(
                     "No Such File",
@@ -564,8 +572,7 @@ class Reconstruction(UIGraph):
                 )
             )
             logging.error("Error while loading '{}': No Such File.".format(filepath))
-            return False
+        except Exception:
-        except Exception as e:
             import traceback
             trace = traceback.format_exc()
             self.error.emit(
@@ -577,20 +584,8 @@ class Reconstruction(UIGraph):
             )
             logging.error("Error while loading '{}'.".format(filepath))
             logging.error(trace)
-            return False
 
-    @Slot(QUrl, result=bool)
+        return False
-    @Slot(QUrl, bool, bool, result=bool)
-    def loadUrl(self, url, setupProjectFile=True, publishOutputs=False):
-        if isinstance(url, (QUrl)):
-            # depending how the QUrl has been initialized,
-            # toLocalFile() may return the local path or an empty string
-            localFile = url.toLocalFile()
-            if not localFile:
-                localFile = url.toString()
-        else:
-            localFile = url
-        return self.load(localFile, setupProjectFile, publishOutputs)
 
     def onGraphChanged(self):
         """ React to the change of the internal graph. """
@@ -860,7 +855,7 @@ class Reconstruction(UIGraph):
                     )
                 )
             else:
-                return self.loadUrl(filesByType["meshroomScenes"][0])
+                return self.load(filesByType["meshroomScenes"][0])