From 7eab289d302ee35562d9625cbb877b57f9fd5053 Mon Sep 17 00:00:00 2001 From: Yann Lanthony Date: Thu, 6 Feb 2025 16:46:04 +0100 Subject: [PATCH] [core] Graph: initial refactoring of graph loading API and logic * API Instead of having a single `load` function that exposes in its API some elements only applicable to initializing a graph from a templates, split it into 2 distinct functions: `load` and `initFromTemplate`. Apply those changes to users of the API (UI, CLI), and simplify Graph wrapper classes to better align with those concepts. * Deserialization Reduce the cognitive complexity of the deserizalization process by splitting it into more atomic functions, while maintaining the current behavior. --- bin/meshroom_batch | 4 +- meshroom/core/graph.py | 195 ++++++++++++++++++-------------- meshroom/core/typing.py | 8 ++ meshroom/ui/graph.py | 18 +-- meshroom/ui/qml/Application.qml | 6 +- meshroom/ui/qml/Homepage.qml | 2 +- meshroom/ui/qml/main.qml | 2 +- meshroom/ui/reconstruction.py | 45 ++++---- 8 files changed, 153 insertions(+), 127 deletions(-) create mode 100644 meshroom/core/typing.py diff --git a/bin/meshroom_batch b/bin/meshroom_batch index 36b8fef6..6bee4c1f 100755 --- a/bin/meshroom_batch +++ b/bin/meshroom_batch @@ -154,10 +154,10 @@ with meshroom.core.graph.GraphModification(graph): # initialize template pipeline loweredPipelineTemplates = dict((k.lower(), v) for k, v in meshroom.core.pipelineTemplates.items()) if args.pipeline.lower() in loweredPipelineTemplates: - graph.load(loweredPipelineTemplates[args.pipeline.lower()], setupProjectFile=False, publishOutputs=True if args.output else False) + graph.initFromTemplate(loweredPipelineTemplates[args.pipeline.lower()], publishOutputs=True if args.output else False) else: # custom pipeline - graph.load(args.pipeline, setupProjectFile=False, publishOutputs=True if args.output else False) + graph.initFromTemplate(args.pipeline, publishOutputs=True if args.output else False) def parseInputs(inputs, uniqueInitNode): """Utility method for parsing the input and inputRecursive arguments.""" diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index da68b288..1f454b20 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -4,6 +4,7 @@ import json import logging import os import re +from typing import Optional import weakref from collections import defaultdict, OrderedDict from contextlib import contextmanager @@ -18,6 +19,7 @@ from meshroom.core.attribute import Attribute, ListAttribute, GroupAttribute from meshroom.core.exception import GraphCompatibilityError, StopGraphVisit, StopBranchVisit from meshroom.core.node import Status, Node, CompatibilityNode from meshroom.core.nodeFactory import nodeFactory +from meshroom.core.typing import PathLike # Replace default encoder to support Enums @@ -149,6 +151,21 @@ def changeTopology(func): return decorator +def blockNodeCallbacks(func): + """ + Graph methods loading serialized graph content must be decorated with 'blockNodeCallbacks', + to avoid attribute changed callbacks defined on node descriptions to be triggered during + this process. + """ + def inner(self, *args, **kwargs): + self._loading = True + try: + return func(self, *args, **kwargs) + finally: + self._loading = False + return inner + + class Graph(BaseObject): """ _________________ _________________ _________________ @@ -260,37 +277,88 @@ class Graph(BaseObject): return self._saving @Slot(str) - def load(self, filepath, setupProjectFile=True, importProject=False, publishOutputs=False): + def load(self, filepath: PathLike): """ - Load a Meshroom graph ".mg" file. + Load a Meshroom Graph ".mg" file in place. Args: - filepath: project filepath to load - setupProjectFile: Store the reference to the project file and setup the cache directory. - If false, it only loads the graph of the project file as a template. - importProject: True if the project that is loaded will be imported in the current graph, instead - of opened. - publishOutputs: True if "Publish" nodes from templates should not be ignored. + filepath: The path to the Meshroom Graph file to load. """ - self._loading = True - try: - return self._load(filepath, setupProjectFile, importProject, publishOutputs) - finally: - self._loading = False + self._deserialize(Graph._loadGraphData(filepath)) + self._setFilepath(filepath) + self._fileDateVersion = os.path.getmtime(filepath) - def _load(self, filepath, setupProjectFile, importProject, publishOutputs): - if not importProject: - self.clear() - with open(filepath) as jsonFile: - fileData = json.load(jsonFile) + def initFromTemplate(self, filepath: PathLike, publishOutputs: bool = False): + """ + Deserialize a template Meshroom Graph ".mg" file in place. - self.header = fileData.get(Graph.IO.Keys.Header, {}) + When initializing from a template, the internal filepath of the graph instance is not set. + Saving the file on disk will require to specify a filepath. - fileVersion = self.header.get(Graph.IO.Keys.FileVersion, "0.0") - # Retro-compatibility for all project files with the previous UID format - if Version(fileVersion) < Version("2.0"): + Args: + filepath: The path to the Meshroom Graph file to load. + publishOutputs: (optional) Whether to keep 'Publish' nodes. + """ + self._deserialize(Graph._loadGraphData(filepath)) + + if not publishOutputs: + for node in [node for node in self.nodes if node.nodeType == "Publish"]: + self.removeNode(node.name) + + @staticmethod + def _loadGraphData(filepath: PathLike) -> dict: + """Deserialize the content of the Meshroom Graph file at `filepath` to a dictionnary.""" + with open(filepath) as file: + graphData = json.load(file) + return graphData + + @blockNodeCallbacks + def _deserialize(self, graphData: dict): + """Deserialize `graphData` in the current Graph instance. + + Args: + graphData: The serialized Graph. + """ + self.clear() + self.header = graphData.get(Graph.IO.Keys.Header, {}) + fileVersion = Version(self.header.get(Graph.IO.Keys.FileVersion, "0.0")) + graphContent = self._normalizeGraphContent(graphData, fileVersion) + isTemplate = self.header.get("template", False) + + with GraphModification(self): + # iterate over nodes sorted by suffix index in their names + for nodeName, nodeData in sorted( + graphContent.items(), key=lambda x: self.getNodeIndexFromName(x[0]) + ): + self._deserializeNode(nodeData, nodeName) + + # Create graph edges by resolving attributes expressions + self._applyExpr() + + # Templates are specific: they contain only the minimal amount of + # serialized data to describe the graph structure. + # They are not meant to be computed: therefore, we can early return here, + # as uid conflict evaluation is only meaningful for nodes with computed data. + if isTemplate: + return + + # By this point, the graph has been fully loaded and an updateInternals has been triggered, so all the + # nodes' links have been resolved and their UID computations are all complete. + # It is now possible to check whether the UIDs stored in the graph file for each node correspond to the ones + # that were computed. + self.updateInternals() + self._evaluateUidConflicts(graphContent) + try: + self._applyExpr() + except Exception as e: + logging.warning(e) + + def _normalizeGraphContent(self, graphData: dict, fileVersion: Version) -> dict: + graphContent = graphData.get(Graph.IO.Keys.Graph, graphData) + + if fileVersion < Version("2.0"): # For internal folders, all "{uid0}" keys should be replaced with "{uid}" - updatedFileData = json.dumps(fileData).replace("{uid0}", "{uid}") + updatedFileData = json.dumps(graphContent).replace("{uid0}", "{uid}") # For fileVersion < 2.0, the nodes' UID is stored as: # "uids": {"0": "hashvalue"} @@ -302,74 +370,25 @@ class Graph(BaseObject): uid = occ.split("\"")[-2] # UID is second to last element newUidStr = r'"uid": "{}"'.format(uid) updatedFileData = updatedFileData.replace(occ, newUidStr) - fileData = json.loads(updatedFileData) + graphContent = json.loads(updatedFileData) - # Older versions of Meshroom files only contained the serialized nodes - graphData = fileData.get(Graph.IO.Keys.Graph, fileData) + return graphContent - if importProject: - self._importedNodes.clear() - graphData = self.updateImportedProject(graphData) + def _deserializeNode(self, nodeData: dict, nodeName: str): + # Retrieve version from + # 1. nodeData: node saved from a CompatibilityNode + # 2. nodesVersion in file header: node saved from a Node + # 3. fallback behavior: default to "0.0" + if "version" not in nodeData: + nodeData["version"] = self._getNodeTypeVersionFromHeader(nodeData["nodeType"], "0.0") + inTemplate = self.header.get("template", False) + node = nodeFactory(nodeData, nodeName, inTemplate=inTemplate) + self._addNode(node, nodeName) + return node - if not isinstance(graphData, dict): - raise RuntimeError('loadGraph error: Graph is not a dict. File: {}'.format(filepath)) - - nodesVersions = self.header.get(Graph.IO.Keys.NodesVersions, {}) - - self._fileDateVersion = os.path.getmtime(filepath) - - # Check whether the file was saved as a template in minimal mode - isTemplate = self.header.get("template", False) - - with GraphModification(self): - # iterate over nodes sorted by suffix index in their names - for nodeName, nodeData in sorted(graphData.items(), key=lambda x: self.getNodeIndexFromName(x[0])): - if not isinstance(nodeData, dict): - raise RuntimeError('loadGraph error: Node is not a dict. File: {}'.format(filepath)) - - # retrieve version from - # 1. nodeData: node saved from a CompatibilityNode - # 2. nodesVersion in file header: node saved from a Node - # 3. fallback to no version "0.0": retro-compatibility - if "version" not in nodeData: - nodeData["version"] = nodesVersions.get(nodeData["nodeType"], "0.0") - - # if the node is a "Publish" node and comes from a template file, it should be ignored - # unless publishOutputs is True - if isTemplate and not publishOutputs and nodeData["nodeType"] == "Publish": - continue - - n = nodeFactory(nodeData, nodeName, inTemplate=isTemplate) - - # Add node to the graph with raw attributes values - self._addNode(n, nodeName) - - if importProject: - self._importedNodes.add(n) - - # Create graph edges by resolving attributes expressions - self._applyExpr() - - if setupProjectFile: - # Update filepath related members - # Note: needs to be done at the end as it will trigger an updateInternals. - self._setFilepath(filepath) - elif not isTemplate: - # If no filepath is being set but the graph is not a template, trigger an updateInternals either way. - self.updateInternals() - - # By this point, the graph has been fully loaded and an updateInternals has been triggered, so all the - # nodes' links have been resolved and their UID computations are all complete. - # It is now possible to check whether the UIDs stored in the graph file for each node correspond to the ones - # that were computed. - if not isTemplate: # UIDs are not stored in templates - self._evaluateUidConflicts(graphData) - try: - self._applyExpr() - except Exception as e: - logging.warning(e) - - return True + def _getNodeTypeVersionFromHeader(self, nodeType: str, default: Optional[str] = None) -> Optional[str]: + nodeVersions = self.header.get(Graph.IO.Keys.NodesVersions, {}) + return nodeVersions.get(nodeType, default) def _evaluateUidConflicts(self, data): """ diff --git a/meshroom/core/typing.py b/meshroom/core/typing.py new file mode 100644 index 00000000..f526fb3e --- /dev/null +++ b/meshroom/core/typing.py @@ -0,0 +1,8 @@ +""" +Common typing aliases used in Meshroom. +""" + +from pathlib import Path +from typing import Union + +PathLike = Union[Path, str] diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index 0cde6b2d..2b699816 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -451,17 +451,21 @@ class UIGraph(QObject): self.stopExecution() self._chunksMonitor.stop() - @Slot(str, result=bool) - def loadGraph(self, filepath, setupProjectFile=True, publishOutputs=False): - g = Graph('') - status = True + @Slot(str) + def loadGraph(self, filepath): + g = Graph("") if filepath: - status = g.load(filepath, setupProjectFile, importProject=False, publishOutputs=publishOutputs) + g.load(filepath) if not os.path.exists(g.cacheDir): os.mkdir(g.cacheDir) - g.fileDateVersion = os.path.getmtime(filepath) self.setGraph(g) - return status + + @Slot(str, bool, result=bool) + def initFromTemplate(self, filepath, publishOutputs=False): + graph = Graph("") + if filepath: + graph.initFromTemplate(filepath, publishOutputs=publishOutputs) + self.setGraph(graph) @Slot(QUrl, result="QVariantList") @Slot(QUrl, QPoint, result="QVariantList") diff --git a/meshroom/ui/qml/Application.qml b/meshroom/ui/qml/Application.qml index a27c1a5c..e96677d7 100644 --- a/meshroom/ui/qml/Application.qml +++ b/meshroom/ui/qml/Application.qml @@ -185,7 +185,7 @@ Page { nameFilters: ["Meshroom Graphs (*.mg)"] onAccepted: { // Open the template as a regular file - if (_reconstruction.loadUrl(currentFile, true, true)) { + if (_reconstruction.load(currentFile)) { MeshroomApp.addRecentProjectFile(currentFile.toString()) } } @@ -400,7 +400,7 @@ Page { text: "Reload File" onClicked: { - _reconstruction.loadUrl(_reconstruction.graph.filepath) + _reconstruction.load(_reconstruction.graph.filepath) fileModifiedDialog.close() } } @@ -705,7 +705,7 @@ Page { MenuItem { onTriggered: ensureSaved(function() { openRecentMenu.dismiss() - if (_reconstruction.loadUrl(modelData["path"])) { + if (_reconstruction.load(modelData["path"])) { MeshroomApp.addRecentProjectFile(modelData["path"]) } else { MeshroomApp.removeRecentProjectFile(modelData["path"]) diff --git a/meshroom/ui/qml/Homepage.qml b/meshroom/ui/qml/Homepage.qml index ee860c44..7708db3c 100644 --- a/meshroom/ui/qml/Homepage.qml +++ b/meshroom/ui/qml/Homepage.qml @@ -389,7 +389,7 @@ Page { } else { // Open project mainStack.push("Application.qml") - if (_reconstruction.loadUrl(modelData["path"])) { + if (_reconstruction.load(modelData["path"])) { MeshroomApp.addRecentProjectFile(modelData["path"]) } else { MeshroomApp.removeRecentProjectFile(modelData["path"]) diff --git a/meshroom/ui/qml/main.qml b/meshroom/ui/qml/main.qml index 16940a74..20c2f81f 100644 --- a/meshroom/ui/qml/main.qml +++ b/meshroom/ui/qml/main.qml @@ -128,7 +128,7 @@ ApplicationWindow { if (mainStack.currentItem instanceof Homepage) { mainStack.push("Application.qml") } - if (_reconstruction.loadUrl(currentFile)) { + if (_reconstruction.load(currentFile)) { MeshroomApp.addRecentProjectFile(currentFile.toString()) } } diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index c774527f..94d926a0 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -5,6 +5,7 @@ import os from collections.abc import Iterable from multiprocessing.pool import ThreadPool from threading import Thread +from typing import Callable from PySide6.QtCore import QObject, Slot, Property, Signal, QUrl, QSizeF, QPoint from PySide6.QtGui import QMatrix4x4, QMatrix3x3, QQuaternion, QVector3D, QVector2D @@ -534,17 +535,24 @@ class Reconstruction(UIGraph): # - correct pipeline name but the case does not match (e.g. panoramaHDR instead of panoramaHdr) # - lowercase pipeline name given through the "New Pipeline" menu loweredPipelineTemplates = dict((k.lower(), v) for k, v in meshroom.core.pipelineTemplates.items()) - if p.lower() in loweredPipelineTemplates: - self.load(loweredPipelineTemplates[p.lower()], setupProjectFile=False) - else: - # use the user-provided default project file - self.load(p, setupProjectFile=False) + filepath = loweredPipelineTemplates.get(p.lower(), p) + return self._loadWithErrorReport(self.initFromTemplate, filepath) @Slot(str, result=bool) - def load(self, filepath, setupProjectFile=True, publishOutputs=False): + @Slot(QUrl, result=bool) + def load(self, url): + if isinstance(url, QUrl): + # depending how the QUrl has been initialized, + # toLocalFile() may return the local path or an empty string + localFile = url.toLocalFile() or url.toString() + else: + localFile = url + return self._loadWithErrorReport(self.loadGraph, localFile) + + def _loadWithErrorReport(self, loadFunction: Callable[[str], None], filepath: str): logging.info(f"Load project file: '{filepath}'") try: - status = super(Reconstruction, self).loadGraph(filepath, setupProjectFile, publishOutputs) + loadFunction(filepath) # warn about pre-release projects being automatically upgraded if Version(self._graph.fileReleaseVersion).major == "0": self.warning.emit(Message( @@ -554,8 +562,8 @@ class Reconstruction(UIGraph): "Open it with the corresponding version of Meshroom to recover your data." )) self.setActive(True) - return status - except FileNotFoundError as e: + return True + except FileNotFoundError: self.error.emit( Message( "No Such File", @@ -564,8 +572,7 @@ class Reconstruction(UIGraph): ) ) logging.error("Error while loading '{}': No Such File.".format(filepath)) - return False - except Exception as e: + except Exception: import traceback trace = traceback.format_exc() self.error.emit( @@ -577,20 +584,8 @@ class Reconstruction(UIGraph): ) logging.error("Error while loading '{}'.".format(filepath)) logging.error(trace) - return False - @Slot(QUrl, result=bool) - @Slot(QUrl, bool, bool, result=bool) - def loadUrl(self, url, setupProjectFile=True, publishOutputs=False): - if isinstance(url, (QUrl)): - # depending how the QUrl has been initialized, - # toLocalFile() may return the local path or an empty string - localFile = url.toLocalFile() - if not localFile: - localFile = url.toString() - else: - localFile = url - return self.load(localFile, setupProjectFile, publishOutputs) + return False def onGraphChanged(self): """ React to the change of the internal graph. """ @@ -860,7 +855,7 @@ class Reconstruction(UIGraph): ) ) else: - return self.loadUrl(filesByType["meshroomScenes"][0]) + return self.load(filesByType["meshroomScenes"][0])