[graph] use 'types' module for data models and signals/slots

* store Nodes and Attributes in Models
* expose key attributes/methods as Properties/Slots
* update command line scripts

meshroom/compute.py

@@ -26,7 +26,7 @@ graph.update()
 
 if args.node:
     # Execute the node
-    node = graph.nodes[args.node]
+    node = graph.node(args.node)
     if node.isAlreadySubmitted():
         print('Error: Node is already submitted with status "{}"'.format(node.status.status.name))
         exit(-1)
@@ -35,6 +35,6 @@ if args.node:
 else:
     startNodes = None
     if args.graph:
-        startNodes = [graph.nodes[args.graph]]
+        startNodes = [graph.nodes(args.graph)]
     pg.execute(graph, startNodes=startNodes, force=args.force)
 

meshroom/processGraph/graph.py

@@ -5,8 +5,6 @@ import json
 import os
 import psutil
 import shutil
-import subprocess
-import time
 import uuid
 from collections import defaultdict
 from enum import Enum  # available by default in python3. For python2: "pip install enum34"
@@ -14,6 +12,7 @@ from pprint import pprint
 
 from . import stats
 from meshroom import processGraph as pg
+from meshroom.types import BaseObject, Model, Slot, Signal, Property
 
 # Replace default encoder to support Enums
 DefaultJSONEncoder = json.JSONEncoder  # store the original one
@@ -49,24 +48,40 @@ def hash(v):
     return hashObject.hexdigest()
 
 
-class Attribute:
+class Attribute(BaseObject):
     """
     """
 
-    def __init__(self, name, node, attributeDesc):
+    def __init__(self, name, node, attributeDesc, parent = None):
-        self.attrName = name
+        super(Attribute, self).__init__(parent)
-        self.node = node
+        self._name = name
-        self._value = attributeDesc.__dict__.get('value', None)
+        self.node = node  # type: Node
         self.attributeDesc = attributeDesc
+        self._value = getattr(attributeDesc, 'value', None)
+        self._label = getattr(attributeDesc, 'label', None)
 
     def absoluteName(self):
-        return '{}.{}.{}'.format(self.node.graph.name, self.node.name, self.attrName)
+        return '{}.{}.{}'.format(self.node.graph.name, self.node.name, self._name)
 
-    def name(self):
+    def fullName(self):
-        """
+        """ Name inside the Graph: nodeName.name """
-        Name inside the Graph.
+        return '{}.{}'.format(self.node.name, self._name)
-        """
+
-        return '{}.{}'.format(self.node.name, self.attrName)
+    def getName(self):
+        """ Attribute name """
+        return self._name
+
+    def getLabel(self):
+        return self._label
+
+    def getValue(self):
+        return self._value
+
+    def setValue(self, value):
+        if self._value == value:
+            return
+        self._value = value
+        self.valueChanged.emit()
 
     def uid(self):
         """
@@ -107,16 +122,21 @@ class Attribute:
             g = self.node.graph
             link = v[1:-1]
             linkNode, linkAttr = link.split('.')
-            g.addEdge(g.nodes[linkNode].attributes[linkAttr], self)
+            g.addEdge(g.node(linkNode).attribute(linkAttr), self)
             self._value = ""
 
     def getExportValue(self):
         value = self._value
         # print('getExportValue: ', self.name(), value, self.isLink())
         if self.isLink():
-            value = '{' + self.node.graph.edges[self].name() + '}'
+            value = '{' + self.node.graph.edges[self].fullName() + '}'
         return value
 
+    name = Property(str, getName, constant=True)
+    label = Property(str, getLabel, constant=True)
+    valueChanged = Signal()
+    value = Property("QVariant", getValue, setValue, notify=valueChanged)
+
 
 class Status(Enum):
     """
@@ -153,19 +173,20 @@ class StatusData:
         self.env = d.get('env', '')
 
 
-class Node:
+class Node(BaseObject):
     """
     """
-    name = None
-    graph = None
 
-    def __init__(self, nodeDesc, **kwargs):
+    def __init__(self, nodeDesc, parent=None, **kwargs):
+        super(Node, self).__init__(parent)
+        self._name = None  # type: str
+        self.graph = None  # type: Graph
         self.nodeDesc = pg.nodesDesc[nodeDesc]()
-        self.attributes = {}
+        self._attributes = Model(parent=self)
         self.attributesPerUid = defaultdict(set)
         self._initFromDesc()
         for k, v in kwargs.items():
-            self.attributes[k]._value = v
+            self.attribute(k)._value = v
         self.status = StatusData(self.name, self.nodeType())
         self.statistics = stats.Statistics()
 
@@ -176,26 +197,36 @@ class Node:
             return object.__getattr__(self, k)
         except AttributeError:
             try:
-                return self.attributes[k]
+                return self.attribute(k)
             except KeyError:
                 raise AttributeError(k)
 
+    def getName(self):
+        return self._name
+
+    @Slot(str, result=Attribute)
+    def attribute(self, name):
+        return self._attributes.get(name)
+
+    def getAttributes(self):
+        return self._attributes
+
     def _initFromDesc(self):
         # Init from class members
         for name, desc in self.nodeDesc.__class__.__dict__.items():
             if issubclass(desc.__class__, pg.desc.Attribute):
-                self.attributes[name] = Attribute(name, self, desc)
+                self._attributes.add(Attribute(name, self, desc))
         # Init from instance members
         for name, desc in self.nodeDesc.__dict__.items():
             if issubclass(desc.__class__, pg.desc.Attribute):
-                self.attributes[name] = Attribute(name, self, desc)
+                self._attributes.add(Attribute(name, self, desc))
         # List attributes per uid
-        for name, attr in self.attributes.items():
+        for attr in self._attributes:
             for uidIndex in attr.attributeDesc.uid:
                 self.attributesPerUid[uidIndex].add(attr)
 
     def _applyExpr(self):
-        for attr in self.attributes.values():
+        for attr in self._attributes:
             attr._applyExpr()
 
     def nodeType(self):
@@ -205,7 +236,7 @@ class Node:
         return self.nodeUid
 
     def _updateUid(self):
-        hashInputParams = [(attr.attrName, attr.uid()) for attr in self.attributes.values() if
+        hashInputParams = [(attr.getName(), attr.uid()) for attr in self._attributes if
                            not attr.attributeDesc.isOutput]
         hashInputParams.sort()
         self.nodeUid = hash(tuple([b for a, b in hashInputParams]))
@@ -215,7 +246,7 @@ class Node:
         return self.graph.getDepth(self)
 
     def toDict(self):
-        attributes = {k: v.getExportValue() for k, v in self.attributes.items()}
+        attributes = {k: v.getExportValue() for k, v in self._attributes.objects.items()}
         return {
             'nodeType': self.nodeType(),
             'attributes': {k: v for k, v in attributes.items() if v is not None},  # filter empty values
@@ -228,17 +259,17 @@ class Node:
             'cache': pg.cacheFolder,
             }
         for uidIndex, associatedAttributes in self.attributesPerUid.items():
-            assAttr = [(a.attrName, a.uid()) for a in associatedAttributes]
+            assAttr = [(a.getName(), a.uid()) for a in associatedAttributes]
             assAttr.sort()
             self._cmdVars['uid{}'.format(uidIndex)] = hash(tuple([b for a, b in assAttr]))
 
-        for name, attr in self.attributes.items():
+        for attr in self._attributes:
             if attr.attributeDesc.isOutput:
                 attr._value = attr.attributeDesc.value.format(
                     nodeType=self.nodeType(),
                     **self._cmdVars)  # self._cmdVars only contains uids at this step
 
-        for name, attr in self.attributes.items():
+        for name, attr in self._attributes.objects.items():
             linkAttr = attr.getLinkParam()
             v = attr._value
             if linkAttr:
@@ -315,7 +346,7 @@ class Node:
 
     def upgradeStatusTo(self, newStatus):
         if int(newStatus.value) <= int(self.status.status.value):
-            print('WARNING: downgrade status on node "{}" from {} to {}'.format(self.name, self.status.status.name,
+            print('WARNING: downgrade status on node "{}" from {} to {}'.format(self._name, self.status.status.name,
                                                                                 newStatus))
         self.status.status = newStatus
         self.saveStatusFile()
@@ -369,6 +400,12 @@ class Node:
     def endSequence(self):
         pass
 
+    def getStatus(self):
+        return self.status
+
+    name = Property(str, getName, constant=True)
+    attributes = Property(BaseObject, getAttributes, constant=True)
+
 
 WHITE = 0
 GRAY = 1
@@ -407,7 +444,7 @@ class Visitor:
         pass
 
 
-class Graph:
+class Graph(BaseObject):
     """
     _________________      _________________      _________________
     |               |      |               |      |               |
@@ -423,10 +460,31 @@ class Graph:
 
     """
 
-    def __init__(self, name):
+    def __init__(self, name, parent=None):
+        super(Graph, self).__init__(parent)
         self.name = name
-        self.nodes = {}
         self.edges = {}  # key/input <- value/output, it is organized this way because key/input can have only one connection.
+        self._nodes = Model(parent=self)
+
+    def clear(self):
+        self._nodes.clear()
+        self.edges = {}
+
+    @Slot(str)
+    def load(self, filepath):
+        self.clear()
+        with open(filepath) as jsonFile:
+            graphData = json.load(jsonFile)
+        if not isinstance(graphData, dict):
+            raise RuntimeError('loadGraph error: Graph is not a dict. File: {}'.format(filepath))
+
+        self.name = os.path.splitext(os.path.basename(filepath))[0]
+        for nodeName, nodeData in graphData.items():
+            if not isinstance(nodeData, dict):
+                raise RuntimeError('loadGraph error: Node is not a dict. File: {}'.format(filepath))
+            n = Node(nodeData['nodeType'], parent=self, **nodeData['attributes'])
+            self.addNode(n, uniqueName=nodeName)
+        self._applyExpr()
 
     def addNode(self, node, uniqueName=None):
         if node.graph is not None and node.graph != self:
@@ -434,15 +492,23 @@ class Graph:
                 'Node "{}" cannot be part of the Graph "{}", as it is already part of the other graph "{}".'.format(
                     node.nodeType(), self.name, node.graph.name))
         if uniqueName:
-            assert uniqueName not in self.nodes
+            assert uniqueName not in self._nodes.objects
-            node.name = uniqueName
+            node._name = uniqueName
         else:
-            node.name = self._createUniqueNodeName(node.nodeType())
+            node._name = self._createUniqueNodeName(node.nodeType())
         node.graph = self
-        self.nodes[node.name] = node
+        self._nodes.add(node)
 
         return node
 
+    def removeNode(self, nodeName):
+        node = self.node(nodeName)
+        self._nodes.pop(nodeName)
+        for attr in node._attributes:
+            if attr in self.edges:
+                self.edges.pop(attr)
+
+    @Slot(str, result=Node)
     def addNewNode(self, nodeType, **kwargs):
         """
 
@@ -451,19 +517,22 @@ class Graph:
         :return:
         :rtype: Node
         """
-        return self.addNode(Node(nodeDesc=nodeType, **kwargs))
+        return self.addNode(Node(nodeDesc=nodeType, parent=self, **kwargs))
 
     def _createUniqueNodeName(self, inputName):
         i = 1
         while i:
             newName = "{name}_{index}".format(name=inputName, index=i)
-            if newName not in self.nodes:
+            if newName not in self._nodes.objects:
                 return newName
             i += 1
 
+    def node(self, nodeName):
+        return self._nodes.get(nodeName)
+
     def getLeaves(self):
         nodesWithOutput = set([outputAttr.node for outputAttr in self.edges.values()])
-        return set(self.nodes.values()) - nodesWithOutput
+        return set(self._nodes) - nodesWithOutput
 
     def addEdge(self, outputAttr, inputAttr):
         assert isinstance(outputAttr, Attribute)
@@ -492,7 +561,7 @@ class Graph:
     def dfs(self, visitor, startNodes=None):
         nodeChildren = self._getNodeEdges()
         colors = {}
-        for u in self.nodes.values():
+        for u in self._nodes:
             colors[u] = WHITE
         time = 0
         if startNodes:
@@ -544,12 +613,17 @@ class Graph:
         return nodes
 
     def _applyExpr(self):
-        for node in self.nodes.values():
+        for node in self._nodes:
             node._applyExpr()
 
     def toDict(self):
-        return {k: node.toDict() for k, node in self.nodes.items()}
+        return {k: node.toDict() for k, node in self._nodes.objects.items()}
 
+    @Slot(result=str)
+    def asString(self):
+        return str(self.toDict())
+
+    @Slot(str)
     def save(self, filepath):
         """
         """
@@ -564,35 +638,32 @@ class Graph:
             node.updateInternals()
 
     def updateStatusFromCache(self):
-        for node in self.nodes.values():
+        for node in self._nodes:
             node.updateStatusFromCache()
 
     def updateStatisticsFromCache(self):
-        for node in self.nodes.values():
+        for node in self._nodes:
             node.updateStatisticsFromCache()
 
     def update(self):
         self.updateInternals()
         self.updateStatusFromCache()
 
+    @property
+    def nodes(self):
+        return self._nodes
+
+    nodes = Property(BaseObject, nodes.fget, constant=True)
+
 
 def loadGraph(filepath):
     """
     """
-    with open(filepath) as jsonFile:
+    graph = Graph("")
-        graphData = json.load(jsonFile)
+    graph.load(filepath)
-    if not isinstance(graphData, dict):
-        raise RuntimeError('loadGraph error: Graph is not a dict. File: {}'.format(filepath))
-
-    graph = Graph(os.path.splitext(os.path.basename(filepath))[0])
-    for nodeName, nodeData in graphData.items():
-        if not isinstance(nodeData, dict):
-            raise RuntimeError('loadGraph error: Node is not a dict. File: {}'.format(filepath))
-        n = Node(nodeData['nodeType'], **nodeData['attributes'])
-        graph.addNode(n, uniqueName=nodeName)
-    graph._applyExpr()
     return graph
 
+
 def getAlreadySubmittedNodes(nodes):
     out = []
     for node in nodes:
@@ -600,6 +671,7 @@ def getAlreadySubmittedNodes(nodes):
             out.append(node)
     return out
 
+
 def execute(graph, startNodes=None, force=False):
     """
     """

meshroom/statistics.py

@@ -56,11 +56,11 @@ if args.node:
     if args.node not in graph.nodes:
         print('ERROR: node "{}" does not exist in file "{}".'.format(args.node, args.graphFile))
         exit(-1)
-    nodes = [graph.nodes[args.node]]
+    nodes = [graph.nodes(args.node)]
 else:
     startNodes = None
     if args.graph:
-        startNodes = [graph.nodes[args.graph]]
+        startNodes = [graph.nodes(args.graph)]
     nodes = graph.dfsNodesOnFinish(startNodes=startNodes)
 
 for node in nodes:

meshroom/status.py

@@ -26,10 +26,10 @@ graph = pg.loadGraph(args.graphFile)
 graph.update()
 
 if args.node:
-    if args.node not in graph.nodes:
+    node = graph.node(args.node)
+    if node is None:
         print('ERROR: node "{}" does not exist in file "{}".'.format(args.node, args.graphFile))
         exit(-1)
-    node = graph.nodes[args.node]
     print('{}: {}'.format(node.name, node.status.status.name))
     if args.verbose:
         print('statusFile: ', node.statusFile())
@@ -37,7 +37,7 @@ if args.node:
 else:
     startNodes = None
     if args.graph:
-        startNodes = [graph.nodes[args.graph]]
+        startNodes = [graph.nodes(args.graph)]
     nodes = graph.dfsNodesOnFinish(startNodes=startNodes)
     for node in nodes:
         print('{}: {}'.format(node.name, node.status.status.name))