[core] split graph.py into separate modules

core.graph is now splitted into: 
  * graph.py
  * node.py
  * attribute.py

bin/meshroom_compute

@@ -2,7 +2,7 @@
 import argparse
 
 import meshroom.core.graph
-from meshroom.core.graph import Status
+from meshroom.core.node import Status
 
 
 parser = argparse.ArgumentParser(description='Execute a Graph of processes.')
@@ -57,5 +57,5 @@ else:
     toNodes = None
     if args.toNode:
         toNodes = graph.findNodes([args.toNode])
-    meshroom.core.graph.execute(graph, toNodes=toNodes, forceCompute=args.forceCompute, forceStatus=args.forceStatus)
+    meshroom.core.graph.executeGraph(graph, toNodes=toNodes, forceCompute=args.forceCompute, forceStatus=args.forceStatus)
 

bin/meshroom_photogrammetry

@@ -61,5 +61,5 @@ toNodes = None
 if args.toNode:
     toNodes = graph.findNodes(args.toNode)
 
-meshroom.core.graph.execute(graph, toNodes=toNodes, forceCompute=args.forceCompute, forceStatus=args.forceStatus)
+meshroom.core.graph.executeGraph(graph, toNodes=toNodes, forceCompute=args.forceCompute, forceStatus=args.forceStatus)
 

meshroom/core/__init__.py

@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+import hashlib
 from contextlib import contextmanager
 import importlib
 import inspect
@@ -26,6 +27,12 @@ nodesDesc = {}
 submitters = {}
 
 
+def hashValue(value):
+    """ Hash 'value' using sha1. """
+    hashObject = hashlib.sha1(str(value).encode('utf-8'))
+    return hashObject.hexdigest()
+
+
 @contextmanager
 def add_to_path(p):
     import sys

meshroom/core/attribute.py

@@ -0,0 +1,402 @@
+#!/usr/bin/env python
+# coding:utf-8
+import collections
+import re
+import weakref
+
+from meshroom.common import BaseObject, Property, Variant, Signal, ListModel, DictModel
+from meshroom.core import desc, pyCompatibility, hashValue
+
+
+def attribute_factory(description, value, isOutput, node, root=None, parent=None):
+    """
+    Create an Attribute based on description type.
+
+    Args:
+        description: the Attribute description
+        value:  value of the Attribute. Will be set if not None.
+        isOutput: whether is Attribute is an output attribute.
+        node (Node): node owning the Attribute. Note that the created Attribute is not added to Node's attributes
+        root: (optional) parent Attribute (must be ListAttribute or GroupAttribute)
+        parent (BaseObject): (optional) the parent BaseObject if any
+    """
+    if isinstance(description, desc.GroupAttribute):
+        cls = GroupAttribute
+    elif isinstance(description, desc.ListAttribute):
+        cls = ListAttribute
+    else:
+        cls = Attribute
+    attr = cls(node, description, isOutput, root, parent)
+    if value is not None:
+        attr.value = value
+    return attr
+
+
+class Attribute(BaseObject):
+    """
+    """
+    stringIsLinkRe = re.compile('^\{[A-Za-z]+[A-Za-z0-9_.]*\}$')
+
+    def __init__(self, node, attributeDesc, isOutput, root=None, parent=None):
+        """
+        Attribute constructor
+
+        Args:
+            node (Node): the Node hosting this Attribute
+            attributeDesc (desc.Attribute): the description of this Attribute
+            isOutput (bool): whether this Attribute is an output of the Node
+            root (Attribute): (optional) the root Attribute (List or Group) containing this one
+            parent (BaseObject): (optional) the parent BaseObject
+        """
+        super(Attribute, self).__init__(parent)
+        self._name = attributeDesc.name
+        self._root = None if root is None else weakref.ref(root)
+        self._node = weakref.ref(node)
+        self.attributeDesc = attributeDesc
+        self._isOutput = isOutput
+        self._value = attributeDesc.value
+        self._label = attributeDesc.label
+
+        # invalidation value for output attributes
+        self._invalidationValue = ""
+
+    @property
+    def node(self):
+        return self._node()
+
+    @property
+    def root(self):
+        return self._root() if self._root else None
+
+    def absoluteName(self):
+        return '{}.{}.{}'.format(self.node.graph.name, self.node.name, self._name)
+
+    def fullName(self):
+        """ Name inside the Graph: nodeName.name """
+        if isinstance(self.root, ListAttribute):
+            return '{}[{}]'.format(self.root.fullName(), self.root.index(self))
+        elif isinstance(self.root, GroupAttribute):
+            return '{}.{}'.format(self.root.fullName(), self._name)
+        return '{}.{}'.format(self.node.name, self._name)
+
+    def asLinkExpr(self):
+        """ Return link expression for this Attribute """
+        return "{" + self.fullName() + "}"
+
+    def getName(self):
+        """ Attribute name """
+        return self._name
+
+    def getType(self):
+        return self.attributeDesc.__class__.__name__
+
+    def getLabel(self):
+        return self._label
+
+    def _get_value(self):
+        return self.getLinkParam().value if self.isLink else self._value
+
+    def _set_value(self, value):
+        if self._value == value:
+            return
+
+        if isinstance(value, Attribute) or Attribute.isLinkExpression(value):
+            # if we set a link to another attribute
+            self._value = value
+        else:
+            # if we set a new value, we use the attribute descriptor validator to check the validity of the value
+            # and apply some conversion if needed
+            convertedValue = self.desc.validateValue(value)
+            self._value = convertedValue
+        # Request graph update when input parameter value is set
+        # and parent node belongs to a graph
+        # Output attributes value are set internally during the update process,
+        # which is why we don't trigger any update in this case
+        # TODO: update only the nodes impacted by this change
+        # TODO: only update the graph if this attribute participates to a UID
+        if self.isInput:
+            self.requestGraphUpdate()
+        self.valueChanged.emit()
+
+    def resetValue(self):
+        self._value = ""
+
+    def requestGraphUpdate(self):
+        if self.node.graph:
+            self.node.graph.markNodesDirty(self.node)
+
+    @property
+    def isOutput(self):
+        return self._isOutput
+
+    @property
+    def isInput(self):
+        return not self._isOutput
+
+    def uid(self, uidIndex=-1):
+        """
+        """
+        # 'uidIndex' should be in 'self.desc.uid' but in the case of linked attribute
+        # it will not be the case (so we cannot have an assert).
+        if self.isOutput:
+            # only dependent on the hash of its value without the cache folder
+            return hashValue(self._invalidationValue)
+        if self.isLink:
+            return self.getLinkParam().uid(uidIndex)
+        if isinstance(self._value, (list, tuple, set,)):
+            # hash of sorted values hashed
+            return hashValue([hashValue(v) for v in sorted(self._value)])
+        return hashValue(self._value)
+
+    @property
+    def isLink(self):
+        """ Whether the attribute is a link to another attribute. """
+        return self.node.graph and self.isInput and self in self.node.graph.edges.keys()
+
+    @staticmethod
+    def isLinkExpression(value):
+        """
+        Return whether the given argument is a link expression.
+        A link expression is a string matching the {nodeName.attrName} pattern.
+        """
+        return isinstance(value, pyCompatibility.basestring) and Attribute.stringIsLinkRe.match(value)
+
+    def getLinkParam(self):
+        return self.node.graph.edge(self).src if self.isLink else None
+
+    def _applyExpr(self):
+        """
+        For string parameters with an expression (when loaded from file),
+        this function convert the expression into a real edge in the graph
+        and clear the string value.
+        """
+        v = self._value
+        g = self.node.graph
+        if not g:
+            return
+        if isinstance(v, Attribute):
+            g.addEdge(v, self)
+            self.resetValue()
+        elif self.isInput and Attribute.isLinkExpression(v):
+            # value is a link to another attribute
+            link = v[1:-1]
+            linkNode, linkAttr = link.split('.')
+            g.addEdge(g.node(linkNode).attribute(linkAttr), self)
+            self.resetValue()
+
+    def getExportValue(self):
+        if self.isLink:
+            return self.getLinkParam().asLinkExpr()
+        if self.isOutput:
+            return self.desc.value
+        return self._value
+
+    def getValueStr(self):
+        if isinstance(self.attributeDesc, desc.ChoiceParam) and not self.attributeDesc.exclusive:
+            assert(isinstance(self.value, collections.Sequence) and not isinstance(self.value, pyCompatibility.basestring))
+            return self.attributeDesc.joinChar.join(self.value)
+        if isinstance(self.attributeDesc, (desc.StringParam, desc.File)):
+            return '"{}"'.format(self.value)
+        return str(self.value)
+
+    def defaultValue(self):
+        return self.desc.value
+
+    def _isDefault(self):
+        return self._value == self.defaultValue()
+
+    def getPrimitiveValue(self, exportDefault=True):
+        return self._value
+
+    name = Property(str, getName, constant=True)
+    label = Property(str, getLabel, constant=True)
+    type = Property(str, getType, constant=True)
+    desc = Property(desc.Attribute, lambda self: self.attributeDesc, constant=True)
+    valueChanged = Signal()
+    value = Property(Variant, _get_value, _set_value, notify=valueChanged)
+    isOutput = Property(bool, isOutput.fget, constant=True)
+    isLinkChanged = Signal()
+    isLink = Property(bool, isLink.fget, notify=isLinkChanged)
+    isDefault = Property(bool, _isDefault, notify=valueChanged)
+
+
+def raiseIfLink(func):
+    """ If Attribute instance is a link, raise a RuntimeError."""
+    def wrapper(attr, *args, **kwargs):
+        if attr.isLink:
+            raise RuntimeError("Can't modify connected Attribute")
+        return func(attr, *args, **kwargs)
+    return wrapper
+
+
+class ListAttribute(Attribute):
+
+    def __init__(self, node, attributeDesc, isOutput, root=None, parent=None):
+        super(ListAttribute, self).__init__(node, attributeDesc, isOutput, root, parent)
+        self._value = ListModel(parent=self)
+
+    def __len__(self):
+        return len(self._value)
+
+    def at(self, idx):
+        """ Returns child attribute at index 'idx' """
+        # implement 'at' rather than '__getitem__'
+        # since the later is called spuriously when object is used in QML
+        return self._value.at(idx)
+
+    def index(self, item):
+        return self._value.indexOf(item)
+
+    def resetValue(self):
+        self._value = ListModel(parent=self)
+
+    def _set_value(self, value):
+        if self.node.graph:
+            self.remove(0, len(self))
+        # Link to another attribute
+        if isinstance(value, ListAttribute) or Attribute.isLinkExpression(value):
+            self._value = value
+        # New value
+        else:
+            self.desc.validateValue(value)
+            self.extend(value)
+        self.requestGraphUpdate()
+
+    @raiseIfLink
+    def append(self, value):
+        self.extend([value])
+
+    @raiseIfLink
+    def insert(self, index, value):
+        values = value if isinstance(value, list) else [value]
+        attrs = [attribute_factory(self.attributeDesc.elementDesc, v, self.isOutput, self.node, self) for v in values]
+        self._value.insert(index, attrs)
+        self.valueChanged.emit()
+        self._applyExpr()
+        self.requestGraphUpdate()
+
+    @raiseIfLink
+    def extend(self, values):
+        self.insert(len(self._value), values)
+
+    @raiseIfLink
+    def remove(self, index, count=1):
+        if self.node.graph:
+            from meshroom.core.graph import GraphModification
+            with GraphModification(self.node.graph):
+                # remove potential links
+                for i in range(index, index + count):
+                    attr = self._value.at(i)
+                    if attr.isLink:
+                        # delete edge if the attribute is linked
+                        self.node.graph.removeEdge(attr)
+        self._value.removeAt(index, count)
+        self.requestGraphUpdate()
+        self.valueChanged.emit()
+
+    def uid(self, uidIndex):
+        if isinstance(self.value, ListModel):
+            uids = []
+            for value in self.value:
+                if uidIndex in value.desc.uid:
+                    uids.append(value.uid(uidIndex))
+            return hashValue(uids)
+        return super(ListAttribute, self).uid(uidIndex)
+
+    def _applyExpr(self):
+        if not self.node.graph:
+            return
+        if isinstance(self._value, ListAttribute) or Attribute.isLinkExpression(self._value):
+            super(ListAttribute, self)._applyExpr()
+        else:
+            for value in self._value:
+                value._applyExpr()
+
+    def getExportValue(self):
+        if self.isLink:
+            return self.getLinkParam().asLinkExpr()
+        return [attr.getExportValue() for attr in self._value]
+
+    def defaultValue(self):
+        return []
+
+    def _isDefault(self):
+        return len(self._value) == 0
+
+    def getPrimitiveValue(self, exportDefault=True):
+        if exportDefault:
+            return [attr.getPrimitiveValue(exportDefault=exportDefault) for attr in self._value]
+        else:
+            return [attr.getPrimitiveValue(exportDefault=exportDefault) for attr in self._value if not attr.isDefault]
+
+    def getValueStr(self):
+        if isinstance(self.value, ListModel):
+            return self.attributeDesc.joinChar.join([v.getValueStr() for v in self.value])
+        return super(ListAttribute, self).getValueStr()
+
+    # Override value property setter
+    value = Property(Variant, Attribute._get_value, _set_value, notify=Attribute.valueChanged)
+    isDefault = Property(bool, _isDefault, notify=Attribute.valueChanged)
+
+
+class GroupAttribute(Attribute):
+
+    def __init__(self, node, attributeDesc, isOutput, root=None, parent=None):
+        super(GroupAttribute, self).__init__(node, attributeDesc, isOutput, root, parent)
+        self._value = DictModel(keyAttrName='name', parent=self)
+
+        subAttributes = []
+        for subAttrDesc in self.attributeDesc.groupDesc:
+            childAttr = attribute_factory(subAttrDesc, None, self.isOutput, self.node, self)
+            subAttributes.append(childAttr)
+            childAttr.valueChanged.connect(self.valueChanged)
+
+        self._value.reset(subAttributes)
+
+    def __getattr__(self, key):
+        try:
+            return super(GroupAttribute, self).__getattr__(key)
+        except AttributeError:
+            try:
+                return self._value.get(key)
+            except KeyError:
+                raise AttributeError(key)
+
+    def _set_value(self, exportedValue):
+        self.desc.validateValue(exportedValue)
+        # set individual child attribute values
+        for key, value in exportedValue.items():
+            self._value.get(key).value = value
+
+    def uid(self, uidIndex):
+        uids = []
+        for k, v in self._value.items():
+            if uidIndex in v.desc.uid:
+                uids.append(v.uid(uidIndex))
+        return hashValue(uids)
+
+    def _applyExpr(self):
+        for value in self._value:
+            value._applyExpr()
+
+    def getExportValue(self):
+        return {key: attr.getExportValue() for key, attr in self._value.objects.items()}
+
+    def _isDefault(self):
+        return all(v.isDefault for v in self._value)
+
+    def defaultValue(self):
+        return {key: attr.defaultValue() for key, attr in self._value.items()}
+
+    def getPrimitiveValue(self, exportDefault=True):
+        if exportDefault:
+            return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items()}
+        else:
+            return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items() if not attr.isDefault}
+
+    def getValueStr(self):
+        return self.attributeDesc.joinChar.join([v.getValueStr() for v in self._value.objects.values()])
+
+    # Override value property
+    value = Property(Variant, Attribute._get_value, _set_value, notify=Attribute.valueChanged)
+    isDefault = Property(bool, _isDefault, notify=Attribute.valueChanged)

meshroom/core/node.py

@@ -0,0 +1,654 @@
+#!/usr/bin/env python
+# coding:utf-8
+import atexit
+import datetime
+import json
+import logging
+import os
+import re
+import shutil
+import time
+import uuid
+from collections import defaultdict
+
+from enum import Enum
+
+import meshroom
+from meshroom.common import Signal, Variant, Property, BaseObject, Slot, ListModel, DictModel
+from meshroom.core import desc, stats, hashValue
+from meshroom.core.attribute import attribute_factory, ListAttribute, GroupAttribute, Attribute
+from meshroom.core.exception import UnknownNodeTypeError
+
+
+class Status(Enum):
+    """
+    """
+    NONE = 0
+    SUBMITTED = 1
+    RUNNING = 2
+    ERROR = 3
+    STOPPED = 4
+    KILLED = 5
+    SUCCESS = 6
+
+
+class ExecMode(Enum):
+    NONE = 0
+    LOCAL = 1
+    EXTERN = 2
+
+
+class StatusData:
+    """
+    """
+    dateTimeFormatting = '%Y-%m-%d %H:%M:%S.%f'
+
+    def __init__(self, nodeName, nodeType, packageName, packageVersion):
+        self.status = Status.NONE
+        self.execMode = ExecMode.NONE
+        self.nodeName = nodeName
+        self.nodeType = nodeType
+        self.packageName = packageName
+        self.packageVersion = packageVersion
+        self.graph = ''
+        self.commandLine = None
+        self.env = None
+        self.startDateTime = ""
+        self.endDateTime = ""
+        self.elapsedTime = 0
+        self.hostname = ""
+        self.sessionUid = meshroom.core.sessionUid
+
+    def reset(self):
+        self.status = Status.NONE
+        self.execMode = ExecMode.NONE
+        self.graph = ''
+        self.commandLine = None
+        self.env = None
+        self.startDateTime = ""
+        self.endDateTime = ""
+        self.elapsedTime = 0
+        self.hostname = ""
+        self.sessionUid = meshroom.core.sessionUid
+
+    def initStartCompute(self):
+        import platform
+        self.sessionUid = meshroom.core.sessionUid
+        self.hostname = platform.node()
+        self.startDateTime = datetime.datetime.now().strftime(self.dateTimeFormatting)
+        # to get datetime obj: datetime.datetime.strptime(obj, self.dateTimeFormatting)
+
+    def initEndCompute(self):
+        self.sessionUid = meshroom.core.sessionUid
+        self.endDateTime = datetime.datetime.now().strftime(self.dateTimeFormatting)
+
+    @property
+    def elapsedTimeStr(self):
+        return str(datetime.timedelta(seconds=self.elapsedTime))
+
+    def toDict(self):
+        d = self.__dict__.copy()
+        d["elapsedTimeStr"] = self.elapsedTimeStr
+        return d
+
+    def fromDict(self, d):
+        self.status = getattr(Status, d.get('status', ''), Status.NONE)
+        self.execMode = getattr(ExecMode, d.get('execMode', ''), ExecMode.NONE)
+        self.nodeName = d.get('nodeName', '')
+        self.nodeType = d.get('nodeType', '')
+        self.packageName = d.get('packageName', '')
+        self.packageVersion = d.get('packageVersion', '')
+        self.graph = d.get('graph', '')
+        self.commandLine = d.get('commandLine', '')
+        self.env = d.get('env', '')
+        self.startDateTime = d.get('startDateTime', '')
+        self.endDateTime = d.get('endDateTime', '')
+        self.elapsedTime = d.get('elapsedTime', 0)
+        self.hostname = d.get('hostname', '')
+        self.sessionUid = d.get('sessionUid', '')
+
+
+runningProcesses = {}
+
+
+@atexit.register
+def clearProcessesStatus():
+    global runningProcesses
+    for k, v in runningProcesses.items():
+        v.upgradeStatusTo(Status.KILLED)
+
+
+class NodeChunk(BaseObject):
+    def __init__(self, node, range, parent=None):
+        super(NodeChunk, self).__init__(parent)
+        self.node = node
+        self.range = range
+        self.status = StatusData(node.name, node.nodeType, node.packageName, node.packageVersion)
+        self.statistics = stats.Statistics()
+        self._subprocess = None
+        # notify update in filepaths when node's internal folder changes
+        self.node.internalFolderChanged.connect(self.nodeFolderChanged)
+
+    @property
+    def index(self):
+        return self.range.iteration
+
+    @property
+    def name(self):
+        if self.range.blockSize:
+            return "{}({})".format(self.node.name, self.index)
+        else:
+            return self.node.name
+
+    @property
+    def statusName(self):
+        return self.status.status.name
+
+    @property
+    def execModeName(self):
+        return self.status.execMode.name
+
+    def updateStatusFromCache(self):
+        """
+        Update node status based on status file content/existence.
+        """
+        statusFile = self.statusFile
+        oldStatus = self.status.status
+        # No status file => reset status to Status.None
+        if not os.path.exists(statusFile):
+            self.status.reset()
+        else:
+            with open(statusFile, 'r') as jsonFile:
+                statusData = json.load(jsonFile)
+            self.status.fromDict(statusData)
+        if oldStatus != self.status.status:
+            self.statusChanged.emit()
+
+    @property
+    def statusFile(self):
+        if self.range.blockSize == 0:
+            return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'status')
+        else:
+            return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.status')
+
+    @property
+    def statisticsFile(self):
+        if self.range.blockSize == 0:
+            return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'statistics')
+        else:
+            return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.statistics')
+
+    @property
+    def logFile(self):
+        if self.range.blockSize == 0:
+            return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'log')
+        else:
+            return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.log')
+
+    def saveStatusFile(self):
+        """
+        Write node status on disk.
+        """
+        data = self.status.toDict()
+        statusFilepath = self.statusFile
+        folder = os.path.dirname(statusFilepath)
+        if not os.path.exists(folder):
+            os.makedirs(folder)
+        statusFilepathWriting = statusFilepath + '.writing.' + str(uuid.uuid4())
+        with open(statusFilepathWriting, 'w') as jsonFile:
+            json.dump(data, jsonFile, indent=4)
+        shutil.move(statusFilepathWriting, statusFilepath)
+
+    def upgradeStatusTo(self, newStatus, execMode=None):
+        if newStatus.value <= self.status.status.value:
+            print('WARNING: downgrade status on node "{}" from {} to {}'.format(self.name, self.status.status,
+                                                                                newStatus))
+        if execMode is not None:
+            self.status.execMode = execMode
+            self.execModeNameChanged.emit()
+        self.status.status = newStatus
+        self.saveStatusFile()
+        self.statusChanged.emit()
+
+    def updateStatisticsFromCache(self):
+        """
+        """
+        oldTimes = self.statistics.times
+        statisticsFile = self.statisticsFile
+        if not os.path.exists(statisticsFile):
+            return
+        with open(statisticsFile, 'r') as jsonFile:
+            statisticsData = json.load(jsonFile)
+        self.statistics.fromDict(statisticsData)
+        if oldTimes != self.statistics.times:
+            self.statisticsChanged.emit()
+
+    def saveStatistics(self):
+        data = self.statistics.toDict()
+        statisticsFilepath = self.statisticsFile
+        folder = os.path.dirname(statisticsFilepath)
+        if not os.path.exists(folder):
+            os.makedirs(folder)
+        statisticsFilepathWriting = statisticsFilepath + '.writing.' + str(uuid.uuid4())
+        with open(statisticsFilepathWriting, 'w') as jsonFile:
+            json.dump(data, jsonFile, indent=4)
+        shutil.move(statisticsFilepathWriting, statisticsFilepath)
+
+    def isAlreadySubmitted(self):
+        return self.status.status in (Status.SUBMITTED, Status.RUNNING)
+
+    def process(self, forceCompute=False):
+        if not forceCompute and self.status.status == Status.SUCCESS:
+            print("Node chunk already computed:", self.name)
+            return
+        global runningProcesses
+        runningProcesses[self.name] = self
+        self.status.initStartCompute()
+        startTime = time.time()
+        self.upgradeStatusTo(Status.RUNNING)
+        self.statThread = stats.StatisticsThread(self)
+        self.statThread.start()
+        try:
+            self.node.nodeDesc.processChunk(self)
+        except Exception as e:
+            self.upgradeStatusTo(Status.ERROR)
+            raise
+        except (KeyboardInterrupt, SystemError, GeneratorExit) as e:
+            self.upgradeStatusTo(Status.STOPPED)
+            raise
+        finally:
+            self.status.initEndCompute()
+            self.status.elapsedTime = time.time() - startTime
+            print(' - elapsed time:', self.status.elapsedTimeStr)
+            # ask and wait for the stats thread to stop
+            self.statThread.stopRequest()
+            self.statThread.join()
+            del runningProcesses[self.name]
+
+        self.upgradeStatusTo(Status.SUCCESS)
+
+    def stopProcess(self):
+        self.node.nodeDesc.stopProcess(self)
+
+    statusChanged = Signal()
+    statusName = Property(str, statusName.fget, notify=statusChanged)
+    execModeNameChanged = Signal()
+    execModeName = Property(str, execModeName.fget, notify=execModeNameChanged)
+    statisticsChanged = Signal()
+
+    nodeFolderChanged = Signal()
+    statusFile = Property(str, statusFile.fget, notify=nodeFolderChanged)
+    logFile = Property(str, logFile.fget, notify=nodeFolderChanged)
+    statisticsFile = Property(str, statisticsFile.fget, notify=nodeFolderChanged)
+
+
+class Node(BaseObject):
+    """
+    """
+
+    # Regexp handling complex attribute names with recursive understanding of Lists and Groups
+    # i.e: a.b, a[0], a[0].b.c[1]
+    attributeRE = re.compile(r'\.?(?P<name>\w+)(?:\[(?P<index>\d+)\])?')
+
+    def __init__(self, nodeDesc, parent=None, **kwargs):
+        """
+        Create a new Node instance based on the given node description.
+        Any other keyword argument will be used to initialize this node's attributes.
+
+        Args:
+            nodeDesc (desc.Node): the node description for this node
+            parent (BaseObject): this Node's parent
+            **kwargs: attributes values
+        """
+        super(Node, self).__init__(parent)
+        self.nodeDesc = nodeDesc
+        self.packageName = self.nodeDesc.packageName
+        self.packageVersion = self.nodeDesc.packageVersion
+
+        self._name = None  # type: str
+        self.graph = None  # type: Graph
+        self.dirty = True  # whether this node's outputs must be re-evaluated on next Graph update
+        self._chunks = ListModel(parent=self)
+        self._cmdVars = {}
+        self._size = 0
+        self._attributes = DictModel(keyAttrName='name', parent=self)
+        self.attributesPerUid = defaultdict(set)
+        self._initFromDesc()
+        for k, v in kwargs.items():
+            self.attribute(k).value = v
+        self._updateChunks()
+
+    def __getattr__(self, k):
+        try:
+            # Throws exception if not in prototype chain
+            # return object.__getattribute__(self, k) # doesn't work in python2
+            return object.__getattr__(self, k)
+        except AttributeError:
+            try:
+                return self.attribute(k)
+            except KeyError:
+                raise AttributeError(k)
+
+    def getName(self):
+        return self._name
+
+
+    @property
+    def packageFullName(self):
+        return '-'.join([self.packageName, self.packageVersion])
+
+    @Slot(str, result=Attribute)
+    def attribute(self, name):
+        att = None
+        # Complex name indicating group or list attribute
+        if '[' in name or '.' in name:
+            p = self.attributeRE.findall(name)
+
+            for n, idx in p:
+                # first step: get root attribute
+                if att is None:
+                    att = self._attributes.get(n)
+                else:
+                    # get child Attribute in Group
+                    assert isinstance(att, GroupAttribute)
+                    att = att.value.get(n)
+                if idx != '':
+                    # get child Attribute in List
+                    assert isinstance(att, ListAttribute)
+                    att = att.value.at(int(idx))
+        else:
+            att = self._attributes.get(name)
+        return att
+
+    def getAttributes(self):
+        return self._attributes
+
+    def _initFromDesc(self):
+        # Init from class and instance members
+
+        for attrDesc in self.nodeDesc.inputs:
+            assert isinstance(attrDesc, meshroom.core.desc.Attribute)
+            self._attributes.add(attribute_factory(attrDesc, None, False, self))
+
+        for attrDesc in self.nodeDesc.outputs:
+            assert isinstance(attrDesc, meshroom.core.desc.Attribute)
+            self._attributes.add(attribute_factory(attrDesc, None, True, self))
+
+        # List attributes per uid
+        for attr in self._attributes:
+            for uidIndex in attr.attributeDesc.uid:
+                self.attributesPerUid[uidIndex].add(attr)
+
+    def _applyExpr(self):
+        for attr in self._attributes:
+            attr._applyExpr()
+
+    @property
+    def nodeType(self):
+        return self.nodeDesc.__class__.__name__
+
+    @property
+    def depth(self):
+        return self.graph.getDepth(self)
+
+    @property
+    def minDepth(self):
+        return self.graph.getDepth(self, minimal=True)
+
+    def toDict(self):
+        attributes = {k: v.getExportValue() for k, v in self._attributes.objects.items() if v.isInput}
+        return {
+            'nodeType': self.nodeType,
+            'packageName': self.packageName,
+            'packageVersion': self.packageVersion,
+            'attributes': {k: v for k, v in attributes.items() if v is not None},  # filter empty values
+        }
+
+    def _buildCmdVars(self, cmdVars):
+        for uidIndex, associatedAttributes in self.attributesPerUid.items():
+            assAttr = [(a.getName(), a.uid(uidIndex)) for a in associatedAttributes]
+            assAttr.sort()
+            cmdVars['uid{}'.format(uidIndex)] = hashValue(tuple([b for a, b in assAttr]))
+
+        # Evaluate input params
+        for name, attr in self._attributes.objects.items():
+            if attr.isOutput:
+                continue  # skip outputs
+            v = attr.getValueStr()
+
+            cmdVars[name] = '--{name} {value}'.format(name=name, value=v)
+            cmdVars[name + 'Value'] = str(v)
+
+            if v is not None and v is not '':
+                cmdVars[attr.attributeDesc.group] = cmdVars.get(attr.attributeDesc.group, '') + \
+                                                          ' ' + cmdVars[name]
+
+        # For updating output attributes invalidation values
+        cmdVarsNoCache = cmdVars.copy()
+        cmdVarsNoCache['cache'] = ''
+
+        # Evaluate output params
+        for name, attr in self._attributes.objects.items():
+            if attr.isInput:
+                continue  # skip inputs
+            attr.value = attr.attributeDesc.value.format(
+                **cmdVars)
+            attr._invalidationValue = attr.attributeDesc.value.format(
+                **cmdVarsNoCache)
+            v = attr.getValueStr()
+
+            cmdVars[name] = '--{name} {value}'.format(name=name, value=v)
+            cmdVars[name + 'Value'] = str(v)
+
+            if v is not None and v is not '':
+                cmdVars[attr.attributeDesc.group] = cmdVars.get(attr.attributeDesc.group, '') + \
+                                                          ' ' + cmdVars[name]
+
+    @property
+    def isParallelized(self):
+        return bool(self.nodeDesc.parallelization)
+
+    @property
+    def nbParallelizationBlocks(self):
+        return len(self._chunks)
+
+    def hasStatus(self, status):
+        if not self._chunks:
+            return False
+        for chunk in self._chunks:
+            if chunk.status.status != status:
+                return False
+        return True
+
+    @Slot()
+    def clearData(self):
+        """ Delete this Node internal folder.
+        Status will be reset to Status.NONE
+        """
+        if os.path.exists(self.internalFolder):
+            shutil.rmtree(self.internalFolder)
+            self.updateStatusFromCache()
+
+    def isAlreadySubmitted(self):
+        for chunk in self._chunks:
+            if chunk.isAlreadySubmitted():
+                return True
+        return False
+
+    def alreadySubmittedChunks(self):
+        return [ch for ch in self._chunks if ch.isAlreadySubmitted()]
+
+    @Slot()
+    def clearSubmittedChunks(self):
+        """ Reset all submitted chunks to Status.NONE. This method should be used to clear inconsistent status
+        if a computation failed without informing the graph.
+
+        Warnings:
+            This must be used with caution. This could lead to inconsistent node status
+            if the graph is still being computed.
+        """
+        for chunk in self.alreadySubmittedChunks():
+            chunk.upgradeStatusTo(Status.NONE, ExecMode.NONE)
+
+    def upgradeStatusTo(self, newStatus):
+        """
+        Upgrade node to the given status and save it on disk.
+        """
+        for chunk in self._chunks:
+            chunk.upgradeStatusTo(newStatus)
+
+    def updateStatisticsFromCache(self):
+        for chunk in self._chunks:
+            chunk.updateStatisticsFromCache()
+
+    def _updateChunks(self):
+        """ Update Node's computation task splitting into NodeChunks based on its description """
+        self.setSize(self.nodeDesc.size.computeSize(self))
+        if self.isParallelized:
+            try:
+                ranges = self.nodeDesc.parallelization.getRanges(self)
+                if len(ranges) != len(self._chunks):
+                    self._chunks.setObjectList([NodeChunk(self, range) for range in ranges])
+                else:
+                    for chunk, range in zip(self._chunks, ranges):
+                        chunk.range = range
+            except RuntimeError:
+                # TODO: set node internal status to error
+                logging.warning("Invalid Parallelization on node {}".format(self._name))
+                self._chunks.clear()
+        else:
+            if len(self._chunks) != 1:
+                self._chunks.setObjectList([NodeChunk(self, desc.Range())])
+            else:
+                self._chunks[0].range = desc.Range()
+
+    def updateInternals(self):
+        """ Update Node's internal parameters and output attributes.
+
+        This method is called when:
+         - an input parameter is modified
+         - the graph main cache directory is changed
+        """
+        # Update chunks splitting
+        self._updateChunks()
+        # Retrieve current internal folder (if possible)
+        try:
+            folder = self.internalFolder
+        except KeyError:
+            folder = ''
+        # Update command variables / output attributes
+        self._cmdVars = {
+            'cache': self.graph.cacheDir,
+            'nodeType': self.nodeType,
+        }
+        self._buildCmdVars(self._cmdVars)
+        # Notify internal folder change if needed
+        if self.internalFolder != folder:
+            self.internalFolderChanged.emit()
+
+    @property
+    def internalFolder(self):
+        return self.nodeDesc.internalFolder.format(**self._cmdVars)
+
+    def updateStatusFromCache(self):
+        """
+        Update node status based on status file content/existence.
+        """
+        for chunk in self._chunks:
+            chunk.updateStatusFromCache()
+
+    def submit(self, forceCompute=False):
+        for chunk in self._chunks:
+            if forceCompute or chunk.status.status != Status.SUCCESS:
+                chunk.upgradeStatusTo(Status.SUBMITTED, ExecMode.EXTERN)
+
+    def beginSequence(self, forceCompute=False):
+        for chunk in self._chunks:
+            if forceCompute or chunk.status.status != Status.SUCCESS:
+                chunk.upgradeStatusTo(Status.SUBMITTED, ExecMode.LOCAL)
+
+    def processIteration(self, iteration):
+        self._chunks[iteration].process()
+
+    def process(self, forceCompute=False):
+        for chunk in self._chunks:
+            chunk.process(forceCompute)
+
+    def endSequence(self):
+        pass
+
+    def getStatus(self):
+        return self.status
+
+    def getChunks(self):
+        return self._chunks
+
+    @property
+    def statusNames(self):
+        return [s.status.name for s in self.status]
+
+    def getSize(self):
+        return self._size
+
+    def setSize(self, value):
+        if self._size == value:
+            return
+        self._size = value
+        self.sizeChanged.emit()
+
+    def __repr__(self):
+        return self.name
+
+    name = Property(str, getName, constant=True)
+    nodeType = Property(str, nodeType.fget, constant=True)
+    attributes = Property(BaseObject, getAttributes, constant=True)
+    internalFolderChanged = Signal()
+    internalFolder = Property(str, internalFolder.fget, notify=internalFolderChanged)
+    depthChanged = Signal()
+    depth = Property(int, depth.fget, notify=depthChanged)
+    minDepth = Property(int, minDepth.fget, notify=depthChanged)
+    chunksChanged = Signal()
+    chunks = Property(Variant, getChunks, notify=chunksChanged)
+    sizeChanged = Signal()
+    size = Property(int, getSize, notify=sizeChanged)
+
+
+def node_factory(nodeType, skipInvalidAttributes=False, **attributes):
+    """
+    Create a new Node of type NodeType and initialize its attributes with given kwargs.
+
+    Args:
+        nodeType (str): name of the node description class
+        skipInvalidAttributes (bool): whether to skip attributes not defined in
+                                      or incompatible with nodeType's description.
+        attributes (): serialized nodes attributes
+
+    Raises:
+        UnknownNodeTypeError if nodeType is unknown
+    """
+    try:
+        nodeDesc = meshroom.core.nodesDesc[nodeType]()
+    except KeyError:
+        # unknown node type
+        raise UnknownNodeTypeError(nodeType)
+
+    if skipInvalidAttributes:
+        # compare given attributes with the ones from node desc
+        descAttrNames = set([attr.name for attr in nodeDesc.inputs])
+        attrNames = set([name for name in attributes.keys()])
+        invalidAttributes = list(attrNames.difference(descAttrNames))
+        commonAttributes = list(attrNames.intersection(descAttrNames))
+        # compare value types for common attributes
+        for attr in [attr for attr in nodeDesc.inputs if attr.name in commonAttributes]:
+            try:
+                attr.validateValue(attributes[attr.name])
+            except:
+                invalidAttributes.append(attr.name)
+
+        if invalidAttributes and skipInvalidAttributes:
+            # filter out invalid attributes
+            logging.info("Skipping invalid attributes initialization for {}: {}".format(nodeType, invalidAttributes))
+            for attr in invalidAttributes:
+                del attributes[attr]
+
+    return Node(nodeDesc, **attributes)

meshroom/ui/commands.py

@@ -4,7 +4,9 @@ from contextlib import contextmanager
 
 from PySide2.QtWidgets import QUndoCommand, QUndoStack
 from PySide2.QtCore import Property, Signal
-from meshroom.core.graph import Node, ListAttribute, Graph, GraphModification, Attribute
+
+from meshroom.core.attribute import ListAttribute, Attribute
+from meshroom.core.graph import GraphModification
 
 
 class UndoCommand(QUndoCommand):

meshroom/ui/graph.py

@@ -1,13 +1,15 @@
 #!/usr/bin/env python
 # coding:utf-8
 import logging
+import os
 from threading import Thread
 
-import os
 from PySide2.QtCore import Slot, QJsonValue, QObject, QUrl, Property, Signal
 
 from meshroom.common.qt import QObjectListModel
-from meshroom.core import graph
+from meshroom.core.attribute import Attribute, ListAttribute
+from meshroom.core.graph import Graph, Edge, submitGraph, executeGraph
+from meshroom.core.node import NodeChunk, Node, Status
 from meshroom.ui import commands
 
 
@@ -71,7 +73,7 @@ class ChunksMonitor(QObject):
                 chunk.updateStatusFromCache()
                 logging.debug("Status for node {} changed: {}".format(chunk.node, chunk.status.status))
 
-    chunkStatusChanged = Signal(graph.NodeChunk, int)
+    chunkStatusChanged = Signal(NodeChunk, int)
 
 
 class UIGraph(QObject):
@@ -83,7 +85,7 @@ class UIGraph(QObject):
     def __init__(self, filepath='', parent=None):
         super(UIGraph, self).__init__(parent)
         self._undoStack = commands.UndoStack(self)
-        self._graph = graph.Graph('', self)
+        self._graph = Graph('', self)
         self._modificationCount = 0
         self._chunksMonitor = ChunksMonitor(parent=self)
         self._chunksMonitor.chunkStatusChanged.connect(self.onChunkStatusChanged)
@@ -126,7 +128,7 @@ class UIGraph(QObject):
         self._undoStack.clear()
 
     def load(self, filepath):
-        g = graph.Graph('')
+        g = Graph('')
         g.load(filepath)
         if not os.path.exists(g.cacheDir):
             os.mkdir(g.cacheDir)
@@ -146,7 +148,7 @@ class UIGraph(QObject):
         self._graph.save()
         self._undoStack.setClean()
 
-    @Slot(graph.Node)
+    @Slot(Node)
     def execute(self, node=None):
         if self.computing:
             return
@@ -157,7 +159,7 @@ class UIGraph(QObject):
     def _execute(self, nodes):
         self.computeStatusChanged.emit()
         try:
-            graph.execute(self._graph, nodes)
+            executeGraph(self._graph, nodes)
         except Exception as e:
             logging.error("Error during Graph execution {}".format(e))
         finally:
@@ -171,23 +173,23 @@ class UIGraph(QObject):
         self._computeThread.join()
         self.computeStatusChanged.emit()
 
-    @Slot(graph.Node)
+    @Slot(Node)
     def submit(self, node=None):
         """ Submit the graph to the default Submitter.
         If a node is specified, submit this node and its uncomputed predecessors.
-        Otherwise, submit the whole graph.
+        Otherwise, submit the whole 
 
         Notes:
             Default submitter is specified using the MESHROOM_DEFAULT_SUBMITTER environment variable.
         """
         self.save()  # graph must be saved before being submitted
         node = [node] if node else None
-        graph.submitGraph(self._graph, os.environ.get('MESHROOM_DEFAULT_SUBMITTER', ''), node)
+        submitGraph(self._graph, os.environ.get('MESHROOM_DEFAULT_SUBMITTER', ''), node)
 
     def onChunkStatusChanged(self, chunk, status):
         # update graph computing status
-        running = any([ch.status.status == graph.Status.RUNNING for ch in self._sortedDFSChunks])
-        submitted = any([ch.status.status == graph.Status.SUBMITTED for ch in self._sortedDFSChunks])
+        running = any([ch.status.status == Status.RUNNING for ch in self._sortedDFSChunks])
+        submitted = any([ch.status.status == Status.SUBMITTED for ch in self._sortedDFSChunks])
         if self._running != running or self._submitted != submitted:
             self._running = running
             self._submitted = submitted
@@ -250,68 +252,68 @@ class UIGraph(QObject):
         """
         return self.push(commands.AddNodeCommand(self._graph, nodeType, **kwargs))
 
-    @Slot(graph.Node)
+    @Slot(Node)
     def removeNode(self, node):
         self.push(commands.RemoveNodeCommand(self._graph, node))
 
-    @Slot(graph.Attribute, graph.Attribute)
+    @Slot(Attribute, Attribute)
     def addEdge(self, src, dst):
-        if isinstance(dst, graph.ListAttribute) and not isinstance(src, graph.ListAttribute):
+        if isinstance(dst, ListAttribute) and not isinstance(src, ListAttribute):
             with self.groupedGraphModification("Insert and Add Edge on {}".format(dst.fullName())):
                 self.appendAttribute(dst)
                 self.push(commands.AddEdgeCommand(self._graph, src, dst.at(-1)))
         else:
             self.push(commands.AddEdgeCommand(self._graph, src, dst))
 
-    @Slot(graph.Edge)
+    @Slot(Edge)
     def removeEdge(self, edge):
-        if isinstance(edge.dst.root, graph.ListAttribute):
+        if isinstance(edge.dst.root, ListAttribute):
             with self.groupedGraphModification("Remove Edge and Delete {}".format(edge.dst.fullName())):
                 self.push(commands.RemoveEdgeCommand(self._graph, edge))
                 self.removeAttribute(edge.dst)
         else:
             self.push(commands.RemoveEdgeCommand(self._graph, edge))
 
-    @Slot(graph.Attribute, "QVariant")
+    @Slot(Attribute, "QVariant")
     def setAttribute(self, attribute, value):
         self.push(commands.SetAttributeCommand(self._graph, attribute, value))
 
-    @Slot(graph.Attribute)
+    @Slot(Attribute)
     def resetAttribute(self, attribute):
         """ Reset 'attribute' to its default value """
         self.push(commands.SetAttributeCommand(self._graph, attribute, attribute.defaultValue()))
 
-    @Slot(graph.Node)
+    @Slot(Node)
     def duplicateNode(self, srcNode, createEdges=True):
         """
         Duplicate 'srcNode'.
 
         Args:
-            srcNode (graph.Node): the node to duplicate
+            srcNode (Node): the node to duplicate
             createEdges (bool): whether to replicate 'srcNode' edges on the duplicated node
 
         Returns:
-            graph.Node: the duplicated node
+            Node: the duplicated node
         """
         serialized = srcNode.toDict()
         with self.groupedGraphModification("Duplicate Node {}".format(srcNode.name)):
             # skip edges: filter out attributes which are links
             if not createEdges:
-                serialized["attributes"] = {k: v for k, v in serialized["attributes"].items() if not graph.isLinkExpression(v)}
+                serialized["attributes"] = {k: v for k, v in serialized["attributes"].items() if not Attribute.isLinkExpression(v)}
             # create a new node of the same type and with the same attributes values
             node = self.addNewNode(serialized["nodeType"], **serialized["attributes"])
         return node
 
-    @Slot(graph.Node, result="QVariantList")
+    @Slot(Node, result="QVariantList")
     def duplicateNodes(self, fromNode):
         """
         Duplicate 'fromNode' and all the following nodes towards graph's leaves.
 
         Args:
-            fromNode (graph.Node): the node to start the duplication from
+            fromNode (Node): the node to start the duplication from
 
         Returns:
-            [graph.Nodes]: the duplicated nodes
+            [Nodes]: the duplicated nodes
         """
         srcNodes, srcEdges = self._graph.nodesFromNode(fromNode)
         srcNodes = srcNodes[1:]  # skip fromNode
@@ -324,7 +326,7 @@ class UIGraph(QObject):
                 duplicate = self.duplicateNode(srcNode, createEdges=False)
                 duplicates[srcNode.name] = duplicate  # original node to duplicate map
                 # get link attributes
-                links = {k: v for k, v in srcNode.toDict()["attributes"].items() if graph.isLinkExpression(v)}
+                links = {k: v for k, v in srcNode.toDict()["attributes"].items() if Attribute.isLinkExpression(v)}
                 for attr, link in links.items():
                     link = link[1:-1]  # remove starting '{' and trailing '}'
                     # get source node and attribute name
@@ -336,7 +338,7 @@ class UIGraph(QObject):
 
         return duplicates.values()
 
-    @Slot(graph.Attribute, QJsonValue)
+    @Slot(Attribute, QJsonValue)
     def appendAttribute(self, attribute, value=QJsonValue()):
         if isinstance(value, QJsonValue):
             if value.isArray():
@@ -347,13 +349,13 @@ class UIGraph(QObject):
             pyValue = value
         self.push(commands.ListAttributeAppendCommand(self._graph, attribute, pyValue))
 
-    @Slot(graph.Attribute)
+    @Slot(Attribute)
     def removeAttribute(self, attribute):
         self.push(commands.ListAttributeRemoveCommand(self._graph, attribute))
 
     undoStack = Property(QObject, lambda self: self._undoStack, constant=True)
     graphChanged = Signal()
-    graph = Property(graph.Graph, lambda self: self._graph, notify=graphChanged)
+    graph = Property(Graph, lambda self: self._graph, notify=graphChanged)
 
     computeStatusChanged = Signal()
     computing = Property(bool, isComputing, notify=computeStatusChanged)

meshroom/ui/reconstruction.py

@@ -6,7 +6,7 @@ from PySide2.QtCore import QObject, Slot, Property, Signal
 
 from meshroom import multiview
 from meshroom.common.qt import QObjectListModel
-from meshroom.core import graph
+from meshroom.core.node import Node, node_factory, Status
 from meshroom.ui.graph import UIGraph
 
 
@@ -256,7 +256,7 @@ class Reconstruction(UIGraph):
         self.setCameraInit(self._cameraInits[idx])
 
     def updateMeshFile(self):
-        if self._endChunk and self._endChunk.status.status == graph.Status.SUCCESS:
+        if self._endChunk and self._endChunk.status.status == Status.SUCCESS:
             self.setMeshFile(self._endChunk.node.outputMesh.value)
         else:
             self.setMeshFile('')
@@ -310,7 +310,7 @@ class Reconstruction(UIGraph):
         """ Get all view Ids involved in the reconstruction. """
         return [vp.viewId.value for node in self._cameraInits for vp in node.viewpoints.value]
 
-    @Slot(QObject, graph.Node)
+    @Slot(QObject, Node)
     def handleFilesDrop(self, drop, cameraInit):
         """ Handle drop events aiming to add images to the Reconstruction.
         Fetching urls from dropEvent is generally expensive in QML/JS (bug ?).
@@ -357,7 +357,7 @@ class Reconstruction(UIGraph):
         #   * create an uninitialized node
         #   * wait for the result before actually creating new nodes in the graph (see onIntrinsicsAvailable)
         attributes = cameraInit.toDict()["attributes"] if cameraInit else {}
-        cameraInitCopy = graph.node_factory("CameraInit", **attributes)
+        cameraInitCopy = node_factory("CameraInit", **attributes)
 
         try:
             self.setBuildingIntrinsics(True)
@@ -490,7 +490,7 @@ class Reconstruction(UIGraph):
     sfmReportChanged = Signal()
     # convenient property for QML binding re-evaluation when sfm report changes
     sfmReport = Property(bool, lambda self: len(self._poses) > 0, notify=sfmReportChanged)
-    sfmAugmented = Signal(graph.Node, graph.Node)
+    sfmAugmented = Signal(Node, Node)
 
     # Signals to propagate high-level messages
     error = Signal(Message)