Make Task Manager with Nodes submitted to a render farm

meshroom/core/graph.py

@@ -1029,6 +1029,7 @@ class Graph(BaseObject):
     def stopExecution(self):
         """ Request graph execution to be stopped by terminating running chunks"""
         for chunk in self.iterChunksByStatus(Status.RUNNING):
+            if not chunk.isExtern():
                 chunk.stopProcess()
 
     @Slot()

meshroom/core/node.py

@@ -307,6 +307,9 @@ class NodeChunk(BaseObject):
         self.upgradeStatusTo(Status.STOPPED)
         self.node.nodeDesc.stopProcess(self)
 
+    def isExtern(self):
+        return self.status.execMode == ExecMode.EXTERN
+
     statusChanged = Signal()
     statusName = Property(str, statusName.fget, notify=statusChanged)
     execModeNameChanged = Signal()
@@ -558,6 +561,7 @@ class BaseNode(BaseObject):
             if the graph is still being computed.
         """
         for chunk in self.alreadySubmittedChunks():
+            if not chunk.isExtern():
                 chunk.upgradeStatusTo(Status.NONE, ExecMode.NONE)
 
     def upgradeStatusTo(self, newStatus):

meshroom/core/taskManager.py

@@ -2,6 +2,7 @@ import logging
 from threading import Thread
 from enum import Enum
 
+import meshroom
 from meshroom.common import BaseObject, DictModel, Property
 
 class State(Enum):
@@ -18,12 +19,14 @@ class TaskThread(Thread):
         self._manager = manager
         self.forceCompute = False
 
+    def isRunning(self):
+        return self._state == State.RUNNING
+
     def run(self):
         self._state = State.RUNNING
 
         for n, node in enumerate(self._manager._nodesToProcess):
             if not node.isFinishedOrRunning():
-                #try:
                 multiChunks = len(node.chunks) > 1
                 for c, chunk in enumerate(node.chunks):
                     if multiChunks:
@@ -50,12 +53,16 @@ class TaskThread(Thread):
 
                                 for nodeD in nodesToDelete:
                                     if nodeD != node:
+                                        try:
                                             self._manager._nodesToProcess.remove(nodeD)
+                                        except:
+                                            # Node already removed (for instance a global clear of _nodesToProcess)
+                                            pass
                                     nodeD.clearSubmittedChunks()
 
-        self._state = State.DEAD
-
         self._manager._nodesToProcess.clear()
+        self._state = State.DEAD
+        self._manager._uigraph.computeStatusChanged.emit()
 
 
 class TaskManager(BaseObject):
@@ -63,16 +70,30 @@ class TaskManager(BaseObject):
         super(TaskManager, self).__init__(parent)
         self._nodes = DictModel(keyAttrName='name', parent=self)
         self._nodesToProcess = DictModel(keyAttrName='name', parent=self)
+        self._nodesExtern = DictModel(keyAttrName='name', parent=self)
         self._graph = None
         self._thread = TaskThread(self)
 
-    def addNodes(self, graph=None, toNodes=None, forceCompute=False, forceStatus=False):
+    def compute(self, graph=None, toNodes=None, uigraph=None, forceCompute=False, forceStatus=False):
         self._graph = graph
-
-        logging.info(self._thread._state)
+        self._uigraph = uigraph
 
         if self._thread._state in (State.IDLE, State.DEAD, State.ERROR, State.STOPPED):
+            try:
                 self._nodes.clear()
+            except:
+                print("Task Manager nodes already empty")
+
+            externEmpty = True
+            for node in self._nodesExtern:
+                if node.isAlreadySubmitted():
+                    externEmpty = False
+                    break
+
+            if not externEmpty:
+                self._nodes.update(self._nodesExtern)
+            else:
+                self._nodesExtern.clear()
 
         if forceCompute:
             nodes, edges = graph.dfsOnFinish(startNodes=toNodes)
@@ -88,7 +109,10 @@ class TaskManager(BaseObject):
         for node in nodes:
             node.beginSequence(forceCompute)
 
+        try:
             self._nodes.update(nodes)
+        except:
+            print("nodes already added to Task Manager")
         self._nodesToProcess.update(nodes)
 
         if self._thread._state == State.IDLE:
@@ -97,6 +121,39 @@ class TaskManager(BaseObject):
             self._thread = TaskThread(self)
             self._thread.start()
 
+    def submit(self, graph=None, submitter=None, toNodes=None):
+        if self._thread._state in (State.IDLE, State.DEAD, State.ERROR, State.STOPPED):
+            self._nodes.clear()
+
+            externEmpty = True
+            for node in self._nodesExtern:
+                if node.isAlreadySubmitted():
+                    externEmpty = False
+                    break
+
+            if not externEmpty:
+                self._nodes.update(self._nodesExtern)
+            else:
+                self._nodesExtern.clear()
+
+        nodesToProcess, edgesToProcess = graph.dfsToProcess(startNodes=toNodes)
+        flowEdges = graph.flowEdges(startNodes=toNodes)
+        edgesToProcess = set(edgesToProcess).intersection(flowEdges)
+
+        sub = meshroom.core.submitters.get(submitter, None)
+        if sub is None:
+            raise RuntimeError("Unknown Submitter : " + submitter)
+
+        try:
+            res = sub.submit(nodesToProcess, edgesToProcess, graph.filepath)
+            if res:
+                for node in nodesToProcess:
+                    node.submit()  # update node status
+            self._nodes.update(nodesToProcess)
+            self._nodesExtern.update(nodesToProcess)
+        except Exception as e:
+            logging.error("Error on submit : {}".format(e))
+
     nodes = Property(BaseObject, lambda self: self._nodes, constant=True)
 
 def getAlreadySubmittedChunks(nodes):

meshroom/ui/graph.py

@@ -352,7 +352,7 @@ class UIGraph(QObject):
     @Slot(Node)
     def execute(self, node=None):
         nodes = [node] if node else None
-        self._taskManager.addNodes(self._graph, nodes)
+        self._taskManager.compute(self._graph, nodes, self)
 
     def _execute(self, nodes):
         self.computeStatusChanged.emit()
@@ -382,7 +382,7 @@ class UIGraph(QObject):
         """
         self.save()  # graph must be saved before being submitted
         node = [node] if node else None
-        submitGraph(self._graph, os.environ.get('MESHROOM_DEFAULT_SUBMITTER', ''), node)
+        self._taskManager.submit(self._graph, os.environ.get('MESHROOM_DEFAULT_SUBMITTER', ''), node)
 
     def updateGraphComputingStatus(self):
         # update graph computing status
@@ -399,11 +399,11 @@ class UIGraph(QObject):
 
     def isComputingExternally(self):
         """ Whether this graph is being computed externally. """
-        return (self._running or self._submitted) and not self.isComputingLocally()
+        return self._submitted
 
     def isComputingLocally(self):
         """ Whether this graph is being computed locally (i.e computation can be stopped). """
-        return self._taskManager._thread.is_alive()
+        return self._taskManager._thread.isRunning()
 
     def push(self, command):
         """ Try and push the given command to the undo stack.

meshroom/ui/qml/main.qml

@@ -462,7 +462,6 @@ ApplicationWindow {
 
                 Row {
                     // disable controls if graph is executed externally
-                    enabled: !_reconstruction.computingExternally
                     Layout.alignment: Qt.AlignHCenter
 
                     Button {
@@ -471,7 +470,6 @@ ApplicationWindow {
                         palette.button: enabled ? buttonColor : disabledPalette.button
                         palette.window: enabled ? buttonColor : disabledPalette.window
                         palette.buttonText: enabled ? "white" : disabledPalette.buttonText
-                        enabled: computeManager.canStartComputation
                         onClicked: computeManager.compute(null)
                     }
                     Button {
@@ -482,7 +480,6 @@ ApplicationWindow {
                     Item { width: 20; height: 1 }
                     Button {
                         visible: _reconstruction.canSubmit
-                        enabled: computeManager.canSubmit
                         text: "Submit"
                         onClicked: computeManager.submit(null)
                     }
@@ -502,13 +499,6 @@ ApplicationWindow {
                 }
             }
 
-            Label {
-                text: "Graph is being computed externally"
-                font.italic: true
-                Layout.alignment: Qt.AlignHCenter
-                visible: _reconstruction.computingExternally
-            }
-
             // "ProgressBar" reflecting status of all the chunks in the graph, in their process order
             NodeChunks {
                 id: chunksListView
@@ -636,23 +626,22 @@ ApplicationWindow {
                 width: Math.round(parent.width * 0.3)
                 node: _reconstruction.selectedNode
                 // Make NodeEditor readOnly when computing
-//                readOnly: graphLocked
                 readOnly: {
                     if(! _reconstruction.computing) {
                         return false;
                     }
 
-                    if(_reconstruction.taskManager.nodes.contains(_reconstruction.selectedNode)) {
-                            return true;
-                    } else {
                     if(_reconstruction.selectedNode.globalStatus == "SUCCESS") {
-                            var nodes = _reconstruction.graph.onlyNodesFromNode(_reconstruction.selectedNode);
+                        var nodes = uigraph.graph.onlyNodesFromNode(_reconstruction.selectedNode);
                         for(var i = 0; i < nodes.length; i++) {
                             if(["SUBMITTED", "RUNNING"].includes(nodes[i].globalStatus) && nodes[i].chunks.at(0).statusNodeName == nodes[i].name) {
                                 return true;
                             }
                         }
-                        }
+                    } else if(["SUBMITTED", "RUNNING"].includes(_reconstruction.selectedNode.globalStatus)) {
+                        return true;
+                    } else {
+                        return false;
                     }
 
                     return false;