[core] parallelization: introduce the notion of node 'size'

* node size is an estimation of the number of element to process on a given node, that may be used for parallelization.
* replace the explicit reference to an attribute on a node (Parallelization.inputListParamName)
* size can be:
  * dynamic: depends on the size of the node connected to a specific Attribute
  * static

meshroom/core/desc.py

@@ -223,8 +223,7 @@ class Range:
 
 
 class Parallelization:
-    def __init__(self, inputListParamName='', staticNbBlocks=0, blockSize=0):
+    def __init__(self, staticNbBlocks=0, blockSize=0):
-        self.inputListParamName = inputListParamName
         self.staticNbBlocks = staticNbBlocks
         self.blockSize = blockSize
 
@@ -234,18 +233,10 @@ class Parallelization:
             node:
         Returns: (blockSize, fullSize, nbBlocks)
         """
-        if self.inputListParamName:
+        size = node.size
-            # Look for this attribute on preceding nodes
+        if self.blockSize:
-            parentNodes, edges = node.graph.dfsOnFinish(startNodes=[node])
+            nbBlocks = int(math.ceil(float(size) / float(self.blockSize)))
-            for parentNode in parentNodes:
+            return (self.blockSize, size, nbBlocks)
-                if self.inputListParamName in parentNode.getAttributes().keys():
-                    fullSize = len(parentNode.attribute(self.inputListParamName))
-                    nbBlocks = int(math.ceil(float(fullSize) / float(self.blockSize)))
-                    return (self.blockSize, fullSize, nbBlocks)
-            # No attribute has been found (parallelization won't work): raise
-            raise RuntimeError(
-                'Cannot find the "inputListParamName": "{}" in the list of input nodes: {} for node: {}'.format(
-                    self.inputListParamName, parentNodes, node.name))
         if self.staticNbBlocks:
             return (1, self.staticNbBlocks, self.staticNbBlocks)
         return None
@@ -262,6 +253,27 @@ class Parallelization:
         return ranges
 
 
+class DynamicNodeSize(object):
+    def __init__(self, param):
+        self._param = param
+
+    def computeSize(self, node):
+        param = node.attribute(self._param)
+        if param.isLink:
+            return param.getLinkParam().node.size
+        if isinstance(param.desc, ListAttribute):
+            return len(param)
+        return 1
+
+
+class StaticNodeSize(object):
+    def __init__(self, size):
+        self._size = size
+
+    def computeSize(self, node):
+        return self._size
+
+
 class Node(object):
     """
     """
@@ -273,6 +285,7 @@ class Node(object):
     packageVersion = ''
     inputs = []
     outputs = []
+    size = StaticNodeSize(1)
     parallelization = None
 
     def __init__(self):

meshroom/core/graph.py

@@ -621,6 +621,7 @@ class Node(BaseObject):
         self.graph = None  # type: Graph
         self._chunks = []
         self._cmdVars = {}
+        self._size = 0
         self._attributes = DictModel(keyAttrName='name', parent=self)
         self.attributesPerUid = defaultdict(set)
         self._initFromDesc()
@@ -795,6 +796,7 @@ class Node(BaseObject):
             chunk.updateStatisticsFromCache()
 
     def updateInternals(self):
+        self.setSize(self.nodeDesc.size.computeSize(self))
         if self.isParallelized:
             try:
                 ranges = self.nodeDesc.parallelization.getRanges(self)
@@ -860,6 +862,15 @@ class Node(BaseObject):
     def statusNames(self):
         return [s.status.name for s in self.status]
 
+    def getSize(self):
+        return self._size
+
+    def setSize(self, value):
+        if self._size == value:
+            return
+        self._size = value
+        self.sizeChanged.emit()
+
     def __repr__(self):
         return self.name
 
@@ -872,6 +883,8 @@ class Node(BaseObject):
     depth = Property(int, depth.fget, notify=depthChanged)
     chunksChanged = Signal()
     chunks = Property(Variant, getChunks, notify=chunksChanged)
+    sizeChanged = Signal()
+    size = Property(int, getSize, notify=sizeChanged)
 
 WHITE = 0
 GRAY = 1

meshroom/nodes/aliceVision/CameraConnection.py

@@ -4,6 +4,7 @@ from meshroom.core import desc
 class CameraConnection(desc.CommandLineNode):
     internalFolder = '{cache}/{nodeType}/{uid0}/'
     commandLine = 'aliceVision_cameraConnection {allParams}'
+    size = desc.DynamicNodeSize('ini')
 
     inputs = [
         desc.File(

meshroom/nodes/aliceVision/CameraInit.py

@@ -45,6 +45,8 @@ class CameraInit(desc.CommandLineNode):
     internalFolder = '{cache}/{nodeType}/{uid0}/'
     commandLine = 'aliceVision_cameraInit {allParams}'
 
+    size = desc.DynamicNodeSize('viewpoints')
+
     inputs = [
         desc.ListAttribute(
             name="viewpoints",

meshroom/nodes/aliceVision/DepthMap.py

@@ -4,7 +4,8 @@ class DepthMap(desc.CommandLineNode):
     internalFolder = '{cache}/{nodeType}/{uid0}/'
     commandLine = 'aliceVision_depthMapEstimation {allParams}'
     gpu = desc.Level.INTENSIVE
-    parallelization = desc.Parallelization(inputListParamName='viewpoints', blockSize=3)
+    size = desc.DynamicNodeSize('ini')
+    parallelization = desc.Parallelization(blockSize=3)
     commandLineRange = '--rangeStart {rangeStart} --rangeSize {rangeBlockSize}'
 
     inputs = [

meshroom/nodes/aliceVision/DepthMapFilter.py

@@ -4,7 +4,8 @@ class DepthMapFilter(desc.CommandLineNode):
     internalFolder = '{cache}/{nodeType}/{uid0}/'
     commandLine = 'aliceVision_depthMapFiltering {allParams}'
     gpu = desc.Level.NORMAL
-    parallelization = desc.Parallelization(inputListParamName='viewpoints', blockSize=10)
+    size = desc.DynamicNodeSize('ini')
+    parallelization = desc.Parallelization(blockSize=10)
     commandLineRange = '--rangeStart {rangeStart} --rangeSize {rangeBlockSize}'
 
     inputs = [

meshroom/nodes/aliceVision/FeatureExtraction.py

@@ -5,7 +5,8 @@ from meshroom.core import desc
 class FeatureExtraction(desc.CommandLineNode):
     internalFolder = '{cache}/{nodeType}/{uid0}/'
     commandLine = 'aliceVision_featureExtraction {allParams}'
-    parallelization = desc.Parallelization(inputListParamName='viewpoints', blockSize=10)
+    size = desc.DynamicNodeSize('input')
+    parallelization = desc.Parallelization(blockSize=10)
     commandLineRange = '--rangeStart {rangeStart} --rangeSize {rangeBlockSize}'
 
     inputs = [

meshroom/nodes/aliceVision/FeatureMatching.py

@@ -5,7 +5,8 @@ from meshroom.core import desc
 class FeatureMatching(desc.CommandLineNode):
     internalFolder = '{cache}/{nodeType}/{uid0}/'
     commandLine = 'aliceVision_featureMatching {allParams}'
-    parallelization = desc.Parallelization(inputListParamName='viewpoints', blockSize=20)
+    size = desc.DynamicNodeSize('input')
+    parallelization = desc.Parallelization(blockSize=20)
     commandLineRange = '--rangeStart {rangeStart} --rangeSize {rangeBlockSize}'
 
     inputs = [

meshroom/nodes/aliceVision/ImageMatching.py

@@ -6,6 +6,7 @@ from meshroom.core import desc
 class ImageMatching(desc.CommandLineNode):
     internalFolder = '{cache}/{nodeType}/{uid0}/'
     commandLine = 'aliceVision_imageMatching {allParams}'
+    size = desc.DynamicNodeSize('input')
 
     inputs = [
         desc.File(

meshroom/nodes/aliceVision/Meshing.py

@@ -3,6 +3,7 @@ from meshroom.core import desc
 class Meshing(desc.CommandLineNode):
     internalFolder = '{cache}/{nodeType}/{uid0}/'
     commandLine = 'aliceVision_meshing {allParams}'
+
     cpu = desc.Level.INTENSIVE
     ram = desc.Level.INTENSIVE
 

meshroom/nodes/aliceVision/PrepareDenseScene.py

@@ -5,6 +5,7 @@ from meshroom.core import desc
 class PrepareDenseScene(desc.CommandLineNode):
     internalFolder = '{cache}/{nodeType}/{uid0}/'
     commandLine = 'aliceVision_prepareDenseScene {allParams}'
+    size = desc.DynamicNodeSize('input')
 
     inputs = [
         desc.File(

meshroom/nodes/aliceVision/Publish.py

@@ -6,6 +6,7 @@ import os
 
 
 class Publish(desc.Node):
+    size = desc.DynamicNodeSize('inputFiles')
     inputs = [
         desc.ListAttribute(
             elementDesc=desc.File(

meshroom/nodes/aliceVision/StructureFromMotion.py

@@ -5,6 +5,7 @@ from meshroom.core import desc
 class StructureFromMotion(desc.CommandLineNode):
     internalFolder = '{cache}/{nodeType}/{uid0}/'
     commandLine = 'aliceVision_incrementalSfM {allParams}'
+    size = desc.DynamicNodeSize('input')
 
     inputs = [
         desc.File(