3pp-mirror/Meshroom
1
0
Fork 0
mirror of https://github.com/alicevision/Meshroom.git synced 2025-07-26 21:17:29 +02:00
Code Issues Projects Releases Packages Wiki Activity

[core] Simplify invalidation status for all the attributes

Browse source 
The UID system based on a UID index is removed and replaced by a single
UID per node.

Attributes will be included in the UID computation if the `invalidate`
is set to `True` in their description. This replaces the `uid=[]` /
`uid=[0]` element of the description.
This commit is contained in:
Candice Bentéjac 2024-09-05 16:48:19 +02:00
parent 248c301c5a
commit 9973298746
4 changed files with 142 additions and 151 deletions
Download patch file Download diff file Expand all files Collapse all files

23
meshroom/core/attribute.py
View file

@ -272,22 +272,21 @@ class Attribute(BaseObject):
def isInput(self): def isInput(self):
return not self._isOutput return not self._isOutput
def uid(self, uidIndex=-1): def uid(self):
""" """
Compute the UID for the attribute.
""" """
# 'uidIndex' should be in 'self.desc.uid' but in the case of linked attribute
# it will not be the case (so we cannot have an assert).
if self.isOutput: if self.isOutput:
if self.desc.isDynamicValue: if self.desc.isDynamicValue:
# If the attribute is a dynamic output, the UID is derived from the node UID. # If the attribute is a dynamic output, the UID is derived from the node UID.
# To guarantee that each output attribute receives a unique ID, we add the attribute name to it. # To guarantee that each output attribute receives a unique ID, we add the attribute name to it.
return hashValue((self.name, self.node._uids.get(uidIndex))) return hashValue((self.name, self.node._uid))
else: else:
# only dependent on the hash of its value without the cache folder # only dependent on the hash of its value without the cache folder
return hashValue(self._invalidationValue) return hashValue(self._invalidationValue)
if self.isLink: if self.isLink:
linkParam = self.getLinkParam(recursive=True) linkParam = self.getLinkParam(recursive=True)
return linkParam.uid(uidIndex) return linkParam.uid()
if isinstance(self._value, (list, tuple, set,)): if isinstance(self._value, (list, tuple, set,)):
# non-exclusive choice param # non-exclusive choice param
# hash of sorted values hashed # hash of sorted values hashed
@ -610,14 +609,14 @@ class ListAttribute(Attribute):
self.requestGraphUpdate() self.requestGraphUpdate()
self.valueChanged.emit() self.valueChanged.emit()
def uid(self, uidIndex): def uid(self):
if isinstance(self.value, ListModel): if isinstance(self.value, ListModel):
uids = [] uids = []
for value in self.value: for value in self.value:
if uidIndex in value.desc.uid: if value.desc.invalidate:
uids.append(value.uid(uidIndex)) uids.append(value.uid())
return hashValue(uids) return hashValue(uids)
return super(ListAttribute, self).uid(uidIndex) return super(ListAttribute, self).uid()
def _applyExpr(self): def _applyExpr(self):
if not self.node.graph: if not self.node.graph:
@ -747,11 +746,11 @@ class GroupAttribute(Attribute):
except KeyError: except KeyError:
return None return None
def uid(self, uidIndex): def uid(self):
uids = [] uids = []
for k, v in self._value.items(): for k, v in self._value.items():
if v.enabled and uidIndex in v.desc.uid: if v.enabled and v.desc.invalidate:
uids.append(v.uid(uidIndex)) uids.append(v.uid())
return hashValue(uids) return hashValue(uids)
def _applyExpr(self): def _applyExpr(self):

71
meshroom/core/desc.py
View file

@ -15,14 +15,14 @@ class Attribute(BaseObject):
""" """
""" """
def __init__(self, name, label, description, value, advanced, semantic, uid, group, enabled, uidIgnoreValue=None, def __init__(self, name, label, description, value, advanced, semantic, invalidate, group, enabled, uidIgnoreValue=None,
validValue=True, errorMessage="", visible=True, exposed=False): validValue=True, errorMessage="", visible=True, exposed=False):
super(Attribute, self).__init__() super(Attribute, self).__init__()
self._name = name self._name = name
self._label = label self._label = label
self._description = description self._description = description
self._value = value self._value = value
self._uid = uid self._invalidate = invalidate
self._group = group self._group = group
self._advanced = advanced self._advanced = advanced
self._enabled = enabled self._enabled = enabled
@ -84,7 +84,7 @@ class Attribute(BaseObject):
# The default value of the attribute's descriptor is None, so it's not an input value, # The default value of the attribute's descriptor is None, so it's not an input value,
# but an output value that is computed during the Node's process execution. # but an output value that is computed during the Node's process execution.
isDynamicValue = Property(bool, lambda self: self._isDynamicValue, constant=True) isDynamicValue = Property(bool, lambda self: self._isDynamicValue, constant=True)
uid = Property(Variant, lambda self: self._uid, constant=True) invalidate = Property(Variant, lambda self: self._invalidate, constant=True)
group = Property(str, lambda self: self._group, constant=True) group = Property(str, lambda self: self._group, constant=True)
advanced = Property(bool, lambda self: self._advanced, constant=True) advanced = Property(bool, lambda self: self._advanced, constant=True)
enabled = Property(Variant, lambda self: self._enabled, constant=True) enabled = Property(Variant, lambda self: self._enabled, constant=True)
@ -114,7 +114,7 @@ class ListAttribute(Attribute):
""" """
self._elementDesc = elementDesc self._elementDesc = elementDesc
self._joinChar = joinChar self._joinChar = joinChar
super(ListAttribute, self).__init__(name=name, label=label, description=description, value=[], uid=(), group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) super(ListAttribute, self).__init__(name=name, label=label, description=description, value=[], invalidate=False, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
def getInstanceType(self): def getInstanceType(self):
# Import within the method to prevent cyclic dependencies # Import within the method to prevent cyclic dependencies
@ -149,7 +149,7 @@ class ListAttribute(Attribute):
return True return True
elementDesc = Property(Attribute, lambda self: self._elementDesc, constant=True) elementDesc = Property(Attribute, lambda self: self._elementDesc, constant=True)
uid = Property(Variant, lambda self: self.elementDesc.uid, constant=True) invalidate = Property(Variant, lambda self: self.elementDesc.invalidate, constant=True)
joinChar = Property(str, lambda self: self._joinChar, constant=True) joinChar = Property(str, lambda self: self._joinChar, constant=True)
@ -162,7 +162,7 @@ class GroupAttribute(Attribute):
self._groupDesc = groupDesc self._groupDesc = groupDesc
self._joinChar = joinChar self._joinChar = joinChar
self._brackets = brackets self._brackets = brackets
super(GroupAttribute, self).__init__(name=name, label=label, description=description, value={}, uid=(), group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) super(GroupAttribute, self).__init__(name=name, label=label, description=description, value={}, invalidate=False, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
def getInstanceType(self): def getInstanceType(self):
# Import within the method to prevent cyclic dependencies # Import within the method to prevent cyclic dependencies
@ -239,14 +239,14 @@ class GroupAttribute(Attribute):
return matchCount > 0 return matchCount > 0
def retrieveChildrenUids(self): def retrieveChildrenInvalidations(self):
allUids = [] allInvalidations = []
for desc in self._groupDesc: for desc in self._groupDesc:
allUids.extend(desc.uid) allInvalidations.append(desc.invalidate)
return allUids return allInvalidations
groupDesc = Property(Variant, lambda self: self._groupDesc, constant=True) groupDesc = Property(Variant, lambda self: self._groupDesc, constant=True)
uid = Property(Variant, retrieveChildrenUids, constant=True) invalidate = Property(Variant, retrieveChildrenInvalidations, constant=True)
joinChar = Property(str, lambda self: self._joinChar, constant=True) joinChar = Property(str, lambda self: self._joinChar, constant=True)
brackets = Property(str, lambda self: self._brackets, constant=True) brackets = Property(str, lambda self: self._brackets, constant=True)
@ -254,16 +254,16 @@ class GroupAttribute(Attribute):
class Param(Attribute): class Param(Attribute):
""" """
""" """
def __init__(self, name, label, description, value, uid, group, advanced, semantic, enabled, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False): def __init__(self, name, label, description, value, invalidate, group, advanced, semantic, enabled, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False):
super(Param, self).__init__(name=name, label=label, description=description, value=value, uid=uid, group=group, advanced=advanced, semantic=semantic, enabled=enabled, super(Param, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled,
uidIgnoreValue=uidIgnoreValue, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) uidIgnoreValue=uidIgnoreValue, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed)
class File(Attribute): class File(Attribute):
""" """
""" """
def __init__(self, name, label, description, value, uid, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=True): def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=True):
super(File, self).__init__(name=name, label=label, description=description, value=value, uid=uid, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) super(File, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
self._valueType = str self._valueType = str
def validateValue(self, value): def validateValue(self, value):
@ -284,8 +284,8 @@ class File(Attribute):
class BoolParam(Param): class BoolParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, uid, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False): def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False):
super(BoolParam, self).__init__(name=name, label=label, description=description, value=value, uid=uid, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) super(BoolParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
self._valueType = bool self._valueType = bool
def validateValue(self, value): def validateValue(self, value):
@ -308,9 +308,9 @@ class BoolParam(Param):
class IntParam(Param): class IntParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, range, uid, group='allParams', advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False): def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False):
self._range = range self._range = range
super(IntParam, self).__init__(name=name, label=label, description=description, value=value, uid=uid, group=group, advanced=advanced, semantic=semantic, enabled=enabled, super(IntParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled,
validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed)
self._valueType = int self._valueType = int
@ -334,9 +334,9 @@ class IntParam(Param):
class FloatParam(Param): class FloatParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, range, uid, group='allParams', advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False): def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False):
self._range = range self._range = range
super(FloatParam, self).__init__(name=name, label=label, description=description, value=value, uid=uid, group=group, advanced=advanced, semantic=semantic, enabled=enabled, super(FloatParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled,
validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed)
self._valueType = float self._valueType = float
@ -358,8 +358,8 @@ class FloatParam(Param):
class PushButtonParam(Param): class PushButtonParam(Param):
""" """
""" """
def __init__(self, name, label, description, uid, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False): def __init__(self, name, label, description, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False):
super(PushButtonParam, self).__init__(name=name, label=label, description=description, value=None, uid=uid, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) super(PushButtonParam, self).__init__(name=name, label=label, description=description, value=None, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
self._valueType = None self._valueType = None
def getInstanceType(self): def getInstanceType(self):
@ -377,10 +377,10 @@ class PushButtonParam(Param):
class ChoiceParam(Param): class ChoiceParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, values, exclusive, uid, group='allParams', joinChar=' ', advanced=False, semantic='', def __init__(self, name, label, description, value, values, exclusive, invalidate, group='allParams', joinChar=' ', advanced=False, semantic='',
enabled=True, validValue=True, errorMessage="", visible=True, exposed=False): enabled=True, validValue=True, errorMessage="", visible=True, exposed=False):
assert values assert values
super(ChoiceParam, self).__init__(name=name, label=label, description=description, value=value, uid=uid, group=group, advanced=advanced, super(ChoiceParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced,
semantic=semantic, enabled=enabled, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) semantic=semantic, enabled=enabled, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed)
self._values = values self._values = values
self._exclusive = exclusive self._exclusive = exclusive
@ -446,8 +446,8 @@ class ChoiceParam(Param):
class StringParam(Param): class StringParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, uid, group='allParams', advanced=False, semantic='', enabled=True, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False): def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False):
super(StringParam, self).__init__(name=name, label=label, description=description, value=value, uid=uid, group=group, advanced=advanced, semantic=semantic, enabled=enabled, super(StringParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled,
uidIgnoreValue=uidIgnoreValue, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) uidIgnoreValue=uidIgnoreValue, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed)
self._valueType = str self._valueType = str
@ -467,8 +467,8 @@ class StringParam(Param):
class ColorParam(Param): class ColorParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, uid, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False): def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False):
super(ColorParam, self).__init__(name=name, label=label, description=description, value=value, uid=uid, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) super(ColorParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
self._valueType = str self._valueType = str
def validateValue(self, value): def validateValue(self, value):
@ -616,7 +616,7 @@ class StaticNodeSize(object):
class Node(object): class Node(object):
""" """
""" """
internalFolder = '{cache}/{nodeType}/{uid0}/' internalFolder = '{cache}/{nodeType}/{uid}/'
cpu = Level.NORMAL cpu = Level.NORMAL
gpu = Level.NONE gpu = Level.NONE
ram = Level.NORMAL ram = Level.NORMAL
@ -631,7 +631,7 @@ class Node(object):
"It is displayed in bold font in the invalidation/comment messages tooltip.", "It is displayed in bold font in the invalidation/comment messages tooltip.",
value="", value="",
semantic="multiline", semantic="multiline",
uid=[0], invalidate=True,
advanced=True, advanced=True,
uidIgnoreValue="", # If the invalidation string is empty, it does not participate to the node's UID uidIgnoreValue="", # If the invalidation string is empty, it does not participate to the node's UID
), ),
@ -642,21 +642,21 @@ class Node(object):
"It is displayed in regular font in the invalidation/comment messages tooltip.", "It is displayed in regular font in the invalidation/comment messages tooltip.",
value="", value="",
semantic="multiline", semantic="multiline",
uid=[], invalidate=False,
), ),
StringParam( StringParam(
name="label", name="label",
label="Node's Label", label="Node's Label",
description="Customize the default label (to replace the technical name of the node instance).", description="Customize the default label (to replace the technical name of the node instance).",
value="", value="",
uid=[], invalidate=False,
), ),
ColorParam( ColorParam(
name="color", name="color",
label="Color", label="Color",
description="Custom color for the node (SVG name or hexadecimal code).", description="Custom color for the node (SVG name or hexadecimal code).",
value="", value="",
uid=[], invalidate=False,
) )
] ]
inputs = [] inputs = []
@ -780,7 +780,7 @@ class CommandLineNode(Node):
finally: finally:
chunk.subprocess = None chunk.subprocess = None
#specific command line node for alicevision apps # Specific command line node for AliceVision apps
class AVCommandLineNode(CommandLineNode): class AVCommandLineNode(CommandLineNode):
cgroupParsed = False cgroupParsed = False
@ -805,7 +805,6 @@ class AVCommandLineNode(CommandLineNode):
AVCommandLineNode.cgroupParsed = True AVCommandLineNode.cgroupParsed = True
def buildCommandLine(self, chunk): def buildCommandLine(self, chunk):
commandLineString = super(AVCommandLineNode, self).buildCommandLine(chunk) commandLineString = super(AVCommandLineNode, self).buildCommandLine(chunk)
return commandLineString + AVCommandLineNode.cmdMem + AVCommandLineNode.cmdCore return commandLineString + AVCommandLineNode.cmdMem + AVCommandLineNode.cmdCore

22
meshroom/core/graph.py
View file

@ -315,8 +315,8 @@ class Graph(BaseObject):
# If no filepath is being set but the graph is not a template, trigger an updateInternals either way. # If no filepath is being set but the graph is not a template, trigger an updateInternals either way.
self.updateInternals() self.updateInternals()
# By this point, the graph has been fully loaded and an updateInternals has been triggered, so all the nodes' # By this point, the graph has been fully loaded and an updateInternals has been triggered, so all the
# links have been resolved and their UID computations are all complete. # nodes' links have been resolved and their UID computations are all complete.
# It is now possible to check whether the UIDs stored in the graph file for each node correspond to the ones # It is now possible to check whether the UIDs stored in the graph file for each node correspond to the ones
# that were computed. # that were computed.
if not isTemplate: # UIDs are not stored in templates if not isTemplate: # UIDs are not stored in templates
@ -342,13 +342,9 @@ class Graph(BaseObject):
for nodeName, nodeData in sorted(data.items(), key=lambda x: self.getNodeIndexFromName(x[0])): for nodeName, nodeData in sorted(data.items(), key=lambda x: self.getNodeIndexFromName(x[0])):
node = self.node(nodeName) node = self.node(nodeName)
savedUid = nodeData.get("uids", {}) # Node's UID from the graph file savedUid = nodeData.get("uid", None)
# JSON enfore keys to be strings, see graphUid = node._uid # Node's UID from the graph itself
# https://docs.python.org/3.8/library/json.html#json.dump
# We know our keys are integers, so we convert them back to int.
savedUid = {int(k): v for k, v in savedUid.items()}
graphUid = node._uids # Node's UID from the graph itself
if savedUid != graphUid and graphUid is not None: if savedUid != graphUid and graphUid is not None:
# Different UIDs, remove the existing node from the graph and replace it with a CompatibilityNode # Different UIDs, remove the existing node from the graph and replace it with a CompatibilityNode
logging.debug("UID conflict detected for {}".format(nodeName)) logging.debug("UID conflict detected for {}".format(nodeName))
@ -1382,7 +1378,7 @@ class Graph(BaseObject):
del graph[nodeName]["internalInputs"] del graph[nodeName]["internalInputs"]
del graph[nodeName]["outputs"] del graph[nodeName]["outputs"]
del graph[nodeName]["uids"] del graph[nodeName]["uid"]
del graph[nodeName]["internalFolder"] del graph[nodeName]["internalFolder"]
del graph[nodeName]["parallelization"] del graph[nodeName]["parallelization"]
@ -1431,13 +1427,13 @@ class Graph(BaseObject):
node.updateStatisticsFromCache() node.updateStatisticsFromCache()
def updateNodesPerUid(self): def updateNodesPerUid(self):
""" Update the duplicate nodes (sharing same uid) list of each node. """ """ Update the duplicate nodes (sharing same UID) list of each node. """
# First step is to construct a map uid/nodes # First step is to construct a map UID/nodes
nodesPerUid = {} nodesPerUid = {}
for node in self.nodes: for node in self.nodes:
uid = node._uids.get(0) uid = node._uid
# We try to add the node to the list corresponding to this uid # We try to add the node to the list corresponding to this UID
try: try:
nodesPerUid.get(uid).append(node) nodesPerUid.get(uid).append(node)
# If it fails because the uid is not in the map, we add it # If it fails because the uid is not in the map, we add it

161
meshroom/core/node.py
View file

@ -477,7 +477,7 @@ class BaseNode(BaseObject):
# i.e: a.b, a[0], a[0].b.c[1] # i.e: a.b, a[0], a[0].b.c[1]
attributeRE = re.compile(r'\.?(?P<name>\w+)(?:\[(?P<index>\d+)\])?') attributeRE = re.compile(r'\.?(?P<name>\w+)(?:\[(?P<index>\d+)\])?')
def __init__(self, nodeType, position=None, parent=None, uids=None, **kwargs): def __init__(self, nodeType, position=None, parent=None, uid=None, **kwargs):
""" """
Create a new Node instance based on the given node description. Create a new Node instance based on the given node description.
Any other keyword argument will be used to initialize this node's attributes. Any other keyword argument will be used to initialize this node's attributes.
@ -502,13 +502,13 @@ class BaseNode(BaseObject):
self.graph = None self.graph = None
self.dirty = True # whether this node's outputs must be re-evaluated on next Graph update self.dirty = True # whether this node's outputs must be re-evaluated on next Graph update
self._chunks = ListModel(parent=self) self._chunks = ListModel(parent=self)
self._uids = uids if uids else {} self._uid = uid
self._cmdVars = {} self._cmdVars = {}
self._size = 0 self._size = 0
self._position = position or Position() self._position = position or Position()
self._attributes = DictModel(keyAttrName='name', parent=self) self._attributes = DictModel(keyAttrName='name', parent=self)
self._internalAttributes = DictModel(keyAttrName='name', parent=self) self._internalAttributes = DictModel(keyAttrName='name', parent=self)
self.attributesPerUid = defaultdict(set) self.invalidatingAttributes = set()
self._alive = True # for QML side to know if the node can be used or is going to be deleted self._alive = True # for QML side to know if the node can be used or is going to be deleted
self._locked = False self._locked = False
self._duplicates = ListModel(parent=self) # list of nodes with the same uid self._duplicates = ListModel(parent=self) # list of nodes with the same uid
@ -699,56 +699,59 @@ class BaseNode(BaseObject):
def toDict(self): def toDict(self):
pass pass
def _computeUids(self): def _computeUid(self):
""" Compute node UIDs by combining associated attributes' UIDs. """ """ Compute node UID by combining associated attributes' UIDs. """
# Get all the attributes associated to a given UID index, specified in node descriptions with "uid=[index]" # If there is no invalidating attribute, then the computation of the UID should not go through as
# For now, the only index that is used is "0", so there will be a single iteration of the loop below # it will only include the node type
for uidIndex, associatedAttributes in self.attributesPerUid.items(): if not self.invalidatingAttributes:
return
# UID is computed by hashing the sorted list of tuple (name, value) of all attributes impacting this UID # UID is computed by hashing the sorted list of tuple (name, value) of all attributes impacting this UID
uidAttributes = [] uidAttributes = []
for a in associatedAttributes: for attr in self.invalidatingAttributes:
if not a.enabled: if not attr.enabled:
continue # disabled params do not contribute to the uid continue # Disabled params do not contribute to the uid
dynamicOutputAttr = a.isLink and a.getLinkParam(recursive=True).desc.isDynamicValue dynamicOutputAttr = attr.isLink and attr.getLinkParam(recursive=True).desc.isDynamicValue
# For dynamic output attributes, the UID does not depend on the attribute value. # For dynamic output attributes, the UID does not depend on the attribute value.
# In particular, when loading a project file, the UIDs are updated first, # In particular, when loading a project file, the UIDs are updated first,
# and the node status and the dynamic output values are not yet loaded, # and the node status and the dynamic output values are not yet loaded,
# so we should not read the attribute value. # so we should not read the attribute value.
if not dynamicOutputAttr and a.value == a.uidIgnoreValue: if not dynamicOutputAttr and attr.value == attr.uidIgnoreValue:
continue # for non-dynamic attributes, check if the value should be ignored continue # For non-dynamic attributes, check if the value should be ignored
uidAttributes.append((a.getName(), a.uid(uidIndex))) uidAttributes.append((attr.getName(), attr.uid()))
uidAttributes.sort() uidAttributes.sort()
# Adding the node type prevents ending up with two identical UIDs for different node types that have the exact same list of attributes
# Adding the node type prevents ending up with two identical UIDs for different node types
# that have the exact same list of attributes
uidAttributes.append(self.nodeType) uidAttributes.append(self.nodeType)
self._uids[uidIndex] = hashValue(uidAttributes) self._uid = hashValue(uidAttributes)
def _buildCmdVars(self): def _buildCmdVars(self):
def _buildAttributeCmdVars(cmdVars, name, attr): def _buildAttributeCmdVars(cmdVars, name, attr):
if attr.enabled: if attr.enabled:
group = attr.attributeDesc.group(attr.node) if isinstance(attr.attributeDesc.group, types.FunctionType) else attr.attributeDesc.group group = attr.attributeDesc.group(attr.node) if isinstance(attr.attributeDesc.group, types.FunctionType) else attr.attributeDesc.group
if group is not None: if group is not None:
# if there is a valid command line "group" # If there is a valid command line "group"
v = attr.getValueStr(withQuotes=True) v = attr.getValueStr(withQuotes=True)
cmdVars[name] = '--{name} {value}'.format(name=name, value=v) cmdVars[name] = "--{name} {value}".format(name=name, value=v)
# xxValue is exposed without quotes to allow to compose expressions # xxValue is exposed without quotes to allow to compose expressions
cmdVars[name + 'Value'] = attr.getValueStr(withQuotes=False) cmdVars[name + "Value"] = attr.getValueStr(withQuotes=False)
# List elements may give a fully empty string and will not be sent to the command line. # List elements may give a fully empty string and will not be sent to the command line.
# String attributes will return only quotes if it is empty and thus will be send to the command line. # String attributes will return only quotes if it is empty and thus will be send to the command line.
# But a List of string containing 1 element, # But a List of string containing 1 element,
# and this element is an empty string will also return quotes and will be send to the command line. # and this element is an empty string will also return quotes and will be sent to the command line.
if v: if v:
cmdVars[group] = cmdVars.get(group, '') + ' ' + cmdVars[name] cmdVars[group] = cmdVars.get(group, "") + " " + cmdVars[name]
elif isinstance(attr, GroupAttribute): elif isinstance(attr, GroupAttribute):
assert isinstance(attr.value, DictModel) assert isinstance(attr.value, DictModel)
# if the GroupAttribute is not set in a single command line argument, # If the GroupAttribute is not set in a single command line argument,
# the sub-attributes may need to be exposed individually # the sub-attributes may need to be exposed individually
for v in attr._value: for v in attr._value:
_buildAttributeCmdVars(cmdVars, v.name, v) _buildAttributeCmdVars(cmdVars, v.name, v)
""" Generate command variables using input attributes and resolved output attributes names and values. """ """ Generate command variables using input attributes and resolved output attributes names and values. """
for uidIndex, value in self._uids.items(): self._cmdVars["uid"] = self._uid
self._cmdVars['uid{}'.format(uidIndex)] = value
# Evaluate input params # Evaluate input params
for name, attr in self._attributes.objects.items(): for name, attr in self._attributes.objects.items():
@ -768,12 +771,14 @@ class BaseNode(BaseObject):
# Apply expressions for File attributes # Apply expressions for File attributes
if attr.attributeDesc.isExpression: if attr.attributeDesc.isExpression:
defaultValue = "" defaultValue = ""
# Do not evaluate expression for disabled attributes (the expression may refer to other attributes that are not defined) # Do not evaluate expression for disabled attributes
# (the expression may refer to other attributes that are not defined)
if attr.enabled: if attr.enabled:
try: try:
defaultValue = attr.defaultValue() defaultValue = attr.defaultValue()
except AttributeError as e: except AttributeError as e:
# If we load an old scene, the lambda associated to the 'value' could try to access other params that could not exist yet # If we load an old scene, the lambda associated to the 'value' could try to access other
# params that could not exist yet
logging.warning('Invalid lambda evaluation for "{nodeName}.{attrName}"'.format(nodeName=self.name, attrName=attr.name)) logging.warning('Invalid lambda evaluation for "{nodeName}.{attrName}"'.format(nodeName=self.name, attrName=attr.name))
if defaultValue is not None: if defaultValue is not None:
try: try:
@ -972,12 +977,13 @@ class BaseNode(BaseObject):
folder = self.internalFolder folder = self.internalFolder
except KeyError: except KeyError:
folder = '' folder = ''
# Update command variables / output attributes # Update command variables / output attributes
self._cmdVars = { self._cmdVars = {
'cache': cacheDir or self.graph.cacheDir, 'cache': cacheDir or self.graph.cacheDir,
'nodeType': self.nodeType, 'nodeType': self.nodeType,
} }
self._computeUids() self._computeUid()
self._buildCmdVars() self._buildCmdVars()
if self.nodeDesc: if self.nodeDesc:
self.nodeDesc.postUpdate(self) self.nodeDesc.postUpdate(self)
@ -1237,16 +1243,15 @@ class BaseNode(BaseObject):
self.setLocked(False) self.setLocked(False)
def updateDuplicates(self, nodesPerUid): def updateDuplicates(self, nodesPerUid):
""" Update the list of duplicate nodes (sharing the same uid). """ """ Update the list of duplicate nodes (sharing the same UID). """
uid = self._uids.get(0) if not nodesPerUid or not self._uid:
if not nodesPerUid or not uid:
if len(self._duplicates) > 0: if len(self._duplicates) > 0:
self._duplicates.clear() self._duplicates.clear()
self._hasDuplicates = False self._hasDuplicates = False
self.hasDuplicatesChanged.emit() self.hasDuplicatesChanged.emit()
return return
newList = [node for node in nodesPerUid.get(uid) if node != self] newList = [node for node in nodesPerUid.get(self._uid) if node != self]
# If number of elements in both lists are identical, # If number of elements in both lists are identical,
# we must check if their content is the same # we must check if their content is the same
@ -1377,8 +1382,8 @@ class Node(BaseNode):
""" """
A standard Graph node based on a node type. A standard Graph node based on a node type.
""" """
def __init__(self, nodeType, position=None, parent=None, uids=None, **kwargs): def __init__(self, nodeType, position=None, parent=None, uid=None, **kwargs):
super(Node, self).__init__(nodeType, position, parent=parent, uids=uids, **kwargs) super(Node, self).__init__(nodeType, position, parent=parent, uid=uid, **kwargs)
if not self.nodeDesc: if not self.nodeDesc:
raise UnknownNodeTypeError(nodeType) raise UnknownNodeTypeError(nodeType)
@ -1401,19 +1406,18 @@ class Node(BaseNode):
if attr.isOutput and attr.desc.semantic == "image": if attr.isOutput and attr.desc.semantic == "image":
attr.enabledChanged.connect(self.outputAttrEnabledChanged) attr.enabledChanged.connect(self.outputAttrEnabledChanged)
# List attributes per uid # List attributes per UID
for attr in self._attributes: for attr in self._attributes:
if attr.isInput: if attr.isInput and attr.attributeDesc.invalidate:
for uidIndex in attr.attributeDesc.uid: self.invalidatingAttributes.add(attr)
self.attributesPerUid[uidIndex].add(attr)
else: else:
if attr.attributeDesc.uid: if attr.attributeDesc.invalidate:
logging.error(f"Output Attribute should not contain a UID: '{nodeType}.{attr.name}'") logging.error(f"Output Attribute should not be invalidating: '{nodeType}.{attr.name}'")
# Add internal attributes with a UID to the list # Add internal attributes with a UID to the list
for attr in self._internalAttributes: for attr in self._internalAttributes:
for uidIndex in attr.attributeDesc.uid: if attr.attributeDesc.invalidate:
self.attributesPerUid[uidIndex].add(attr) self.invalidatingAttributes.add(attr)
self.optionalCallOnDescriptor("onNodeCreated") self.optionalCallOnDescriptor("onNodeCreated")
@ -1497,7 +1501,7 @@ class Node(BaseNode):
'size': self.size, 'size': self.size,
'split': self.nbParallelizationBlocks 'split': self.nbParallelizationBlocks
}, },
'uids': self._uids, 'uid': self._uid,
'internalFolder': self._internalFolder, 'internalFolder': self._internalFolder,
'inputs': {k: v for k, v in inputs.items() if v is not None}, # filter empty values 'inputs': {k: v for k, v in inputs.items() if v is not None}, # filter empty values
'internalInputs': {k: v for k, v in internalInputs.items() if v is not None}, 'internalInputs': {k: v for k, v in internalInputs.items() if v is not None},
@ -1539,7 +1543,7 @@ class CompatibilityIssue(Enum):
UnknownNodeType = 1 # the node type has no corresponding description class UnknownNodeType = 1 # the node type has no corresponding description class
VersionConflict = 2 # mismatch between node's description version and serialized node data VersionConflict = 2 # mismatch between node's description version and serialized node data
DescriptionConflict = 3 # mismatch between node's description attributes and serialized node data DescriptionConflict = 3 # mismatch between node's description attributes and serialized node data
UidConflict = 4 # mismatch between computed uids and uids stored in serialized node data UidConflict = 4 # mismatch between computed UIDs and UIDs stored in serialized node data
class CompatibilityNode(BaseNode): class CompatibilityNode(BaseNode):
@ -1552,7 +1556,7 @@ class CompatibilityNode(BaseNode):
super(CompatibilityNode, self).__init__(nodeType, position, parent) super(CompatibilityNode, self).__init__(nodeType, position, parent)
self.issue = issue self.issue = issue
# make a deepcopy of nodeDict to handle CompatibilityNode duplication # Make a deepcopy of nodeDict to handle CompatibilityNode duplication
# and be able to change modified inputs (see CompatibilityNode.toDict) # and be able to change modified inputs (see CompatibilityNode.toDict)
self.nodeDict = copy.deepcopy(nodeDict) self.nodeDict = copy.deepcopy(nodeDict)
self.version = Version(self.nodeDict.get("version", None)) self.version = Version(self.nodeDict.get("version", None))
@ -1561,30 +1565,26 @@ class CompatibilityNode(BaseNode):
self._internalInputs = self.nodeDict.get("internalInputs", {}) self._internalInputs = self.nodeDict.get("internalInputs", {})
self.outputs = self.nodeDict.get("outputs", {}) self.outputs = self.nodeDict.get("outputs", {})
self._internalFolder = self.nodeDict.get("internalFolder", "") self._internalFolder = self.nodeDict.get("internalFolder", "")
self._uids = self.nodeDict.get("uids", {}) self._uid = self.nodeDict.get("uid", None)
# JSON enfore keys to be strings, see
# https://docs.python.org/3.8/library/json.html#json.dump
# We know our keys are integers, so we convert them back to int.
self._uids = {int(k): v for k, v in self._uids.items()}
# restore parallelization settings # Restore parallelization settings
self.parallelization = self.nodeDict.get("parallelization", {}) self.parallelization = self.nodeDict.get("parallelization", {})
self.splitCount = self.parallelization.get("split", 1) self.splitCount = self.parallelization.get("split", 1)
self.setSize(self.parallelization.get("size", 1)) self.setSize(self.parallelization.get("size", 1))
# create input attributes # Create input attributes
for attrName, value in self._inputs.items(): for attrName, value in self._inputs.items():
self._addAttribute(attrName, value, isOutput=False) self._addAttribute(attrName, value, isOutput=False)
# create outputs attributes # Create outputs attributes
for attrName, value in self.outputs.items(): for attrName, value in self.outputs.items():
self._addAttribute(attrName, value, isOutput=True) self._addAttribute(attrName, value, isOutput=True)
# create internal attributes # Create internal attributes
for attrName, value in self._internalInputs.items(): for attrName, value in self._internalInputs.items():
self._addAttribute(attrName, value, isOutput=False, internalAttr=True) self._addAttribute(attrName, value, isOutput=False, internalAttr=True)
# create NodeChunks matching serialized parallelization settings # Create NodeChunks matching serialized parallelization settings
self._chunks.setObjectList([ self._chunks.setObjectList([
NodeChunk(self, desc.Range(i, blockSize=self.parallelization.get("blockSize", 0))) NodeChunk(self, desc.Range(i, blockSize=self.parallelization.get("blockSize", 0)))
for i in range(self.splitCount) for i in range(self.splitCount)
@ -1609,7 +1609,7 @@ class CompatibilityNode(BaseNode):
params = { params = {
"name": attrName, "label": attrName, "name": attrName, "label": attrName,
"description": "Incompatible parameter", "description": "Incompatible parameter",
"value": value, "uid": (), "value": value, "invalidate": False,
"group": "incompatible" "group": "incompatible"
} }
if isinstance(value, bool): if isinstance(value, bool):
@ -1626,10 +1626,10 @@ class CompatibilityNode(BaseNode):
# List/GroupAttribute: recursively build descriptions # List/GroupAttribute: recursively build descriptions
elif isinstance(value, (list, dict)): elif isinstance(value, (list, dict)):
del params["value"] del params["value"]
del params["uid"] del params["invalidate"]
attrDesc = None attrDesc = None
if isinstance(value, list): if isinstance(value, list):
elt = value[0] if value else "" # fallback: empty string value if list is empty elt = value[0] if value else "" # Fallback: empty string value if list is empty
eltDesc = CompatibilityNode.attributeDescFromValue("element", elt, isOutput) eltDesc = CompatibilityNode.attributeDescFromValue("element", elt, isOutput)
attrDesc = desc.ListAttribute(elementDesc=eltDesc, **params) attrDesc = desc.ListAttribute(elementDesc=eltDesc, **params)
elif isinstance(value, dict): elif isinstance(value, dict):
@ -1638,10 +1638,10 @@ class CompatibilityNode(BaseNode):
eltDesc = CompatibilityNode.attributeDescFromValue(key, value, isOutput) eltDesc = CompatibilityNode.attributeDescFromValue(key, value, isOutput)
groupDesc.append(eltDesc) groupDesc.append(eltDesc)
attrDesc = desc.GroupAttribute(groupDesc=groupDesc, **params) attrDesc = desc.GroupAttribute(groupDesc=groupDesc, **params)
# override empty default value with # Override empty default value with
attrDesc._value = value attrDesc._value = value
return attrDesc return attrDesc
# handle any other type of parameters as Strings # Handle any other type of parameters as Strings
return desc.StringParam(**params) return desc.StringParam(**params)
@staticmethod @staticmethod
@ -1760,7 +1760,7 @@ class CompatibilityNode(BaseNode):
Return a new Node instance based on original node type with common inputs initialized. Return a new Node instance based on original node type with common inputs initialized.
""" """
if not self.canUpgrade: if not self.canUpgrade:
raise NodeUpgradeError(self.name, "no matching node type") raise NodeUpgradeError(self.name, "No matching node type")
# inputs matching current type description # inputs matching current type description
commonInputs = [] commonInputs = []
@ -1819,23 +1819,19 @@ def nodeFactory(nodeDict, name=None, template=False, uidConflict=False):
""" """
nodeType = nodeDict["nodeType"] nodeType = nodeDict["nodeType"]
# retro-compatibility: inputs were previously saved as "attributes" # Retro-compatibility: inputs were previously saved as "attributes"
if "inputs" not in nodeDict and "attributes" in nodeDict: if "inputs" not in nodeDict and "attributes" in nodeDict:
nodeDict["inputs"] = nodeDict["attributes"] nodeDict["inputs"] = nodeDict["attributes"]
del nodeDict["attributes"] del nodeDict["attributes"]
# get node inputs/outputs # Get node inputs/outputs
inputs = nodeDict.get("inputs", {}) inputs = nodeDict.get("inputs", {})
internalInputs = nodeDict.get("internalInputs", {}) internalInputs = nodeDict.get("internalInputs", {})
outputs = nodeDict.get("outputs", {}) outputs = nodeDict.get("outputs", {})
version = nodeDict.get("version", None) version = nodeDict.get("version", None)
internalFolder = nodeDict.get("internalFolder", None) internalFolder = nodeDict.get("internalFolder", None)
position = Position(*nodeDict.get("position", [])) position = Position(*nodeDict.get("position", []))
uids = nodeDict.get("uids", {}) uid = nodeDict.get("uid", None)
# JSON enfore keys to be strings, see
# https://docs.python.org/3.8/library/json.html#json.dump
# We know our keys are integers, so we convert them back to int.
uids = {int(k): v for k, v in uids.items()}
compatibilityIssue = None compatibilityIssue = None
@ -1843,21 +1839,22 @@ def nodeFactory(nodeDict, name=None, template=False, uidConflict=False):
try: try:
nodeDesc = meshroom.core.nodesDesc[nodeType] nodeDesc = meshroom.core.nodesDesc[nodeType]
except KeyError: except KeyError:
# unknown node type # Unknown node type
compatibilityIssue = CompatibilityIssue.UnknownNodeType compatibilityIssue = CompatibilityIssue.UnknownNodeType
if uidConflict: # Unknown node type should take precedence over UID conflict, as it cannot be resolved
if uidConflict and nodeDesc:
compatibilityIssue = CompatibilityIssue.UidConflict compatibilityIssue = CompatibilityIssue.UidConflict
if nodeDesc and not uidConflict: # if uidConflict, there is no need to look for another compatibility issue if nodeDesc and not uidConflict: # if uidConflict, there is no need to look for another compatibility issue
# compare serialized node version with current node version # Compare serialized node version with current node version
currentNodeVersion = meshroom.core.nodeVersion(nodeDesc) currentNodeVersion = meshroom.core.nodeVersion(nodeDesc)
# if both versions are available, check for incompatibility in major version # If both versions are available, check for incompatibility in major version
if version and currentNodeVersion and Version(version).major != Version(currentNodeVersion).major: if version and currentNodeVersion and Version(version).major != Version(currentNodeVersion).major:
compatibilityIssue = CompatibilityIssue.VersionConflict compatibilityIssue = CompatibilityIssue.VersionConflict
# in other cases, check attributes compatibility between serialized node and its description # In other cases, check attributes compatibility between serialized node and its description
else: else:
# check that the node has the exact same set of inputs/outputs as its description, except # Check that the node has the exact same set of inputs/outputs as its description, except
# if the node is described in a template file, in which only non-default parameters are saved; # if the node is described in a template file, in which only non-default parameters are saved;
# do not perform that check for internal attributes because there is no point in # do not perform that check for internal attributes because there is no point in
# raising compatibility issues if their number differs: in that case, it is only useful # raising compatibility issues if their number differs: in that case, it is only useful
@ -1866,47 +1863,47 @@ def nodeFactory(nodeDict, name=None, template=False, uidConflict=False):
sorted([attr.name for attr in nodeDesc.outputs if not attr.isDynamicValue]) != sorted(outputs.keys())): sorted([attr.name for attr in nodeDesc.outputs if not attr.isDynamicValue]) != sorted(outputs.keys())):
compatibilityIssue = CompatibilityIssue.DescriptionConflict compatibilityIssue = CompatibilityIssue.DescriptionConflict
# check whether there are any internal attributes that are invalidating in the node description: if there # Check whether there are any internal attributes that are invalidating in the node description: if there
# are, then check that these internal attributes are part of nodeDict; if they are not, a compatibility # are, then check that these internal attributes are part of nodeDict; if they are not, a compatibility
# issue must be raised to warn the user, as this will automatically change the node's UID # issue must be raised to warn the user, as this will automatically change the node's UID
if not template: if not template:
invalidatingIntInputs = [] invalidatingIntInputs = []
for attr in nodeDesc.internalInputs: for attr in nodeDesc.internalInputs:
if attr.uid == [0]: if attr.invalidate:
invalidatingIntInputs.append(attr.name) invalidatingIntInputs.append(attr.name)
for attr in invalidatingIntInputs: for attr in invalidatingIntInputs:
if attr not in internalInputs.keys(): if attr not in internalInputs.keys():
compatibilityIssue = CompatibilityIssue.DescriptionConflict compatibilityIssue = CompatibilityIssue.DescriptionConflict
break break
# verify that all inputs match their descriptions # Verify that all inputs match their descriptions
for attrName, value in inputs.items(): for attrName, value in inputs.items():
if not CompatibilityNode.attributeDescFromName(nodeDesc.inputs, attrName, value): if not CompatibilityNode.attributeDescFromName(nodeDesc.inputs, attrName, value):
compatibilityIssue = CompatibilityIssue.DescriptionConflict compatibilityIssue = CompatibilityIssue.DescriptionConflict
break break
# verify that all internal inputs match their description # Verify that all internal inputs match their description
for attrName, value in internalInputs.items(): for attrName, value in internalInputs.items():
if not CompatibilityNode.attributeDescFromName(nodeDesc.internalInputs, attrName, value): if not CompatibilityNode.attributeDescFromName(nodeDesc.internalInputs, attrName, value):
compatibilityIssue = CompatibilityIssue.DescriptionConflict compatibilityIssue = CompatibilityIssue.DescriptionConflict
break break
# verify that all outputs match their descriptions # Verify that all outputs match their descriptions
for attrName, value in outputs.items(): for attrName, value in outputs.items():
if not CompatibilityNode.attributeDescFromName(nodeDesc.outputs, attrName, value): if not CompatibilityNode.attributeDescFromName(nodeDesc.outputs, attrName, value):
compatibilityIssue = CompatibilityIssue.DescriptionConflict compatibilityIssue = CompatibilityIssue.DescriptionConflict
break break
if compatibilityIssue is None: if compatibilityIssue is None:
node = Node(nodeType, position, uids=uids, **inputs, **internalInputs, **outputs) node = Node(nodeType, position, uid=uid, **inputs, **internalInputs, **outputs)
else: else:
logging.debug("Compatibility issue detected for node '{}': {}".format(name, compatibilityIssue.name)) logging.debug("Compatibility issue detected for node '{}': {}".format(name, compatibilityIssue.name))
node = CompatibilityNode(nodeType, nodeDict, position, compatibilityIssue) node = CompatibilityNode(nodeType, nodeDict, position, compatibilityIssue)
# retro-compatibility: no internal folder saved # Retro-compatibility: no internal folder saved
# can't spawn meaningful CompatibilityNode with precomputed outputs # can't spawn meaningful CompatibilityNode with precomputed outputs
# => automatically try to perform node upgrade # => automatically try to perform node upgrade
if not internalFolder and nodeDesc: if not internalFolder and nodeDesc:
logging.warning("No serialized output data: performing automatic upgrade on '{}'".format(name)) logging.warning("No serialized output data: performing automatic upgrade on '{}'".format(name))
node = node.upgrade() node = node.upgrade()
elif template: # if the node comes from a template file and there is a conflict, it should be upgraded anyway elif template: # If the node comes from a template file and there is a conflict, it should be upgraded anyway
node = node.upgrade() node = node.upgrade()
return node return node