3pp-mirror/Meshroom
1
0
Fork 0
mirror of https://github.com/alicevision/Meshroom.git synced 2025-05-28 16:36:32 +02:00
Code Issues Projects Releases Packages Wiki Activity

Linting: Clean-up files

Browse source
This commit is contained in:
Candice Bentéjac 2024-09-06 13:11:04 +02:00
parent d67062d39d
commit 41e885d9ff
22 changed files with 298 additions and 198 deletions
Download patch file Download diff file Expand all files Collapse all files

1
docs/source/_ext/meshroom_doc.py
View file

@ -11,7 +11,6 @@
# Note: # Note:
# for now this tool focuses only on meshroom nodes # for now this tool focuses only on meshroom nodes
from docutils import nodes
from docutils.parsers.rst import Directive from docutils.parsers.rst import Directive
from utils import md_to_docutils from utils import md_to_docutils

1
docs/source/conf.py
View file

@ -35,7 +35,6 @@ templates_path = ['_templates']
exclude_patterns = [] exclude_patterns = []
# -- Options for HTML output ------------------------------------------------- # -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output

3
meshroom/__init__.py
View file

@ -4,10 +4,12 @@ import logging
import os import os
import sys import sys
class VersionStatus(Enum): class VersionStatus(Enum):
release = 1 release = 1
develop = 2 develop = 2
__version__ = "2024.1.0" __version__ = "2024.1.0"
# Always increase the minor version when switching from release to develop. # Always increase the minor version when switching from release to develop.
__version_status__ = VersionStatus.develop __version_status__ = VersionStatus.develop
@ -56,6 +58,7 @@ logStringToPython = {
} }
logging.getLogger().setLevel(logStringToPython[os.environ.get('MESHROOM_VERBOSE', 'warning')]) logging.getLogger().setLevel(logStringToPython[os.environ.get('MESHROOM_VERBOSE', 'warning')])
def setupEnvironment(backend=Backend.STANDALONE): def setupEnvironment(backend=Backend.STANDALONE):
""" """
Setup environment for Meshroom to work in a prebuilt, standalone configuration. Setup environment for Meshroom to work in a prebuilt, standalone configuration.

6
meshroom/common/__init__.py
View file

@ -7,10 +7,12 @@ Warning: A call to `init(Backend.XXX)` is required to choose the backend before
from enum import Enum from enum import Enum
class Backend(Enum): class Backend(Enum):
STANDALONE = 1 STANDALONE = 1
PYSIDE = 2 PYSIDE = 2
DictModel = None DictModel = None
ListModel = None ListModel = None
Slot = None Slot = None
@ -21,6 +23,7 @@ Variant = None
VariantList = None VariantList = None
JSValue = None JSValue = None
def init(backend): def init(backend):
global DictModel, ListModel, Slot, Signal, Property, BaseObject, Variant, VariantList, JSValue global DictModel, ListModel, Slot, Signal, Property, BaseObject, Variant, VariantList, JSValue
if backend == Backend.PYSIDE: if backend == Backend.PYSIDE:
@ -30,5 +33,6 @@ def init(backend):
# Core types # Core types
from .core import DictModel, ListModel, Slot, Signal, Property, BaseObject, Variant, VariantList, JSValue from .core import DictModel, ListModel, Slot, Signal, Property, BaseObject, Variant, VariantList, JSValue
# default initialization
# Default initialization
init(Backend.STANDALONE) init(Backend.STANDALONE)

1
meshroom/common/core.py
View file

@ -1,5 +1,6 @@
from . import PySignal from . import PySignal
class CoreDictModel: class CoreDictModel:
def __init__(self, keyAttrName, **kwargs): def __init__(self, keyAttrName, **kwargs):

1
meshroom/common/qt.py
View file

@ -1,6 +1,7 @@
from PySide2 import QtCore, QtQml from PySide2 import QtCore, QtQml
import shiboken2 import shiboken2
class QObjectListModel(QtCore.QAbstractListModel): class QObjectListModel(QtCore.QAbstractListModel):
""" """
QObjectListModel provides a more powerful, but still easy to use, alternative to using QObjectListModel provides a more powerful, but still easy to use, alternative to using

8
meshroom/core/__init__.py
View file

@ -1,11 +1,8 @@
from __future__ import print_function
import hashlib import hashlib
from contextlib import contextmanager from contextlib import contextmanager
import importlib import importlib
import inspect import inspect
import os import os
import re
import tempfile import tempfile
import uuid import uuid
import logging import logging
@ -18,7 +15,7 @@ try:
import encodings.ascii import encodings.ascii
import encodings.idna import encodings.idna
import encodings.utf_8 import encodings.utf_8
except: except Exception:
pass pass
from meshroom.core.submitter import BaseSubmitter from meshroom.core.submitter import BaseSubmitter
@ -330,6 +327,7 @@ def loadPipelineTemplates(folder):
if file.endswith(".mg") and file not in pipelineTemplates: if file.endswith(".mg") and file not in pipelineTemplates:
pipelineTemplates[os.path.splitext(file)[0]] = os.path.join(folder, file) pipelineTemplates[os.path.splitext(file)[0]] = os.path.join(folder, file)
def initNodes(): def initNodes():
meshroomFolder = os.path.dirname(os.path.dirname(__file__)) meshroomFolder = os.path.dirname(os.path.dirname(__file__))
additionalNodesPath = os.environ.get("MESHROOM_NODES_PATH", "").split(os.pathsep) additionalNodesPath = os.environ.get("MESHROOM_NODES_PATH", "").split(os.pathsep)
@ -339,12 +337,14 @@ def initNodes():
for f in nodesFolders: for f in nodesFolders:
loadAllNodes(folder=f) loadAllNodes(folder=f)
def initSubmitters(): def initSubmitters():
meshroomFolder = os.path.dirname(os.path.dirname(__file__)) meshroomFolder = os.path.dirname(os.path.dirname(__file__))
subs = loadSubmitters(os.environ.get("MESHROOM_SUBMITTERS_PATH", meshroomFolder), 'submitters') subs = loadSubmitters(os.environ.get("MESHROOM_SUBMITTERS_PATH", meshroomFolder), 'submitters')
for sub in subs: for sub in subs:
registerSubmitter(sub()) registerSubmitter(sub())
def initPipelines(): def initPipelines():
meshroomFolder = os.path.dirname(os.path.dirname(__file__)) meshroomFolder = os.path.dirname(os.path.dirname(__file__))
# Load pipeline templates: check in the default folder and any folder the user might have # Load pipeline templates: check in the default folder and any folder the user might have

40
meshroom/core/attribute.py
View file

@ -138,7 +138,7 @@ class Attribute(BaseObject):
if isinstance(self.desc.enabled, types.FunctionType): if isinstance(self.desc.enabled, types.FunctionType):
try: try:
return self.desc.enabled(self.node) return self.desc.enabled(self.node)
except: except Exception:
# Node implementation may fail due to version mismatch # Node implementation may fail due to version mismatch
return True return True
return self.attributeDesc.enabled return self.attributeDesc.enabled
@ -325,7 +325,8 @@ class Attribute(BaseObject):
return False return False
# if the attribute is a ListAttribute, we need to check if any of its elements has output connections # if the attribute is a ListAttribute, we need to check if any of its elements has output connections
if isinstance(self, ListAttribute): if isinstance(self, ListAttribute):
return next((edge for edge in self.node.graph.edges.values() if edge.src == self), None) is not None or any(attr.hasOutputConnections for attr in self._value if hasattr(attr, 'hasOutputConnections')) return next((edge for edge in self.node.graph.edges.values() if edge.src == self), None) is not None or \
any(attr.hasOutputConnections for attr in self._value if hasattr(attr, 'hasOutputConnections'))
return next((edge for edge in self.node.graph.edges.values() if edge.src == self), None) is not None return next((edge for edge in self.node.graph.edges.values() if edge.src == self), None) is not None
def _applyExpr(self): def _applyExpr(self):
@ -348,7 +349,8 @@ class Attribute(BaseObject):
try: try:
g.addEdge(g.node(linkNode).attribute(linkAttr), self) g.addEdge(g.node(linkNode).attribute(linkAttr), self)
except KeyError as err: except KeyError as err:
logging.warning('Connect Attribute from Expression failed.\nExpression: "{exp}"\nError: "{err}".'.format(exp=v, err=err)) logging.warning('Connect Attribute from Expression failed.')
logging.warning('Expression: "{exp}"\nError: "{err}".'.format(exp=v, err=err))
self.resetToDefaultValue() self.resetToDefaultValue()
def getExportValue(self): def getExportValue(self):
@ -376,8 +378,8 @@ class Attribute(BaseObject):
''' '''
# ChoiceParam with multiple values should be combined # ChoiceParam with multiple values should be combined
if isinstance(self.attributeDesc, desc.ChoiceParam) and not self.attributeDesc.exclusive: if isinstance(self.attributeDesc, desc.ChoiceParam) and not self.attributeDesc.exclusive:
# ensure value is a list as expected # Ensure value is a list as expected
assert(isinstance(self.value, Sequence) and not isinstance(self.value, str)) assert (isinstance(self.value, Sequence) and not isinstance(self.value, str))
v = self.attributeDesc.joinChar.join(self.getEvalValue()) v = self.attributeDesc.joinChar.join(self.getEvalValue())
if withQuotes and v: if withQuotes and v:
return '"{}"'.format(v) return '"{}"'.format(v)
@ -393,8 +395,9 @@ class Attribute(BaseObject):
return self.desc.value(self) return self.desc.value(self)
except Exception as e: except Exception as e:
if not self.node.isCompatibilityNode: if not self.node.isCompatibilityNode:
# log message only if we are not in compatibility mode # Log message only if we are not in compatibility mode
logging.warning("Failed to evaluate default value (node lambda) for attribute '{}': {}".format(self.name, e)) logging.warning("Failed to evaluate default value (node lambda) for attribute '{}': {}".
format(self.name, e))
return None return None
# Need to force a copy, for the case where the value is a list (avoid reference to the desc value) # Need to force a copy, for the case where the value is a list (avoid reference to the desc value)
return copy.copy(self.desc.value) return copy.copy(self.desc.value)
@ -409,7 +412,6 @@ class Attribute(BaseObject):
# Emit if the enable status has changed # Emit if the enable status has changed
self.setEnabled(self.getEnabled()) self.setEnabled(self.getEnabled())
name = Property(str, getName, constant=True) name = Property(str, getName, constant=True)
fullName = Property(str, getFullName, constant=True) fullName = Property(str, getFullName, constant=True)
fullNameToNode = Property(str, getFullNameToNode, constant=True) fullNameToNode = Property(str, getFullNameToNode, constant=True)
@ -423,11 +425,11 @@ class Attribute(BaseObject):
baseType = Property(str, getType, constant=True) baseType = Property(str, getType, constant=True)
isReadOnly = Property(bool, _isReadOnly, constant=True) isReadOnly = Property(bool, _isReadOnly, constant=True)
# description of the attribute # Description of the attribute
descriptionChanged = Signal() descriptionChanged = Signal()
description = Property(str, _get_description, _set_description, notify=descriptionChanged) description = Property(str, _get_description, _set_description, notify=descriptionChanged)
# definition of the attribute # Definition of the attribute
desc = Property(desc.Attribute, lambda self: self.attributeDesc, constant=True) desc = Property(desc.Attribute, lambda self: self.attributeDesc, constant=True)
valueChanged = Signal() valueChanged = Signal()
@ -460,6 +462,7 @@ def raiseIfLink(func):
return func(attr, *args, **kwargs) return func(attr, *args, **kwargs)
return wrapper return wrapper
class PushButtonParam(Attribute): class PushButtonParam(Attribute):
def __init__(self, node, attributeDesc, isOutput, root=None, parent=None): def __init__(self, node, attributeDesc, isOutput, root=None, parent=None):
super(PushButtonParam, self).__init__(node, attributeDesc, isOutput, root, parent) super(PushButtonParam, self).__init__(node, attributeDesc, isOutput, root, parent)
@ -468,6 +471,7 @@ class PushButtonParam(Attribute):
def clicked(self): def clicked(self):
self.node.onAttributeClicked(self) self.node.onAttributeClicked(self)
class ChoiceParam(Attribute): class ChoiceParam(Attribute):
def __init__(self, node, attributeDesc, isOutput, root=None, parent=None): def __init__(self, node, attributeDesc, isOutput, root=None, parent=None):
@ -489,7 +493,8 @@ class ChoiceParam(Attribute):
value = value.split(',') value = value.split(',')
if not isinstance(value, Iterable): if not isinstance(value, Iterable):
raise ValueError('Non exclusive ChoiceParam value should be iterable (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) raise ValueError("Non exclusive ChoiceParam value should be iterable (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
return [self.conformValue(v) for v in value] return [self.conformValue(v) for v in value]
def setValues(self, values): def setValues(self, values):
@ -523,7 +528,7 @@ class ListAttribute(Attribute):
def at(self, idx): def at(self, idx):
""" Returns child attribute at index 'idx' """ """ Returns child attribute at index 'idx' """
# implement 'at' rather than '__getitem__' # Implement 'at' rather than '__getitem__'
# since the later is called spuriously when object is used in QML # since the later is called spuriously when object is used in QML
return self._value.at(idx) return self._value.at(idx)
@ -560,7 +565,8 @@ class ListAttribute(Attribute):
if isinstance(exportedValues, ListAttribute) or Attribute.isLinkExpression(exportedValues): if isinstance(exportedValues, ListAttribute) or Attribute.isLinkExpression(exportedValues):
self._set_value(exportedValues) self._set_value(exportedValues)
return return
raise RuntimeError("ListAttribute.upgradeValue: the given value is of type " + str(type(exportedValues)) + " but a 'list' is expected.") raise RuntimeError("ListAttribute.upgradeValue: the given value is of type " +
str(type(exportedValues)) + " but a 'list' is expected.")
attrs = [] attrs = []
for v in exportedValues: for v in exportedValues:
@ -645,7 +651,7 @@ class ListAttribute(Attribute):
return [attr.getPrimitiveValue(exportDefault=exportDefault) for attr in self._value if not attr.isDefault] return [attr.getPrimitiveValue(exportDefault=exportDefault) for attr in self._value if not attr.isDefault]
def getValueStr(self, withQuotes=True): def getValueStr(self, withQuotes=True):
assert(isinstance(self.value, ListModel)) assert isinstance(self.value, ListModel)
if self.attributeDesc.joinChar == ' ': if self.attributeDesc.joinChar == ' ':
return self.attributeDesc.joinChar.join([v.getValueStr(withQuotes=withQuotes) for v in self.value]) return self.attributeDesc.joinChar.join([v.getValueStr(withQuotes=withQuotes) for v in self.value])
else: else:
@ -770,7 +776,8 @@ class GroupAttribute(Attribute):
if exportDefault: if exportDefault:
return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items()} return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items()}
else: else:
return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items() if not attr.isDefault} return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items()
if not attr.isDefault}
def getValueStr(self, withQuotes=True): def getValueStr(self, withQuotes=True):
# add brackets if requested # add brackets if requested
@ -787,7 +794,8 @@ class GroupAttribute(Attribute):
spaceSep = self.attributeDesc.joinChar == ' ' spaceSep = self.attributeDesc.joinChar == ' '
# sort values based on child attributes group description order # sort values based on child attributes group description order
sortedSubValues = [self._value.get(attr.name).getValueStr(withQuotes=spaceSep) for attr in self.attributeDesc.groupDesc] sortedSubValues = [self._value.get(attr.name).getValueStr(withQuotes=spaceSep)
for attr in self.attributeDesc.groupDesc]
s = self.attributeDesc.joinChar.join(sortedSubValues) s = self.attributeDesc.joinChar.join(sortedSubValues)
if withQuotes and not spaceSep: if withQuotes and not spaceSep:

32
meshroom/core/cgroup.py
View file

@ -3,21 +3,22 @@
import os import os
#Try to retrieve limits of memory for the current process' cgroup
# Try to retrieve limits of memory for the current process' cgroup
def getCgroupMemorySize(): def getCgroupMemorySize():
#first of all, get pid of process # First of all, get pid of process
pid = os.getpid() pid = os.getpid()
#Get cgroup associated with pid # Get cgroup associated with pid
filename = f"/proc/{pid}/cgroup" filename = f"/proc/{pid}/cgroup"
cgroup = None cgroup = None
try: try:
with open(filename, "r") as f : with open(filename, "r") as f:
#cgroup file is a ':' separated table # cgroup file is a ':' separated table
#lookup a line where the second field is "memory" # lookup a line where the second field is "memory"
lines = f.readlines() lines = f.readlines()
for line in lines: for line in lines:
tokens = line.rstrip("\r\n").split(":") tokens = line.rstrip("\r\n").split(":")
@ -34,7 +35,7 @@ def getCgroupMemorySize():
size = -1 size = -1
filename = f"/sys/fs/cgroup/memory/{cgroup}/memory.limit_in_bytes" filename = f"/sys/fs/cgroup/memory/{cgroup}/memory.limit_in_bytes"
try: try:
with open(filename, "r") as f : with open(filename, "r") as f:
value = f.read().rstrip("\r\n") value = f.read().rstrip("\r\n")
if value.isnumeric(): if value.isnumeric():
size = int(value) size = int(value)
@ -43,6 +44,7 @@ def getCgroupMemorySize():
return size return size
def parseNumericList(numericListString): def parseNumericList(numericListString):
nList = [] nList = []
@ -58,21 +60,22 @@ def parseNumericList(numericListString):
return nList return nList
#Try to retrieve limits of cores for the current process' cgroup
# Try to retrieve limits of cores for the current process' cgroup
def getCgroupCpuCount(): def getCgroupCpuCount():
#first of all, get pid of process # First of all, get pid of process
pid = os.getpid() pid = os.getpid()
#Get cgroup associated with pid # Get cgroup associated with pid
filename = f"/proc/{pid}/cgroup" filename = f"/proc/{pid}/cgroup"
cgroup = None cgroup = None
try: try:
with open(filename, "r") as f : with open(filename, "r") as f:
#cgroup file is a ':' separated table # cgroup file is a ':' separated table
#lookup a line where the second field is "memory" # lookup a line where the second field is "memory"
lines = f.readlines() lines = f.readlines()
for line in lines: for line in lines:
tokens = line.rstrip("\r\n").split(":") tokens = line.rstrip("\r\n").split(":")
@ -89,7 +92,7 @@ def getCgroupCpuCount():
size = -1 size = -1
filename = f"/sys/fs/cgroup/cpuset/{cgroup}/cpuset.cpus" filename = f"/sys/fs/cgroup/cpuset/{cgroup}/cpuset.cpus"
try: try:
with open(filename, "r") as f : with open(filename, "r") as f:
value = f.read().rstrip("\r\n") value = f.read().rstrip("\r\n")
nlist = parseNumericList(value) nlist = parseNumericList(value)
size = len(nlist) size = len(nlist)
@ -98,4 +101,3 @@ def getCgroupCpuCount():
pass pass
return size return size

176
meshroom/core/desc.py
View file

@ -11,6 +11,7 @@ import ast
import distutils.util import distutils.util
import shlex import shlex
class Attribute(BaseObject): class Attribute(BaseObject):
""" """
""" """
@ -32,7 +33,8 @@ class Attribute(BaseObject):
self._errorMessage = errorMessage self._errorMessage = errorMessage
self._visible = visible self._visible = visible
self._exposed = exposed self._exposed = exposed
self._isExpression = (isinstance(self._value, str) and "{" in self._value) or isinstance(self._value, types.FunctionType) self._isExpression = (isinstance(self._value, str) and "{" in self._value) \
or isinstance(self._value, types.FunctionType)
self._isDynamicValue = (self._value is None) self._isDynamicValue = (self._value is None)
self._valueType = None self._valueType = None
@ -48,7 +50,8 @@ class Attribute(BaseObject):
Raises: Raises:
ValueError: if value does not have the proper type ValueError: if value does not have the proper type
""" """
raise NotImplementedError("Attribute.validateValue is an abstract function that should be implemented in the derived class.") raise NotImplementedError("Attribute.validateValue is an abstract function that should be "
"implemented in the derived class.")
def checkValueTypes(self): def checkValueTypes(self):
""" Returns the attribute's name if the default value's type is invalid or if the range's type (when available) """ Returns the attribute's name if the default value's type is invalid or if the range's type (when available)
@ -57,7 +60,8 @@ class Attribute(BaseObject):
Returns: Returns:
string: the attribute's name if the default value's or range's type is invalid, empty string otherwise string: the attribute's name if the default value's or range's type is invalid, empty string otherwise
""" """
raise NotImplementedError("Attribute.checkValueTypes is an abstract function that should be implemented in the derived class.") raise NotImplementedError("Attribute.checkValueTypes is an abstract function that should be implemented in the "
"derived class.")
def matchDescription(self, value, strict=True): def matchDescription(self, value, strict=True):
""" Returns whether the value perfectly match attribute's description. """ Returns whether the value perfectly match attribute's description.
@ -108,13 +112,16 @@ class Attribute(BaseObject):
class ListAttribute(Attribute): class ListAttribute(Attribute):
""" A list of Attributes """ """ A list of Attributes """
def __init__(self, elementDesc, name, label, description, group='allParams', advanced=False, semantic='', enabled=True, joinChar=' ', visible=True, exposed=False): def __init__(self, elementDesc, name, label, description, group='allParams', advanced=False, semantic='',
enabled=True, joinChar=' ', visible=True, exposed=False):
""" """
:param elementDesc: the Attribute description of elements to store in that list :param elementDesc: the Attribute description of elements to store in that list
""" """
self._elementDesc = elementDesc self._elementDesc = elementDesc
self._joinChar = joinChar self._joinChar = joinChar
super(ListAttribute, self).__init__(name=name, label=label, description=description, value=[], invalidate=False, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) super(ListAttribute, self).__init__(name=name, label=label, description=description, value=[],
invalidate=False, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, visible=visible, exposed=exposed)
def getInstanceType(self): def getInstanceType(self):
# Import within the method to prevent cyclic dependencies # Import within the method to prevent cyclic dependencies
@ -126,14 +133,16 @@ class ListAttribute(Attribute):
return value return value
if JSValue is not None and isinstance(value, JSValue): if JSValue is not None and isinstance(value, JSValue):
# Note: we could use isArray(), property("length").toInt() to retrieve all values # Note: we could use isArray(), property("length").toInt() to retrieve all values
raise ValueError("ListAttribute.validateValue: cannot recognize QJSValue. Please, use JSON.stringify(value) in QML.") raise ValueError("ListAttribute.validateValue: cannot recognize QJSValue. "
"Please, use JSON.stringify(value) in QML.")
if isinstance(value, str): if isinstance(value, str):
# Alternative solution to set values from QML is to convert values to JSON string # Alternative solution to set values from QML is to convert values to JSON string
# In this case, it works with all data types # In this case, it works with all data types
value = ast.literal_eval(value) value = ast.literal_eval(value)
if not isinstance(value, (list, tuple)): if not isinstance(value, (list, tuple)):
raise ValueError('ListAttribute only supports list/tuple input values (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) raise ValueError("ListAttribute only supports list/tuple input values "
"(param:{}, value:{}, type:{})".format(self.name, value, type(value)))
return value return value
def checkValueTypes(self): def checkValueTypes(self):
@ -155,14 +164,17 @@ class ListAttribute(Attribute):
class GroupAttribute(Attribute): class GroupAttribute(Attribute):
""" A macro Attribute composed of several Attributes """ """ A macro Attribute composed of several Attributes """
def __init__(self, groupDesc, name, label, description, group='allParams', advanced=False, semantic='', enabled=True, joinChar=' ', brackets=None, visible=True, exposed=False): def __init__(self, groupDesc, name, label, description, group='allParams', advanced=False, semantic='',
enabled=True, joinChar=' ', brackets=None, visible=True, exposed=False):
""" """
:param groupDesc: the description of the Attributes composing this group :param groupDesc: the description of the Attributes composing this group
""" """
self._groupDesc = groupDesc self._groupDesc = groupDesc
self._joinChar = joinChar self._joinChar = joinChar
self._brackets = brackets self._brackets = brackets
super(GroupAttribute, self).__init__(name=name, label=label, description=description, value={}, invalidate=False, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) super(GroupAttribute, self).__init__(name=name, label=label, description=description, value={},
invalidate=False, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, visible=visible, exposed=exposed)
def getInstanceType(self): def getInstanceType(self):
# Import within the method to prevent cyclic dependencies # Import within the method to prevent cyclic dependencies
@ -170,12 +182,13 @@ class GroupAttribute(Attribute):
return GroupAttribute return GroupAttribute
def validateValue(self, value): def validateValue(self, value):
""" Ensure value is compatible with the group description and convert value if needed. """
if value is None: if value is None:
return value return value
""" Ensure value is compatible with the group description and convert value if needed. """
if JSValue is not None and isinstance(value, JSValue): if JSValue is not None and isinstance(value, JSValue):
# Note: we could use isArray(), property("length").toInt() to retrieve all values # Note: we could use isArray(), property("length").toInt() to retrieve all values
raise ValueError("GroupAttribute.validateValue: cannot recognize QJSValue. Please, use JSON.stringify(value) in QML.") raise ValueError("GroupAttribute.validateValue: cannot recognize QJSValue. "
"Please, use JSON.stringify(value) in QML.")
if isinstance(value, str): if isinstance(value, str):
# Alternative solution to set values from QML is to convert values to JSON string # Alternative solution to set values from QML is to convert values to JSON string
# In this case, it works with all data types # In this case, it works with all data types
@ -188,12 +201,16 @@ class GroupAttribute(Attribute):
if self._groupDesc and value.keys(): if self._groupDesc and value.keys():
commonKeys = set(value.keys()).intersection([attr.name for attr in self._groupDesc]) commonKeys = set(value.keys()).intersection([attr.name for attr in self._groupDesc])
if not commonKeys: if not commonKeys:
raise ValueError(f'Value contains no key that matches with the group description (name={self.name}, values={value.keys()}, desc={[attr.name for attr in self._groupDesc]})') raise ValueError(f"Value contains no key that matches with the group description "
f"(name={self.name}, values={value.keys()}, "
f"desc={[attr.name for attr in self._groupDesc]})")
elif isinstance(value, (list, tuple, set)): elif isinstance(value, (list, tuple, set)):
if len(value) != len(self._groupDesc): if len(value) != len(self._groupDesc):
raise ValueError('Value contains incoherent number of values: desc size: {}, value size: {}'.format(len(self._groupDesc), len(value))) raise ValueError("Value contains incoherent number of values: desc size: {}, value size: {}".
format(len(self._groupDesc), len(value)))
else: else:
raise ValueError('GroupAttribute only supports dict/list/tuple input values (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) raise ValueError("GroupAttribute only supports dict/list/tuple input values (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
return value return value
@ -254,24 +271,31 @@ class GroupAttribute(Attribute):
class Param(Attribute): class Param(Attribute):
""" """
""" """
def __init__(self, name, label, description, value, invalidate, group, advanced, semantic, enabled, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False): def __init__(self, name, label, description, value, invalidate, group, advanced, semantic, enabled,
super(Param, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False):
uidIgnoreValue=uidIgnoreValue, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) super(Param, self).__init__(name=name, label=label, description=description, value=value,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, uidIgnoreValue=uidIgnoreValue, validValue=validValue,
errorMessage=errorMessage, visible=visible, exposed=exposed)
class File(Attribute): class File(Attribute):
""" """
""" """
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=True): def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='',
super(File, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) enabled=True, visible=True, exposed=True):
super(File, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate,
group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible,
exposed=exposed)
self._valueType = str self._valueType = str
def validateValue(self, value): def validateValue(self, value):
if value is None: if value is None:
return value return value
if not isinstance(value, str): if not isinstance(value, str):
raise ValueError('File only supports string input (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) raise ValueError("File only supports string input (param:{}, value:{}, type:{})".
return os.path.normpath(value).replace('\\', '/') if value else '' format(self.name, value, type(value)))
return os.path.normpath(value).replace("\\", "/") if value else ""
def checkValueTypes(self): def checkValueTypes(self):
# Some File values are functions generating a string: check whether the value is a string or if it # Some File values are functions generating a string: check whether the value is a string or if it
@ -284,8 +308,11 @@ class File(Attribute):
class BoolParam(Param): class BoolParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False): def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='',
super(BoolParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) enabled=True, visible=True, exposed=False):
super(BoolParam, self).__init__(name=name, label=label, description=description, value=value,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, visible=visible, exposed=exposed)
self._valueType = bool self._valueType = bool
def validateValue(self, value): def validateValue(self, value):
@ -296,8 +323,9 @@ class BoolParam(Param):
# use distutils.util.strtobool to handle (1/0, true/false, on/off, y/n) # use distutils.util.strtobool to handle (1/0, true/false, on/off, y/n)
return bool(distutils.util.strtobool(value)) return bool(distutils.util.strtobool(value))
return bool(value) return bool(value)
except: except Exception:
raise ValueError('BoolParam only supports bool value (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) raise ValueError("BoolParam only supports bool value (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
def checkValueTypes(self): def checkValueTypes(self):
if not isinstance(self.value, bool): if not isinstance(self.value, bool):
@ -308,20 +336,24 @@ class BoolParam(Param):
class IntParam(Param): class IntParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False): def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False,
semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False):
self._range = range self._range = range
super(IntParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, super(IntParam, self).__init__(name=name, label=label, description=description, value=value,
validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, validValue=validValue, errorMessage=errorMessage,
visible=visible, exposed=exposed)
self._valueType = int self._valueType = int
def validateValue(self, value): def validateValue(self, value):
if value is None: if value is None:
return value return value
# handle unsigned int values that are translated to int by shiboken and may overflow # Handle unsigned int values that are translated to int by shiboken and may overflow
try: try:
return int(value) return int(value)
except: except Exception:
raise ValueError('IntParam only supports int value (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) raise ValueError("IntParam only supports int value (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
def checkValueTypes(self): def checkValueTypes(self):
if not isinstance(self.value, int) or (self.range and not all([isinstance(r, int) for r in self.range])): if not isinstance(self.value, int) or (self.range and not all([isinstance(r, int) for r in self.range])):
@ -334,10 +366,13 @@ class IntParam(Param):
class FloatParam(Param): class FloatParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False): def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False,
semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False):
self._range = range self._range = range
super(FloatParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, super(FloatParam, self).__init__(name=name, label=label, description=description, value=value,
validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, validValue=validValue, errorMessage=errorMessage,
visible=visible, exposed=exposed)
self._valueType = float self._valueType = float
def validateValue(self, value): def validateValue(self, value):
@ -345,8 +380,9 @@ class FloatParam(Param):
return value return value
try: try:
return float(value) return float(value)
except: except Exception:
raise ValueError('FloatParam only supports float value (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) raise ValueError("FloatParam only supports float value (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
def checkValueTypes(self): def checkValueTypes(self):
if not isinstance(self.value, float) or (self.range and not all([isinstance(r, float) for r in self.range])): if not isinstance(self.value, float) or (self.range and not all([isinstance(r, float) for r in self.range])):
@ -355,11 +391,15 @@ class FloatParam(Param):
range = Property(VariantList, lambda self: self._range, constant=True) range = Property(VariantList, lambda self: self._range, constant=True)
class PushButtonParam(Param): class PushButtonParam(Param):
""" """
""" """
def __init__(self, name, label, description, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False): def __init__(self, name, label, description, invalidate, group='allParams', advanced=False, semantic='',
super(PushButtonParam, self).__init__(name=name, label=label, description=description, value=None, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) enabled=True, visible=True, exposed=False):
super(PushButtonParam, self).__init__(name=name, label=label, description=description, value=None,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, visible=visible, exposed=exposed)
self._valueType = None self._valueType = None
def getInstanceType(self): def getInstanceType(self):
@ -377,11 +417,14 @@ class PushButtonParam(Param):
class ChoiceParam(Param): class ChoiceParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, values, exclusive, invalidate, group='allParams', joinChar=' ', advanced=False, semantic='', def __init__(self, name, label, description, value, values, exclusive, invalidate, group='allParams', joinChar=' ',
enabled=True, validValue=True, errorMessage="", visible=True, exposed=False): advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True,
exposed=False):
assert values assert values
super(ChoiceParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, super(ChoiceParam, self).__init__(name=name, label=label, description=description, value=value,
semantic=semantic, enabled=enabled, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, validValue=validValue, errorMessage=errorMessage,
visible=visible, exposed=exposed)
self._values = values self._values = values
self._exclusive = exclusive self._exclusive = exclusive
self._joinChar = joinChar self._joinChar = joinChar
@ -416,7 +459,8 @@ class ChoiceParam(Param):
value = value.split(',') value = value.split(',')
if not isinstance(value, Iterable): if not isinstance(value, Iterable):
raise ValueError('Non exclusive ChoiceParam value should be iterable (param: {}, value: {}, type: {}).'.format(self.name, value, type(value))) raise ValueError("Non-exclusive ChoiceParam value should be iterable (param: {}, value: {}, type: {}).".
format(self.name, value, type(value)))
return [self.conformValue(v) for v in value] return [self.conformValue(v) for v in value]
@ -428,7 +472,7 @@ class ChoiceParam(Param):
# If the choices are not exclusive, check that 'value' is a list, and check that it does not contain values that # If the choices are not exclusive, check that 'value' is a list, and check that it does not contain values that
# are not available # are not available
elif not self.exclusive and (not isinstance(self._value, list) or elif not self.exclusive and (not isinstance(self._value, list) or
not all(val in self._values for val in self._value)): not all(val in self._values for val in self._value)):
return self.name return self.name
# If the choices are exclusive, the value should NOT be a list but it can contain any value that is not in the # If the choices are exclusive, the value should NOT be a list but it can contain any value that is not in the
@ -446,16 +490,20 @@ class ChoiceParam(Param):
class StringParam(Param): class StringParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False): def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='',
super(StringParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, enabled=True, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False):
uidIgnoreValue=uidIgnoreValue, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) super(StringParam, self).__init__(name=name, label=label, description=description, value=value,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, uidIgnoreValue=uidIgnoreValue, validValue=validValue,
errorMessage=errorMessage, visible=visible, exposed=exposed)
self._valueType = str self._valueType = str
def validateValue(self, value): def validateValue(self, value):
if value is None: if value is None:
return value return value
if not isinstance(value, str): if not isinstance(value, str):
raise ValueError('StringParam value should be a string (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) raise ValueError("StringParam value should be a string (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
return value return value
def checkValueTypes(self): def checkValueTypes(self):
@ -467,8 +515,11 @@ class StringParam(Param):
class ColorParam(Param): class ColorParam(Param):
""" """
""" """
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False): def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='',
super(ColorParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) enabled=True, visible=True, exposed=False):
super(ColorParam, self).__init__(name=name, label=label, description=description, value=value,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, visible=visible, exposed=exposed)
self._valueType = str self._valueType = str
def validateValue(self, value): def validateValue(self, value):
@ -683,6 +734,7 @@ class Node(object):
BaseNode.updateInternals BaseNode.updateInternals
""" """
pass pass
@classmethod @classmethod
def postUpdate(cls, node): def postUpdate(cls, node):
""" Method call after node's internal update on invalidation. """ Method call after node's internal update on invalidation.
@ -700,6 +752,7 @@ class Node(object):
def processChunk(self, chunk): def processChunk(self, chunk):
raise NotImplementedError('No processChunk implementation on node: "{}"'.format(chunk.node.name)) raise NotImplementedError('No processChunk implementation on node: "{}"'.format(chunk.node.name))
class InputNode(Node): class InputNode(Node):
""" """
Node that does not need to be processed, it is just a placeholder for inputs. Node that does not need to be processed, it is just a placeholder for inputs.
@ -724,12 +777,14 @@ class CommandLineNode(Node):
def buildCommandLine(self, chunk): def buildCommandLine(self, chunk):
cmdPrefix = '' cmdPrefix = ''
# if rez available in env, we use it # If rez available in env, we use it
if 'REZ_ENV' in os.environ and chunk.node.packageVersion: if "REZ_ENV" in os.environ and chunk.node.packageVersion:
# if the node package is already in the environment, we don't need a new dedicated rez environment # If the node package is already in the environment, we don't need a new dedicated rez environment
alreadyInEnv = os.environ.get('REZ_{}_VERSION'.format(chunk.node.packageName.upper()), "").startswith(chunk.node.packageVersion) alreadyInEnv = os.environ.get("REZ_{}_VERSION".format(chunk.node.packageName.upper()),
"").startswith(chunk.node.packageVersion)
if not alreadyInEnv: if not alreadyInEnv:
cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get('REZ_ENV'), packageFullName=chunk.node.packageFullName) cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get("REZ_ENV"),
packageFullName=chunk.node.packageFullName)
cmdSuffix = '' cmdSuffix = ''
if chunk.node.isParallelized and chunk.node.size > 1: if chunk.node.isParallelized and chunk.node.size > 1:
@ -738,12 +793,12 @@ class CommandLineNode(Node):
return cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix return cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix
def stopProcess(self, chunk): def stopProcess(self, chunk):
# the same node could exists several times in the graph and # The same node could exists several times in the graph and
# only one would have the running subprocess; ignore all others # only one would have the running subprocess; ignore all others
if not hasattr(chunk, "subprocess"): if not hasattr(chunk, "subprocess"):
return return
if chunk.subprocess: if chunk.subprocess:
# kill process tree # Kill process tree
processes = chunk.subprocess.children(recursive=True) + [chunk.subprocess] processes = chunk.subprocess.children(recursive=True) + [chunk.subprocess]
try: try:
for process in processes: for process in processes:
@ -761,7 +816,7 @@ class CommandLineNode(Node):
print(' - logFile: {}'.format(chunk.logFile)) print(' - logFile: {}'.format(chunk.logFile))
chunk.subprocess = psutil.Popen(shlex.split(cmd), stdout=logF, stderr=logF, cwd=chunk.node.internalFolder) chunk.subprocess = psutil.Popen(shlex.split(cmd), stdout=logF, stderr=logF, cwd=chunk.node.internalFolder)
# store process static info into the status file # Store process static info into the status file
# chunk.status.env = node.proc.environ() # chunk.status.env = node.proc.environ()
# chunk.status.createTime = node.proc.create_time() # chunk.status.createTime = node.proc.create_time()
@ -775,11 +830,12 @@ class CommandLineNode(Node):
with open(chunk.logFile, 'r') as logF: with open(chunk.logFile, 'r') as logF:
logContent = ''.join(logF.readlines()) logContent = ''.join(logF.readlines())
raise RuntimeError('Error on node "{}":\nLog:\n{}'.format(chunk.name, logContent)) raise RuntimeError('Error on node "{}":\nLog:\n{}'.format(chunk.name, logContent))
except: except Exception:
raise raise
finally: finally:
chunk.subprocess = None chunk.subprocess = None
# Specific command line node for AliceVision apps # Specific command line node for AliceVision apps
class AVCommandLineNode(CommandLineNode): class AVCommandLineNode(CommandLineNode):
@ -809,7 +865,7 @@ class AVCommandLineNode(CommandLineNode):
return commandLineString + AVCommandLineNode.cmdMem + AVCommandLineNode.cmdCore return commandLineString + AVCommandLineNode.cmdMem + AVCommandLineNode.cmdCore
# Test abstract node
class InitNode(object): class InitNode(object):
def __init__(self): def __init__(self):
super(InitNode, self).__init__() super(InitNode, self).__init__()

50
meshroom/core/graph.py
View file

@ -22,6 +22,7 @@ from meshroom.core.node import nodeFactory, Status, Node, CompatibilityNode
DefaultJSONEncoder = json.JSONEncoder # store the original one DefaultJSONEncoder = json.JSONEncoder # store the original one
class MyJSONEncoder(DefaultJSONEncoder): # declare a new one with Enum support class MyJSONEncoder(DefaultJSONEncoder): # declare a new one with Enum support
def default(self, obj): def default(self, obj):
if isinstance(obj, Enum): if isinstance(obj, Enum):
@ -220,7 +221,8 @@ class Graph(BaseObject):
self._computationBlocked = {} self._computationBlocked = {}
self._canComputeLeaves = True self._canComputeLeaves = True
self._nodes = DictModel(keyAttrName='name', parent=self) self._nodes = DictModel(keyAttrName='name', parent=self)
self._edges = DictModel(keyAttrName='dst', parent=self) # use dst attribute as unique key since it can only have one input connection # Edges: use dst attribute as unique key since it can only have one input connection
self._edges = DictModel(keyAttrName='dst', parent=self)
self._importedNodes = DictModel(keyAttrName='name', parent=self) self._importedNodes = DictModel(keyAttrName='name', parent=self)
self._compatibilityNodes = DictModel(keyAttrName='name', parent=self) self._compatibilityNodes = DictModel(keyAttrName='name', parent=self)
self.cacheDir = meshroom.core.defaultCacheFolder self.cacheDir = meshroom.core.defaultCacheFolder
@ -682,9 +684,12 @@ class Graph(BaseObject):
""" """
Remove the node identified by 'nodeName' from the graph. Remove the node identified by 'nodeName' from the graph.
Returns: Returns:
- a dictionary containing the incoming edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} - a dictionary containing the incoming edges removed by this operation:
- a dictionary containing the outgoing edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute prior to the removal of all edges: - a dictionary containing the outgoing edges removed by this operation:
{dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute
prior to the removal of all edges:
{dstAttr.getFullNameToNode(), (dstAttr.root.getFullNameToNode(), dstAttr.index, dstAttr.value)} {dstAttr.getFullNameToNode(), (dstAttr.root.getFullNameToNode(), dstAttr.index, dstAttr.value)}
""" """
node = self.node(nodeName) node = self.node(nodeName)
@ -695,14 +700,18 @@ class Graph(BaseObject):
# Remove all edges arriving to and starting from this node # Remove all edges arriving to and starting from this node
with GraphModification(self): with GraphModification(self):
# Two iterations over the outgoing edges are necessary: # Two iterations over the outgoing edges are necessary:
# - the first one is used to collect all the information about the edges while they are all there (overall context) # - the first one is used to collect all the information about the edges while they are all there
# - once we have collected all the information, the edges (and perhaps the entries in ListAttributes) can actually be removed # (overall context)
# - once we have collected all the information, the edges (and perhaps the entries in ListAttributes) can
# actually be removed
for edge in self.nodeOutEdges(node): for edge in self.nodeOutEdges(node):
outEdges[edge.dst.getFullNameToNode()] = edge.src.getFullNameToNode() outEdges[edge.dst.getFullNameToNode()] = edge.src.getFullNameToNode()
if isinstance(edge.dst.root, ListAttribute): if isinstance(edge.dst.root, ListAttribute):
index = edge.dst.root.index(edge.dst) index = edge.dst.root.index(edge.dst)
outListAttributes[edge.dst.getFullNameToNode()] = (edge.dst.root.getFullNameToNode(), index, edge.dst.value if edge.dst.value else None) outListAttributes[edge.dst.getFullNameToNode()] = (edge.dst.root.getFullNameToNode(),
index, edge.dst.value
if edge.dst.value else None)
for edge in self.nodeOutEdges(node): for edge in self.nodeOutEdges(node):
self.removeEdge(edge.dst) self.removeEdge(edge.dst)
@ -763,9 +772,12 @@ class Graph(BaseObject):
Returns: Returns:
- the upgraded (newly created) node - the upgraded (newly created) node
- a dictionary containing the incoming edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} - a dictionary containing the incoming edges removed by this operation:
- a dictionary containing the outgoing edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute prior to the removal of all edges: - a dictionary containing the outgoing edges removed by this operation:
{dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute
prior to the removal of all edges:
{dstAttr.getFullNameToNode(), (dstAttr.root.getFullNameToNode(), dstAttr.index, dstAttr.value)} {dstAttr.getFullNameToNode(), (dstAttr.root.getFullNameToNode(), dstAttr.index, dstAttr.value)}
""" """
node = self.node(nodeName) node = self.node(nodeName)
@ -834,7 +846,7 @@ class Graph(BaseObject):
""" """
try: try:
return int(name.split('_')[-1]) return int(name.split('_')[-1])
except: except Exception:
return -1 return -1
@staticmethod @staticmethod
@ -970,7 +982,8 @@ class Graph(BaseObject):
def dfs(self, visitor, startNodes=None, longestPathFirst=False): def dfs(self, visitor, startNodes=None, longestPathFirst=False):
# Default direction (visitor.reverse=False): from node to root # Default direction (visitor.reverse=False): from node to root
# Reverse direction (visitor.reverse=True): from node to leaves # Reverse direction (visitor.reverse=True): from node to leaves
nodeChildren = self._getOutputEdgesPerNode(visitor.dependenciesOnly) if visitor.reverse else self._getInputEdgesPerNode(visitor.dependenciesOnly) nodeChildren = self._getOutputEdgesPerNode(visitor.dependenciesOnly) \
if visitor.reverse else self._getInputEdgesPerNode(visitor.dependenciesOnly)
# Initialize color map # Initialize color map
colors = {} colors = {}
for u in self._nodes: for u in self._nodes:
@ -979,9 +992,11 @@ class Graph(BaseObject):
if longestPathFirst and visitor.reverse: if longestPathFirst and visitor.reverse:
# Because we have no knowledge of the node's count between a node and its leaves, # Because we have no knowledge of the node's count between a node and its leaves,
# it is not possible to handle this case at the moment # it is not possible to handle this case at the moment
raise NotImplementedError("Graph.dfs(): longestPathFirst=True and visitor.reverse=True are not compatible yet.") raise NotImplementedError("Graph.dfs(): longestPathFirst=True and visitor.reverse=True are not "
"compatible yet.")
nodes = startNodes or (self.getRootNodes(visitor.dependenciesOnly) if visitor.reverse else self.getLeafNodes(visitor.dependenciesOnly)) nodes = startNodes or (self.getRootNodes(visitor.dependenciesOnly)
if visitor.reverse else self.getLeafNodes(visitor.dependenciesOnly))
if longestPathFirst: if longestPathFirst:
# Graph topology must be known and node depths up-to-date # Graph topology must be known and node depths up-to-date
@ -1302,7 +1317,6 @@ class Graph(BaseObject):
self.dfs(visitor=visitor, startNodes=[startNode]) self.dfs(visitor=visitor, startNodes=[startNode])
return visitor.canCompute + (2 * visitor.canSubmit) return visitor.canCompute + (2 * visitor.canSubmit)
def _applyExpr(self): def _applyExpr(self):
with GraphModification(self): with GraphModification(self):
for node in self._nodes: for node in self._nodes:
@ -1600,14 +1614,15 @@ class Graph(BaseObject):
if node.hasAttribute('verbose'): if node.hasAttribute('verbose'):
try: try:
node.verbose.value = v node.verbose.value = v
except: except Exception:
pass pass
nodes = Property(BaseObject, nodes.fget, constant=True) nodes = Property(BaseObject, nodes.fget, constant=True)
edges = Property(BaseObject, edges.fget, constant=True) edges = Property(BaseObject, edges.fget, constant=True)
filepathChanged = Signal() filepathChanged = Signal()
filepath = Property(str, lambda self: self._filepath, notify=filepathChanged) filepath = Property(str, lambda self: self._filepath, notify=filepathChanged)
fileReleaseVersion = Property(str, lambda self: self.header.get(Graph.IO.Keys.ReleaseVersion, "0.0"), notify=filepathChanged) fileReleaseVersion = Property(str, lambda self: self.header.get(Graph.IO.Keys.ReleaseVersion, "0.0"),
notify=filepathChanged)
fileDateVersion = Property(float, fileDateVersion.fget, fileDateVersion.fset, notify=filepathChanged) fileDateVersion = Property(float, fileDateVersion.fget, fileDateVersion.fset, notify=filepathChanged)
cacheDirChanged = Signal() cacheDirChanged = Signal()
cacheDir = Property(str, cacheDir.fget, cacheDir.fset, notify=cacheDirChanged) cacheDir = Property(str, cacheDir.fget, cacheDir.fset, notify=cacheDirChanged)
@ -1721,4 +1736,3 @@ def submit(graphFile, submitter, toNode=None, submitLabel="{projectName}"):
graph = loadGraph(graphFile) graph = loadGraph(graphFile)
toNodes = graph.findNodes(toNode) if toNode else None toNodes = graph.findNodes(toNode) if toNode else None
submitGraph(graph, submitter, toNodes, submitLabel=submitLabel) submitGraph(graph, submitter, toNodes, submitLabel=submitLabel)

103
meshroom/core/node.py
View file

@ -12,7 +12,7 @@ import shutil
import time import time
import types import types
import uuid import uuid
from collections import defaultdict, namedtuple from collections import namedtuple
from enum import Enum from enum import Enum
import meshroom import meshroom
@ -33,7 +33,7 @@ def renameWritingToFinalPath(writingFilepath, filepath):
for i in range(20): for i in range(20):
try: try:
os.remove(filepath) os.remove(filepath)
# if remove is successful, we can stop the iterations # If remove is successful, we can stop the iterations
break break
except WindowsError: except WindowsError:
pass pass
@ -50,7 +50,7 @@ class Status(Enum):
STOPPED = 4 STOPPED = 4
KILLED = 5 KILLED = 5
SUCCESS = 6 SUCCESS = 6
INPUT = 7 # special status for input nodes INPUT = 7 # Special status for input nodes
class ExecMode(Enum): class ExecMode(Enum):
@ -252,7 +252,7 @@ class NodeChunk(BaseObject):
self.statistics = stats.Statistics() self.statistics = stats.Statistics()
self.statusFileLastModTime = -1 self.statusFileLastModTime = -1
self._subprocess = None self._subprocess = None
# notify update in filepaths when node's internal folder changes # Notify update in filepaths when node's internal folder changes
self.node.internalFolderChanged.connect(self.nodeFolderChanged) self.node.internalFolderChanged.connect(self.nodeFolderChanged)
self.execModeNameChanged.connect(self.node.globalExecModeChanged) self.execModeNameChanged.connect(self.node.globalExecModeChanged)
@ -296,7 +296,7 @@ class NodeChunk(BaseObject):
statusData = json.load(jsonFile) statusData = json.load(jsonFile)
self.status.fromDict(statusData) self.status.fromDict(statusData)
self.statusFileLastModTime = os.path.getmtime(statusFile) self.statusFileLastModTime = os.path.getmtime(statusFile)
except Exception as e: except Exception:
self.statusFileLastModTime = -1 self.statusFileLastModTime = -1
self.status.reset() self.status.reset()
@ -306,23 +306,23 @@ class NodeChunk(BaseObject):
@property @property
def statusFile(self): def statusFile(self):
if self.range.blockSize == 0: if self.range.blockSize == 0:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'status') return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "status")
else: else:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.status') return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".status")
@property @property
def statisticsFile(self): def statisticsFile(self):
if self.range.blockSize == 0: if self.range.blockSize == 0:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'statistics') return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "statistics")
else: else:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.statistics') return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".statistics")
@property @property
def logFile(self): def logFile(self):
if self.range.blockSize == 0: if self.range.blockSize == 0:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'log') return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "log")
else: else:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.log') return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".log")
def saveStatusFile(self): def saveStatusFile(self):
""" """
@ -333,7 +333,7 @@ class NodeChunk(BaseObject):
folder = os.path.dirname(statusFilepath) folder = os.path.dirname(statusFilepath)
try: try:
os.makedirs(folder) os.makedirs(folder)
except Exception as e: except Exception:
pass pass
statusFilepathWriting = getWritingFilepath(statusFilepath) statusFilepathWriting = getWritingFilepath(statusFilepath)
@ -343,8 +343,8 @@ class NodeChunk(BaseObject):
def upgradeStatusTo(self, newStatus, execMode=None): def upgradeStatusTo(self, newStatus, execMode=None):
if newStatus.value <= self._status.status.value: if newStatus.value <= self._status.status.value:
logging.warning('Downgrade status on node "{}" from {} to {}'.format(self.name, self._status.status, logging.warning("Downgrade status on node '{}' from {} to {}".
newStatus)) format(self.name, self._status.status, newStatus))
if newStatus == Status.SUBMITTED: if newStatus == Status.SUBMITTED:
self._status = StatusData(self.node.name, self.node.nodeType, self.node.packageName, self.node.packageVersion) self._status = StatusData(self.node.name, self.node.nodeType, self.node.packageName, self.node.packageVersion)
@ -411,11 +411,11 @@ class NodeChunk(BaseObject):
self.statThread.start() self.statThread.start()
try: try:
self.node.nodeDesc.processChunk(self) self.node.nodeDesc.processChunk(self)
except Exception as e: except Exception:
if self._status.status != Status.STOPPED: if self._status.status != Status.STOPPED:
exceptionStatus = Status.ERROR exceptionStatus = Status.ERROR
raise raise
except (KeyboardInterrupt, SystemError, GeneratorExit) as e: except (KeyboardInterrupt, SystemError, GeneratorExit):
exceptionStatus = Status.STOPPED exceptionStatus = Status.STOPPED
raise raise
finally: finally:
@ -423,8 +423,8 @@ class NodeChunk(BaseObject):
self._status.elapsedTime = time.time() - startTime self._status.elapsedTime = time.time() - startTime
if exceptionStatus is not None: if exceptionStatus is not None:
self.upgradeStatusTo(exceptionStatus) self.upgradeStatusTo(exceptionStatus)
logging.info(' - elapsed time: {}'.format(self._status.elapsedTimeStr)) logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr))
# ask and wait for the stats thread to stop # Ask and wait for the stats thread to stop
self.statThread.stopRequest() self.statThread.stopRequest()
self.statThread.join() self.statThread.join()
self.statistics = stats.Statistics() self.statistics = stats.Statistics()
@ -462,9 +462,9 @@ class NodeChunk(BaseObject):
elapsedTime = Property(float, lambda self: self._status.elapsedTime, notify=statusChanged) elapsedTime = Property(float, lambda self: self._status.elapsedTime, notify=statusChanged)
# simple structure for storing node position # Simple structure for storing node position
Position = namedtuple("Position", ["x", "y"]) Position = namedtuple("Position", ["x", "y"])
# initialize default coordinates values to 0 # Initialize default coordinates values to 0
Position.__new__.__defaults__ = (0,) * len(Position._fields) Position.__new__.__defaults__ = (0,) * len(Position._fields)
@ -729,7 +729,8 @@ class BaseNode(BaseObject):
def _buildCmdVars(self): def _buildCmdVars(self):
def _buildAttributeCmdVars(cmdVars, name, attr): def _buildAttributeCmdVars(cmdVars, name, attr):
if attr.enabled: if attr.enabled:
group = attr.attributeDesc.group(attr.node) if isinstance(attr.attributeDesc.group, types.FunctionType) else attr.attributeDesc.group group = attr.attributeDesc.group(attr.node) \
if isinstance(attr.attributeDesc.group, types.FunctionType) else attr.attributeDesc.group
if group is not None: if group is not None:
# If there is a valid command line "group" # If there is a valid command line "group"
v = attr.getValueStr(withQuotes=True) v = attr.getValueStr(withQuotes=True)
@ -776,18 +777,25 @@ class BaseNode(BaseObject):
if attr.enabled: if attr.enabled:
try: try:
defaultValue = attr.defaultValue() defaultValue = attr.defaultValue()
except AttributeError as e: except AttributeError:
# If we load an old scene, the lambda associated to the 'value' could try to access other # If we load an old scene, the lambda associated to the 'value' could try to access other
# params that could not exist yet # params that could not exist yet
logging.warning('Invalid lambda evaluation for "{nodeName}.{attrName}"'.format(nodeName=self.name, attrName=attr.name)) logging.warning('Invalid lambda evaluation for "{nodeName}.{attrName}"'.
format(nodeName=self.name, attrName=attr.name))
if defaultValue is not None: if defaultValue is not None:
try: try:
attr.value = defaultValue.format(**self._cmdVars) attr.value = defaultValue.format(**self._cmdVars)
attr._invalidationValue = defaultValue.format(**cmdVarsNoCache) attr._invalidationValue = defaultValue.format(**cmdVarsNoCache)
except KeyError as e: except KeyError as e:
logging.warning('Invalid expression with missing key on "{nodeName}.{attrName}" with value "{defaultValue}".\nError: {err}'.format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue, err=str(e))) logging.warning('Invalid expression with missing key on "{nodeName}.{attrName}" with '
'value "{defaultValue}".\nError: {err}'.
format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue,
err=str(e)))
except ValueError as e: except ValueError as e:
logging.warning('Invalid expression value on "{nodeName}.{attrName}" with value "{defaultValue}".\nError: {err}'.format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue, err=str(e))) logging.warning('Invalid expression value on "{nodeName}.{attrName}" with value '
'"{defaultValue}".\nError: {err}'.
format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue,
err=str(e)))
v = attr.getValueStr(withQuotes=True) v = attr.getValueStr(withQuotes=True)
@ -1074,9 +1082,11 @@ class BaseNode(BaseObject):
self.attribute(output.name).value = data[output.name] self.attribute(output.name).value = data[output.name]
else: else:
if not self.hasAttribute(output.name): if not self.hasAttribute(output.name):
logging.warning(f"loadOutputAttr: Missing dynamic output attribute. Node={self.name}, Attribute={output.name}") logging.warning(f"loadOutputAttr: Missing dynamic output attribute. Node={self.name}, "
f"Attribute={output.name}")
if output.name not in data: if output.name not in data:
logging.warning(f"loadOutputAttr: Missing dynamic output value in file. Node={self.name}, Attribute={output.name}, File={valuesFile}, Data keys={data.keys()}") logging.warning(f"loadOutputAttr: Missing dynamic output value in file. Node={self.name}, "
f"Attribute={output.name}, File={valuesFile}, Data keys={data.keys()}")
def saveOutputAttr(self): def saveOutputAttr(self):
""" Save output attributes with dynamic values into a values.json file. """ Save output attributes with dynamic values into a values.json file.
@ -1272,7 +1282,6 @@ class BaseNode(BaseObject):
self._hasDuplicates = bool(len(newList)) self._hasDuplicates = bool(len(newList))
self.hasDuplicatesChanged.emit() self.hasDuplicatesChanged.emit()
def statusInThisSession(self): def statusInThisSession(self):
if not self._chunks: if not self._chunks:
return False return False
@ -1297,7 +1306,8 @@ class BaseNode(BaseObject):
def hasImageOutputAttribute(self): def hasImageOutputAttribute(self):
""" """
Return True if at least one attribute has the 'image' semantic (and can thus be loaded in the 2D Viewer), False otherwise. Return True if at least one attribute has the 'image' semantic (and can thus be loaded in the 2D Viewer),
False otherwise.
""" """
for attr in self._attributes: for attr in self._attributes:
if attr.enabled and attr.isOutput and attr.desc.semantic == "image": if attr.enabled and attr.isOutput and attr.desc.semantic == "image":
@ -1306,10 +1316,12 @@ class BaseNode(BaseObject):
def hasSequenceOutputAttribute(self): def hasSequenceOutputAttribute(self):
""" """
Return True if at least one attribute has the 'sequence' semantic (and can thus be loaded in the 2D Viewer), False otherwise. Return True if at least one attribute has the 'sequence' semantic (and can thus be loaded in the 2D Viewer),
False otherwise.
""" """
for attr in self._attributes: for attr in self._attributes:
if attr.enabled and attr.isOutput and (attr.desc.semantic == "sequence" or attr.desc.semantic == "imageList"): if attr.enabled and attr.isOutput and (attr.desc.semantic == "sequence" or
attr.desc.semantic == "imageList"):
return True return True
return False return False
@ -1326,7 +1338,6 @@ class BaseNode(BaseObject):
return True return True
return False return False
name = Property(str, getName, constant=True) name = Property(str, getName, constant=True)
defaultLabel = Property(str, getDefaultLabel, constant=True) defaultLabel = Property(str, getDefaultLabel, constant=True)
nodeType = Property(str, nodeType.fget, constant=True) nodeType = Property(str, nodeType.fget, constant=True)
@ -1356,8 +1367,10 @@ class BaseNode(BaseObject):
globalStatus = Property(str, lambda self: self.getGlobalStatus().name, notify=globalStatusChanged) globalStatus = Property(str, lambda self: self.getGlobalStatus().name, notify=globalStatusChanged)
fusedStatus = Property(StatusData, getFusedStatus, notify=globalStatusChanged) fusedStatus = Property(StatusData, getFusedStatus, notify=globalStatusChanged)
elapsedTime = Property(float, lambda self: self.getFusedStatus().elapsedTime, notify=globalStatusChanged) elapsedTime = Property(float, lambda self: self.getFusedStatus().elapsedTime, notify=globalStatusChanged)
recursiveElapsedTime = Property(float, lambda self: self.getRecursiveFusedStatus().elapsedTime, notify=globalStatusChanged) recursiveElapsedTime = Property(float, lambda self: self.getRecursiveFusedStatus().elapsedTime,
isCompatibilityNode = Property(bool, lambda self: self._isCompatibilityNode(), constant=True) # need lambda to evaluate the virtual function notify=globalStatusChanged)
# isCompatibilityNode: need lambda to evaluate the virtual function
isCompatibilityNode = Property(bool, lambda self: self._isCompatibilityNode(), constant=True)
isInputNode = Property(bool, lambda self: self._isInputNode(), constant=True) isInputNode = Property(bool, lambda self: self._isInputNode(), constant=True)
globalExecModeChanged = Signal() globalExecModeChanged = Signal()
@ -1378,6 +1391,7 @@ class BaseNode(BaseObject):
hasSequenceOutput = Property(bool, hasSequenceOutputAttribute, notify=outputAttrEnabledChanged) hasSequenceOutput = Property(bool, hasSequenceOutputAttribute, notify=outputAttrEnabledChanged)
has3DOutput = Property(bool, has3DOutputAttribute, notify=outputAttrEnabledChanged) has3DOutput = Property(bool, has3DOutputAttribute, notify=outputAttrEnabledChanged)
class Node(BaseNode): class Node(BaseNode):
""" """
A standard Graph node based on a node type. A standard Graph node based on a node type.
@ -1399,7 +1413,8 @@ class Node(BaseNode):
self._attributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=True, node=self)) self._attributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=True, node=self))
for attrDesc in self.nodeDesc.internalInputs: for attrDesc in self.nodeDesc.internalInputs:
self._internalAttributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=False, node=self)) self._internalAttributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=False,
node=self))
# Declare events for specific output attributes # Declare events for specific output attributes
for attr in self._attributes: for attr in self._attributes:
@ -1421,7 +1436,6 @@ class Node(BaseNode):
self.optionalCallOnDescriptor("onNodeCreated") self.optionalCallOnDescriptor("onNodeCreated")
def optionalCallOnDescriptor(self, methodName, *args, **kwargs): def optionalCallOnDescriptor(self, methodName, *args, **kwargs):
""" Call of optional method defined in the descriptor. """ Call of optional method defined in the descriptor.
Available method names are: Available method names are:
@ -1432,7 +1446,7 @@ class Node(BaseNode):
if callable(m): if callable(m):
try: try:
m(self, *args, **kwargs) m(self, *args, **kwargs)
except Exception as e: except Exception:
import traceback import traceback
# Format error strings with all the provided arguments # Format error strings with all the provided arguments
argsStr = ", ".join(str(arg) for arg in args) argsStr = ", ".join(str(arg) for arg in args)
@ -1443,7 +1457,8 @@ class Node(BaseNode):
finalErrStr += ", " finalErrStr += ", "
finalErrStr += kwargsStr finalErrStr += kwargsStr
logging.error("Error on call to '{}' (with args: '{}') for node type {}".format(methodName, finalErrStr, self.nodeType)) logging.error("Error on call to '{}' (with args: '{}') for node type {}".
format(methodName, finalErrStr, self.nodeType))
logging.error(traceback.format_exc()) logging.error(traceback.format_exc())
def setAttributeValues(self, values): def setAttributeValues(self, values):
@ -1491,7 +1506,8 @@ class Node(BaseNode):
def toDict(self): def toDict(self):
inputs = {k: v.getExportValue() for k, v in self._attributes.objects.items() if v.isInput} inputs = {k: v.getExportValue() for k, v in self._attributes.objects.items() if v.isInput}
internalInputs = {k: v.getExportValue() for k, v in self._internalAttributes.objects.items()} internalInputs = {k: v.getExportValue() for k, v in self._internalAttributes.objects.items()}
outputs = ({k: v.getExportValue() for k, v in self._attributes.objects.items() if v.isOutput and not v.desc.isDynamicValue}) outputs = ({k: v.getExportValue() for k, v in self._attributes.objects.items()
if v.isOutput and not v.desc.isDynamicValue})
return { return {
'nodeType': self.nodeType, 'nodeType': self.nodeType,
@ -1788,7 +1804,8 @@ class CompatibilityNode(BaseNode):
upgradedAttrValues = attrValues upgradedAttrValues = attrValues
if not isinstance(upgradedAttrValues, dict): if not isinstance(upgradedAttrValues, dict):
logging.error("Error in the upgrade implementation of the node: {}. The return type is incorrect.".format(self.name)) logging.error("Error in the upgrade implementation of the node: {}. The return type is incorrect.".
format(self.name))
upgradedAttrValues = attrValues upgradedAttrValues = attrValues
node.upgradeAttributeValues(upgradedAttrValues) node.upgradeAttributeValues(upgradedAttrValues)
@ -1859,8 +1876,10 @@ def nodeFactory(nodeDict, name=None, template=False, uidConflict=False):
# do not perform that check for internal attributes because there is no point in # do not perform that check for internal attributes because there is no point in
# raising compatibility issues if their number differs: in that case, it is only useful # raising compatibility issues if their number differs: in that case, it is only useful
# if some internal attributes do not exist or are invalid # if some internal attributes do not exist or are invalid
if not template and (sorted([attr.name for attr in nodeDesc.inputs if not isinstance(attr, desc.PushButtonParam)]) != sorted(inputs.keys()) or \ if not template and (sorted([attr.name for attr in nodeDesc.inputs
sorted([attr.name for attr in nodeDesc.outputs if not attr.isDynamicValue]) != sorted(outputs.keys())): if not isinstance(attr, desc.PushButtonParam)]) != sorted(inputs.keys()) or
sorted([attr.name for attr in nodeDesc.outputs if not attr.isDynamicValue]) !=
sorted(outputs.keys())):
compatibilityIssue = CompatibilityIssue.DescriptionConflict compatibilityIssue = CompatibilityIssue.DescriptionConflict
# Check whether there are any internal attributes that are invalidating in the node description: if there # Check whether there are any internal attributes that are invalidating in the node description: if there

11
meshroom/core/stats.py
View file

@ -6,7 +6,6 @@ import time
import threading import threading
import platform import platform
import os import os
import sys
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
@ -56,7 +55,7 @@ class ComputerStatistics:
# If the platform is Windows and nvidia-smi # If the platform is Windows and nvidia-smi
self.nvidia_smi = spawn.find_executable('nvidia-smi') self.nvidia_smi = spawn.find_executable('nvidia-smi')
if self.nvidia_smi is None: if self.nvidia_smi is None:
# could not be found from the environment path, # Could not be found from the environment path,
# try to find it from system drive with default installation path # try to find it from system drive with default installation path
default_nvidia_smi = "%s\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe" % os.environ['systemdrive'] default_nvidia_smi = "%s\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe" % os.environ['systemdrive']
if os.path.isfile(default_nvidia_smi): if os.path.isfile(default_nvidia_smi):
@ -77,7 +76,8 @@ class ComputerStatistics:
def update(self): def update(self):
try: try:
self.initOnFirstTime() self.initOnFirstTime()
self._addKV('cpuUsage', psutil.cpu_percent(percpu=True)) # interval=None => non-blocking (percentage since last call) # Interval=None => non-blocking (percentage since last call)
self._addKV('cpuUsage', psutil.cpu_percent(percpu=True))
self._addKV('ramUsage', psutil.virtual_memory().percent) self._addKV('ramUsage', psutil.virtual_memory().percent)
self._addKV('swapUsage', psutil.swap_memory().percent) self._addKV('swapUsage', psutil.swap_memory().percent)
self._addKV('vramUsage', 0) self._addKV('vramUsage', 0)
@ -91,7 +91,7 @@ class ComputerStatistics:
return return
try: try:
p = subprocess.Popen([self.nvidia_smi, "-q", "-x"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p = subprocess.Popen([self.nvidia_smi, "-q", "-x"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
xmlGpu, stdError = p.communicate(timeout=10) # 10 seconds xmlGpu, stdError = p.communicate(timeout=10) # 10 seconds
smiTree = ET.fromstring(xmlGpu) smiTree = ET.fromstring(xmlGpu)
gpuTree = smiTree.find('gpu') gpuTree = smiTree.find('gpu')
@ -140,6 +140,7 @@ class ComputerStatistics:
for k, v in d.items(): for k, v in d.items():
setattr(self, k, v) setattr(self, k, v)
class ProcStatistics: class ProcStatistics:
staticKeys = [ staticKeys = [
'pid', 'pid',
@ -201,7 +202,7 @@ class ProcStatistics:
for k, v in data.items(): for k, v in data.items():
self._addKV(k, v) self._addKV(k, v)
## Note: Do not collect stats about open files for now, # Note: Do not collect stats about open files for now,
# as there is bug in psutil-5.7.2 on Windows which crashes the application. # as there is bug in psutil-5.7.2 on Windows which crashes the application.
# https://github.com/giampaolo/psutil/issues/1763 # https://github.com/giampaolo/psutil/issues/1763
# #

15
meshroom/core/taskManager.py
View file

@ -211,11 +211,8 @@ class TaskManager(BaseObject):
chunksName = [node.name for node in chunksInConflict] chunksName = [node.name for node in chunksInConflict]
# Warning: Syntax and terms are parsed on QML side to recognize the error # Warning: Syntax and terms are parsed on QML side to recognize the error
# Syntax : [Context] ErrorType: ErrorMessage # Syntax : [Context] ErrorType: ErrorMessage
msg = '[COMPUTATION] Already Submitted:\n' \ msg = '[COMPUTATION] Already Submitted:\nWARNING - Some nodes are already submitted with status: ' \
'WARNING - Some nodes are already submitted with status: {}\nNodes: {}'.format( '{}\nNodes: {}'.format(', '.join(chunksStatus), ', '.join(chunksName))
', '.join(chunksStatus),
', '.join(chunksName)
)
if forceStatus: if forceStatus:
logging.warning(msg) logging.warning(msg)
@ -325,8 +322,9 @@ class TaskManager(BaseObject):
raise RuntimeError("[{}] Duplicates Issue:\n" raise RuntimeError("[{}] Duplicates Issue:\n"
"Cannot compute because there are some duplicate nodes to process:\n\n" "Cannot compute because there are some duplicate nodes to process:\n\n"
"First match: '{}' and '{}'\n\n" "First match: '{}' and '{}'\n\n"
"There can be other duplicate nodes in the list. Please, check the graph and try again.".format( "There can be other duplicate nodes in the list. "
context, node.nameToLabel(node.name), node.nameToLabel(duplicate.name))) "Please, check the graph and try again.".
format(context, node.nameToLabel(node.name), node.nameToLabel(duplicate.name)))
def checkNodesDependencies(self, graph, toNodes, context): def checkNodesDependencies(self, graph, toNodes, context):
""" """
@ -366,7 +364,8 @@ class TaskManager(BaseObject):
# Warning: Syntax and terms are parsed on QML side to recognize the error # Warning: Syntax and terms are parsed on QML side to recognize the error
# Syntax : [Context] ErrorType: ErrorMessage # Syntax : [Context] ErrorType: ErrorMessage
raise RuntimeWarning("[{}] Unresolved dependencies:\n" raise RuntimeWarning("[{}] Unresolved dependencies:\n"
"Some nodes cannot be computed in LOCAL/submitted in EXTERN because of unresolved dependencies.\n\n" "Some nodes cannot be computed in LOCAL/submitted in EXTERN because of "
"unresolved dependencies.\n\n"
"Nodes which are ready will be processed.".format(context)) "Nodes which are ready will be processed.".format(context))
def raiseImpossibleProcess(self, context): def raiseImpossibleProcess(self, context):

7
meshroom/multiview.py
View file

@ -1,7 +1,5 @@
import os import os
from meshroom.core.graph import Graph, GraphModification
# Supported image extensions # Supported image extensions
imageExtensions = ( imageExtensions = (
# bmp: # bmp:
@ -37,7 +35,10 @@ imageExtensions = (
# ptex: # ptex:
'.ptex', '.ptx', '.ptex', '.ptx',
# raw: # raw:
'.bay', '.bmq', '.cr2', '.cr3', '.crw', '.cs1', '.dc2', '.dcr', '.dng', '.erf', '.fff', '.k25', '.kdc', '.mdc', '.mos', '.mrw', '.nef', '.orf', '.pef', '.pxn', '.raf', '.raw', '.rdc', '.sr2', '.srf', '.x3f', '.arw', '.3fr', '.cine', '.ia', '.kc2', '.mef', '.nrw', '.qtk', '.rw2', '.sti', '.rwl', '.srw', '.drf', '.dsc', '.cap', '.iiq', '.rwz', '.bay', '.bmq', '.cr2', '.cr3', '.crw', '.cs1', '.dc2', '.dcr', '.dng', '.erf', '.fff', '.k25', '.kdc', '.mdc',
'.mos', '.mrw', '.nef', '.orf', '.pef', '.pxn', '.raf', '.raw', '.rdc', '.sr2', '.srf', '.x3f', '.arw', '.3fr',
'.cine', '.ia', '.kc2', '.mef', '.nrw', '.qtk', '.rw2', '.sti', '.rwl', '.srw', '.drf', '.dsc', '.cap', '.iiq',
'.rwz',
# rla: # rla:
'.rla', '.rla',
# sgi: # sgi:

2
meshroom/ui/utils.py
View file

@ -5,7 +5,7 @@ from PySide2.QtCore import QFileSystemWatcher, QUrl, Slot, QTimer, Property, QOb
from PySide2.QtQml import QQmlApplicationEngine from PySide2.QtQml import QQmlApplicationEngine
try: try:
from PySide2 import shiboken2 from PySide2 import shiboken2
except: except Exception:
import shiboken2 import shiboken2

5
setupInitScriptUnix.py
View file

@ -1,4 +1,4 @@
#------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# ConsoleSetLibPath.py # ConsoleSetLibPath.py
# Initialization script for cx_Freeze which manipulates the path so that the # Initialization script for cx_Freeze which manipulates the path so that the
# directory in which the executable is found is searched for extensions but # directory in which the executable is found is searched for extensions but
@ -6,7 +6,7 @@
# manipulated first, however, to ensure that shared libraries found in the # manipulated first, however, to ensure that shared libraries found in the
# target directory are found. This requires a restart of the executable because # target directory are found. This requires a restart of the executable because
# the environment variable LD_LIBRARY_PATH is only checked at startup. # the environment variable LD_LIBRARY_PATH is only checked at startup.
#------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
import os import os
import sys import sys
@ -40,4 +40,3 @@ def run(*args):
moduleName = args[0] moduleName = args[0]
code = importer.get_code(moduleName) code = importer.get_code(moduleName)
exec(code, m.__dict__) exec(code, m.__dict__)

16
tests/test_compatibility.py
View file

@ -1,7 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
# coding:utf-8 # coding:utf-8
import tempfile import tempfile
import os import os
import copy import copy
@ -34,7 +33,7 @@ SampleGroupV2 = [
) )
] ]
#SampleGroupV3 is SampleGroupV2 with one more int parameter # SampleGroupV3 is SampleGroupV2 with one more int parameter
SampleGroupV3 = [ SampleGroupV3 = [
desc.IntParam(name="a", label="a", description="", value=0, invalidate=True, range=None), desc.IntParam(name="a", label="a", description="", value=0, invalidate=True, range=None),
desc.IntParam(name="notInSampleGroupV2", label="notInSampleGroupV2", description="", value=0, invalidate=True, range=None), desc.IntParam(name="notInSampleGroupV2", label="notInSampleGroupV2", description="", value=0, invalidate=True, range=None),
@ -70,6 +69,7 @@ class SampleNodeV2(desc.Node):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
] ]
class SampleNodeV3(desc.Node): class SampleNodeV3(desc.Node):
""" """
Changes from V3: Changes from V3:
@ -82,6 +82,7 @@ class SampleNodeV3(desc.Node):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
] ]
class SampleNodeV4(desc.Node): class SampleNodeV4(desc.Node):
""" """
Changes from V3: Changes from V3:
@ -115,6 +116,7 @@ class SampleNodeV5(desc.Node):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
] ]
class SampleNodeV6(desc.Node): class SampleNodeV6(desc.Node):
""" """
Changes from V5: Changes from V5:
@ -131,6 +133,7 @@ class SampleNodeV6(desc.Node):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
] ]
class SampleInputNodeV1(desc.InputNode): class SampleInputNodeV1(desc.InputNode):
""" Version 1 Sample Input Node """ """ Version 1 Sample Input Node """
inputs = [ inputs = [
@ -140,6 +143,7 @@ class SampleInputNodeV1(desc.InputNode):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
] ]
class SampleInputNodeV2(desc.InputNode): class SampleInputNodeV2(desc.InputNode):
""" Changes from V1: """ Changes from V1:
* 'path' has been renamed to 'in' * 'path' has been renamed to 'in'
@ -151,6 +155,7 @@ class SampleInputNodeV2(desc.InputNode):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
] ]
def test_unknown_node_type(): def test_unknown_node_type():
""" """
Test compatibility behavior for unknown node type. Test compatibility behavior for unknown node type.
@ -351,6 +356,7 @@ def test_upgradeAllNodes():
unregisterNodeType(SampleNodeV1) unregisterNodeType(SampleNodeV1)
unregisterNodeType(SampleInputNodeV1) unregisterNodeType(SampleInputNodeV1)
def test_conformUpgrade(): def test_conformUpgrade():
registerNodeType(SampleNodeV5) registerNodeType(SampleNodeV5)
registerNodeType(SampleNodeV6) registerNodeType(SampleNodeV6)
@ -389,9 +395,3 @@ def test_conformUpgrade():
unregisterNodeType(SampleNodeV5) unregisterNodeType(SampleNodeV5)
unregisterNodeType(SampleNodeV6) unregisterNodeType(SampleNodeV6)

3
tests/test_graph.py
View file

@ -142,7 +142,6 @@ def test_transitive_reduction():
(tC.output, tE.input3), (tC.output, tE.input3),
(tD.output, tE.input2), (tD.output, tE.input2),
) )
edgesScore = graph.dfsMaxEdgeLength()
flowEdges = graph.flowEdges() flowEdges = graph.flowEdges()
flowEdgesRes = [(tB, tA), flowEdgesRes = [(tB, tA),
@ -153,7 +152,7 @@ def test_transitive_reduction():
] ]
assert set(flowEdgesRes) == set(flowEdges) assert set(flowEdgesRes) == set(flowEdges)
assert len(graph._nodesMinMaxDepths) == len(graph.nodes) assert len(graph._nodesMinMaxDepths) == len(graph.nodes)
for node, (minDepth, maxDepth) in graph._nodesMinMaxDepths.items(): for node, (minDepth, maxDepth) in graph._nodesMinMaxDepths.items():
assert node.depth == maxDepth assert node.depth == maxDepth

2
tests/test_invalidation.py
View file

@ -59,5 +59,3 @@ def test_inputLinkInvalidation():
graph.addEdges((n1.input, n2.input)) graph.addEdges((n1.input, n2.input))
assert n1.input.uid() == n2.input.uid() assert n1.input.uid() == n2.input.uid()
assert n1.output.value == n2.output.value assert n1.output.value == n2.output.value

2
tests/test_model.py
View file

@ -28,7 +28,7 @@ def test_DictModel_add_remove():
assert len(m.values()) == 1 assert len(m.values()) == 1
assert m.get("DummyNode_1") == node assert m.get("DummyNode_1") == node
assert m.get("something") == None assert m.get("something") is None
with pytest.raises(KeyError): with pytest.raises(KeyError):
m.getr("something") m.getr("something")

11
tests/test_nodeCommandLineFormatting.py
View file

@ -1,11 +1,7 @@
#!/usr/bin/env python #!/usr/bin/env python
# coding:utf-8 # coding:utf-8
import os
import tempfile
import meshroom.multiview import meshroom.multiview
from meshroom.core.graph import Graph from meshroom.core.graph import Graph
from meshroom.core.node import Node
def test_formatting_listOfFiles(): def test_formatting_listOfFiles():
@ -17,7 +13,8 @@ def test_formatting_listOfFiles():
n1 = graph.addNewNode('CameraInit') n1 = graph.addNewNode('CameraInit')
n1.viewpoints.extend([{'path': image} for image in inputImages]) n1.viewpoints.extend([{'path': image} for image in inputImages])
# viewId, poseId, path, intrinsicId, rigId, subPoseId, metadata # viewId, poseId, path, intrinsicId, rigId, subPoseId, metadata
assert n1.viewpoints.getValueStr() == '-1 -1 "/non/existing/fileA" -1 -1 -1 "" -1 -1 "/non/existing/with space/fileB" -1 -1 -1 ""' assert n1.viewpoints.getValueStr() == \
'-1 -1 "/non/existing/fileA" -1 -1 -1 "" -1 -1 "/non/existing/with space/fileB" -1 -1 -1 ""'
graph = Graph('') graph = Graph('')
n1 = graph.addNewNode('ImageMatching') n1 = graph.addNewNode('ImageMatching')
@ -61,7 +58,8 @@ def test_formatting_strings():
n2.featuresFolders.extend('') n2.featuresFolders.extend('')
n2._buildCmdVars() # prepare vars for command line creation n2._buildCmdVars() # prepare vars for command line creation
assert n2.featuresFolders.getValueStr() == '"" ""', 'A list with 2 empty strings should generate quotes' assert n2.featuresFolders.getValueStr() == '"" ""', 'A list with 2 empty strings should generate quotes'
assert n2._cmdVars[name + 'Value'] == ' ', 'The Value is always only the value, so 2 empty with the space separator in the middle' assert n2._cmdVars[name + 'Value'] == ' ', \
'The Value is always only the value, so 2 empty with the space separator in the middle'
def test_formatting_groups(): def test_formatting_groups():
@ -78,4 +76,3 @@ def test_formatting_groups():
name = 'noiseFilter' name = 'noiseFilter'
assert n3.noiseFilter.getValueStr() == '"False:uniform:0.0:1.0:True"' assert n3.noiseFilter.getValueStr() == '"False:uniform:0.0:1.0:True"'
assert n3._cmdVars[name + 'Value'] == 'False:uniform:0.0:1.0:True' assert n3._cmdVars[name + 'Value'] == 'False:uniform:0.0:1.0:True'