diff --git a/docs/source/_ext/meshroom_doc.py b/docs/source/_ext/meshroom_doc.py index d5151ef7..93a864cc 100644 --- a/docs/source/_ext/meshroom_doc.py +++ b/docs/source/_ext/meshroom_doc.py @@ -11,7 +11,6 @@ # Note: # for now this tool focuses only on meshroom nodes -from docutils import nodes from docutils.parsers.rst import Directive from utils import md_to_docutils diff --git a/docs/source/conf.py b/docs/source/conf.py index bd6d5cac..378ca801 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -35,7 +35,6 @@ templates_path = ['_templates'] exclude_patterns = [] - # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/meshroom/__init__.py b/meshroom/__init__.py index 51a05ee3..dfe9f616 100644 --- a/meshroom/__init__.py +++ b/meshroom/__init__.py @@ -4,10 +4,12 @@ import logging import os import sys + class VersionStatus(Enum): release = 1 develop = 2 + __version__ = "2024.1.0" # Always increase the minor version when switching from release to develop. __version_status__ = VersionStatus.develop @@ -56,6 +58,7 @@ logStringToPython = { } logging.getLogger().setLevel(logStringToPython[os.environ.get('MESHROOM_VERBOSE', 'warning')]) + def setupEnvironment(backend=Backend.STANDALONE): """ Setup environment for Meshroom to work in a prebuilt, standalone configuration. diff --git a/meshroom/common/__init__.py b/meshroom/common/__init__.py index db73fe05..f9d41f7f 100644 --- a/meshroom/common/__init__.py +++ b/meshroom/common/__init__.py @@ -7,10 +7,12 @@ Warning: A call to `init(Backend.XXX)` is required to choose the backend before from enum import Enum + class Backend(Enum): STANDALONE = 1 PYSIDE = 2 + DictModel = None ListModel = None Slot = None @@ -21,6 +23,7 @@ Variant = None VariantList = None JSValue = None + def init(backend): global DictModel, ListModel, Slot, Signal, Property, BaseObject, Variant, VariantList, JSValue if backend == Backend.PYSIDE: @@ -30,5 +33,6 @@ def init(backend): # Core types from .core import DictModel, ListModel, Slot, Signal, Property, BaseObject, Variant, VariantList, JSValue -# default initialization + +# Default initialization init(Backend.STANDALONE) diff --git a/meshroom/common/core.py b/meshroom/common/core.py index e9e20579..2f3ad487 100644 --- a/meshroom/common/core.py +++ b/meshroom/common/core.py @@ -1,5 +1,6 @@ from . import PySignal + class CoreDictModel: def __init__(self, keyAttrName, **kwargs): diff --git a/meshroom/common/qt.py b/meshroom/common/qt.py index 26f51270..438a7537 100644 --- a/meshroom/common/qt.py +++ b/meshroom/common/qt.py @@ -1,6 +1,7 @@ from PySide2 import QtCore, QtQml import shiboken2 + class QObjectListModel(QtCore.QAbstractListModel): """ QObjectListModel provides a more powerful, but still easy to use, alternative to using diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 0f6af870..54dc73cf 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -1,11 +1,8 @@ -from __future__ import print_function - import hashlib from contextlib import contextmanager import importlib import inspect import os -import re import tempfile import uuid import logging @@ -18,7 +15,7 @@ try: import encodings.ascii import encodings.idna import encodings.utf_8 -except: +except Exception: pass from meshroom.core.submitter import BaseSubmitter @@ -330,6 +327,7 @@ def loadPipelineTemplates(folder): if file.endswith(".mg") and file not in pipelineTemplates: pipelineTemplates[os.path.splitext(file)[0]] = os.path.join(folder, file) + def initNodes(): meshroomFolder = os.path.dirname(os.path.dirname(__file__)) additionalNodesPath = os.environ.get("MESHROOM_NODES_PATH", "").split(os.pathsep) @@ -339,12 +337,14 @@ def initNodes(): for f in nodesFolders: loadAllNodes(folder=f) + def initSubmitters(): meshroomFolder = os.path.dirname(os.path.dirname(__file__)) subs = loadSubmitters(os.environ.get("MESHROOM_SUBMITTERS_PATH", meshroomFolder), 'submitters') for sub in subs: registerSubmitter(sub()) + def initPipelines(): meshroomFolder = os.path.dirname(os.path.dirname(__file__)) # Load pipeline templates: check in the default folder and any folder the user might have diff --git a/meshroom/core/attribute.py b/meshroom/core/attribute.py index 25282e9c..6bd7380c 100644 --- a/meshroom/core/attribute.py +++ b/meshroom/core/attribute.py @@ -138,7 +138,7 @@ class Attribute(BaseObject): if isinstance(self.desc.enabled, types.FunctionType): try: return self.desc.enabled(self.node) - except: + except Exception: # Node implementation may fail due to version mismatch return True return self.attributeDesc.enabled @@ -325,7 +325,8 @@ class Attribute(BaseObject): return False # if the attribute is a ListAttribute, we need to check if any of its elements has output connections if isinstance(self, ListAttribute): - return next((edge for edge in self.node.graph.edges.values() if edge.src == self), None) is not None or any(attr.hasOutputConnections for attr in self._value if hasattr(attr, 'hasOutputConnections')) + return next((edge for edge in self.node.graph.edges.values() if edge.src == self), None) is not None or \ + any(attr.hasOutputConnections for attr in self._value if hasattr(attr, 'hasOutputConnections')) return next((edge for edge in self.node.graph.edges.values() if edge.src == self), None) is not None def _applyExpr(self): @@ -348,7 +349,8 @@ class Attribute(BaseObject): try: g.addEdge(g.node(linkNode).attribute(linkAttr), self) except KeyError as err: - logging.warning('Connect Attribute from Expression failed.\nExpression: "{exp}"\nError: "{err}".'.format(exp=v, err=err)) + logging.warning('Connect Attribute from Expression failed.') + logging.warning('Expression: "{exp}"\nError: "{err}".'.format(exp=v, err=err)) self.resetToDefaultValue() def getExportValue(self): @@ -376,8 +378,8 @@ class Attribute(BaseObject): ''' # ChoiceParam with multiple values should be combined if isinstance(self.attributeDesc, desc.ChoiceParam) and not self.attributeDesc.exclusive: - # ensure value is a list as expected - assert(isinstance(self.value, Sequence) and not isinstance(self.value, str)) + # Ensure value is a list as expected + assert (isinstance(self.value, Sequence) and not isinstance(self.value, str)) v = self.attributeDesc.joinChar.join(self.getEvalValue()) if withQuotes and v: return '"{}"'.format(v) @@ -393,8 +395,9 @@ class Attribute(BaseObject): return self.desc.value(self) except Exception as e: if not self.node.isCompatibilityNode: - # log message only if we are not in compatibility mode - logging.warning("Failed to evaluate default value (node lambda) for attribute '{}': {}".format(self.name, e)) + # Log message only if we are not in compatibility mode + logging.warning("Failed to evaluate default value (node lambda) for attribute '{}': {}". + format(self.name, e)) return None # Need to force a copy, for the case where the value is a list (avoid reference to the desc value) return copy.copy(self.desc.value) @@ -409,7 +412,6 @@ class Attribute(BaseObject): # Emit if the enable status has changed self.setEnabled(self.getEnabled()) - name = Property(str, getName, constant=True) fullName = Property(str, getFullName, constant=True) fullNameToNode = Property(str, getFullNameToNode, constant=True) @@ -423,11 +425,11 @@ class Attribute(BaseObject): baseType = Property(str, getType, constant=True) isReadOnly = Property(bool, _isReadOnly, constant=True) - # description of the attribute + # Description of the attribute descriptionChanged = Signal() description = Property(str, _get_description, _set_description, notify=descriptionChanged) - # definition of the attribute + # Definition of the attribute desc = Property(desc.Attribute, lambda self: self.attributeDesc, constant=True) valueChanged = Signal() @@ -460,6 +462,7 @@ def raiseIfLink(func): return func(attr, *args, **kwargs) return wrapper + class PushButtonParam(Attribute): def __init__(self, node, attributeDesc, isOutput, root=None, parent=None): super(PushButtonParam, self).__init__(node, attributeDesc, isOutput, root, parent) @@ -468,6 +471,7 @@ class PushButtonParam(Attribute): def clicked(self): self.node.onAttributeClicked(self) + class ChoiceParam(Attribute): def __init__(self, node, attributeDesc, isOutput, root=None, parent=None): @@ -489,7 +493,8 @@ class ChoiceParam(Attribute): value = value.split(',') if not isinstance(value, Iterable): - raise ValueError('Non exclusive ChoiceParam value should be iterable (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) + raise ValueError("Non exclusive ChoiceParam value should be iterable (param:{}, value:{}, type:{})". + format(self.name, value, type(value))) return [self.conformValue(v) for v in value] def setValues(self, values): @@ -523,7 +528,7 @@ class ListAttribute(Attribute): def at(self, idx): """ Returns child attribute at index 'idx' """ - # implement 'at' rather than '__getitem__' + # Implement 'at' rather than '__getitem__' # since the later is called spuriously when object is used in QML return self._value.at(idx) @@ -560,7 +565,8 @@ class ListAttribute(Attribute): if isinstance(exportedValues, ListAttribute) or Attribute.isLinkExpression(exportedValues): self._set_value(exportedValues) return - raise RuntimeError("ListAttribute.upgradeValue: the given value is of type " + str(type(exportedValues)) + " but a 'list' is expected.") + raise RuntimeError("ListAttribute.upgradeValue: the given value is of type " + + str(type(exportedValues)) + " but a 'list' is expected.") attrs = [] for v in exportedValues: @@ -645,7 +651,7 @@ class ListAttribute(Attribute): return [attr.getPrimitiveValue(exportDefault=exportDefault) for attr in self._value if not attr.isDefault] def getValueStr(self, withQuotes=True): - assert(isinstance(self.value, ListModel)) + assert isinstance(self.value, ListModel) if self.attributeDesc.joinChar == ' ': return self.attributeDesc.joinChar.join([v.getValueStr(withQuotes=withQuotes) for v in self.value]) else: @@ -770,7 +776,8 @@ class GroupAttribute(Attribute): if exportDefault: return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items()} else: - return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items() if not attr.isDefault} + return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items() + if not attr.isDefault} def getValueStr(self, withQuotes=True): # add brackets if requested @@ -787,7 +794,8 @@ class GroupAttribute(Attribute): spaceSep = self.attributeDesc.joinChar == ' ' # sort values based on child attributes group description order - sortedSubValues = [self._value.get(attr.name).getValueStr(withQuotes=spaceSep) for attr in self.attributeDesc.groupDesc] + sortedSubValues = [self._value.get(attr.name).getValueStr(withQuotes=spaceSep) + for attr in self.attributeDesc.groupDesc] s = self.attributeDesc.joinChar.join(sortedSubValues) if withQuotes and not spaceSep: diff --git a/meshroom/core/cgroup.py b/meshroom/core/cgroup.py index e35e45f1..7ecde930 100755 --- a/meshroom/core/cgroup.py +++ b/meshroom/core/cgroup.py @@ -3,21 +3,22 @@ import os -#Try to retrieve limits of memory for the current process' cgroup + +# Try to retrieve limits of memory for the current process' cgroup def getCgroupMemorySize(): - #first of all, get pid of process + # First of all, get pid of process pid = os.getpid() - #Get cgroup associated with pid + # Get cgroup associated with pid filename = f"/proc/{pid}/cgroup" cgroup = None try: - with open(filename, "r") as f : + with open(filename, "r") as f: - #cgroup file is a ':' separated table - #lookup a line where the second field is "memory" + # cgroup file is a ':' separated table + # lookup a line where the second field is "memory" lines = f.readlines() for line in lines: tokens = line.rstrip("\r\n").split(":") @@ -34,7 +35,7 @@ def getCgroupMemorySize(): size = -1 filename = f"/sys/fs/cgroup/memory/{cgroup}/memory.limit_in_bytes" try: - with open(filename, "r") as f : + with open(filename, "r") as f: value = f.read().rstrip("\r\n") if value.isnumeric(): size = int(value) @@ -43,6 +44,7 @@ def getCgroupMemorySize(): return size + def parseNumericList(numericListString): nList = [] @@ -58,21 +60,22 @@ def parseNumericList(numericListString): return nList -#Try to retrieve limits of cores for the current process' cgroup + +# Try to retrieve limits of cores for the current process' cgroup def getCgroupCpuCount(): - #first of all, get pid of process + # First of all, get pid of process pid = os.getpid() - #Get cgroup associated with pid + # Get cgroup associated with pid filename = f"/proc/{pid}/cgroup" cgroup = None try: - with open(filename, "r") as f : + with open(filename, "r") as f: - #cgroup file is a ':' separated table - #lookup a line where the second field is "memory" + # cgroup file is a ':' separated table + # lookup a line where the second field is "memory" lines = f.readlines() for line in lines: tokens = line.rstrip("\r\n").split(":") @@ -89,7 +92,7 @@ def getCgroupCpuCount(): size = -1 filename = f"/sys/fs/cgroup/cpuset/{cgroup}/cpuset.cpus" try: - with open(filename, "r") as f : + with open(filename, "r") as f: value = f.read().rstrip("\r\n") nlist = parseNumericList(value) size = len(nlist) @@ -98,4 +101,3 @@ def getCgroupCpuCount(): pass return size - diff --git a/meshroom/core/desc.py b/meshroom/core/desc.py index 1d9aa93e..ce5b35ab 100644 --- a/meshroom/core/desc.py +++ b/meshroom/core/desc.py @@ -11,6 +11,7 @@ import ast import distutils.util import shlex + class Attribute(BaseObject): """ """ @@ -32,7 +33,8 @@ class Attribute(BaseObject): self._errorMessage = errorMessage self._visible = visible self._exposed = exposed - self._isExpression = (isinstance(self._value, str) and "{" in self._value) or isinstance(self._value, types.FunctionType) + self._isExpression = (isinstance(self._value, str) and "{" in self._value) \ + or isinstance(self._value, types.FunctionType) self._isDynamicValue = (self._value is None) self._valueType = None @@ -48,7 +50,8 @@ class Attribute(BaseObject): Raises: ValueError: if value does not have the proper type """ - raise NotImplementedError("Attribute.validateValue is an abstract function that should be implemented in the derived class.") + raise NotImplementedError("Attribute.validateValue is an abstract function that should be " + "implemented in the derived class.") def checkValueTypes(self): """ Returns the attribute's name if the default value's type is invalid or if the range's type (when available) @@ -57,7 +60,8 @@ class Attribute(BaseObject): Returns: string: the attribute's name if the default value's or range's type is invalid, empty string otherwise """ - raise NotImplementedError("Attribute.checkValueTypes is an abstract function that should be implemented in the derived class.") + raise NotImplementedError("Attribute.checkValueTypes is an abstract function that should be implemented in the " + "derived class.") def matchDescription(self, value, strict=True): """ Returns whether the value perfectly match attribute's description. @@ -108,13 +112,16 @@ class Attribute(BaseObject): class ListAttribute(Attribute): """ A list of Attributes """ - def __init__(self, elementDesc, name, label, description, group='allParams', advanced=False, semantic='', enabled=True, joinChar=' ', visible=True, exposed=False): + def __init__(self, elementDesc, name, label, description, group='allParams', advanced=False, semantic='', + enabled=True, joinChar=' ', visible=True, exposed=False): """ :param elementDesc: the Attribute description of elements to store in that list """ self._elementDesc = elementDesc self._joinChar = joinChar - super(ListAttribute, self).__init__(name=name, label=label, description=description, value=[], invalidate=False, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) + super(ListAttribute, self).__init__(name=name, label=label, description=description, value=[], + invalidate=False, group=group, advanced=advanced, semantic=semantic, + enabled=enabled, visible=visible, exposed=exposed) def getInstanceType(self): # Import within the method to prevent cyclic dependencies @@ -126,14 +133,16 @@ class ListAttribute(Attribute): return value if JSValue is not None and isinstance(value, JSValue): # Note: we could use isArray(), property("length").toInt() to retrieve all values - raise ValueError("ListAttribute.validateValue: cannot recognize QJSValue. Please, use JSON.stringify(value) in QML.") + raise ValueError("ListAttribute.validateValue: cannot recognize QJSValue. " + "Please, use JSON.stringify(value) in QML.") if isinstance(value, str): # Alternative solution to set values from QML is to convert values to JSON string # In this case, it works with all data types value = ast.literal_eval(value) if not isinstance(value, (list, tuple)): - raise ValueError('ListAttribute only supports list/tuple input values (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) + raise ValueError("ListAttribute only supports list/tuple input values " + "(param:{}, value:{}, type:{})".format(self.name, value, type(value))) return value def checkValueTypes(self): @@ -155,14 +164,17 @@ class ListAttribute(Attribute): class GroupAttribute(Attribute): """ A macro Attribute composed of several Attributes """ - def __init__(self, groupDesc, name, label, description, group='allParams', advanced=False, semantic='', enabled=True, joinChar=' ', brackets=None, visible=True, exposed=False): + def __init__(self, groupDesc, name, label, description, group='allParams', advanced=False, semantic='', + enabled=True, joinChar=' ', brackets=None, visible=True, exposed=False): """ :param groupDesc: the description of the Attributes composing this group """ self._groupDesc = groupDesc self._joinChar = joinChar self._brackets = brackets - super(GroupAttribute, self).__init__(name=name, label=label, description=description, value={}, invalidate=False, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) + super(GroupAttribute, self).__init__(name=name, label=label, description=description, value={}, + invalidate=False, group=group, advanced=advanced, semantic=semantic, + enabled=enabled, visible=visible, exposed=exposed) def getInstanceType(self): # Import within the method to prevent cyclic dependencies @@ -170,12 +182,13 @@ class GroupAttribute(Attribute): return GroupAttribute def validateValue(self, value): + """ Ensure value is compatible with the group description and convert value if needed. """ if value is None: return value - """ Ensure value is compatible with the group description and convert value if needed. """ if JSValue is not None and isinstance(value, JSValue): # Note: we could use isArray(), property("length").toInt() to retrieve all values - raise ValueError("GroupAttribute.validateValue: cannot recognize QJSValue. Please, use JSON.stringify(value) in QML.") + raise ValueError("GroupAttribute.validateValue: cannot recognize QJSValue. " + "Please, use JSON.stringify(value) in QML.") if isinstance(value, str): # Alternative solution to set values from QML is to convert values to JSON string # In this case, it works with all data types @@ -188,12 +201,16 @@ class GroupAttribute(Attribute): if self._groupDesc and value.keys(): commonKeys = set(value.keys()).intersection([attr.name for attr in self._groupDesc]) if not commonKeys: - raise ValueError(f'Value contains no key that matches with the group description (name={self.name}, values={value.keys()}, desc={[attr.name for attr in self._groupDesc]})') + raise ValueError(f"Value contains no key that matches with the group description " + f"(name={self.name}, values={value.keys()}, " + f"desc={[attr.name for attr in self._groupDesc]})") elif isinstance(value, (list, tuple, set)): if len(value) != len(self._groupDesc): - raise ValueError('Value contains incoherent number of values: desc size: {}, value size: {}'.format(len(self._groupDesc), len(value))) + raise ValueError("Value contains incoherent number of values: desc size: {}, value size: {}". + format(len(self._groupDesc), len(value))) else: - raise ValueError('GroupAttribute only supports dict/list/tuple input values (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) + raise ValueError("GroupAttribute only supports dict/list/tuple input values (param:{}, value:{}, type:{})". + format(self.name, value, type(value))) return value @@ -254,24 +271,31 @@ class GroupAttribute(Attribute): class Param(Attribute): """ """ - def __init__(self, name, label, description, value, invalidate, group, advanced, semantic, enabled, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False): - super(Param, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, - uidIgnoreValue=uidIgnoreValue, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) + def __init__(self, name, label, description, value, invalidate, group, advanced, semantic, enabled, + uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False): + super(Param, self).__init__(name=name, label=label, description=description, value=value, + invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, + enabled=enabled, uidIgnoreValue=uidIgnoreValue, validValue=validValue, + errorMessage=errorMessage, visible=visible, exposed=exposed) class File(Attribute): """ """ - def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=True): - super(File, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) + def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', + enabled=True, visible=True, exposed=True): + super(File, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, + group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, + exposed=exposed) self._valueType = str def validateValue(self, value): if value is None: return value if not isinstance(value, str): - raise ValueError('File only supports string input (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) - return os.path.normpath(value).replace('\\', '/') if value else '' + raise ValueError("File only supports string input (param:{}, value:{}, type:{})". + format(self.name, value, type(value))) + return os.path.normpath(value).replace("\\", "/") if value else "" def checkValueTypes(self): # Some File values are functions generating a string: check whether the value is a string or if it @@ -284,8 +308,11 @@ class File(Attribute): class BoolParam(Param): """ """ - def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False): - super(BoolParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) + def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', + enabled=True, visible=True, exposed=False): + super(BoolParam, self).__init__(name=name, label=label, description=description, value=value, + invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, + enabled=enabled, visible=visible, exposed=exposed) self._valueType = bool def validateValue(self, value): @@ -296,8 +323,9 @@ class BoolParam(Param): # use distutils.util.strtobool to handle (1/0, true/false, on/off, y/n) return bool(distutils.util.strtobool(value)) return bool(value) - except: - raise ValueError('BoolParam only supports bool value (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) + except Exception: + raise ValueError("BoolParam only supports bool value (param:{}, value:{}, type:{})". + format(self.name, value, type(value))) def checkValueTypes(self): if not isinstance(self.value, bool): @@ -308,20 +336,24 @@ class BoolParam(Param): class IntParam(Param): """ """ - def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False): + def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False, + semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False): self._range = range - super(IntParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, - validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) + super(IntParam, self).__init__(name=name, label=label, description=description, value=value, + invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, + enabled=enabled, validValue=validValue, errorMessage=errorMessage, + visible=visible, exposed=exposed) self._valueType = int def validateValue(self, value): if value is None: return value - # handle unsigned int values that are translated to int by shiboken and may overflow + # Handle unsigned int values that are translated to int by shiboken and may overflow try: return int(value) - except: - raise ValueError('IntParam only supports int value (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) + except Exception: + raise ValueError("IntParam only supports int value (param:{}, value:{}, type:{})". + format(self.name, value, type(value))) def checkValueTypes(self): if not isinstance(self.value, int) or (self.range and not all([isinstance(r, int) for r in self.range])): @@ -334,10 +366,13 @@ class IntParam(Param): class FloatParam(Param): """ """ - def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False): + def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False, + semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False): self._range = range - super(FloatParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, - validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) + super(FloatParam, self).__init__(name=name, label=label, description=description, value=value, + invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, + enabled=enabled, validValue=validValue, errorMessage=errorMessage, + visible=visible, exposed=exposed) self._valueType = float def validateValue(self, value): @@ -345,8 +380,9 @@ class FloatParam(Param): return value try: return float(value) - except: - raise ValueError('FloatParam only supports float value (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) + except Exception: + raise ValueError("FloatParam only supports float value (param:{}, value:{}, type:{})". + format(self.name, value, type(value))) def checkValueTypes(self): if not isinstance(self.value, float) or (self.range and not all([isinstance(r, float) for r in self.range])): @@ -355,11 +391,15 @@ class FloatParam(Param): range = Property(VariantList, lambda self: self._range, constant=True) + class PushButtonParam(Param): """ """ - def __init__(self, name, label, description, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False): - super(PushButtonParam, self).__init__(name=name, label=label, description=description, value=None, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) + def __init__(self, name, label, description, invalidate, group='allParams', advanced=False, semantic='', + enabled=True, visible=True, exposed=False): + super(PushButtonParam, self).__init__(name=name, label=label, description=description, value=None, + invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, + enabled=enabled, visible=visible, exposed=exposed) self._valueType = None def getInstanceType(self): @@ -377,11 +417,14 @@ class PushButtonParam(Param): class ChoiceParam(Param): """ """ - def __init__(self, name, label, description, value, values, exclusive, invalidate, group='allParams', joinChar=' ', advanced=False, semantic='', - enabled=True, validValue=True, errorMessage="", visible=True, exposed=False): + def __init__(self, name, label, description, value, values, exclusive, invalidate, group='allParams', joinChar=' ', + advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True, + exposed=False): assert values - super(ChoiceParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, - semantic=semantic, enabled=enabled, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) + super(ChoiceParam, self).__init__(name=name, label=label, description=description, value=value, + invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, + enabled=enabled, validValue=validValue, errorMessage=errorMessage, + visible=visible, exposed=exposed) self._values = values self._exclusive = exclusive self._joinChar = joinChar @@ -416,7 +459,8 @@ class ChoiceParam(Param): value = value.split(',') if not isinstance(value, Iterable): - raise ValueError('Non exclusive ChoiceParam value should be iterable (param: {}, value: {}, type: {}).'.format(self.name, value, type(value))) + raise ValueError("Non-exclusive ChoiceParam value should be iterable (param: {}, value: {}, type: {}).". + format(self.name, value, type(value))) return [self.conformValue(v) for v in value] @@ -428,7 +472,7 @@ class ChoiceParam(Param): # If the choices are not exclusive, check that 'value' is a list, and check that it does not contain values that # are not available elif not self.exclusive and (not isinstance(self._value, list) or - not all(val in self._values for val in self._value)): + not all(val in self._values for val in self._value)): return self.name # If the choices are exclusive, the value should NOT be a list but it can contain any value that is not in the @@ -446,16 +490,20 @@ class ChoiceParam(Param): class StringParam(Param): """ """ - def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False): - super(StringParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, - uidIgnoreValue=uidIgnoreValue, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed) + def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', + enabled=True, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False): + super(StringParam, self).__init__(name=name, label=label, description=description, value=value, + invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, + enabled=enabled, uidIgnoreValue=uidIgnoreValue, validValue=validValue, + errorMessage=errorMessage, visible=visible, exposed=exposed) self._valueType = str def validateValue(self, value): if value is None: return value if not isinstance(value, str): - raise ValueError('StringParam value should be a string (param:{}, value:{}, type:{})'.format(self.name, value, type(value))) + raise ValueError("StringParam value should be a string (param:{}, value:{}, type:{})". + format(self.name, value, type(value))) return value def checkValueTypes(self): @@ -467,8 +515,11 @@ class StringParam(Param): class ColorParam(Param): """ """ - def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False): - super(ColorParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed) + def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', + enabled=True, visible=True, exposed=False): + super(ColorParam, self).__init__(name=name, label=label, description=description, value=value, + invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, + enabled=enabled, visible=visible, exposed=exposed) self._valueType = str def validateValue(self, value): @@ -683,6 +734,7 @@ class Node(object): BaseNode.updateInternals """ pass + @classmethod def postUpdate(cls, node): """ Method call after node's internal update on invalidation. @@ -700,6 +752,7 @@ class Node(object): def processChunk(self, chunk): raise NotImplementedError('No processChunk implementation on node: "{}"'.format(chunk.node.name)) + class InputNode(Node): """ Node that does not need to be processed, it is just a placeholder for inputs. @@ -724,12 +777,14 @@ class CommandLineNode(Node): def buildCommandLine(self, chunk): cmdPrefix = '' - # if rez available in env, we use it - if 'REZ_ENV' in os.environ and chunk.node.packageVersion: - # if the node package is already in the environment, we don't need a new dedicated rez environment - alreadyInEnv = os.environ.get('REZ_{}_VERSION'.format(chunk.node.packageName.upper()), "").startswith(chunk.node.packageVersion) + # If rez available in env, we use it + if "REZ_ENV" in os.environ and chunk.node.packageVersion: + # If the node package is already in the environment, we don't need a new dedicated rez environment + alreadyInEnv = os.environ.get("REZ_{}_VERSION".format(chunk.node.packageName.upper()), + "").startswith(chunk.node.packageVersion) if not alreadyInEnv: - cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get('REZ_ENV'), packageFullName=chunk.node.packageFullName) + cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get("REZ_ENV"), + packageFullName=chunk.node.packageFullName) cmdSuffix = '' if chunk.node.isParallelized and chunk.node.size > 1: @@ -738,12 +793,12 @@ class CommandLineNode(Node): return cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix def stopProcess(self, chunk): - # the same node could exists several times in the graph and + # The same node could exists several times in the graph and # only one would have the running subprocess; ignore all others if not hasattr(chunk, "subprocess"): return if chunk.subprocess: - # kill process tree + # Kill process tree processes = chunk.subprocess.children(recursive=True) + [chunk.subprocess] try: for process in processes: @@ -761,7 +816,7 @@ class CommandLineNode(Node): print(' - logFile: {}'.format(chunk.logFile)) chunk.subprocess = psutil.Popen(shlex.split(cmd), stdout=logF, stderr=logF, cwd=chunk.node.internalFolder) - # store process static info into the status file + # Store process static info into the status file # chunk.status.env = node.proc.environ() # chunk.status.createTime = node.proc.create_time() @@ -775,11 +830,12 @@ class CommandLineNode(Node): with open(chunk.logFile, 'r') as logF: logContent = ''.join(logF.readlines()) raise RuntimeError('Error on node "{}":\nLog:\n{}'.format(chunk.name, logContent)) - except: + except Exception: raise finally: chunk.subprocess = None + # Specific command line node for AliceVision apps class AVCommandLineNode(CommandLineNode): @@ -809,7 +865,7 @@ class AVCommandLineNode(CommandLineNode): return commandLineString + AVCommandLineNode.cmdMem + AVCommandLineNode.cmdCore -# Test abstract node + class InitNode(object): def __init__(self): super(InitNode, self).__init__() diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index 7253a3ad..b5a5c36e 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -22,6 +22,7 @@ from meshroom.core.node import nodeFactory, Status, Node, CompatibilityNode DefaultJSONEncoder = json.JSONEncoder # store the original one + class MyJSONEncoder(DefaultJSONEncoder): # declare a new one with Enum support def default(self, obj): if isinstance(obj, Enum): @@ -220,7 +221,8 @@ class Graph(BaseObject): self._computationBlocked = {} self._canComputeLeaves = True self._nodes = DictModel(keyAttrName='name', parent=self) - self._edges = DictModel(keyAttrName='dst', parent=self) # use dst attribute as unique key since it can only have one input connection + # Edges: use dst attribute as unique key since it can only have one input connection + self._edges = DictModel(keyAttrName='dst', parent=self) self._importedNodes = DictModel(keyAttrName='name', parent=self) self._compatibilityNodes = DictModel(keyAttrName='name', parent=self) self.cacheDir = meshroom.core.defaultCacheFolder @@ -682,9 +684,12 @@ class Graph(BaseObject): """ Remove the node identified by 'nodeName' from the graph. Returns: - - a dictionary containing the incoming edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} - - a dictionary containing the outgoing edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} - - a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute prior to the removal of all edges: + - a dictionary containing the incoming edges removed by this operation: + {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} + - a dictionary containing the outgoing edges removed by this operation: + {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} + - a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute + prior to the removal of all edges: {dstAttr.getFullNameToNode(), (dstAttr.root.getFullNameToNode(), dstAttr.index, dstAttr.value)} """ node = self.node(nodeName) @@ -695,14 +700,18 @@ class Graph(BaseObject): # Remove all edges arriving to and starting from this node with GraphModification(self): # Two iterations over the outgoing edges are necessary: - # - the first one is used to collect all the information about the edges while they are all there (overall context) - # - once we have collected all the information, the edges (and perhaps the entries in ListAttributes) can actually be removed + # - the first one is used to collect all the information about the edges while they are all there + # (overall context) + # - once we have collected all the information, the edges (and perhaps the entries in ListAttributes) can + # actually be removed for edge in self.nodeOutEdges(node): outEdges[edge.dst.getFullNameToNode()] = edge.src.getFullNameToNode() if isinstance(edge.dst.root, ListAttribute): index = edge.dst.root.index(edge.dst) - outListAttributes[edge.dst.getFullNameToNode()] = (edge.dst.root.getFullNameToNode(), index, edge.dst.value if edge.dst.value else None) + outListAttributes[edge.dst.getFullNameToNode()] = (edge.dst.root.getFullNameToNode(), + index, edge.dst.value + if edge.dst.value else None) for edge in self.nodeOutEdges(node): self.removeEdge(edge.dst) @@ -763,9 +772,12 @@ class Graph(BaseObject): Returns: - the upgraded (newly created) node - - a dictionary containing the incoming edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} - - a dictionary containing the outgoing edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} - - a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute prior to the removal of all edges: + - a dictionary containing the incoming edges removed by this operation: + {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} + - a dictionary containing the outgoing edges removed by this operation: + {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()} + - a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute + prior to the removal of all edges: {dstAttr.getFullNameToNode(), (dstAttr.root.getFullNameToNode(), dstAttr.index, dstAttr.value)} """ node = self.node(nodeName) @@ -834,7 +846,7 @@ class Graph(BaseObject): """ try: return int(name.split('_')[-1]) - except: + except Exception: return -1 @staticmethod @@ -970,7 +982,8 @@ class Graph(BaseObject): def dfs(self, visitor, startNodes=None, longestPathFirst=False): # Default direction (visitor.reverse=False): from node to root # Reverse direction (visitor.reverse=True): from node to leaves - nodeChildren = self._getOutputEdgesPerNode(visitor.dependenciesOnly) if visitor.reverse else self._getInputEdgesPerNode(visitor.dependenciesOnly) + nodeChildren = self._getOutputEdgesPerNode(visitor.dependenciesOnly) \ + if visitor.reverse else self._getInputEdgesPerNode(visitor.dependenciesOnly) # Initialize color map colors = {} for u in self._nodes: @@ -979,9 +992,11 @@ class Graph(BaseObject): if longestPathFirst and visitor.reverse: # Because we have no knowledge of the node's count between a node and its leaves, # it is not possible to handle this case at the moment - raise NotImplementedError("Graph.dfs(): longestPathFirst=True and visitor.reverse=True are not compatible yet.") + raise NotImplementedError("Graph.dfs(): longestPathFirst=True and visitor.reverse=True are not " + "compatible yet.") - nodes = startNodes or (self.getRootNodes(visitor.dependenciesOnly) if visitor.reverse else self.getLeafNodes(visitor.dependenciesOnly)) + nodes = startNodes or (self.getRootNodes(visitor.dependenciesOnly) + if visitor.reverse else self.getLeafNodes(visitor.dependenciesOnly)) if longestPathFirst: # Graph topology must be known and node depths up-to-date @@ -1302,7 +1317,6 @@ class Graph(BaseObject): self.dfs(visitor=visitor, startNodes=[startNode]) return visitor.canCompute + (2 * visitor.canSubmit) - def _applyExpr(self): with GraphModification(self): for node in self._nodes: @@ -1600,14 +1614,15 @@ class Graph(BaseObject): if node.hasAttribute('verbose'): try: node.verbose.value = v - except: + except Exception: pass nodes = Property(BaseObject, nodes.fget, constant=True) edges = Property(BaseObject, edges.fget, constant=True) filepathChanged = Signal() filepath = Property(str, lambda self: self._filepath, notify=filepathChanged) - fileReleaseVersion = Property(str, lambda self: self.header.get(Graph.IO.Keys.ReleaseVersion, "0.0"), notify=filepathChanged) + fileReleaseVersion = Property(str, lambda self: self.header.get(Graph.IO.Keys.ReleaseVersion, "0.0"), + notify=filepathChanged) fileDateVersion = Property(float, fileDateVersion.fget, fileDateVersion.fset, notify=filepathChanged) cacheDirChanged = Signal() cacheDir = Property(str, cacheDir.fget, cacheDir.fset, notify=cacheDirChanged) @@ -1721,4 +1736,3 @@ def submit(graphFile, submitter, toNode=None, submitLabel="{projectName}"): graph = loadGraph(graphFile) toNodes = graph.findNodes(toNode) if toNode else None submitGraph(graph, submitter, toNodes, submitLabel=submitLabel) - diff --git a/meshroom/core/node.py b/meshroom/core/node.py index f55ebcf1..64166748 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -12,7 +12,7 @@ import shutil import time import types import uuid -from collections import defaultdict, namedtuple +from collections import namedtuple from enum import Enum import meshroom @@ -33,7 +33,7 @@ def renameWritingToFinalPath(writingFilepath, filepath): for i in range(20): try: os.remove(filepath) - # if remove is successful, we can stop the iterations + # If remove is successful, we can stop the iterations break except WindowsError: pass @@ -50,7 +50,7 @@ class Status(Enum): STOPPED = 4 KILLED = 5 SUCCESS = 6 - INPUT = 7 # special status for input nodes + INPUT = 7 # Special status for input nodes class ExecMode(Enum): @@ -252,7 +252,7 @@ class NodeChunk(BaseObject): self.statistics = stats.Statistics() self.statusFileLastModTime = -1 self._subprocess = None - # notify update in filepaths when node's internal folder changes + # Notify update in filepaths when node's internal folder changes self.node.internalFolderChanged.connect(self.nodeFolderChanged) self.execModeNameChanged.connect(self.node.globalExecModeChanged) @@ -296,7 +296,7 @@ class NodeChunk(BaseObject): statusData = json.load(jsonFile) self.status.fromDict(statusData) self.statusFileLastModTime = os.path.getmtime(statusFile) - except Exception as e: + except Exception: self.statusFileLastModTime = -1 self.status.reset() @@ -306,23 +306,23 @@ class NodeChunk(BaseObject): @property def statusFile(self): if self.range.blockSize == 0: - return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'status') + return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "status") else: - return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.status') + return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".status") @property def statisticsFile(self): if self.range.blockSize == 0: - return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'statistics') + return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "statistics") else: - return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.statistics') + return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".statistics") @property def logFile(self): if self.range.blockSize == 0: - return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'log') + return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "log") else: - return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.log') + return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".log") def saveStatusFile(self): """ @@ -333,7 +333,7 @@ class NodeChunk(BaseObject): folder = os.path.dirname(statusFilepath) try: os.makedirs(folder) - except Exception as e: + except Exception: pass statusFilepathWriting = getWritingFilepath(statusFilepath) @@ -343,8 +343,8 @@ class NodeChunk(BaseObject): def upgradeStatusTo(self, newStatus, execMode=None): if newStatus.value <= self._status.status.value: - logging.warning('Downgrade status on node "{}" from {} to {}'.format(self.name, self._status.status, - newStatus)) + logging.warning("Downgrade status on node '{}' from {} to {}". + format(self.name, self._status.status, newStatus)) if newStatus == Status.SUBMITTED: self._status = StatusData(self.node.name, self.node.nodeType, self.node.packageName, self.node.packageVersion) @@ -411,11 +411,11 @@ class NodeChunk(BaseObject): self.statThread.start() try: self.node.nodeDesc.processChunk(self) - except Exception as e: + except Exception: if self._status.status != Status.STOPPED: exceptionStatus = Status.ERROR raise - except (KeyboardInterrupt, SystemError, GeneratorExit) as e: + except (KeyboardInterrupt, SystemError, GeneratorExit): exceptionStatus = Status.STOPPED raise finally: @@ -423,8 +423,8 @@ class NodeChunk(BaseObject): self._status.elapsedTime = time.time() - startTime if exceptionStatus is not None: self.upgradeStatusTo(exceptionStatus) - logging.info(' - elapsed time: {}'.format(self._status.elapsedTimeStr)) - # ask and wait for the stats thread to stop + logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr)) + # Ask and wait for the stats thread to stop self.statThread.stopRequest() self.statThread.join() self.statistics = stats.Statistics() @@ -462,9 +462,9 @@ class NodeChunk(BaseObject): elapsedTime = Property(float, lambda self: self._status.elapsedTime, notify=statusChanged) -# simple structure for storing node position +# Simple structure for storing node position Position = namedtuple("Position", ["x", "y"]) -# initialize default coordinates values to 0 +# Initialize default coordinates values to 0 Position.__new__.__defaults__ = (0,) * len(Position._fields) @@ -729,7 +729,8 @@ class BaseNode(BaseObject): def _buildCmdVars(self): def _buildAttributeCmdVars(cmdVars, name, attr): if attr.enabled: - group = attr.attributeDesc.group(attr.node) if isinstance(attr.attributeDesc.group, types.FunctionType) else attr.attributeDesc.group + group = attr.attributeDesc.group(attr.node) \ + if isinstance(attr.attributeDesc.group, types.FunctionType) else attr.attributeDesc.group if group is not None: # If there is a valid command line "group" v = attr.getValueStr(withQuotes=True) @@ -776,18 +777,25 @@ class BaseNode(BaseObject): if attr.enabled: try: defaultValue = attr.defaultValue() - except AttributeError as e: + except AttributeError: # If we load an old scene, the lambda associated to the 'value' could try to access other # params that could not exist yet - logging.warning('Invalid lambda evaluation for "{nodeName}.{attrName}"'.format(nodeName=self.name, attrName=attr.name)) + logging.warning('Invalid lambda evaluation for "{nodeName}.{attrName}"'. + format(nodeName=self.name, attrName=attr.name)) if defaultValue is not None: try: attr.value = defaultValue.format(**self._cmdVars) attr._invalidationValue = defaultValue.format(**cmdVarsNoCache) except KeyError as e: - logging.warning('Invalid expression with missing key on "{nodeName}.{attrName}" with value "{defaultValue}".\nError: {err}'.format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue, err=str(e))) + logging.warning('Invalid expression with missing key on "{nodeName}.{attrName}" with ' + 'value "{defaultValue}".\nError: {err}'. + format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue, + err=str(e))) except ValueError as e: - logging.warning('Invalid expression value on "{nodeName}.{attrName}" with value "{defaultValue}".\nError: {err}'.format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue, err=str(e))) + logging.warning('Invalid expression value on "{nodeName}.{attrName}" with value ' + '"{defaultValue}".\nError: {err}'. + format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue, + err=str(e))) v = attr.getValueStr(withQuotes=True) @@ -1074,9 +1082,11 @@ class BaseNode(BaseObject): self.attribute(output.name).value = data[output.name] else: if not self.hasAttribute(output.name): - logging.warning(f"loadOutputAttr: Missing dynamic output attribute. Node={self.name}, Attribute={output.name}") + logging.warning(f"loadOutputAttr: Missing dynamic output attribute. Node={self.name}, " + f"Attribute={output.name}") if output.name not in data: - logging.warning(f"loadOutputAttr: Missing dynamic output value in file. Node={self.name}, Attribute={output.name}, File={valuesFile}, Data keys={data.keys()}") + logging.warning(f"loadOutputAttr: Missing dynamic output value in file. Node={self.name}, " + f"Attribute={output.name}, File={valuesFile}, Data keys={data.keys()}") def saveOutputAttr(self): """ Save output attributes with dynamic values into a values.json file. @@ -1272,7 +1282,6 @@ class BaseNode(BaseObject): self._hasDuplicates = bool(len(newList)) self.hasDuplicatesChanged.emit() - def statusInThisSession(self): if not self._chunks: return False @@ -1297,7 +1306,8 @@ class BaseNode(BaseObject): def hasImageOutputAttribute(self): """ - Return True if at least one attribute has the 'image' semantic (and can thus be loaded in the 2D Viewer), False otherwise. + Return True if at least one attribute has the 'image' semantic (and can thus be loaded in the 2D Viewer), + False otherwise. """ for attr in self._attributes: if attr.enabled and attr.isOutput and attr.desc.semantic == "image": @@ -1306,10 +1316,12 @@ class BaseNode(BaseObject): def hasSequenceOutputAttribute(self): """ - Return True if at least one attribute has the 'sequence' semantic (and can thus be loaded in the 2D Viewer), False otherwise. + Return True if at least one attribute has the 'sequence' semantic (and can thus be loaded in the 2D Viewer), + False otherwise. """ for attr in self._attributes: - if attr.enabled and attr.isOutput and (attr.desc.semantic == "sequence" or attr.desc.semantic == "imageList"): + if attr.enabled and attr.isOutput and (attr.desc.semantic == "sequence" or + attr.desc.semantic == "imageList"): return True return False @@ -1326,7 +1338,6 @@ class BaseNode(BaseObject): return True return False - name = Property(str, getName, constant=True) defaultLabel = Property(str, getDefaultLabel, constant=True) nodeType = Property(str, nodeType.fget, constant=True) @@ -1356,8 +1367,10 @@ class BaseNode(BaseObject): globalStatus = Property(str, lambda self: self.getGlobalStatus().name, notify=globalStatusChanged) fusedStatus = Property(StatusData, getFusedStatus, notify=globalStatusChanged) elapsedTime = Property(float, lambda self: self.getFusedStatus().elapsedTime, notify=globalStatusChanged) - recursiveElapsedTime = Property(float, lambda self: self.getRecursiveFusedStatus().elapsedTime, notify=globalStatusChanged) - isCompatibilityNode = Property(bool, lambda self: self._isCompatibilityNode(), constant=True) # need lambda to evaluate the virtual function + recursiveElapsedTime = Property(float, lambda self: self.getRecursiveFusedStatus().elapsedTime, + notify=globalStatusChanged) + # isCompatibilityNode: need lambda to evaluate the virtual function + isCompatibilityNode = Property(bool, lambda self: self._isCompatibilityNode(), constant=True) isInputNode = Property(bool, lambda self: self._isInputNode(), constant=True) globalExecModeChanged = Signal() @@ -1378,6 +1391,7 @@ class BaseNode(BaseObject): hasSequenceOutput = Property(bool, hasSequenceOutputAttribute, notify=outputAttrEnabledChanged) has3DOutput = Property(bool, has3DOutputAttribute, notify=outputAttrEnabledChanged) + class Node(BaseNode): """ A standard Graph node based on a node type. @@ -1399,7 +1413,8 @@ class Node(BaseNode): self._attributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=True, node=self)) for attrDesc in self.nodeDesc.internalInputs: - self._internalAttributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=False, node=self)) + self._internalAttributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=False, + node=self)) # Declare events for specific output attributes for attr in self._attributes: @@ -1421,7 +1436,6 @@ class Node(BaseNode): self.optionalCallOnDescriptor("onNodeCreated") - def optionalCallOnDescriptor(self, methodName, *args, **kwargs): """ Call of optional method defined in the descriptor. Available method names are: @@ -1432,7 +1446,7 @@ class Node(BaseNode): if callable(m): try: m(self, *args, **kwargs) - except Exception as e: + except Exception: import traceback # Format error strings with all the provided arguments argsStr = ", ".join(str(arg) for arg in args) @@ -1443,7 +1457,8 @@ class Node(BaseNode): finalErrStr += ", " finalErrStr += kwargsStr - logging.error("Error on call to '{}' (with args: '{}') for node type {}".format(methodName, finalErrStr, self.nodeType)) + logging.error("Error on call to '{}' (with args: '{}') for node type {}". + format(methodName, finalErrStr, self.nodeType)) logging.error(traceback.format_exc()) def setAttributeValues(self, values): @@ -1491,7 +1506,8 @@ class Node(BaseNode): def toDict(self): inputs = {k: v.getExportValue() for k, v in self._attributes.objects.items() if v.isInput} internalInputs = {k: v.getExportValue() for k, v in self._internalAttributes.objects.items()} - outputs = ({k: v.getExportValue() for k, v in self._attributes.objects.items() if v.isOutput and not v.desc.isDynamicValue}) + outputs = ({k: v.getExportValue() for k, v in self._attributes.objects.items() + if v.isOutput and not v.desc.isDynamicValue}) return { 'nodeType': self.nodeType, @@ -1788,7 +1804,8 @@ class CompatibilityNode(BaseNode): upgradedAttrValues = attrValues if not isinstance(upgradedAttrValues, dict): - logging.error("Error in the upgrade implementation of the node: {}. The return type is incorrect.".format(self.name)) + logging.error("Error in the upgrade implementation of the node: {}. The return type is incorrect.". + format(self.name)) upgradedAttrValues = attrValues node.upgradeAttributeValues(upgradedAttrValues) @@ -1859,8 +1876,10 @@ def nodeFactory(nodeDict, name=None, template=False, uidConflict=False): # do not perform that check for internal attributes because there is no point in # raising compatibility issues if their number differs: in that case, it is only useful # if some internal attributes do not exist or are invalid - if not template and (sorted([attr.name for attr in nodeDesc.inputs if not isinstance(attr, desc.PushButtonParam)]) != sorted(inputs.keys()) or \ - sorted([attr.name for attr in nodeDesc.outputs if not attr.isDynamicValue]) != sorted(outputs.keys())): + if not template and (sorted([attr.name for attr in nodeDesc.inputs + if not isinstance(attr, desc.PushButtonParam)]) != sorted(inputs.keys()) or + sorted([attr.name for attr in nodeDesc.outputs if not attr.isDynamicValue]) != + sorted(outputs.keys())): compatibilityIssue = CompatibilityIssue.DescriptionConflict # Check whether there are any internal attributes that are invalidating in the node description: if there diff --git a/meshroom/core/stats.py b/meshroom/core/stats.py index 085b8409..7b031d3c 100644 --- a/meshroom/core/stats.py +++ b/meshroom/core/stats.py @@ -6,7 +6,6 @@ import time import threading import platform import os -import sys import xml.etree.ElementTree as ET @@ -56,7 +55,7 @@ class ComputerStatistics: # If the platform is Windows and nvidia-smi self.nvidia_smi = spawn.find_executable('nvidia-smi') if self.nvidia_smi is None: - # could not be found from the environment path, + # Could not be found from the environment path, # try to find it from system drive with default installation path default_nvidia_smi = "%s\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe" % os.environ['systemdrive'] if os.path.isfile(default_nvidia_smi): @@ -77,7 +76,8 @@ class ComputerStatistics: def update(self): try: self.initOnFirstTime() - self._addKV('cpuUsage', psutil.cpu_percent(percpu=True)) # interval=None => non-blocking (percentage since last call) + # Interval=None => non-blocking (percentage since last call) + self._addKV('cpuUsage', psutil.cpu_percent(percpu=True)) self._addKV('ramUsage', psutil.virtual_memory().percent) self._addKV('swapUsage', psutil.swap_memory().percent) self._addKV('vramUsage', 0) @@ -91,7 +91,7 @@ class ComputerStatistics: return try: p = subprocess.Popen([self.nvidia_smi, "-q", "-x"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - xmlGpu, stdError = p.communicate(timeout=10) # 10 seconds + xmlGpu, stdError = p.communicate(timeout=10) # 10 seconds smiTree = ET.fromstring(xmlGpu) gpuTree = smiTree.find('gpu') @@ -140,6 +140,7 @@ class ComputerStatistics: for k, v in d.items(): setattr(self, k, v) + class ProcStatistics: staticKeys = [ 'pid', @@ -201,7 +202,7 @@ class ProcStatistics: for k, v in data.items(): self._addKV(k, v) - ## Note: Do not collect stats about open files for now, + # Note: Do not collect stats about open files for now, # as there is bug in psutil-5.7.2 on Windows which crashes the application. # https://github.com/giampaolo/psutil/issues/1763 # diff --git a/meshroom/core/taskManager.py b/meshroom/core/taskManager.py index e902c62b..901cee37 100644 --- a/meshroom/core/taskManager.py +++ b/meshroom/core/taskManager.py @@ -211,11 +211,8 @@ class TaskManager(BaseObject): chunksName = [node.name for node in chunksInConflict] # Warning: Syntax and terms are parsed on QML side to recognize the error # Syntax : [Context] ErrorType: ErrorMessage - msg = '[COMPUTATION] Already Submitted:\n' \ - 'WARNING - Some nodes are already submitted with status: {}\nNodes: {}'.format( - ', '.join(chunksStatus), - ', '.join(chunksName) - ) + msg = '[COMPUTATION] Already Submitted:\nWARNING - Some nodes are already submitted with status: ' \ + '{}\nNodes: {}'.format(', '.join(chunksStatus), ', '.join(chunksName)) if forceStatus: logging.warning(msg) @@ -325,8 +322,9 @@ class TaskManager(BaseObject): raise RuntimeError("[{}] Duplicates Issue:\n" "Cannot compute because there are some duplicate nodes to process:\n\n" "First match: '{}' and '{}'\n\n" - "There can be other duplicate nodes in the list. Please, check the graph and try again.".format( - context, node.nameToLabel(node.name), node.nameToLabel(duplicate.name))) + "There can be other duplicate nodes in the list. " + "Please, check the graph and try again.". + format(context, node.nameToLabel(node.name), node.nameToLabel(duplicate.name))) def checkNodesDependencies(self, graph, toNodes, context): """ @@ -366,7 +364,8 @@ class TaskManager(BaseObject): # Warning: Syntax and terms are parsed on QML side to recognize the error # Syntax : [Context] ErrorType: ErrorMessage raise RuntimeWarning("[{}] Unresolved dependencies:\n" - "Some nodes cannot be computed in LOCAL/submitted in EXTERN because of unresolved dependencies.\n\n" + "Some nodes cannot be computed in LOCAL/submitted in EXTERN because of " + "unresolved dependencies.\n\n" "Nodes which are ready will be processed.".format(context)) def raiseImpossibleProcess(self, context): diff --git a/meshroom/multiview.py b/meshroom/multiview.py index b08653d7..bf5f4224 100644 --- a/meshroom/multiview.py +++ b/meshroom/multiview.py @@ -1,7 +1,5 @@ import os -from meshroom.core.graph import Graph, GraphModification - # Supported image extensions imageExtensions = ( # bmp: @@ -37,7 +35,10 @@ imageExtensions = ( # ptex: '.ptex', '.ptx', # raw: - '.bay', '.bmq', '.cr2', '.cr3', '.crw', '.cs1', '.dc2', '.dcr', '.dng', '.erf', '.fff', '.k25', '.kdc', '.mdc', '.mos', '.mrw', '.nef', '.orf', '.pef', '.pxn', '.raf', '.raw', '.rdc', '.sr2', '.srf', '.x3f', '.arw', '.3fr', '.cine', '.ia', '.kc2', '.mef', '.nrw', '.qtk', '.rw2', '.sti', '.rwl', '.srw', '.drf', '.dsc', '.cap', '.iiq', '.rwz', + '.bay', '.bmq', '.cr2', '.cr3', '.crw', '.cs1', '.dc2', '.dcr', '.dng', '.erf', '.fff', '.k25', '.kdc', '.mdc', + '.mos', '.mrw', '.nef', '.orf', '.pef', '.pxn', '.raf', '.raw', '.rdc', '.sr2', '.srf', '.x3f', '.arw', '.3fr', + '.cine', '.ia', '.kc2', '.mef', '.nrw', '.qtk', '.rw2', '.sti', '.rwl', '.srw', '.drf', '.dsc', '.cap', '.iiq', + '.rwz', # rla: '.rla', # sgi: diff --git a/meshroom/ui/utils.py b/meshroom/ui/utils.py index 700bb1b4..e09408ea 100755 --- a/meshroom/ui/utils.py +++ b/meshroom/ui/utils.py @@ -5,7 +5,7 @@ from PySide2.QtCore import QFileSystemWatcher, QUrl, Slot, QTimer, Property, QOb from PySide2.QtQml import QQmlApplicationEngine try: from PySide2 import shiboken2 -except: +except Exception: import shiboken2 diff --git a/setupInitScriptUnix.py b/setupInitScriptUnix.py index 75e1844c..82e70f59 100755 --- a/setupInitScriptUnix.py +++ b/setupInitScriptUnix.py @@ -1,4 +1,4 @@ -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # ConsoleSetLibPath.py # Initialization script for cx_Freeze which manipulates the path so that the # directory in which the executable is found is searched for extensions but @@ -6,7 +6,7 @@ # manipulated first, however, to ensure that shared libraries found in the # target directory are found. This requires a restart of the executable because # the environment variable LD_LIBRARY_PATH is only checked at startup. -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ import os import sys @@ -40,4 +40,3 @@ def run(*args): moduleName = args[0] code = importer.get_code(moduleName) exec(code, m.__dict__) - diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index 01383ae7..265946cb 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # coding:utf-8 import tempfile - import os import copy @@ -34,7 +33,7 @@ SampleGroupV2 = [ ) ] -#SampleGroupV3 is SampleGroupV2 with one more int parameter +# SampleGroupV3 is SampleGroupV2 with one more int parameter SampleGroupV3 = [ desc.IntParam(name="a", label="a", description="", value=0, invalidate=True, range=None), desc.IntParam(name="notInSampleGroupV2", label="notInSampleGroupV2", description="", value=0, invalidate=True, range=None), @@ -70,6 +69,7 @@ class SampleNodeV2(desc.Node): desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) ] + class SampleNodeV3(desc.Node): """ Changes from V3: @@ -82,6 +82,7 @@ class SampleNodeV3(desc.Node): desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) ] + class SampleNodeV4(desc.Node): """ Changes from V3: @@ -115,6 +116,7 @@ class SampleNodeV5(desc.Node): desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) ] + class SampleNodeV6(desc.Node): """ Changes from V5: @@ -131,6 +133,7 @@ class SampleNodeV6(desc.Node): desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) ] + class SampleInputNodeV1(desc.InputNode): """ Version 1 Sample Input Node """ inputs = [ @@ -140,6 +143,7 @@ class SampleInputNodeV1(desc.InputNode): desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) ] + class SampleInputNodeV2(desc.InputNode): """ Changes from V1: * 'path' has been renamed to 'in' @@ -151,6 +155,7 @@ class SampleInputNodeV2(desc.InputNode): desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False) ] + def test_unknown_node_type(): """ Test compatibility behavior for unknown node type. @@ -351,6 +356,7 @@ def test_upgradeAllNodes(): unregisterNodeType(SampleNodeV1) unregisterNodeType(SampleInputNodeV1) + def test_conformUpgrade(): registerNodeType(SampleNodeV5) registerNodeType(SampleNodeV6) @@ -389,9 +395,3 @@ def test_conformUpgrade(): unregisterNodeType(SampleNodeV5) unregisterNodeType(SampleNodeV6) - - - - - - diff --git a/tests/test_graph.py b/tests/test_graph.py index 6960258d..003bc6ed 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -142,7 +142,6 @@ def test_transitive_reduction(): (tC.output, tE.input3), (tD.output, tE.input2), ) - edgesScore = graph.dfsMaxEdgeLength() flowEdges = graph.flowEdges() flowEdgesRes = [(tB, tA), @@ -153,7 +152,7 @@ def test_transitive_reduction(): ] assert set(flowEdgesRes) == set(flowEdges) - assert len(graph._nodesMinMaxDepths) == len(graph.nodes) + assert len(graph._nodesMinMaxDepths) == len(graph.nodes) for node, (minDepth, maxDepth) in graph._nodesMinMaxDepths.items(): assert node.depth == maxDepth diff --git a/tests/test_invalidation.py b/tests/test_invalidation.py index cb4c0ba6..6e189a64 100644 --- a/tests/test_invalidation.py +++ b/tests/test_invalidation.py @@ -59,5 +59,3 @@ def test_inputLinkInvalidation(): graph.addEdges((n1.input, n2.input)) assert n1.input.uid() == n2.input.uid() assert n1.output.value == n2.output.value - - diff --git a/tests/test_model.py b/tests/test_model.py index 18492123..1c05c3ca 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -28,7 +28,7 @@ def test_DictModel_add_remove(): assert len(m.values()) == 1 assert m.get("DummyNode_1") == node - assert m.get("something") == None + assert m.get("something") is None with pytest.raises(KeyError): m.getr("something") diff --git a/tests/test_nodeCommandLineFormatting.py b/tests/test_nodeCommandLineFormatting.py index 5b46968d..8c85c9c5 100644 --- a/tests/test_nodeCommandLineFormatting.py +++ b/tests/test_nodeCommandLineFormatting.py @@ -1,11 +1,7 @@ #!/usr/bin/env python # coding:utf-8 -import os -import tempfile - import meshroom.multiview from meshroom.core.graph import Graph -from meshroom.core.node import Node def test_formatting_listOfFiles(): @@ -17,7 +13,8 @@ def test_formatting_listOfFiles(): n1 = graph.addNewNode('CameraInit') n1.viewpoints.extend([{'path': image} for image in inputImages]) # viewId, poseId, path, intrinsicId, rigId, subPoseId, metadata - assert n1.viewpoints.getValueStr() == '-1 -1 "/non/existing/fileA" -1 -1 -1 "" -1 -1 "/non/existing/with space/fileB" -1 -1 -1 ""' + assert n1.viewpoints.getValueStr() == \ + '-1 -1 "/non/existing/fileA" -1 -1 -1 "" -1 -1 "/non/existing/with space/fileB" -1 -1 -1 ""' graph = Graph('') n1 = graph.addNewNode('ImageMatching') @@ -61,7 +58,8 @@ def test_formatting_strings(): n2.featuresFolders.extend('') n2._buildCmdVars() # prepare vars for command line creation assert n2.featuresFolders.getValueStr() == '"" ""', 'A list with 2 empty strings should generate quotes' - assert n2._cmdVars[name + 'Value'] == ' ', 'The Value is always only the value, so 2 empty with the space separator in the middle' + assert n2._cmdVars[name + 'Value'] == ' ', \ + 'The Value is always only the value, so 2 empty with the space separator in the middle' def test_formatting_groups(): @@ -78,4 +76,3 @@ def test_formatting_groups(): name = 'noiseFilter' assert n3.noiseFilter.getValueStr() == '"False:uniform:0.0:1.0:True"' assert n3._cmdVars[name + 'Value'] == 'False:uniform:0.0:1.0:True' -