3pp-mirror/Meshroom
1
0
Fork 0
mirror of https://github.com/alicevision/Meshroom.git synced 2025-06-03 03:11:56 +02:00
Code Issues Projects Releases Packages Wiki Activity

Linting: Clean-up files

Browse source
This commit is contained in:
Candice Bentéjac 2024-09-06 13:11:04 +02:00
parent d67062d39d
commit 41e885d9ff
22 changed files with 298 additions and 198 deletions
Download patch file Download diff file Expand all files Collapse all files

1
docs/source/_ext/meshroom_doc.py
View file

@ -11,7 +11,6 @@
# Note:
# for now this tool focuses only on meshroom nodes
from docutils import nodes
from docutils.parsers.rst import Directive
from utils import md_to_docutils

1
docs/source/conf.py
View file

@ -35,7 +35,6 @@ templates_path = ['_templates']
exclude_patterns = []
# -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output

3
meshroom/__init__.py
View file

@ -4,10 +4,12 @@ import logging
import os
import sys
class VersionStatus(Enum):
release = 1
develop = 2
__version__ = "2024.1.0"
# Always increase the minor version when switching from release to develop.
__version_status__ = VersionStatus.develop
@ -56,6 +58,7 @@ logStringToPython = {
}
logging.getLogger().setLevel(logStringToPython[os.environ.get('MESHROOM_VERBOSE', 'warning')])
def setupEnvironment(backend=Backend.STANDALONE):
"""
Setup environment for Meshroom to work in a prebuilt, standalone configuration.

6
meshroom/common/__init__.py
View file

@ -7,10 +7,12 @@ Warning: A call to `init(Backend.XXX)` is required to choose the backend before
from enum import Enum
class Backend(Enum):
STANDALONE = 1
PYSIDE = 2
DictModel = None
ListModel = None
Slot = None
@ -21,6 +23,7 @@ Variant = None
VariantList = None
JSValue = None
def init(backend):
global DictModel, ListModel, Slot, Signal, Property, BaseObject, Variant, VariantList, JSValue
if backend == Backend.PYSIDE:
@ -30,5 +33,6 @@ def init(backend):
# Core types
from .core import DictModel, ListModel, Slot, Signal, Property, BaseObject, Variant, VariantList, JSValue
# default initialization
# Default initialization
init(Backend.STANDALONE)

1
meshroom/common/core.py
View file

@ -1,5 +1,6 @@
from . import PySignal
class CoreDictModel:
def __init__(self, keyAttrName, **kwargs):

1
meshroom/common/qt.py
View file

@ -1,6 +1,7 @@
from PySide2 import QtCore, QtQml
import shiboken2
class QObjectListModel(QtCore.QAbstractListModel):
"""
QObjectListModel provides a more powerful, but still easy to use, alternative to using

8
meshroom/core/__init__.py
View file

@ -1,11 +1,8 @@
from __future__ import print_function
import hashlib
from contextlib import contextmanager
import importlib
import inspect
import os
import re
import tempfile
import uuid
import logging
@ -18,7 +15,7 @@ try:
import encodings.ascii
import encodings.idna
import encodings.utf_8
except:
except Exception:
pass
from meshroom.core.submitter import BaseSubmitter
@ -330,6 +327,7 @@ def loadPipelineTemplates(folder):
if file.endswith(".mg") and file not in pipelineTemplates:
pipelineTemplates[os.path.splitext(file)[0]] = os.path.join(folder, file)
def initNodes():
meshroomFolder = os.path.dirname(os.path.dirname(__file__))
additionalNodesPath = os.environ.get("MESHROOM_NODES_PATH", "").split(os.pathsep)
@ -339,12 +337,14 @@ def initNodes():
for f in nodesFolders:
loadAllNodes(folder=f)
def initSubmitters():
meshroomFolder = os.path.dirname(os.path.dirname(__file__))
subs = loadSubmitters(os.environ.get("MESHROOM_SUBMITTERS_PATH", meshroomFolder), 'submitters')
for sub in subs:
registerSubmitter(sub())
def initPipelines():
meshroomFolder = os.path.dirname(os.path.dirname(__file__))
# Load pipeline templates: check in the default folder and any folder the user might have

40
meshroom/core/attribute.py
View file

@ -138,7 +138,7 @@ class Attribute(BaseObject):
if isinstance(self.desc.enabled, types.FunctionType):
try:
return self.desc.enabled(self.node)
except:
except Exception:
# Node implementation may fail due to version mismatch
return True
return self.attributeDesc.enabled
@ -325,7 +325,8 @@ class Attribute(BaseObject):
return False
# if the attribute is a ListAttribute, we need to check if any of its elements has output connections
if isinstance(self, ListAttribute):
return next((edge for edge in self.node.graph.edges.values() if edge.src == self), None) is not None or any(attr.hasOutputConnections for attr in self._value if hasattr(attr, 'hasOutputConnections'))
return next((edge for edge in self.node.graph.edges.values() if edge.src == self), None) is not None or \
any(attr.hasOutputConnections for attr in self._value if hasattr(attr, 'hasOutputConnections'))
return next((edge for edge in self.node.graph.edges.values() if edge.src == self), None) is not None
def _applyExpr(self):
@ -348,7 +349,8 @@ class Attribute(BaseObject):
try:
g.addEdge(g.node(linkNode).attribute(linkAttr), self)
except KeyError as err:
logging.warning('Connect Attribute from Expression failed.\nExpression: "{exp}"\nError: "{err}".'.format(exp=v, err=err))
logging.warning('Connect Attribute from Expression failed.')
logging.warning('Expression: "{exp}"\nError: "{err}".'.format(exp=v, err=err))
self.resetToDefaultValue()
def getExportValue(self):
@ -376,8 +378,8 @@ class Attribute(BaseObject):
'''
# ChoiceParam with multiple values should be combined
if isinstance(self.attributeDesc, desc.ChoiceParam) and not self.attributeDesc.exclusive:
# ensure value is a list as expected
assert(isinstance(self.value, Sequence) and not isinstance(self.value, str))
# Ensure value is a list as expected
assert (isinstance(self.value, Sequence) and not isinstance(self.value, str))
v = self.attributeDesc.joinChar.join(self.getEvalValue())
if withQuotes and v:
return '"{}"'.format(v)
@ -393,8 +395,9 @@ class Attribute(BaseObject):
return self.desc.value(self)
except Exception as e:
if not self.node.isCompatibilityNode:
# log message only if we are not in compatibility mode
logging.warning("Failed to evaluate default value (node lambda) for attribute '{}': {}".format(self.name, e))
# Log message only if we are not in compatibility mode
logging.warning("Failed to evaluate default value (node lambda) for attribute '{}': {}".
format(self.name, e))
return None
# Need to force a copy, for the case where the value is a list (avoid reference to the desc value)
return copy.copy(self.desc.value)
@ -409,7 +412,6 @@ class Attribute(BaseObject):
# Emit if the enable status has changed
self.setEnabled(self.getEnabled())
name = Property(str, getName, constant=True)
fullName = Property(str, getFullName, constant=True)
fullNameToNode = Property(str, getFullNameToNode, constant=True)
@ -423,11 +425,11 @@ class Attribute(BaseObject):
baseType = Property(str, getType, constant=True)
isReadOnly = Property(bool, _isReadOnly, constant=True)
# description of the attribute
# Description of the attribute
descriptionChanged = Signal()
description = Property(str, _get_description, _set_description, notify=descriptionChanged)
# definition of the attribute
# Definition of the attribute
desc = Property(desc.Attribute, lambda self: self.attributeDesc, constant=True)
valueChanged = Signal()
@ -460,6 +462,7 @@ def raiseIfLink(func):
return func(attr, *args, **kwargs)
return wrapper
class PushButtonParam(Attribute):
def __init__(self, node, attributeDesc, isOutput, root=None, parent=None):
super(PushButtonParam, self).__init__(node, attributeDesc, isOutput, root, parent)
@ -468,6 +471,7 @@ class PushButtonParam(Attribute):
def clicked(self):
self.node.onAttributeClicked(self)
class ChoiceParam(Attribute):
def __init__(self, node, attributeDesc, isOutput, root=None, parent=None):
@ -489,7 +493,8 @@ class ChoiceParam(Attribute):
value = value.split(',')
if not isinstance(value, Iterable):
raise ValueError('Non exclusive ChoiceParam value should be iterable (param:{}, value:{}, type:{})'.format(self.name, value, type(value)))
raise ValueError("Non exclusive ChoiceParam value should be iterable (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
return [self.conformValue(v) for v in value]
def setValues(self, values):
@ -523,7 +528,7 @@ class ListAttribute(Attribute):
def at(self, idx):
""" Returns child attribute at index 'idx' """
# implement 'at' rather than '__getitem__'
# Implement 'at' rather than '__getitem__'
# since the later is called spuriously when object is used in QML
return self._value.at(idx)
@ -560,7 +565,8 @@ class ListAttribute(Attribute):
if isinstance(exportedValues, ListAttribute) or Attribute.isLinkExpression(exportedValues):
self._set_value(exportedValues)
return
raise RuntimeError("ListAttribute.upgradeValue: the given value is of type " + str(type(exportedValues)) + " but a 'list' is expected.")
raise RuntimeError("ListAttribute.upgradeValue: the given value is of type " +
str(type(exportedValues)) + " but a 'list' is expected.")
attrs = []
for v in exportedValues:
@ -645,7 +651,7 @@ class ListAttribute(Attribute):
return [attr.getPrimitiveValue(exportDefault=exportDefault) for attr in self._value if not attr.isDefault]
def getValueStr(self, withQuotes=True):
assert(isinstance(self.value, ListModel))
assert isinstance(self.value, ListModel)
if self.attributeDesc.joinChar == ' ':
return self.attributeDesc.joinChar.join([v.getValueStr(withQuotes=withQuotes) for v in self.value])
else:
@ -770,7 +776,8 @@ class GroupAttribute(Attribute):
if exportDefault:
return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items()}
else:
return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items() if not attr.isDefault}
return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items()
if not attr.isDefault}
def getValueStr(self, withQuotes=True):
# add brackets if requested
@ -787,7 +794,8 @@ class GroupAttribute(Attribute):
spaceSep = self.attributeDesc.joinChar == ' '
# sort values based on child attributes group description order
sortedSubValues = [self._value.get(attr.name).getValueStr(withQuotes=spaceSep) for attr in self.attributeDesc.groupDesc]
sortedSubValues = [self._value.get(attr.name).getValueStr(withQuotes=spaceSep)
for attr in self.attributeDesc.groupDesc]
s = self.attributeDesc.joinChar.join(sortedSubValues)
if withQuotes and not spaceSep:

32
meshroom/core/cgroup.py
View file

@ -3,21 +3,22 @@
import os
#Try to retrieve limits of memory for the current process' cgroup
# Try to retrieve limits of memory for the current process' cgroup
def getCgroupMemorySize():
#first of all, get pid of process
# First of all, get pid of process
pid = os.getpid()
#Get cgroup associated with pid
# Get cgroup associated with pid
filename = f"/proc/{pid}/cgroup"
cgroup = None
try:
with open(filename, "r") as f :
with open(filename, "r") as f:
#cgroup file is a ':' separated table
#lookup a line where the second field is "memory"
# cgroup file is a ':' separated table
# lookup a line where the second field is "memory"
lines = f.readlines()
for line in lines:
tokens = line.rstrip("\r\n").split(":")
@ -34,7 +35,7 @@ def getCgroupMemorySize():
size = -1
filename = f"/sys/fs/cgroup/memory/{cgroup}/memory.limit_in_bytes"
try:
with open(filename, "r") as f :
with open(filename, "r") as f:
value = f.read().rstrip("\r\n")
if value.isnumeric():
size = int(value)
@ -43,6 +44,7 @@ def getCgroupMemorySize():
return size
def parseNumericList(numericListString):
nList = []
@ -58,21 +60,22 @@ def parseNumericList(numericListString):
return nList
#Try to retrieve limits of cores for the current process' cgroup
# Try to retrieve limits of cores for the current process' cgroup
def getCgroupCpuCount():
#first of all, get pid of process
# First of all, get pid of process
pid = os.getpid()
#Get cgroup associated with pid
# Get cgroup associated with pid
filename = f"/proc/{pid}/cgroup"
cgroup = None
try:
with open(filename, "r") as f :
with open(filename, "r") as f:
#cgroup file is a ':' separated table
#lookup a line where the second field is "memory"
# cgroup file is a ':' separated table
# lookup a line where the second field is "memory"
lines = f.readlines()
for line in lines:
tokens = line.rstrip("\r\n").split(":")
@ -89,7 +92,7 @@ def getCgroupCpuCount():
size = -1
filename = f"/sys/fs/cgroup/cpuset/{cgroup}/cpuset.cpus"
try:
with open(filename, "r") as f :
with open(filename, "r") as f:
value = f.read().rstrip("\r\n")
nlist = parseNumericList(value)
size = len(nlist)
@ -98,4 +101,3 @@ def getCgroupCpuCount():
pass
return size

174
meshroom/core/desc.py
View file

@ -11,6 +11,7 @@ import ast
import distutils.util
import shlex
class Attribute(BaseObject):
"""
"""
@ -32,7 +33,8 @@ class Attribute(BaseObject):
self._errorMessage = errorMessage
self._visible = visible
self._exposed = exposed
self._isExpression = (isinstance(self._value, str) and "{" in self._value) or isinstance(self._value, types.FunctionType)
self._isExpression = (isinstance(self._value, str) and "{" in self._value) \
or isinstance(self._value, types.FunctionType)
self._isDynamicValue = (self._value is None)
self._valueType = None
@ -48,7 +50,8 @@ class Attribute(BaseObject):
Raises:
ValueError: if value does not have the proper type
"""
raise NotImplementedError("Attribute.validateValue is an abstract function that should be implemented in the derived class.")
raise NotImplementedError("Attribute.validateValue is an abstract function that should be "
"implemented in the derived class.")
def checkValueTypes(self):
""" Returns the attribute's name if the default value's type is invalid or if the range's type (when available)
@ -57,7 +60,8 @@ class Attribute(BaseObject):
Returns:
string: the attribute's name if the default value's or range's type is invalid, empty string otherwise
"""
raise NotImplementedError("Attribute.checkValueTypes is an abstract function that should be implemented in the derived class.")
raise NotImplementedError("Attribute.checkValueTypes is an abstract function that should be implemented in the "
"derived class.")
def matchDescription(self, value, strict=True):
""" Returns whether the value perfectly match attribute's description.
@ -108,13 +112,16 @@ class Attribute(BaseObject):
class ListAttribute(Attribute):
""" A list of Attributes """
def __init__(self, elementDesc, name, label, description, group='allParams', advanced=False, semantic='', enabled=True, joinChar=' ', visible=True, exposed=False):
def __init__(self, elementDesc, name, label, description, group='allParams', advanced=False, semantic='',
enabled=True, joinChar=' ', visible=True, exposed=False):
"""
:param elementDesc: the Attribute description of elements to store in that list
"""
self._elementDesc = elementDesc
self._joinChar = joinChar
super(ListAttribute, self).__init__(name=name, label=label, description=description, value=[], invalidate=False, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
super(ListAttribute, self).__init__(name=name, label=label, description=description, value=[],
invalidate=False, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, visible=visible, exposed=exposed)
def getInstanceType(self):
# Import within the method to prevent cyclic dependencies
@ -126,14 +133,16 @@ class ListAttribute(Attribute):
return value
if JSValue is not None and isinstance(value, JSValue):
# Note: we could use isArray(), property("length").toInt() to retrieve all values
raise ValueError("ListAttribute.validateValue: cannot recognize QJSValue. Please, use JSON.stringify(value) in QML.")
raise ValueError("ListAttribute.validateValue: cannot recognize QJSValue. "
"Please, use JSON.stringify(value) in QML.")
if isinstance(value, str):
# Alternative solution to set values from QML is to convert values to JSON string
# In this case, it works with all data types
value = ast.literal_eval(value)
if not isinstance(value, (list, tuple)):
raise ValueError('ListAttribute only supports list/tuple input values (param:{}, value:{}, type:{})'.format(self.name, value, type(value)))
raise ValueError("ListAttribute only supports list/tuple input values "
"(param:{}, value:{}, type:{})".format(self.name, value, type(value)))
return value
def checkValueTypes(self):
@ -155,14 +164,17 @@ class ListAttribute(Attribute):
class GroupAttribute(Attribute):
""" A macro Attribute composed of several Attributes """
def __init__(self, groupDesc, name, label, description, group='allParams', advanced=False, semantic='', enabled=True, joinChar=' ', brackets=None, visible=True, exposed=False):
def __init__(self, groupDesc, name, label, description, group='allParams', advanced=False, semantic='',
enabled=True, joinChar=' ', brackets=None, visible=True, exposed=False):
"""
:param groupDesc: the description of the Attributes composing this group
"""
self._groupDesc = groupDesc
self._joinChar = joinChar
self._brackets = brackets
super(GroupAttribute, self).__init__(name=name, label=label, description=description, value={}, invalidate=False, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
super(GroupAttribute, self).__init__(name=name, label=label, description=description, value={},
invalidate=False, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, visible=visible, exposed=exposed)
def getInstanceType(self):
# Import within the method to prevent cyclic dependencies
@ -170,12 +182,13 @@ class GroupAttribute(Attribute):
return GroupAttribute
def validateValue(self, value):
""" Ensure value is compatible with the group description and convert value if needed. """
if value is None:
return value
""" Ensure value is compatible with the group description and convert value if needed. """
if JSValue is not None and isinstance(value, JSValue):
# Note: we could use isArray(), property("length").toInt() to retrieve all values
raise ValueError("GroupAttribute.validateValue: cannot recognize QJSValue. Please, use JSON.stringify(value) in QML.")
raise ValueError("GroupAttribute.validateValue: cannot recognize QJSValue. "
"Please, use JSON.stringify(value) in QML.")
if isinstance(value, str):
# Alternative solution to set values from QML is to convert values to JSON string
# In this case, it works with all data types
@ -188,12 +201,16 @@ class GroupAttribute(Attribute):
if self._groupDesc and value.keys():
commonKeys = set(value.keys()).intersection([attr.name for attr in self._groupDesc])
if not commonKeys:
raise ValueError(f'Value contains no key that matches with the group description (name={self.name}, values={value.keys()}, desc={[attr.name for attr in self._groupDesc]})')
raise ValueError(f"Value contains no key that matches with the group description "
f"(name={self.name}, values={value.keys()}, "
f"desc={[attr.name for attr in self._groupDesc]})")
elif isinstance(value, (list, tuple, set)):
if len(value) != len(self._groupDesc):
raise ValueError('Value contains incoherent number of values: desc size: {}, value size: {}'.format(len(self._groupDesc), len(value)))
raise ValueError("Value contains incoherent number of values: desc size: {}, value size: {}".
format(len(self._groupDesc), len(value)))
else:
raise ValueError('GroupAttribute only supports dict/list/tuple input values (param:{}, value:{}, type:{})'.format(self.name, value, type(value)))
raise ValueError("GroupAttribute only supports dict/list/tuple input values (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
return value
@ -254,24 +271,31 @@ class GroupAttribute(Attribute):
class Param(Attribute):
"""
"""
def __init__(self, name, label, description, value, invalidate, group, advanced, semantic, enabled, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False):
super(Param, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled,
uidIgnoreValue=uidIgnoreValue, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed)
def __init__(self, name, label, description, value, invalidate, group, advanced, semantic, enabled,
uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False):
super(Param, self).__init__(name=name, label=label, description=description, value=value,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, uidIgnoreValue=uidIgnoreValue, validValue=validValue,
errorMessage=errorMessage, visible=visible, exposed=exposed)
class File(Attribute):
"""
"""
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=True):
super(File, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='',
enabled=True, visible=True, exposed=True):
super(File, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate,
group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible,
exposed=exposed)
self._valueType = str
def validateValue(self, value):
if value is None:
return value
if not isinstance(value, str):
raise ValueError('File only supports string input (param:{}, value:{}, type:{})'.format(self.name, value, type(value)))
return os.path.normpath(value).replace('\\', '/') if value else ''
raise ValueError("File only supports string input (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
return os.path.normpath(value).replace("\\", "/") if value else ""
def checkValueTypes(self):
# Some File values are functions generating a string: check whether the value is a string or if it
@ -284,8 +308,11 @@ class File(Attribute):
class BoolParam(Param):
"""
"""
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False):
super(BoolParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='',
enabled=True, visible=True, exposed=False):
super(BoolParam, self).__init__(name=name, label=label, description=description, value=value,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, visible=visible, exposed=exposed)
self._valueType = bool
def validateValue(self, value):
@ -296,8 +323,9 @@ class BoolParam(Param):
# use distutils.util.strtobool to handle (1/0, true/false, on/off, y/n)
return bool(distutils.util.strtobool(value))
return bool(value)
except:
raise ValueError('BoolParam only supports bool value (param:{}, value:{}, type:{})'.format(self.name, value, type(value)))
except Exception:
raise ValueError("BoolParam only supports bool value (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
def checkValueTypes(self):
if not isinstance(self.value, bool):
@ -308,20 +336,24 @@ class BoolParam(Param):
class IntParam(Param):
"""
"""
def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False):
def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False,
semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False):
self._range = range
super(IntParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled,
validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed)
super(IntParam, self).__init__(name=name, label=label, description=description, value=value,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, validValue=validValue, errorMessage=errorMessage,
visible=visible, exposed=exposed)
self._valueType = int
def validateValue(self, value):
if value is None:
return value
# handle unsigned int values that are translated to int by shiboken and may overflow
# Handle unsigned int values that are translated to int by shiboken and may overflow
try:
return int(value)
except:
raise ValueError('IntParam only supports int value (param:{}, value:{}, type:{})'.format(self.name, value, type(value)))
except Exception:
raise ValueError("IntParam only supports int value (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
def checkValueTypes(self):
if not isinstance(self.value, int) or (self.range and not all([isinstance(r, int) for r in self.range])):
@ -334,10 +366,13 @@ class IntParam(Param):
class FloatParam(Param):
"""
"""
def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False):
def __init__(self, name, label, description, value, range, invalidate, group='allParams', advanced=False,
semantic='', enabled=True, validValue=True, errorMessage="", visible=True, exposed=False):
self._range = range
super(FloatParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled,
validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed)
super(FloatParam, self).__init__(name=name, label=label, description=description, value=value,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, validValue=validValue, errorMessage=errorMessage,
visible=visible, exposed=exposed)
self._valueType = float
def validateValue(self, value):
@ -345,8 +380,9 @@ class FloatParam(Param):
return value
try:
return float(value)
except:
raise ValueError('FloatParam only supports float value (param:{}, value:{}, type:{})'.format(self.name, value, type(value)))
except Exception:
raise ValueError("FloatParam only supports float value (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
def checkValueTypes(self):
if not isinstance(self.value, float) or (self.range and not all([isinstance(r, float) for r in self.range])):
@ -355,11 +391,15 @@ class FloatParam(Param):
range = Property(VariantList, lambda self: self._range, constant=True)
class PushButtonParam(Param):
"""
"""
def __init__(self, name, label, description, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False):
super(PushButtonParam, self).__init__(name=name, label=label, description=description, value=None, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
def __init__(self, name, label, description, invalidate, group='allParams', advanced=False, semantic='',
enabled=True, visible=True, exposed=False):
super(PushButtonParam, self).__init__(name=name, label=label, description=description, value=None,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, visible=visible, exposed=exposed)
self._valueType = None
def getInstanceType(self):
@ -377,11 +417,14 @@ class PushButtonParam(Param):
class ChoiceParam(Param):
"""
"""
def __init__(self, name, label, description, value, values, exclusive, invalidate, group='allParams', joinChar=' ', advanced=False, semantic='',
enabled=True, validValue=True, errorMessage="", visible=True, exposed=False):
def __init__(self, name, label, description, value, values, exclusive, invalidate, group='allParams', joinChar=' ',
advanced=False, semantic='', enabled=True, validValue=True, errorMessage="", visible=True,
exposed=False):
assert values
super(ChoiceParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced,
semantic=semantic, enabled=enabled, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed)
super(ChoiceParam, self).__init__(name=name, label=label, description=description, value=value,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, validValue=validValue, errorMessage=errorMessage,
visible=visible, exposed=exposed)
self._values = values
self._exclusive = exclusive
self._joinChar = joinChar
@ -416,7 +459,8 @@ class ChoiceParam(Param):
value = value.split(',')
if not isinstance(value, Iterable):
raise ValueError('Non exclusive ChoiceParam value should be iterable (param: {}, value: {}, type: {}).'.format(self.name, value, type(value)))
raise ValueError("Non-exclusive ChoiceParam value should be iterable (param: {}, value: {}, type: {}).".
format(self.name, value, type(value)))
return [self.conformValue(v) for v in value]
@ -446,16 +490,20 @@ class ChoiceParam(Param):
class StringParam(Param):
"""
"""
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False):
super(StringParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled,
uidIgnoreValue=uidIgnoreValue, validValue=validValue, errorMessage=errorMessage, visible=visible, exposed=exposed)
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='',
enabled=True, uidIgnoreValue=None, validValue=True, errorMessage="", visible=True, exposed=False):
super(StringParam, self).__init__(name=name, label=label, description=description, value=value,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, uidIgnoreValue=uidIgnoreValue, validValue=validValue,
errorMessage=errorMessage, visible=visible, exposed=exposed)
self._valueType = str
def validateValue(self, value):
if value is None:
return value
if not isinstance(value, str):
raise ValueError('StringParam value should be a string (param:{}, value:{}, type:{})'.format(self.name, value, type(value)))
raise ValueError("StringParam value should be a string (param:{}, value:{}, type:{})".
format(self.name, value, type(value)))
return value
def checkValueTypes(self):
@ -467,8 +515,11 @@ class StringParam(Param):
class ColorParam(Param):
"""
"""
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='', enabled=True, visible=True, exposed=False):
super(ColorParam, self).__init__(name=name, label=label, description=description, value=value, invalidate=invalidate, group=group, advanced=advanced, semantic=semantic, enabled=enabled, visible=visible, exposed=exposed)
def __init__(self, name, label, description, value, invalidate, group='allParams', advanced=False, semantic='',
enabled=True, visible=True, exposed=False):
super(ColorParam, self).__init__(name=name, label=label, description=description, value=value,
invalidate=invalidate, group=group, advanced=advanced, semantic=semantic,
enabled=enabled, visible=visible, exposed=exposed)
self._valueType = str
def validateValue(self, value):
@ -683,6 +734,7 @@ class Node(object):
BaseNode.updateInternals
"""
pass
@classmethod
def postUpdate(cls, node):
""" Method call after node's internal update on invalidation.
@ -700,6 +752,7 @@ class Node(object):
def processChunk(self, chunk):
raise NotImplementedError('No processChunk implementation on node: "{}"'.format(chunk.node.name))
class InputNode(Node):
"""
Node that does not need to be processed, it is just a placeholder for inputs.
@ -724,12 +777,14 @@ class CommandLineNode(Node):
def buildCommandLine(self, chunk):
cmdPrefix = ''
# if rez available in env, we use it
if 'REZ_ENV' in os.environ and chunk.node.packageVersion:
# if the node package is already in the environment, we don't need a new dedicated rez environment
alreadyInEnv = os.environ.get('REZ_{}_VERSION'.format(chunk.node.packageName.upper()), "").startswith(chunk.node.packageVersion)
# If rez available in env, we use it
if "REZ_ENV" in os.environ and chunk.node.packageVersion:
# If the node package is already in the environment, we don't need a new dedicated rez environment
alreadyInEnv = os.environ.get("REZ_{}_VERSION".format(chunk.node.packageName.upper()),
"").startswith(chunk.node.packageVersion)
if not alreadyInEnv:
cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get('REZ_ENV'), packageFullName=chunk.node.packageFullName)
cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get("REZ_ENV"),
packageFullName=chunk.node.packageFullName)
cmdSuffix = ''
if chunk.node.isParallelized and chunk.node.size > 1:
@ -738,12 +793,12 @@ class CommandLineNode(Node):
return cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix
def stopProcess(self, chunk):
# the same node could exists several times in the graph and
# The same node could exists several times in the graph and
# only one would have the running subprocess; ignore all others
if not hasattr(chunk, "subprocess"):
return
if chunk.subprocess:
# kill process tree
# Kill process tree
processes = chunk.subprocess.children(recursive=True) + [chunk.subprocess]
try:
for process in processes:
@ -761,7 +816,7 @@ class CommandLineNode(Node):
print(' - logFile: {}'.format(chunk.logFile))
chunk.subprocess = psutil.Popen(shlex.split(cmd), stdout=logF, stderr=logF, cwd=chunk.node.internalFolder)
# store process static info into the status file
# Store process static info into the status file
# chunk.status.env = node.proc.environ()
# chunk.status.createTime = node.proc.create_time()
@ -775,11 +830,12 @@ class CommandLineNode(Node):
with open(chunk.logFile, 'r') as logF:
logContent = ''.join(logF.readlines())
raise RuntimeError('Error on node "{}":\nLog:\n{}'.format(chunk.name, logContent))
except:
except Exception:
raise
finally:
chunk.subprocess = None
# Specific command line node for AliceVision apps
class AVCommandLineNode(CommandLineNode):
@ -809,7 +865,7 @@ class AVCommandLineNode(CommandLineNode):
return commandLineString + AVCommandLineNode.cmdMem + AVCommandLineNode.cmdCore
# Test abstract node
class InitNode(object):
def __init__(self):
super(InitNode, self).__init__()

50
meshroom/core/graph.py
View file

@ -22,6 +22,7 @@ from meshroom.core.node import nodeFactory, Status, Node, CompatibilityNode
DefaultJSONEncoder = json.JSONEncoder # store the original one
class MyJSONEncoder(DefaultJSONEncoder): # declare a new one with Enum support
def default(self, obj):
if isinstance(obj, Enum):
@ -220,7 +221,8 @@ class Graph(BaseObject):
self._computationBlocked = {}
self._canComputeLeaves = True
self._nodes = DictModel(keyAttrName='name', parent=self)
self._edges = DictModel(keyAttrName='dst', parent=self) # use dst attribute as unique key since it can only have one input connection
# Edges: use dst attribute as unique key since it can only have one input connection
self._edges = DictModel(keyAttrName='dst', parent=self)
self._importedNodes = DictModel(keyAttrName='name', parent=self)
self._compatibilityNodes = DictModel(keyAttrName='name', parent=self)
self.cacheDir = meshroom.core.defaultCacheFolder
@ -682,9 +684,12 @@ class Graph(BaseObject):
"""
Remove the node identified by 'nodeName' from the graph.
Returns:
- a dictionary containing the incoming edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the outgoing edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute prior to the removal of all edges:
- a dictionary containing the incoming edges removed by this operation:
{dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the outgoing edges removed by this operation:
{dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute
prior to the removal of all edges:
{dstAttr.getFullNameToNode(), (dstAttr.root.getFullNameToNode(), dstAttr.index, dstAttr.value)}
"""
node = self.node(nodeName)
@ -695,14 +700,18 @@ class Graph(BaseObject):
# Remove all edges arriving to and starting from this node
with GraphModification(self):
# Two iterations over the outgoing edges are necessary:
# - the first one is used to collect all the information about the edges while they are all there (overall context)
# - once we have collected all the information, the edges (and perhaps the entries in ListAttributes) can actually be removed
# - the first one is used to collect all the information about the edges while they are all there
# (overall context)
# - once we have collected all the information, the edges (and perhaps the entries in ListAttributes) can
# actually be removed
for edge in self.nodeOutEdges(node):
outEdges[edge.dst.getFullNameToNode()] = edge.src.getFullNameToNode()
if isinstance(edge.dst.root, ListAttribute):
index = edge.dst.root.index(edge.dst)
outListAttributes[edge.dst.getFullNameToNode()] = (edge.dst.root.getFullNameToNode(), index, edge.dst.value if edge.dst.value else None)
outListAttributes[edge.dst.getFullNameToNode()] = (edge.dst.root.getFullNameToNode(),
index, edge.dst.value
if edge.dst.value else None)
for edge in self.nodeOutEdges(node):
self.removeEdge(edge.dst)
@ -763,9 +772,12 @@ class Graph(BaseObject):
Returns:
- the upgraded (newly created) node
- a dictionary containing the incoming edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the outgoing edges removed by this operation: {dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute prior to the removal of all edges:
- a dictionary containing the incoming edges removed by this operation:
{dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the outgoing edges removed by this operation:
{dstAttr.getFullNameToNode(), srcAttr.getFullNameToNode()}
- a dictionary containing the values, indices and keys of attributes that were connected to a ListAttribute
prior to the removal of all edges:
{dstAttr.getFullNameToNode(), (dstAttr.root.getFullNameToNode(), dstAttr.index, dstAttr.value)}
"""
node = self.node(nodeName)
@ -834,7 +846,7 @@ class Graph(BaseObject):
"""
try:
return int(name.split('_')[-1])
except:
except Exception:
return -1
@staticmethod
@ -970,7 +982,8 @@ class Graph(BaseObject):
def dfs(self, visitor, startNodes=None, longestPathFirst=False):
# Default direction (visitor.reverse=False): from node to root
# Reverse direction (visitor.reverse=True): from node to leaves
nodeChildren = self._getOutputEdgesPerNode(visitor.dependenciesOnly) if visitor.reverse else self._getInputEdgesPerNode(visitor.dependenciesOnly)
nodeChildren = self._getOutputEdgesPerNode(visitor.dependenciesOnly) \
if visitor.reverse else self._getInputEdgesPerNode(visitor.dependenciesOnly)
# Initialize color map
colors = {}
for u in self._nodes:
@ -979,9 +992,11 @@ class Graph(BaseObject):
if longestPathFirst and visitor.reverse:
# Because we have no knowledge of the node's count between a node and its leaves,
# it is not possible to handle this case at the moment
raise NotImplementedError("Graph.dfs(): longestPathFirst=True and visitor.reverse=True are not compatible yet.")
raise NotImplementedError("Graph.dfs(): longestPathFirst=True and visitor.reverse=True are not "
"compatible yet.")
nodes = startNodes or (self.getRootNodes(visitor.dependenciesOnly) if visitor.reverse else self.getLeafNodes(visitor.dependenciesOnly))
nodes = startNodes or (self.getRootNodes(visitor.dependenciesOnly)
if visitor.reverse else self.getLeafNodes(visitor.dependenciesOnly))
if longestPathFirst:
# Graph topology must be known and node depths up-to-date
@ -1302,7 +1317,6 @@ class Graph(BaseObject):
self.dfs(visitor=visitor, startNodes=[startNode])
return visitor.canCompute + (2 * visitor.canSubmit)
def _applyExpr(self):
with GraphModification(self):
for node in self._nodes:
@ -1600,14 +1614,15 @@ class Graph(BaseObject):
if node.hasAttribute('verbose'):
try:
node.verbose.value = v
except:
except Exception:
pass
nodes = Property(BaseObject, nodes.fget, constant=True)
edges = Property(BaseObject, edges.fget, constant=True)
filepathChanged = Signal()
filepath = Property(str, lambda self: self._filepath, notify=filepathChanged)
fileReleaseVersion = Property(str, lambda self: self.header.get(Graph.IO.Keys.ReleaseVersion, "0.0"), notify=filepathChanged)
fileReleaseVersion = Property(str, lambda self: self.header.get(Graph.IO.Keys.ReleaseVersion, "0.0"),
notify=filepathChanged)
fileDateVersion = Property(float, fileDateVersion.fget, fileDateVersion.fset, notify=filepathChanged)
cacheDirChanged = Signal()
cacheDir = Property(str, cacheDir.fget, cacheDir.fset, notify=cacheDirChanged)
@ -1721,4 +1736,3 @@ def submit(graphFile, submitter, toNode=None, submitLabel="{projectName}"):
graph = loadGraph(graphFile)
toNodes = graph.findNodes(toNode) if toNode else None
submitGraph(graph, submitter, toNodes, submitLabel=submitLabel)

103
meshroom/core/node.py
View file

@ -12,7 +12,7 @@ import shutil
import time
import types
import uuid
from collections import defaultdict, namedtuple
from collections import namedtuple
from enum import Enum
import meshroom
@ -33,7 +33,7 @@ def renameWritingToFinalPath(writingFilepath, filepath):
for i in range(20):
try:
os.remove(filepath)
# if remove is successful, we can stop the iterations
# If remove is successful, we can stop the iterations
break
except WindowsError:
pass
@ -50,7 +50,7 @@ class Status(Enum):
STOPPED = 4
KILLED = 5
SUCCESS = 6
INPUT = 7 # special status for input nodes
INPUT = 7 # Special status for input nodes
class ExecMode(Enum):
@ -252,7 +252,7 @@ class NodeChunk(BaseObject):
self.statistics = stats.Statistics()
self.statusFileLastModTime = -1
self._subprocess = None
# notify update in filepaths when node's internal folder changes
# Notify update in filepaths when node's internal folder changes
self.node.internalFolderChanged.connect(self.nodeFolderChanged)
self.execModeNameChanged.connect(self.node.globalExecModeChanged)
@ -296,7 +296,7 @@ class NodeChunk(BaseObject):
statusData = json.load(jsonFile)
self.status.fromDict(statusData)
self.statusFileLastModTime = os.path.getmtime(statusFile)
except Exception as e:
except Exception:
self.statusFileLastModTime = -1
self.status.reset()
@ -306,23 +306,23 @@ class NodeChunk(BaseObject):
@property
def statusFile(self):
if self.range.blockSize == 0:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'status')
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "status")
else:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.status')
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".status")
@property
def statisticsFile(self):
if self.range.blockSize == 0:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'statistics')
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "statistics")
else:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.statistics')
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".statistics")
@property
def logFile(self):
if self.range.blockSize == 0:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, 'log')
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "log")
else:
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + '.log')
return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".log")
def saveStatusFile(self):
"""
@ -333,7 +333,7 @@ class NodeChunk(BaseObject):
folder = os.path.dirname(statusFilepath)
try:
os.makedirs(folder)
except Exception as e:
except Exception:
pass
statusFilepathWriting = getWritingFilepath(statusFilepath)
@ -343,8 +343,8 @@ class NodeChunk(BaseObject):
def upgradeStatusTo(self, newStatus, execMode=None):
if newStatus.value <= self._status.status.value:
logging.warning('Downgrade status on node "{}" from {} to {}'.format(self.name, self._status.status,
newStatus))
logging.warning("Downgrade status on node '{}' from {} to {}".
format(self.name, self._status.status, newStatus))
if newStatus == Status.SUBMITTED:
self._status = StatusData(self.node.name, self.node.nodeType, self.node.packageName, self.node.packageVersion)
@ -411,11 +411,11 @@ class NodeChunk(BaseObject):
self.statThread.start()
try:
self.node.nodeDesc.processChunk(self)
except Exception as e:
except Exception:
if self._status.status != Status.STOPPED:
exceptionStatus = Status.ERROR
raise
except (KeyboardInterrupt, SystemError, GeneratorExit) as e:
except (KeyboardInterrupt, SystemError, GeneratorExit):
exceptionStatus = Status.STOPPED
raise
finally:
@ -423,8 +423,8 @@ class NodeChunk(BaseObject):
self._status.elapsedTime = time.time() - startTime
if exceptionStatus is not None:
self.upgradeStatusTo(exceptionStatus)
logging.info(' - elapsed time: {}'.format(self._status.elapsedTimeStr))
# ask and wait for the stats thread to stop
logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr))
# Ask and wait for the stats thread to stop
self.statThread.stopRequest()
self.statThread.join()
self.statistics = stats.Statistics()
@ -462,9 +462,9 @@ class NodeChunk(BaseObject):
elapsedTime = Property(float, lambda self: self._status.elapsedTime, notify=statusChanged)
# simple structure for storing node position
# Simple structure for storing node position
Position = namedtuple("Position", ["x", "y"])
# initialize default coordinates values to 0
# Initialize default coordinates values to 0
Position.__new__.__defaults__ = (0,) * len(Position._fields)
@ -729,7 +729,8 @@ class BaseNode(BaseObject):
def _buildCmdVars(self):
def _buildAttributeCmdVars(cmdVars, name, attr):
if attr.enabled:
group = attr.attributeDesc.group(attr.node) if isinstance(attr.attributeDesc.group, types.FunctionType) else attr.attributeDesc.group
group = attr.attributeDesc.group(attr.node) \
if isinstance(attr.attributeDesc.group, types.FunctionType) else attr.attributeDesc.group
if group is not None:
# If there is a valid command line "group"
v = attr.getValueStr(withQuotes=True)
@ -776,18 +777,25 @@ class BaseNode(BaseObject):
if attr.enabled:
try:
defaultValue = attr.defaultValue()
except AttributeError as e:
except AttributeError:
# If we load an old scene, the lambda associated to the 'value' could try to access other
# params that could not exist yet
logging.warning('Invalid lambda evaluation for "{nodeName}.{attrName}"'.format(nodeName=self.name, attrName=attr.name))
logging.warning('Invalid lambda evaluation for "{nodeName}.{attrName}"'.
format(nodeName=self.name, attrName=attr.name))
if defaultValue is not None:
try:
attr.value = defaultValue.format(**self._cmdVars)
attr._invalidationValue = defaultValue.format(**cmdVarsNoCache)
except KeyError as e:
logging.warning('Invalid expression with missing key on "{nodeName}.{attrName}" with value "{defaultValue}".\nError: {err}'.format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue, err=str(e)))
logging.warning('Invalid expression with missing key on "{nodeName}.{attrName}" with '
'value "{defaultValue}".\nError: {err}'.
format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue,
err=str(e)))
except ValueError as e:
logging.warning('Invalid expression value on "{nodeName}.{attrName}" with value "{defaultValue}".\nError: {err}'.format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue, err=str(e)))
logging.warning('Invalid expression value on "{nodeName}.{attrName}" with value '
'"{defaultValue}".\nError: {err}'.
format(nodeName=self.name, attrName=attr.name, defaultValue=defaultValue,
err=str(e)))
v = attr.getValueStr(withQuotes=True)
@ -1074,9 +1082,11 @@ class BaseNode(BaseObject):
self.attribute(output.name).value = data[output.name]
else:
if not self.hasAttribute(output.name):
logging.warning(f"loadOutputAttr: Missing dynamic output attribute. Node={self.name}, Attribute={output.name}")
logging.warning(f"loadOutputAttr: Missing dynamic output attribute. Node={self.name}, "
f"Attribute={output.name}")
if output.name not in data:
logging.warning(f"loadOutputAttr: Missing dynamic output value in file. Node={self.name}, Attribute={output.name}, File={valuesFile}, Data keys={data.keys()}")
logging.warning(f"loadOutputAttr: Missing dynamic output value in file. Node={self.name}, "
f"Attribute={output.name}, File={valuesFile}, Data keys={data.keys()}")
def saveOutputAttr(self):
""" Save output attributes with dynamic values into a values.json file.
@ -1272,7 +1282,6 @@ class BaseNode(BaseObject):
self._hasDuplicates = bool(len(newList))
self.hasDuplicatesChanged.emit()
def statusInThisSession(self):
if not self._chunks:
return False
@ -1297,7 +1306,8 @@ class BaseNode(BaseObject):
def hasImageOutputAttribute(self):
"""
Return True if at least one attribute has the 'image' semantic (and can thus be loaded in the 2D Viewer), False otherwise.
Return True if at least one attribute has the 'image' semantic (and can thus be loaded in the 2D Viewer),
False otherwise.
"""
for attr in self._attributes:
if attr.enabled and attr.isOutput and attr.desc.semantic == "image":
@ -1306,10 +1316,12 @@ class BaseNode(BaseObject):
def hasSequenceOutputAttribute(self):
"""
Return True if at least one attribute has the 'sequence' semantic (and can thus be loaded in the 2D Viewer), False otherwise.
Return True if at least one attribute has the 'sequence' semantic (and can thus be loaded in the 2D Viewer),
False otherwise.
"""
for attr in self._attributes:
if attr.enabled and attr.isOutput and (attr.desc.semantic == "sequence" or attr.desc.semantic == "imageList"):
if attr.enabled and attr.isOutput and (attr.desc.semantic == "sequence" or
attr.desc.semantic == "imageList"):
return True
return False
@ -1326,7 +1338,6 @@ class BaseNode(BaseObject):
return True
return False
name = Property(str, getName, constant=True)
defaultLabel = Property(str, getDefaultLabel, constant=True)
nodeType = Property(str, nodeType.fget, constant=True)
@ -1356,8 +1367,10 @@ class BaseNode(BaseObject):
globalStatus = Property(str, lambda self: self.getGlobalStatus().name, notify=globalStatusChanged)
fusedStatus = Property(StatusData, getFusedStatus, notify=globalStatusChanged)
elapsedTime = Property(float, lambda self: self.getFusedStatus().elapsedTime, notify=globalStatusChanged)
recursiveElapsedTime = Property(float, lambda self: self.getRecursiveFusedStatus().elapsedTime, notify=globalStatusChanged)
isCompatibilityNode = Property(bool, lambda self: self._isCompatibilityNode(), constant=True) # need lambda to evaluate the virtual function
recursiveElapsedTime = Property(float, lambda self: self.getRecursiveFusedStatus().elapsedTime,
notify=globalStatusChanged)
# isCompatibilityNode: need lambda to evaluate the virtual function
isCompatibilityNode = Property(bool, lambda self: self._isCompatibilityNode(), constant=True)
isInputNode = Property(bool, lambda self: self._isInputNode(), constant=True)
globalExecModeChanged = Signal()
@ -1378,6 +1391,7 @@ class BaseNode(BaseObject):
hasSequenceOutput = Property(bool, hasSequenceOutputAttribute, notify=outputAttrEnabledChanged)
has3DOutput = Property(bool, has3DOutputAttribute, notify=outputAttrEnabledChanged)
class Node(BaseNode):
"""
A standard Graph node based on a node type.
@ -1399,7 +1413,8 @@ class Node(BaseNode):
self._attributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=True, node=self))
for attrDesc in self.nodeDesc.internalInputs:
self._internalAttributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=False, node=self))
self._internalAttributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=False,
node=self))
# Declare events for specific output attributes
for attr in self._attributes:
@ -1421,7 +1436,6 @@ class Node(BaseNode):
self.optionalCallOnDescriptor("onNodeCreated")
def optionalCallOnDescriptor(self, methodName, *args, **kwargs):
""" Call of optional method defined in the descriptor.
Available method names are:
@ -1432,7 +1446,7 @@ class Node(BaseNode):
if callable(m):
try:
m(self, *args, **kwargs)
except Exception as e:
except Exception:
import traceback
# Format error strings with all the provided arguments
argsStr = ", ".join(str(arg) for arg in args)
@ -1443,7 +1457,8 @@ class Node(BaseNode):
finalErrStr += ", "
finalErrStr += kwargsStr
logging.error("Error on call to '{}' (with args: '{}') for node type {}".format(methodName, finalErrStr, self.nodeType))
logging.error("Error on call to '{}' (with args: '{}') for node type {}".
format(methodName, finalErrStr, self.nodeType))
logging.error(traceback.format_exc())
def setAttributeValues(self, values):
@ -1491,7 +1506,8 @@ class Node(BaseNode):
def toDict(self):
inputs = {k: v.getExportValue() for k, v in self._attributes.objects.items() if v.isInput}
internalInputs = {k: v.getExportValue() for k, v in self._internalAttributes.objects.items()}
outputs = ({k: v.getExportValue() for k, v in self._attributes.objects.items() if v.isOutput and not v.desc.isDynamicValue})
outputs = ({k: v.getExportValue() for k, v in self._attributes.objects.items()
if v.isOutput and not v.desc.isDynamicValue})
return {
'nodeType': self.nodeType,
@ -1788,7 +1804,8 @@ class CompatibilityNode(BaseNode):
upgradedAttrValues = attrValues
if not isinstance(upgradedAttrValues, dict):
logging.error("Error in the upgrade implementation of the node: {}. The return type is incorrect.".format(self.name))
logging.error("Error in the upgrade implementation of the node: {}. The return type is incorrect.".
format(self.name))
upgradedAttrValues = attrValues
node.upgradeAttributeValues(upgradedAttrValues)
@ -1859,8 +1876,10 @@ def nodeFactory(nodeDict, name=None, template=False, uidConflict=False):
# do not perform that check for internal attributes because there is no point in
# raising compatibility issues if their number differs: in that case, it is only useful
# if some internal attributes do not exist or are invalid
if not template and (sorted([attr.name for attr in nodeDesc.inputs if not isinstance(attr, desc.PushButtonParam)]) != sorted(inputs.keys()) or \
sorted([attr.name for attr in nodeDesc.outputs if not attr.isDynamicValue]) != sorted(outputs.keys())):
if not template and (sorted([attr.name for attr in nodeDesc.inputs
if not isinstance(attr, desc.PushButtonParam)]) != sorted(inputs.keys()) or
sorted([attr.name for attr in nodeDesc.outputs if not attr.isDynamicValue]) !=
sorted(outputs.keys())):
compatibilityIssue = CompatibilityIssue.DescriptionConflict
# Check whether there are any internal attributes that are invalidating in the node description: if there

9
meshroom/core/stats.py
View file

@ -6,7 +6,6 @@ import time
import threading
import platform
import os
import sys
import xml.etree.ElementTree as ET
@ -56,7 +55,7 @@ class ComputerStatistics:
# If the platform is Windows and nvidia-smi
self.nvidia_smi = spawn.find_executable('nvidia-smi')
if self.nvidia_smi is None:
# could not be found from the environment path,
# Could not be found from the environment path,
# try to find it from system drive with default installation path
default_nvidia_smi = "%s\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe" % os.environ['systemdrive']
if os.path.isfile(default_nvidia_smi):
@ -77,7 +76,8 @@ class ComputerStatistics:
def update(self):
try:
self.initOnFirstTime()
self._addKV('cpuUsage', psutil.cpu_percent(percpu=True)) # interval=None => non-blocking (percentage since last call)
# Interval=None => non-blocking (percentage since last call)
self._addKV('cpuUsage', psutil.cpu_percent(percpu=True))
self._addKV('ramUsage', psutil.virtual_memory().percent)
self._addKV('swapUsage', psutil.swap_memory().percent)
self._addKV('vramUsage', 0)
@ -140,6 +140,7 @@ class ComputerStatistics:
for k, v in d.items():
setattr(self, k, v)
class ProcStatistics:
staticKeys = [
'pid',
@ -201,7 +202,7 @@ class ProcStatistics:
for k, v in data.items():
self._addKV(k, v)
## Note: Do not collect stats about open files for now,
# Note: Do not collect stats about open files for now,
# as there is bug in psutil-5.7.2 on Windows which crashes the application.
# https://github.com/giampaolo/psutil/issues/1763
#

15
meshroom/core/taskManager.py
View file

@ -211,11 +211,8 @@ class TaskManager(BaseObject):
chunksName = [node.name for node in chunksInConflict]
# Warning: Syntax and terms are parsed on QML side to recognize the error
# Syntax : [Context] ErrorType: ErrorMessage
msg = '[COMPUTATION] Already Submitted:\n' \
'WARNING - Some nodes are already submitted with status: {}\nNodes: {}'.format(
', '.join(chunksStatus),
', '.join(chunksName)
)
msg = '[COMPUTATION] Already Submitted:\nWARNING - Some nodes are already submitted with status: ' \
'{}\nNodes: {}'.format(', '.join(chunksStatus), ', '.join(chunksName))
if forceStatus:
logging.warning(msg)
@ -325,8 +322,9 @@ class TaskManager(BaseObject):
raise RuntimeError("[{}] Duplicates Issue:\n"
"Cannot compute because there are some duplicate nodes to process:\n\n"
"First match: '{}' and '{}'\n\n"
"There can be other duplicate nodes in the list. Please, check the graph and try again.".format(
context, node.nameToLabel(node.name), node.nameToLabel(duplicate.name)))
"There can be other duplicate nodes in the list. "
"Please, check the graph and try again.".
format(context, node.nameToLabel(node.name), node.nameToLabel(duplicate.name)))
def checkNodesDependencies(self, graph, toNodes, context):
"""
@ -366,7 +364,8 @@ class TaskManager(BaseObject):
# Warning: Syntax and terms are parsed on QML side to recognize the error
# Syntax : [Context] ErrorType: ErrorMessage
raise RuntimeWarning("[{}] Unresolved dependencies:\n"
"Some nodes cannot be computed in LOCAL/submitted in EXTERN because of unresolved dependencies.\n\n"
"Some nodes cannot be computed in LOCAL/submitted in EXTERN because of "
"unresolved dependencies.\n\n"
"Nodes which are ready will be processed.".format(context))
def raiseImpossibleProcess(self, context):

7
meshroom/multiview.py
View file

@ -1,7 +1,5 @@
import os
from meshroom.core.graph import Graph, GraphModification
# Supported image extensions
imageExtensions = (
# bmp:
@ -37,7 +35,10 @@ imageExtensions = (
# ptex:
'.ptex', '.ptx',
# raw:
'.bay', '.bmq', '.cr2', '.cr3', '.crw', '.cs1', '.dc2', '.dcr', '.dng', '.erf', '.fff', '.k25', '.kdc', '.mdc', '.mos', '.mrw', '.nef', '.orf', '.pef', '.pxn', '.raf', '.raw', '.rdc', '.sr2', '.srf', '.x3f', '.arw', '.3fr', '.cine', '.ia', '.kc2', '.mef', '.nrw', '.qtk', '.rw2', '.sti', '.rwl', '.srw', '.drf', '.dsc', '.cap', '.iiq', '.rwz',
'.bay', '.bmq', '.cr2', '.cr3', '.crw', '.cs1', '.dc2', '.dcr', '.dng', '.erf', '.fff', '.k25', '.kdc', '.mdc',
'.mos', '.mrw', '.nef', '.orf', '.pef', '.pxn', '.raf', '.raw', '.rdc', '.sr2', '.srf', '.x3f', '.arw', '.3fr',
'.cine', '.ia', '.kc2', '.mef', '.nrw', '.qtk', '.rw2', '.sti', '.rwl', '.srw', '.drf', '.dsc', '.cap', '.iiq',
'.rwz',
# rla:
'.rla',
# sgi:

2
meshroom/ui/utils.py
View file

@ -5,7 +5,7 @@ from PySide2.QtCore import QFileSystemWatcher, QUrl, Slot, QTimer, Property, QOb
from PySide2.QtQml import QQmlApplicationEngine
try:
from PySide2 import shiboken2
except:
except Exception:
import shiboken2

5
setupInitScriptUnix.py
View file

@ -1,4 +1,4 @@
#------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# ConsoleSetLibPath.py
# Initialization script for cx_Freeze which manipulates the path so that the
# directory in which the executable is found is searched for extensions but
@ -6,7 +6,7 @@
# manipulated first, however, to ensure that shared libraries found in the
# target directory are found. This requires a restart of the executable because
# the environment variable LD_LIBRARY_PATH is only checked at startup.
#------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
import os
import sys
@ -40,4 +40,3 @@ def run(*args):
moduleName = args[0]
code = importer.get_code(moduleName)
exec(code, m.__dict__)

16
tests/test_compatibility.py
View file

@ -1,7 +1,6 @@
#!/usr/bin/env python
# coding:utf-8
import tempfile
import os
import copy
@ -34,7 +33,7 @@ SampleGroupV2 = [
)
]
#SampleGroupV3 is SampleGroupV2 with one more int parameter
# SampleGroupV3 is SampleGroupV2 with one more int parameter
SampleGroupV3 = [
desc.IntParam(name="a", label="a", description="", value=0, invalidate=True, range=None),
desc.IntParam(name="notInSampleGroupV2", label="notInSampleGroupV2", description="", value=0, invalidate=True, range=None),
@ -70,6 +69,7 @@ class SampleNodeV2(desc.Node):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
]
class SampleNodeV3(desc.Node):
"""
Changes from V3:
@ -82,6 +82,7 @@ class SampleNodeV3(desc.Node):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
]
class SampleNodeV4(desc.Node):
"""
Changes from V3:
@ -115,6 +116,7 @@ class SampleNodeV5(desc.Node):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
]
class SampleNodeV6(desc.Node):
"""
Changes from V5:
@ -131,6 +133,7 @@ class SampleNodeV6(desc.Node):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
]
class SampleInputNodeV1(desc.InputNode):
""" Version 1 Sample Input Node """
inputs = [
@ -140,6 +143,7 @@ class SampleInputNodeV1(desc.InputNode):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
]
class SampleInputNodeV2(desc.InputNode):
""" Changes from V1:
* 'path' has been renamed to 'in'
@ -151,6 +155,7 @@ class SampleInputNodeV2(desc.InputNode):
desc.File(name='output', label='Output', description='', value=desc.Node.internalFolder, invalidate=False)
]
def test_unknown_node_type():
"""
Test compatibility behavior for unknown node type.
@ -351,6 +356,7 @@ def test_upgradeAllNodes():
unregisterNodeType(SampleNodeV1)
unregisterNodeType(SampleInputNodeV1)
def test_conformUpgrade():
registerNodeType(SampleNodeV5)
registerNodeType(SampleNodeV6)
@ -389,9 +395,3 @@ def test_conformUpgrade():
unregisterNodeType(SampleNodeV5)
unregisterNodeType(SampleNodeV6)

1
tests/test_graph.py
View file

@ -142,7 +142,6 @@ def test_transitive_reduction():
(tC.output, tE.input3),
(tD.output, tE.input2),
)
edgesScore = graph.dfsMaxEdgeLength()
flowEdges = graph.flowEdges()
flowEdgesRes = [(tB, tA),

2
tests/test_invalidation.py
View file

@ -59,5 +59,3 @@ def test_inputLinkInvalidation():
graph.addEdges((n1.input, n2.input))
assert n1.input.uid() == n2.input.uid()
assert n1.output.value == n2.output.value

2
tests/test_model.py
View file

@ -28,7 +28,7 @@ def test_DictModel_add_remove():
assert len(m.values()) == 1
assert m.get("DummyNode_1") == node
assert m.get("something") == None
assert m.get("something") is None
with pytest.raises(KeyError):
m.getr("something")

11
tests/test_nodeCommandLineFormatting.py
View file

@ -1,11 +1,7 @@
#!/usr/bin/env python
# coding:utf-8
import os
import tempfile
import meshroom.multiview
from meshroom.core.graph import Graph
from meshroom.core.node import Node
def test_formatting_listOfFiles():
@ -17,7 +13,8 @@ def test_formatting_listOfFiles():
n1 = graph.addNewNode('CameraInit')
n1.viewpoints.extend([{'path': image} for image in inputImages])
# viewId, poseId, path, intrinsicId, rigId, subPoseId, metadata
assert n1.viewpoints.getValueStr() == '-1 -1 "/non/existing/fileA" -1 -1 -1 "" -1 -1 "/non/existing/with space/fileB" -1 -1 -1 ""'
assert n1.viewpoints.getValueStr() == \
'-1 -1 "/non/existing/fileA" -1 -1 -1 "" -1 -1 "/non/existing/with space/fileB" -1 -1 -1 ""'
graph = Graph('')
n1 = graph.addNewNode('ImageMatching')
@ -61,7 +58,8 @@ def test_formatting_strings():
n2.featuresFolders.extend('')
n2._buildCmdVars() # prepare vars for command line creation
assert n2.featuresFolders.getValueStr() == '"" ""', 'A list with 2 empty strings should generate quotes'
assert n2._cmdVars[name + 'Value'] == ' ', 'The Value is always only the value, so 2 empty with the space separator in the middle'
assert n2._cmdVars[name + 'Value'] == ' ', \
'The Value is always only the value, so 2 empty with the space separator in the middle'
def test_formatting_groups():
@ -78,4 +76,3 @@ def test_formatting_groups():
name = 'noiseFilter'
assert n3.noiseFilter.getValueStr() == '"False:uniform:0.0:1.0:True"'
assert n3._cmdVars[name + 'Value'] == 'False:uniform:0.0:1.0:True'