mirror of
https://github.com/alicevision/Meshroom.git
synced 2025-07-23 19:47:39 +02:00
Add chunk notion for parallelization and implement specific updateInternals in CameraInit node
* Add chunk notion for parallelization * Allows Node desc to implement custom updateInternals * CameraInit node implement a specific updateInternals to update the input image list * FeatureExtraction, FeatureMatching, DepthMap, DepthMapFilter: implement parallelization
This commit is contained in:
parent
39f6ef3d64
commit
1e4f8f8a61
14 changed files with 614 additions and 271 deletions
|
@ -1,10 +1,10 @@
|
|||
from meshroom.common import BaseObject, Property, Variant
|
||||
from enum import Enum # available by default in python3. For python2: "pip install enum34"
|
||||
import collections
|
||||
import math
|
||||
import os
|
||||
import psutil
|
||||
|
||||
|
||||
class Attribute(BaseObject):
|
||||
"""
|
||||
"""
|
||||
|
@ -181,6 +181,76 @@ class Level(Enum):
|
|||
INTENSIVE = 2
|
||||
|
||||
|
||||
class Range:
|
||||
def __init__(self, iteration=0, blockSize=0, fullSize=0):
|
||||
self.iteration = iteration
|
||||
self.blockSize = blockSize
|
||||
self.fullSize = fullSize
|
||||
|
||||
@property
|
||||
def start(self):
|
||||
return self.iteration * self.blockSize
|
||||
|
||||
@property
|
||||
def effectiveBlockSize(self):
|
||||
remaining = (self.fullSize - self.start) + 1
|
||||
return self.blockSize if remaining >= self.blockSize else remaining
|
||||
|
||||
@property
|
||||
def end(self):
|
||||
return self.start + self.effectiveBlockSize
|
||||
|
||||
@property
|
||||
def last(self):
|
||||
return self.end - 1
|
||||
|
||||
def toDict(self):
|
||||
return {
|
||||
"rangeIteration": self.iteration,
|
||||
"rangeStart": self.start,
|
||||
"rangeEnd": self.end,
|
||||
"rangeLast": self.last,
|
||||
"rangeBlockSize": self.effectiveBlockSize,
|
||||
"rangeFullSize": self.fullSize,
|
||||
}
|
||||
|
||||
|
||||
class Parallelization:
|
||||
def __init__(self, inputListParamName='', staticNbBlocks=0, blockSize=0):
|
||||
self.inputListParamName = inputListParamName
|
||||
self.staticNbBlocks = staticNbBlocks
|
||||
self.blockSize = blockSize
|
||||
|
||||
def getSizes(self, node):
|
||||
"""
|
||||
Args:
|
||||
node:
|
||||
Returns: (blockSize, fullSize, nbBlocks)
|
||||
"""
|
||||
if self.inputListParamName:
|
||||
parentNodes, edges = node.graph.dfsOnFinish(startNodes=[node])
|
||||
for parentNode in parentNodes:
|
||||
if self.inputListParamName in parentNode.getAttributes().keys():
|
||||
fullSize = len(parentNode.attribute(self.inputListParamName))
|
||||
nbBlocks = int(math.ceil(float(fullSize) / float(self.blockSize)))
|
||||
return (self.blockSize, fullSize, nbBlocks)
|
||||
raise RuntimeError('Cannot find the "inputListParamName": "{}" in the list of input nodes: {} for node: {}'.format(self.inputListParamName, parentNodes, node.name))
|
||||
if self.staticNbBlocks:
|
||||
return (1, self.staticNbBlocks, self.staticNbBlocks)
|
||||
return None
|
||||
|
||||
def getRange(self, node, iteration):
|
||||
blockSize, fullSize, nbBlocks = self.getSizes(node)
|
||||
return Range(iteration=iteration, blockSize=blockSize, fullSize=fullSize)
|
||||
|
||||
def getRanges(self, node):
|
||||
blockSize, fullSize, nbBlocks = self.getSizes(node)
|
||||
ranges = []
|
||||
for i in range(nbBlocks):
|
||||
ranges.append(Range(iteration=i, blockSize=blockSize, fullSize=fullSize))
|
||||
return ranges
|
||||
|
||||
|
||||
class Node(object):
|
||||
"""
|
||||
"""
|
||||
|
@ -192,56 +262,67 @@ class Node(object):
|
|||
packageVersion = ''
|
||||
inputs = []
|
||||
outputs = []
|
||||
parallelization = None
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def updateInternals(self, node):
|
||||
pass
|
||||
|
||||
def stop(self, node):
|
||||
pass
|
||||
|
||||
def process(self, node):
|
||||
raise NotImplementedError('No process implementation on this node')
|
||||
def processChunk(self, node, range):
|
||||
raise NotImplementedError('No process implementation on node: "{}"'.format(node.name))
|
||||
|
||||
|
||||
class CommandLineNode(Node):
|
||||
"""
|
||||
"""
|
||||
internalFolder = '{cache}/{nodeType}/{uid0}/'
|
||||
commandLine = '' # need to be defined on the node
|
||||
parallelization = None
|
||||
commandLineRange = ''
|
||||
|
||||
def buildCommandLine(self, node):
|
||||
def buildCommandLine(self, chunk):
|
||||
cmdPrefix = ''
|
||||
if 'REZ_ENV' in os.environ:
|
||||
cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get('REZ_ENV'), packageFullName=node.packageFullName)
|
||||
return cmdPrefix + node.nodeDesc.commandLine.format(**node._cmdVars)
|
||||
cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get('REZ_ENV'), packageFullName=chunk.node.packageFullName)
|
||||
cmdSuffix = ''
|
||||
if chunk.range:
|
||||
cmdSuffix = ' ' + self.commandLineRange.format(**chunk.range.toDict())
|
||||
return cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix
|
||||
|
||||
def stop(self, node):
|
||||
if node.subprocess:
|
||||
node.subprocess.terminate()
|
||||
|
||||
def process(self, node):
|
||||
def processChunk(self, chunk):
|
||||
try:
|
||||
with open(node.logFile(), 'w') as logF:
|
||||
cmd = self.buildCommandLine(node)
|
||||
with open(chunk.logFile(), 'w') as logF:
|
||||
cmd = self.buildCommandLine(chunk)
|
||||
print(' - commandLine:', cmd)
|
||||
print(' - logFile:', node.logFile())
|
||||
node.subprocess = psutil.Popen(cmd, stdout=logF, stderr=logF, shell=True)
|
||||
print(' - logFile:', chunk.logFile())
|
||||
chunk.subprocess = psutil.Popen(cmd, stdout=logF, stderr=logF, shell=True)
|
||||
|
||||
# store process static info into the status file
|
||||
node.status.commandLine = cmd
|
||||
# node.status.env = node.proc.environ()
|
||||
# node.status.createTime = node.proc.create_time()
|
||||
chunk.status.commandLine = cmd
|
||||
# chunk.status.env = node.proc.environ()
|
||||
# chunk.status.createTime = node.proc.create_time()
|
||||
|
||||
node.statThread.proc = node.subprocess
|
||||
stdout, stderr = node.subprocess.communicate()
|
||||
node.subprocess.wait()
|
||||
chunk.statThread.proc = chunk.subprocess
|
||||
stdout, stderr = chunk.subprocess.communicate()
|
||||
chunk.subprocess.wait()
|
||||
|
||||
node.status.returnCode = node.subprocess.returncode
|
||||
chunk.status.returnCode = chunk.subprocess.returncode
|
||||
|
||||
if node.subprocess.returncode != 0:
|
||||
with open(node.logFile(), 'r') as logF:
|
||||
if chunk.subprocess.returncode != 0:
|
||||
with open(chunk.logFile(), 'r') as logF:
|
||||
logContent = ''.join(logF.readlines())
|
||||
raise RuntimeError('Error on node "{}":\nLog:\n{}'.format(node.name, logContent))
|
||||
raise RuntimeError('Error on node "{}":\nLog:\n{}'.format(chunk.name, logContent))
|
||||
except:
|
||||
raise
|
||||
finally:
|
||||
node.subprocess = None
|
||||
chunk.subprocess = None
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue