From 374b9abb770a2b4f01e8e7541b6412ae6e94501a Mon Sep 17 00:00:00 2001 From: mugulmd Date: Tue, 17 Jan 2023 02:53:03 -0800 Subject: [PATCH] [ui] ThumbnailCache: replace thread pool with custom thread spawning This is so we can control in which order the requests are handled, as we want LIFO ordering instead of FIFO for smoother interaction. --- meshroom/ui/components/thumbnail.py | 192 ++++++++++++++++------------ 1 file changed, 108 insertions(+), 84 deletions(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index f0c4983d..4575d3c2 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -9,7 +9,7 @@ import stat import hashlib import time import logging -from multiprocessing.pool import ThreadPool +from threading import Thread class ThumbnailCache(QObject): @@ -55,8 +55,10 @@ class ThumbnailCache(QObject): # - an identifier for the caller, e.g. the component that sent the request (useful for faster dispatch in QML) thumbnailCreated = Signal(QUrl, int) - # Thread pool for running createThumbnail asynchronously on a fixed number of worker threads - pool = ThreadPool(processes=3) + # Threads info and LIFO structure for running createThumbnail asynchronously + maxWorkerThreads = 3 + activeWorkerThreads = 0 + requests = [] @staticmethod def initialize(): @@ -85,86 +87,6 @@ class ThumbnailCache(QObject): # Make sure the thumbnail directory exists before writing into it os.makedirs(ThumbnailCache.thumbnailDir, exist_ok=True) - @staticmethod - def thumbnailPath(imgPath): - """Use SHA1 hashing to associate a unique thumbnail to an image. - - Args: - imgPath (str): filepath to the input image - - Returns: - str: filepath to the corresponding thumbnail - """ - digest = hashlib.sha1(imgPath.encode('utf-8')).hexdigest() - path = os.path.join(ThumbnailCache.thumbnailDir, f'{digest}.jpg') - return path - - @Slot(QUrl, int, result=QUrl) - def thumbnail(self, imgSource, callerID): - """Retrieve the filepath of the thumbnail corresponding to a given image. - - If the thumbnail does not exist on disk, it will be created asynchronously. - When this is done, the thumbnailCreated signal is emitted. - - Args: - imgSource (QUrl): location of the input image - callerID (int): identifier for the object that requested the thumbnail - - Returns: - QUrl: location of the corresponding thumbnail if it exists, otherwise None - """ - if not imgSource.isValid(): - return None - - imgPath = imgSource.toLocalFile() - path = ThumbnailCache.thumbnailPath(imgPath) - source = QUrl.fromLocalFile(path) - - # Check if thumbnail already exists - if os.path.exists(path): - # Update last modification time - Path(path).touch(exist_ok=True) - return source - - # Thumbnail does not exist - # create it in a worker thread to avoid UI freeze - ThumbnailCache.pool.apply_async(self.createThumbnail, args=(imgSource,callerID,)) - return None - - def createThumbnail(self, imgSource, callerID): - """Load an image, resize it to thumbnail dimensions and save the result in the cache directory. - - Args: - imgSource (QUrl): location of the input image - callerID (int): identifier for the object that requested the thumbnail - """ - imgPath = imgSource.toLocalFile() - path = ThumbnailCache.thumbnailPath(imgPath) - logging.debug(f'[ThumbnailCache] Creating thumbnail {path} for image {imgPath}') - - # Initialize image reader object - reader = QImageReader() - reader.setFileName(imgPath) - reader.setAutoTransform(True) - - # Read image and check for potential errors - img = reader.read() - if img.isNull(): - logging.error(f'[ThumbnailCache] Error when reading image: {reader.errorString()}') - return - - # Scale image while preserving aspect ratio - thumbnail = img.scaled(ThumbnailCache.thumbnailSize, aspectMode=Qt.KeepAspectRatio) - - # Write thumbnail to disk and check for potential errors - writer = QImageWriter(path) - success = writer.write(thumbnail) - if not success: - logging.error(f'[ThumbnailCache] Error when writing thumbnail: {writer.errorString()}') - - # Notify listeners - self.thumbnailCreated.emit(imgSource, callerID) - @staticmethod def clean(): """Scan the thumbnail directory and: @@ -216,7 +138,8 @@ class ThumbnailCache(QObject): # Check if number of thumbnails on disk exceeds limit if len(remaining) > ThumbnailCache.maxThumbnailsOnDisk: nbToRemove = len(remaining) - ThumbnailCache.maxThumbnailsOnDisk - logging.debug(f'[ThumbnailCache] Too many thumbnails: {len(remaining)} remaining, {nbToRemove} will be removed') + logging.debug( + f'[ThumbnailCache] Too many thumbnails: {len(remaining)} remaining, {nbToRemove} will be removed') # Sort by last usage order and remove excess remaining.sort(key=lambda elt: elt[1]) @@ -227,3 +150,104 @@ class ThumbnailCache(QObject): os.remove(path) except FileNotFoundError: logging.error(f'[ThumbnailCache] Tried to remove {path} but this file does not exist') + + @staticmethod + def thumbnailPath(imgPath): + """Use SHA1 hashing to associate a unique thumbnail to an image. + + Args: + imgPath (str): filepath to the input image + + Returns: + str: filepath to the corresponding thumbnail + """ + digest = hashlib.sha1(imgPath.encode('utf-8')).hexdigest() + path = os.path.join(ThumbnailCache.thumbnailDir, f'{digest}.jpg') + return path + + @Slot(QUrl, int, result=QUrl) + def thumbnail(self, imgSource, callerID): + """Retrieve the filepath of the thumbnail corresponding to a given image. + + If the thumbnail does not exist on disk, it will be created asynchronously. + When this is done, the thumbnailCreated signal is emitted. + + Args: + imgSource (QUrl): location of the input image + callerID (int): identifier for the object that requested the thumbnail + + Returns: + QUrl: location of the corresponding thumbnail if it exists, otherwise None + """ + if not imgSource.isValid(): + return None + + imgPath = imgSource.toLocalFile() + path = ThumbnailCache.thumbnailPath(imgPath) + source = QUrl.fromLocalFile(path) + + # Check if thumbnail already exists + if os.path.exists(path): + # Update last modification time + Path(path).touch(exist_ok=True) + return source + + # Thumbnail does not exist + # Create request and start a thread if needed + ThumbnailCache.requests.append((imgSource, callerID)) + if ThumbnailCache.activeWorkerThreads < ThumbnailCache.maxWorkerThreads: + thread = Thread(target=self.handleRequestsAsync) + thread.start() + + return None + + def createThumbnail(self, imgSource, callerID): + """Load an image, resize it to thumbnail dimensions and save the result in the cache directory. + + Args: + imgSource (QUrl): location of the input image + callerID (int): identifier for the object that requested the thumbnail + """ + imgPath = imgSource.toLocalFile() + path = ThumbnailCache.thumbnailPath(imgPath) + logging.debug(f'[ThumbnailCache] Creating thumbnail {path} for image {imgPath}') + + # Initialize image reader object + reader = QImageReader() + reader.setFileName(imgPath) + reader.setAutoTransform(True) + + # Read image and check for potential errors + img = reader.read() + if img.isNull(): + logging.error(f'[ThumbnailCache] Error when reading image: {reader.errorString()}') + return + + # Scale image while preserving aspect ratio + thumbnail = img.scaled(ThumbnailCache.thumbnailSize, aspectMode=Qt.KeepAspectRatio) + + # Write thumbnail to disk and check for potential errors + writer = QImageWriter(path) + success = writer.write(thumbnail) + if not success: + logging.error(f'[ThumbnailCache] Error when writing thumbnail: {writer.errorString()}') + + # Notify listeners + self.thumbnailCreated.emit(imgSource, callerID) + + def handleRequestsAsync(self): + """Process thumbnail creation requests in LIFO order. + + This method is only meant to be called by worker threads, + hence it also takes care of registering/unregistering the calling thread. + """ + # Register worker thread + ThumbnailCache.activeWorkerThreads += 1 + try: + while True: + req = ThumbnailCache.requests.pop() + self.createThumbnail(req[0], req[1]) + except IndexError: + # No more request to process + # Unregister worker thread + ThumbnailCache.activeWorkerThreads -= 1