From b0b0dd92440f0b454559b4d7c886d90591b8e4aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1lm=C3=A1n=20Tarnay?= Date: Mon, 29 Mar 2010 23:05:39 +0200 Subject: [PATCH 01/12] OOP Image --- src/trimage/trimage.py | 106 ++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/src/trimage/trimage.py b/src/trimage/trimage.py index 1942f95..8bf9894 100755 --- a/src/trimage/trimage.py +++ b/src/trimage/trimage.py @@ -106,21 +106,14 @@ class StartQT4(QMainWindow): self.showapp = False dirpath = path.abspath(directory) imagedir = listdir(directory) - filelist = [] - for image in imagedir: - image = path.join(dirpath, image) - if path.isfile(image) and self.checkname(image): - filelist.append(image) + filelist = [path.join(dirpath, image) for image in imagedir] self.delegator(filelist) def file_from_cmd(self, image): """Get the file and send it to compress_file""" self.showapp = False - image = path.abspath(image) - filecmdlist = [] - if self.checkname(image): - filecmdlist.append(image) - self.delegator(filecmdlist) + filelist = [path.abspath(image)] + self.delegator(filelist) def file_drop(self, images): """ @@ -161,12 +154,13 @@ class StartQT4(QMainWindow): """ delegatorlist = [] for image in images: - if self.checkname(image): - delegatorlist.append((image, QIcon(image))) + image=Image(image) + if image.valid: + delegatorlist.append(image) self.imagelist.append(("Compressing...", "", "", "", image, QIcon(QPixmap(self.ui.get_image("pixmaps/compressing.gif"))))) else: - sys.stderr.write("[error] %s not an image file\n" % image) + print >>sys.stderr, u"[error] %s not a supported image file" % image.fullpath self.update_table() self.thread.compress_file(delegatorlist, self.showapp, self.verbose, @@ -208,10 +202,6 @@ class StartQT4(QMainWindow): Helper functions """ - def checkname(self, name): - """Check if the file is a jpg or png.""" - return determinetype(name) in ["jpeg", "png"] - def enable_recompress(self): """Enable the recompress button.""" self.ui.recompress.setEnabled(True) @@ -287,6 +277,38 @@ class TriTableModel(QAbstractTableModel): return QVariant(self.header[col]) return QVariant() +class Image: + def __init__(self, fullpath): + self.valid = False + self.fullpath = fullpath + if path.isfile(self.fullpath): + self.filetype = determinetype(self.fullpath) + if self.filetype in ["jpeg", "png"]: + oldfile = QFileInfo(self.fullpath) + self.shortname = oldfile.fileName() + self.oldfilesize = oldfile.size() + self.icon = QIcon(self.fullpath) + self.valid = True + + def _determinetype(self): + filetype=determinetype(self.fullpath) + if filetype in ["jpeg", "png"]: + self.filetype=filetype + else: + self.filetype=None + return self.filetype + + def compress(self): + if not self.valid: + raise "Tried to compress invalid image (unsupported format or not file)" + runString = { + "jpeg": u"jpegoptim -f --strip-all '%(file)s'", + "png" : u"optipng -force -o7 '%(file)s'; advpng -z4 '%(file)s'"} + retcode = call(runString[self.filetype] % {"file": self.fullpath}, + shell = True, stdout=PIPE) + if retcode == 0: + self.newfilesize = QFile(self.fullpath).size() + return not retcode class Worker(QThread): @@ -311,55 +333,31 @@ class Worker(QThread): def run(self): """Compress the given file, get data from it and call update_table.""" while self.showapp or not self.toProcess.empty(): - #gather old file data - filename, icon = self.toProcess.get() - oldfile = QFileInfo(filename) - name = oldfile.fileName() - oldfilesize = oldfile.size() - oldfilesizestr = size(oldfilesize, system=alternative) - - filetype = determinetype(filename) - #decide which tool to use - if filetype is "jpeg": - runString = u"jpegoptim -f --strip-all '%(file)s'" - elif filetype is "png": - runString = (u"optipng -force -o7 '%(file)s'; advpng -z4 '%(file)s'") - else: - sys.stderr.write("[error] %s not an image file" % filename) - - try: - retcode = call(runString % {"file": filename}, shell=True, - stdout=PIPE) - runfile = retcode - except OSError as e: - runfile = e - - if runfile == 0: - #gather new file data - newfile = QFile(filename) - newfilesize = newfile.size() - newfilesizestr = size(newfilesize, system=alternative) - + image = self.toProcess.get() + success=image.compress() + if success: #calculate ratio and make a nice string - ratio = 100 - (float(newfilesize) / float(oldfilesize) * 100) + oldfilesizestr = size(image.oldfilesize, system=alternative) + newfilesizestr = size(image.newfilesize, system=alternative) + ratio = 100 - (float(image.newfilesize) / float(image.oldfilesize) * 100) ratiostr = "%.1f%%" % ratio # append current image to list - for i, image in enumerate(self.imagelist): - if image[4] == filename: - self.imagelist.remove(image) - self.imagelist.insert(i, (name, oldfilesizestr, - newfilesizestr, ratiostr, filename, icon)) + for i, listitem in enumerate(self.imagelist): + if listitem[4] == image: + self.imagelist.remove(listitem) + self.imagelist.insert(i, (image.shortname, oldfilesizestr, + newfilesizestr, ratiostr, image.fullpath, image.icon)) self.emit(SIGNAL("updateUi")) if not self.showapp and self.verbose: # we work via the commandline - print("File: " + filename + ", Old Size: " + print("File: " + image.fullpath + ", Old Size: " + oldfilesizestr + ", New Size: " + newfilesizestr + ", Ratio: " + ratiostr) else: - sys.stderr.write("[error] %s" % runfile) + print >>sys.stderr, u"[error] %s could not be compressed" % image.fullpath class TrimageTableView(QTableView): From ecfc4af6a3c9e4cbf969e768bfc6429e64f0489e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1lm=C3=A1n=20Tarnay?= Date: Tue, 30 Mar 2010 00:26:29 +0200 Subject: [PATCH 02/12] "and" relationship between the two png compressor commands --- src/trimage/trimage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trimage/trimage.py b/src/trimage/trimage.py index 8bf9894..e09710f 100755 --- a/src/trimage/trimage.py +++ b/src/trimage/trimage.py @@ -303,7 +303,7 @@ class Image: raise "Tried to compress invalid image (unsupported format or not file)" runString = { "jpeg": u"jpegoptim -f --strip-all '%(file)s'", - "png" : u"optipng -force -o7 '%(file)s'; advpng -z4 '%(file)s'"} + "png" : u"optipng -force -o7 '%(file)s'&&advpng -z4 '%(file)s'"} retcode = call(runString[self.filetype] % {"file": self.fullpath}, shell = True, stdout=PIPE) if retcode == 0: From a3a31a7e9336ebdc8b8be4f73b5c6e4d4536ca69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1lm=C3=A1n=20Tarnay?= Date: Tue, 30 Mar 2010 02:21:00 +0200 Subject: [PATCH 03/12] Parallel processing --- src/trimage/ThreadPool.py | 262 ++++++++++++++++++++++++++++++++++++++ src/trimage/trimage.py | 22 ++-- 2 files changed, 274 insertions(+), 10 deletions(-) create mode 100644 src/trimage/ThreadPool.py diff --git a/src/trimage/ThreadPool.py b/src/trimage/ThreadPool.py new file mode 100644 index 0000000..c7387d8 --- /dev/null +++ b/src/trimage/ThreadPool.py @@ -0,0 +1,262 @@ +''' +ThreadPool Implementation + +@author: Morten Holdflod Moeller - morten@holdflod.dk +@license: LGPL v3 + +(A buggy line commented out by Kalman Tarnay... +see: http://code.google.com/p/pythonthreadpool/issues/detail?id=4 ) +''' + +from __future__ import with_statement +from threading import Thread, RLock +from time import sleep +from Queue import Queue, Empty +import logging + +class ThreadPoolMixIn: + """Mix-in class to handle each request in a new thread from the ThreadPool.""" + + def __init__(self, threadpool=None): + if (threadpool == None): + threadpool = ThreadPool() + self.__private_threadpool = True + else: + self.__private_threadpool = False + + self.__threadpool = threadpool + + def process_request_thread(self, request, client_address): + """Same as in BaseServer but as a thread. + + In addition, exception handling is done here. + + """ + try: + self.finish_request(request, client_address) + self.close_request(request) + except: + self.handle_error(request, client_address) #IGNORE:W0702 + self.close_request(request) + + def process_request(self, request, client_address): + self.__threadpool.add_job(self.process_request_thread, [request, client_address]) + + def shutdown(self): + if (self.__private_threadpool): self.__threadpool.shutdown() + + +class AddJobException(Exception): + ''' + Exceptoion raised when a Job could not be added + to the queue + ''' + def __init__(self, msg): + Exception.__init__(self, msg) + + +class ThreadPool: + ''' + The class implementing the ThreadPool. + + Instantiate and add jobs using add_job(func, args_list) + ''' + + class Job: #IGNORE:R0903 + ''' + Class encapsulating a job to be handled + by ThreadPool workers + ''' + def __init__(self, function, args, return_callback=None): + self.callable = function + self.arguments = args + self.return_callback = return_callback + + def execute(self): + ''' + Called to execute the function + ''' + try: + return_value = self.callable(*self.arguments) #IGNORE:W0142 + except Exception, excep: #IGNORE:W0703 + logger = logging.getLogger("threadpool.worker") + logger.warning("A job in the ThreadPool raised an exception: " + excep) + #else do nothing cause we don't know what to do... + return + + try: + if (self.return_callback != None): + self.return_callback(return_value) + except Exception, _: #IGNORE:W0703 everything could go wrong... + logger = logging.getLogger('threadpool') + logger.warning('Error while delivering return value to callback function') + + class Worker(Thread): + ''' + A worker thread handling jobs in the thread pool + job queue + ''' + + def __init__(self, pool): + Thread.__init__(self) + + if (not isinstance(pool, ThreadPool)): + raise TypeError("pool is not a ThreadPool instance") + + self.pool = pool + + self.alive = True + self.start() + + def run(self): + ''' + The workers main-loop getting jobs from queue + and executing them + ''' + while self.alive: + job = self.pool.get_job() + if (job != None): + job.execute() + else: + self.alive = False + + self.pool.punch_out() + + def __init__(self, max_workers = 5, kill_workers_after = 3): + if (not isinstance(max_workers, int)): + raise TypeError("max_workers is not an int") + if (max_workers < 1): + raise ValueError('max_workers must be >= 1') + + if (not isinstance(kill_workers_after, int)): + raise TypeError("kill_workers_after is not an int") + + self.__max_workers = max_workers + self.__kill_workers_after = kill_workers_after + + # This Queue is assumed Thread Safe + self.__jobs = Queue() + + self.__active_workers_lock = RLock() + self.__active_workers = 0 + + self.__shutting_down = False + logger = logging.getLogger('threadpool') + logger.info('started') + + def shutdown(self, wait_for_workers_period = 1, clean_shutdown_reties = 5): + if (not isinstance(clean_shutdown_reties, int)): + raise TypeError("clean_shutdown_reties is not an int") + if (not clean_shutdown_reties >= 0): + raise ValueError('clean_shutdown_reties must be >= 0') + + if (not isinstance(wait_for_workers_period, int)): + raise TypeError("wait_for_workers_period is not an int") + if (not wait_for_workers_period >= 0): + raise ValueError('wait_for_workers_period must be >= 0') + + logger = logging.getLogger("threadpool") + logger.info("shutting down") + + with self.__active_workers_lock: + self.__shutting_down = True + self.__max_workers = 0 + self.__kill_workers_after = 0 + + retries_left = clean_shutdown_reties + while (retries_left > 0): + + with self.__active_workers_lock: + logger.info("waiting for workers to shut down (%i), %i workers left"%(retries_left, self.__active_workers)) + if (self.__active_workers > 0): + retries_left -= 1 + else: + retries_left = 0 + + sleep(wait_for_workers_period) + + + with self.__active_workers_lock: + if (self.__active_workers > 0): + logger.warning("shutdown stopped waiting. Still %i active workers"%self.__active_workers) + clean_shutdown = False + else: + clean_shutdown = True + + logger.info("shutdown complete") + + return clean_shutdown + + def punch_out(self): + ''' + Called by worker to update worker count + when the worker is shutting down + ''' + with self.__active_workers_lock: + self.__active_workers -= 1 + + def __new_worker(self): + ''' + Adding a new worker thread to the thread pool + ''' + with self.__active_workers_lock: + ThreadPool.Worker(self) + self.__active_workers += 1 + + def add_job(self, function, args = None, return_callback=None): + ''' + Put new job into queue + ''' + + if (not callable(function)): + raise TypeError("function is not a callable") + if (not ( args == None or isinstance(args, list))): + raise TypeError("args is not a list") + if (not (return_callback == None or callable(return_callback))): + raise TypeError("return_callback is not a callable") + + if (args == None): + args = [] + + job = ThreadPool.Job(function, args, return_callback) + + with self.__active_workers_lock: + if (self.__shutting_down): + raise AddJobException("ThreadPool is shutting down") + + try: + start_new_worker = False + if (self.__active_workers < self.__max_workers): + #DIY fixed.... FIXME + #http://code.google.com/p/pythonthreadpool/issues/detail?id=4 + #if (self.__active_workers == 0 or not self.__jobs.empty()): + start_new_worker = True + + self.__jobs.put(job) + + if (start_new_worker): + self.__new_worker() + + except Exception: + raise AddJobException("Could not add job") + + + def get_job(self): + ''' + Retrieve next job from queue + workers die (and should) when + returning None + ''' + + job = None + try: + if (self.__kill_workers_after < 0): + job = self.__jobs.get(True) + elif (self.__kill_workers_after == 0): + job = self.__jobs.get(False) + else: + job = self.__jobs.get(True, self.__kill_workers_after) + except Empty: + job = None + + return job diff --git a/src/trimage/trimage.py b/src/trimage/trimage.py index e09710f..216ea26 100755 --- a/src/trimage/trimage.py +++ b/src/trimage/trimage.py @@ -13,6 +13,8 @@ from hurry.filesize import * from imghdr import what as determinetype from Queue import Queue +from ThreadPool import ThreadPool +from multiprocessing import cpu_count from ui import Ui_trimage @@ -308,23 +310,23 @@ class Image: shell = True, stdout=PIPE) if retcode == 0: self.newfilesize = QFile(self.fullpath).size() - return not retcode + self.retcode=retcode + return self class Worker(QThread): def __init__(self, parent=None): QThread.__init__(self, parent) - self.exiting = False - self.toProcess=Queue() + self.toDisplay=Queue() + self.threadpool = ThreadPool(max_workers=cpu_count()) def __del__(self): - self.exiting = True - self.wait() + self.threadpool.shutdown() def compress_file(self, images, showapp, verbose, imagelist): """Start the worker thread.""" for image in images: - self.toProcess.put(image) + self.threadpool.add_job(image.compress, None, return_callback=self.toDisplay.put) self.showapp = showapp self.verbose = verbose self.imagelist = imagelist @@ -332,10 +334,10 @@ class Worker(QThread): def run(self): """Compress the given file, get data from it and call update_table.""" - while self.showapp or not self.toProcess.empty(): - image = self.toProcess.get() - success=image.compress() - if success: + tp = self.threadpool + while self.showapp or not (tp.__active_workers==0 and tp.__jobs.empty()): + image = self.toDisplay.get() + if image.retcode==0: #calculate ratio and make a nice string oldfilesizestr = size(image.oldfilesize, system=alternative) newfilesizestr = size(image.newfilesize, system=alternative) From c9126ae19622eb22659f9e190080467ad4eb3d58 Mon Sep 17 00:00:00 2001 From: Kilian Valkhof Date: Wed, 31 Mar 2010 17:44:29 +0200 Subject: [PATCH 04/12] compress image, remove unneeded code, add more docstrings --- src/trimage/trimage.py | 34 +++++++--------------------------- website/ubuntu.png | Bin 1101 -> 1088 bytes 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/src/trimage/trimage.py b/src/trimage/trimage.py index 216ea26..7df6cf4 100755 --- a/src/trimage/trimage.py +++ b/src/trimage/trimage.py @@ -281,18 +281,20 @@ class TriTableModel(QAbstractTableModel): class Image: def __init__(self, fullpath): + """ gather image information. """ self.valid = False self.fullpath = fullpath - if path.isfile(self.fullpath): + if path.isfile(self.fullpath): self.filetype = determinetype(self.fullpath) if self.filetype in ["jpeg", "png"]: oldfile = QFileInfo(self.fullpath) self.shortname = oldfile.fileName() - self.oldfilesize = oldfile.size() - self.icon = QIcon(self.fullpath) + self.oldfilesize = oldfile.size() + self.icon = QIcon(self.fullpath) self.valid = True def _determinetype(self): + """ Determine the filetype of the file using imghdr. """ filetype=determinetype(self.fullpath) if filetype in ["jpeg", "png"]: self.filetype=filetype @@ -301,6 +303,7 @@ class Image: return self.filetype def compress(self): + """ Compress the image and return it to the thread. """ if not self.valid: raise "Tried to compress invalid image (unsupported format or not file)" runString = { @@ -346,7 +349,7 @@ class Worker(QThread): # append current image to list for i, listitem in enumerate(self.imagelist): - if listitem[4] == image: + if listitem[4] == image: self.imagelist.remove(listitem) self.imagelist.insert(i, (image.shortname, oldfilesizestr, newfilesizestr, ratiostr, image.fullpath, image.icon)) @@ -361,29 +364,6 @@ class Worker(QThread): else: print >>sys.stderr, u"[error] %s could not be compressed" % image.fullpath - -class TrimageTableView(QTableView): - """Init the table drop event.""" - def __init__(self, parent=None): - super(TrimageTableView, self).__init__(parent) - self.setAcceptDrops(True) - - def dragEnterEvent(self, event): - if event.mimeData().hasFormat("text/uri-list"): - event.accept() - else: - event.ignore() - - def dragMoveEvent(self, event): - event.accept() - - def dropEvent(self, event): - files = str(event.mimeData().data("text/uri-list")).strip().split() - for i, file in enumerate(files): - files[i] = QUrl(QString(file)).toLocalFile() - files=[i.toUtf8().decode("utf-8") for i in files] - self.emit(SIGNAL("fileDropEvent"), (files)) - if __name__ == "__main__": app = QApplication(sys.argv) myapp = StartQT4() diff --git a/website/ubuntu.png b/website/ubuntu.png index 0b2e82c8d04cfba0a449f55819093156c3583c05..3e77ad4940ef22aa903766ffff2c7810747cd707 100644 GIT binary patch delta 438 zcmX@hae!mPRHpiAo-U3d8t0P}6hwWtmc=!CuWRq0=$T=<>&3cvYfgOVVXivfFQ4A& zJ~`Voa1wLV&0`Lp*HlDS{^Xg><4{yIQ8DrcW>DJ*+2C9k)R$VE?s}9VuJur%#{mFWVEohHpw~ z%SXOrOsd6#Y)2VXO~smKM)S*_`}T<|vTH(2vWmKTJX6Y>&EXf;-4S~`wdakUe`x5_ z1ckXACQj9nt>Jc?o-Fiz9hS#_$xmVb*^F;mE}WCTzP|pREhx3Cs3tHw)X7Gk#WBSXcCpUrx*uMU%a($a9ARK;i%7a9kbLWofbidhmtHF$I`s0KH z`PZ+!*e`fAJ-AQzA=~M~tNQCRZ}iM-v9UNfY1!&Ud-go~w&=9kO|NbaZ@(8TR(j{M yjhM?Wyyxw?Gt)w)f64D3PYfQgbc;1>GO(I8inY(;WoBStVDNPHb6Mw<&;$U9_RISK delta 451 zcmX@Wah7AlRHpjHo-U3d6}P5d_-P&*D01NAU!UtX(|;-@w`rUd2vpiO{pHkCr>C?m zIyr%dlT&kZNXCSK7ow>>i+22<_IBO7>h!%^@A2>3|7P>`^tqo8H`bb$7$rtcx7|MF z<;`;qVXI@!*T-o6Ep`!LX|B=Dwyx_~?skZScRk~6>lyq{ziYkTAXcBk9X-ita`kzC_qJ9%oXnQq|8Y;XsI1u|8zy-@t&JQrf@*5^h?sD{ z>^yKXYR}g*0{ ziGQl>hWWA4(Mhpy-p;AAo+y>jxKdp-@Lv@BKkbR9PdlbRmOr|Yo7JpQETSUd1p@;E NgQu&X%Q~loCIGp$+WP Date: Wed, 31 Mar 2010 22:54:56 +0200 Subject: [PATCH 05/12] ThreadPool.py r15 bugfix release from ThreadPool.py's author http://code.google.com/p/pythonthreadpool/issues/detail?id=4 --- src/trimage/ThreadPool.py | 62 +++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/src/trimage/ThreadPool.py b/src/trimage/ThreadPool.py index c7387d8..5c64409 100644 --- a/src/trimage/ThreadPool.py +++ b/src/trimage/ThreadPool.py @@ -3,9 +3,6 @@ ThreadPool Implementation @author: Morten Holdflod Moeller - morten@holdflod.dk @license: LGPL v3 - -(A buggy line commented out by Kalman Tarnay... -see: http://code.google.com/p/pythonthreadpool/issues/detail?id=4 ) ''' from __future__ import with_statement @@ -13,6 +10,17 @@ from threading import Thread, RLock from time import sleep from Queue import Queue, Empty import logging +import sys + +class NullHandler(logging.Handler): + def emit(self, record): + pass + +h = sys.stderr +logging.getLogger('threadpool').addHandler(h) +logging.getLogger('threadpool.worker').addHandler(h) + + class ThreadPoolMixIn: """Mix-in class to handle each request in a new thread from the ThreadPool.""" @@ -114,9 +122,12 @@ class ThreadPool: and executing them ''' while self.alive: + #print self.pool.__active_worker_count, self.pool.__worker_count job = self.pool.get_job() if (job != None): + self.pool.worker_active() job.execute() + self.pool.worker_inactive() else: self.alive = False @@ -137,8 +148,9 @@ class ThreadPool: # This Queue is assumed Thread Safe self.__jobs = Queue() - self.__active_workers_lock = RLock() - self.__active_workers = 0 + self.__worker_count_lock = RLock() + self.__worker_count = 0 + self.__active_worker_count = 0 self.__shutting_down = False logger = logging.getLogger('threadpool') @@ -158,7 +170,7 @@ class ThreadPool: logger = logging.getLogger("threadpool") logger.info("shutting down") - with self.__active_workers_lock: + with self.__worker_count_lock: self.__shutting_down = True self.__max_workers = 0 self.__kill_workers_after = 0 @@ -166,9 +178,9 @@ class ThreadPool: retries_left = clean_shutdown_reties while (retries_left > 0): - with self.__active_workers_lock: - logger.info("waiting for workers to shut down (%i), %i workers left"%(retries_left, self.__active_workers)) - if (self.__active_workers > 0): + with self.__worker_count_lock: + logger.info("waiting for workers to shut down (%i), %i workers left"%(retries_left, self.__worker_count)) + if (self.__worker_count > 0): retries_left -= 1 else: retries_left = 0 @@ -176,9 +188,9 @@ class ThreadPool: sleep(wait_for_workers_period) - with self.__active_workers_lock: - if (self.__active_workers > 0): - logger.warning("shutdown stopped waiting. Still %i active workers"%self.__active_workers) + with self.__worker_count_lock: + if (self.__worker_count > 0): + logger.warning("shutdown stopped waiting. Still %i active workers"%self.__worker_count) clean_shutdown = False else: clean_shutdown = True @@ -192,16 +204,24 @@ class ThreadPool: Called by worker to update worker count when the worker is shutting down ''' - with self.__active_workers_lock: - self.__active_workers -= 1 + with self.__worker_count_lock: + self.__worker_count -= 1 def __new_worker(self): ''' Adding a new worker thread to the thread pool ''' - with self.__active_workers_lock: + with self.__worker_count_lock: ThreadPool.Worker(self) - self.__active_workers += 1 + self.__worker_count += 1 + + def worker_active(self): + with self.__worker_count_lock: + self.__active_worker_count = self.__active_worker_count + 1 + + def worker_inactive(self): + with self.__worker_count_lock: + self.__active_worker_count = self.__active_worker_count - 1 def add_job(self, function, args = None, return_callback=None): ''' @@ -220,16 +240,14 @@ class ThreadPool: job = ThreadPool.Job(function, args, return_callback) - with self.__active_workers_lock: + with self.__worker_count_lock: if (self.__shutting_down): raise AddJobException("ThreadPool is shutting down") - + try: start_new_worker = False - if (self.__active_workers < self.__max_workers): - #DIY fixed.... FIXME - #http://code.google.com/p/pythonthreadpool/issues/detail?id=4 - #if (self.__active_workers == 0 or not self.__jobs.empty()): + if (self.__worker_count < self.__max_workers): + if (self.__active_worker_count == self.__worker_count): start_new_worker = True self.__jobs.put(job) From 2c9599813ad15472fea0089e86911501bec33296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1lm=C3=A1n=20Tarnay?= Date: Thu, 1 Apr 2010 00:41:12 +0200 Subject: [PATCH 06/12] temporary workaround for pythonthreadpool bug: http://code.google.com/p/pythonthreadpool/issues/detail?id=5 --- src/trimage/trimage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/trimage/trimage.py b/src/trimage/trimage.py index 216ea26..a8963c4 100755 --- a/src/trimage/trimage.py +++ b/src/trimage/trimage.py @@ -1,5 +1,5 @@ #!/usr/bin/python - +import time import sys import errno from os import listdir @@ -326,6 +326,7 @@ class Worker(QThread): def compress_file(self, images, showapp, verbose, imagelist): """Start the worker thread.""" for image in images: + time.sleep(0.05) #FIXME: Workaround http://code.google.com/p/pythonthreadpool/issues/detail?id=5 self.threadpool.add_job(image.compress, None, return_callback=self.toDisplay.put) self.showapp = showapp self.verbose = verbose From 5782698dda2baf3039a8f6fde08c3998058284cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1lm=C3=A1n=20Tarnay?= Date: Fri, 2 Apr 2010 07:18:18 +0200 Subject: [PATCH 07/12] Lots of fixes/changes there - ImageRow - Recompress fixed - Adding a file again recompresses it - Status indicators - cli working again - more crash resistant --- src/trimage/trimage.py | 127 ++++++++++++++++++++++++++--------------- 1 file changed, 81 insertions(+), 46 deletions(-) diff --git a/src/trimage/trimage.py b/src/trimage/trimage.py index a8963c4..0a8b4cd 100755 --- a/src/trimage/trimage.py +++ b/src/trimage/trimage.py @@ -132,19 +132,11 @@ class StartQT4(QMainWindow): # this is a fix for file dialog differentiating between cases "Image files (*.png *.jpg *.jpeg *.PNG *.JPG *.JPEG)") - imagelist = [] - for i, image in enumerate(images): - imagelist.append(unicode(image)) - - self.delegator(imagelist) + self.delegator([unicode(fullpath) for fullpath in images]) def recompress_files(self): """Send each file in the current file list to compress_file again.""" - newimagelist = [] - for image in self.imagelist: - newimagelist.append(image[4]) - self.imagelist = [] - self.delegator(newimagelist) + self.delegator([row.image.fullpath for row in self.imagelist]) """ Compress functions @@ -155,14 +147,21 @@ class StartQT4(QMainWindow): Recieve all images, check them and send them to the worker thread. """ delegatorlist = [] - for image in images: - image=Image(image) - if image.valid: - delegatorlist.append(image) - self.imagelist.append(("Compressing...", "", "", "", image, - QIcon(QPixmap(self.ui.get_image("pixmaps/compressing.gif"))))) - else: - print >>sys.stderr, u"[error] %s not a supported image file" % image.fullpath + for fullpath in images: + try: # do not add already existing images again, recompress them instead + image=(i.image for i in self.imagelist + if i.image.fullpath == fullpath).next() + if image.compressed: + image.reset() + image.recompression=True + delegatorlist.append(image) + except StopIteration: + image=Image(fullpath) + if image.valid: + delegatorlist.append(image) + self.imagelist.append(ImageRow(image,QIcon(QPixmap(self.ui.get_image("pixmaps/compressing.gif"))))) + else: + print >>sys.stderr, u"[error] %s not a supported image file" % image.fullpath self.update_table() self.thread.compress_file(delegatorlist, self.showapp, self.verbose, @@ -267,7 +266,7 @@ class TriTableModel(QAbstractTableModel): return QVariant(data) elif index.column() == 0 and role == Qt.DecorationRole: # decorate column 0 with an icon of the image itself - f_icon = self.imagelist[index.row()][5] + f_icon = self.imagelist[index.row()][4] return QVariant(f_icon) else: return QVariant() @@ -278,18 +277,51 @@ class TriTableModel(QAbstractTableModel): role == Qt.DecorationRole): return QVariant(self.header[col]) return QVariant() + +class ImageRow: + def __init__(self, image, waitingIcon=None): + self.image=image + d={ + 'shortname': lambda i: self.statusStr() % i.shortname, + 'oldfilesizestr': lambda i: size(i.oldfilesize, system=alternative) if i.compressed else "", + 'newfilesizestr': lambda i: size(i.newfilesize, system=alternative) if i.compressed else "", + 'ratiostr': lambda i: + "%.1f%%" % (100 - (float(i.newfilesize) / i.oldfilesize * 100)) if i.compressed else "", + 'icon': lambda i: i.icon if i.compressed else waitingIcon, + + 'fullpath': lambda i: i.fullpath, #only used by cli + } + for i,n in enumerate(['shortname','oldfilesizestr','newfilesizestr','ratiostr','icon']): + d[i]=d[n] + + self.d = d + + def statusStr(self): + if self.image.failed: + return "ERROR: %s" + if self.image.compressing: + return "In Progress..." + if not self.image.compressed and self.image.recompression: + return "Queued for recompression..." + if not self.image.compressed: + return "Queued..." + return "%s" + + def __getitem__(self, key): + return self.d[key](self.image) class Image: def __init__(self, fullpath): self.valid = False + self.reset() self.fullpath = fullpath if path.isfile(self.fullpath): self.filetype = determinetype(self.fullpath) if self.filetype in ["jpeg", "png"]: oldfile = QFileInfo(self.fullpath) self.shortname = oldfile.fileName() - self.oldfilesize = oldfile.size() - self.icon = QIcon(self.fullpath) + self.oldfilesize = oldfile.size() + self.icon = QIcon(self.fullpath) self.valid = True def _determinetype(self): @@ -300,19 +332,35 @@ class Image: self.filetype=None return self.filetype + def reset(self): + self.failed = False + self.compressed = False + self.compressing = False + self.recompression= False + def compress(self): if not self.valid: raise "Tried to compress invalid image (unsupported format or not file)" + self.reset() + self.compressing=True runString = { "jpeg": u"jpegoptim -f --strip-all '%(file)s'", "png" : u"optipng -force -o7 '%(file)s'&&advpng -z4 '%(file)s'"} - retcode = call(runString[self.filetype] % {"file": self.fullpath}, - shell = True, stdout=PIPE) + try: + retcode = call(runString[self.filetype] % {"file": self.fullpath}, + shell = True, stdout=PIPE) + except: + retcode = -1 if retcode == 0: self.newfilesize = QFile(self.fullpath).size() + self.compressed=True + else: + self.failed=True + self.compressing=False self.retcode=retcode return self + class Worker(QThread): def __init__(self, parent=None): @@ -336,31 +384,18 @@ class Worker(QThread): def run(self): """Compress the given file, get data from it and call update_table.""" tp = self.threadpool - while self.showapp or not (tp.__active_workers==0 and tp.__jobs.empty()): + while self.showapp or not (tp._ThreadPool__active_worker_count==0 and tp._ThreadPool__jobs.empty()): image = self.toDisplay.get() - if image.retcode==0: - #calculate ratio and make a nice string - oldfilesizestr = size(image.oldfilesize, system=alternative) - newfilesizestr = size(image.newfilesize, system=alternative) - ratio = 100 - (float(image.newfilesize) / float(image.oldfilesize) * 100) - ratiostr = "%.1f%%" % ratio + self.emit(SIGNAL("updateUi")) - # append current image to list - for i, listitem in enumerate(self.imagelist): - if listitem[4] == image: - self.imagelist.remove(listitem) - self.imagelist.insert(i, (image.shortname, oldfilesizestr, - newfilesizestr, ratiostr, image.fullpath, image.icon)) - - self.emit(SIGNAL("updateUi")) - - if not self.showapp and self.verbose: - # we work via the commandline - print("File: " + image.fullpath + ", Old Size: " - + oldfilesizestr + ", New Size: " + newfilesizestr - + ", Ratio: " + ratiostr) - else: - print >>sys.stderr, u"[error] %s could not be compressed" % image.fullpath + if not self.showapp and self.verbose: # we work via the commandline + if image.retcode==0: + ir=ImageRow(image) + print("File: " + ir['fullpath'] + ", Old Size: " + + ir['oldfilesizestr'] + ", New Size: " + ir['newfilesizestr'] + + ", Ratio: " + ir['ratiostr']) + else: + print >>sys.stderr, u"[error] %s could not be compressed" % image.fullpath class TrimageTableView(QTableView): From 97956d7e14c85e462aedf12d00562f0693c53793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1lm=C3=A1n=20Tarnay?= Date: Fri, 2 Apr 2010 07:22:50 +0200 Subject: [PATCH 08/12] removed multiprocessing from todo --- resources/todo | 4 ---- 1 file changed, 4 deletions(-) diff --git a/resources/todo b/resources/todo index 43dbe42..a215d43 100644 --- a/resources/todo +++ b/resources/todo @@ -15,10 +15,6 @@ todo app wise todo else - figure out how to make mac and win versions (someone else :) <- via gui2exe -todo later -- use multiprocessing lib to take advantage of multicore/multi-CPU to compress - multiple files simultaneously (threads have issues in Python; see "GIL") - =========================================== later versions: animate compressing.gif From cf9d98027c56930cb57286ed48991d7e4b3db581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1lm=C3=A1n=20Tarnay?= Date: Fri, 2 Apr 2010 07:24:00 +0200 Subject: [PATCH 09/12] changed hint text for recompress button to indicate that it recompresses all --- resources/window.ui | 2 +- src/trimage/ui.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/window.ui b/resources/window.ui index 066032f..bf5310d 100644 --- a/resources/window.ui +++ b/resources/window.ui @@ -127,7 +127,7 @@ PointingHandCursor - Recompress selected images + Recompress all images &Recompress diff --git a/src/trimage/ui.py b/src/trimage/ui.py index d644299..86caee8 100644 --- a/src/trimage/ui.py +++ b/src/trimage/ui.py @@ -153,7 +153,7 @@ class Ui_trimage(object): "Drag and drop images onto the table", None, QApplication.UnicodeUTF8)) self.recompress.setToolTip(QApplication.translate("trimage", - "Recompress selected images", None, QApplication.UnicodeUTF8)) + "Recompress all images", None, QApplication.UnicodeUTF8)) self.recompress.setText(QApplication.translate("trimage", "&Recompress", None, QApplication.UnicodeUTF8)) self.recompress.setShortcut(QApplication.translate("trimage", From 51691351d354268f5c8a1a736770bf02234c7b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1lm=C3=A1n=20Tarnay?= Date: Fri, 2 Apr 2010 07:25:10 +0200 Subject: [PATCH 10/12] swapped my first name and last name --- website/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/index.html b/website/index.html index b439856..2bd205d 100644 --- a/website/index.html +++ b/website/index.html @@ -123,7 +123,7 @@
  • Neil Wallace
  • Jeroen Goudsmit
  • -
  • Kálmán Tarnay
  • +
  • Tarnay Kálmán
From 29be211a9df1aeee4a55644479fa3249525aafac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1lm=C3=A1n=20Tarnay?= Date: Fri, 2 Apr 2010 07:27:16 +0200 Subject: [PATCH 11/12] license --- trimage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trimage b/trimage index 05e6139..6906ad3 100644 --- a/trimage +++ b/trimage @@ -1,6 +1,6 @@ #!/usr/bin/env python # -#Copyright (c) 2010 Kilian Valkhof, Paul Chaplin +#Copyright (c) 2010 Kilian Valkhof, Paul Chaplin, Tarnay Kálmán # #Permission is hereby granted, free of charge, to any person #obtaining a copy of this software and associated documentation From 1106ca482c121f3ac9eb6ecede89521f2e312ff3 Mon Sep 17 00:00:00 2001 From: Kilian Valkhof Date: Fri, 2 Apr 2010 13:51:19 +0200 Subject: [PATCH 12/12] pep8 rewriting, version update, todo editing --- resources/todo | 8 +++ setup.py | 2 +- src/trimage/trimage.py | 107 ++++++++++++++++++++++++----------------- website/index.html | 8 +-- 4 files changed, 73 insertions(+), 52 deletions(-) diff --git a/resources/todo b/resources/todo index a215d43..d602f42 100644 --- a/resources/todo +++ b/resources/todo @@ -33,3 +33,11 @@ later versions: again would currently try to recompress all 100, when only 10 would be worthy of trying to compress further. + +1.1.0 changes: +- use multiprocessing for images +- more robust file handling +- re-adding images now results in recompressing them +- compressing message now shows filename +- wider array of status messages in the table + diff --git a/setup.py b/setup.py index 4ffd086..21635fa 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from distutils.core import setup setup(name = "trimage", - version = "1.0.0b3", + version = "1.1.0b", description = "Trimage image compressor - A cross-platform tool for optimizing PNG and JPG files", author = "Kilian Valkhof, Paul Chaplin", author_email = "help@trimage.org", diff --git a/src/trimage/trimage.py b/src/trimage/trimage.py index c054d61..6f77c4d 100755 --- a/src/trimage/trimage.py +++ b/src/trimage/trimage.py @@ -18,7 +18,7 @@ from multiprocessing import cpu_count from ui import Ui_trimage -VERSION = "1.0.0b3" +VERSION = "1.1.0b" class StartQT4(QMainWindow): @@ -148,20 +148,21 @@ class StartQT4(QMainWindow): """ delegatorlist = [] for fullpath in images: - try: # do not add already existing images again, recompress them instead - image=(i.image for i in self.imagelist + try: # recompress images already in the list + image = (i.image for i in self.imagelist if i.image.fullpath == fullpath).next() if image.compressed: - image.reset() - image.recompression=True - delegatorlist.append(image) + image.reset() + image.recompression = True + delegatorlist.append(image) except StopIteration: - image=Image(fullpath) + image = Image(fullpath) if image.valid: delegatorlist.append(image) - self.imagelist.append(ImageRow(image,QIcon(QPixmap(self.ui.get_image("pixmaps/compressing.gif"))))) + icon = QIcon(QPixmap(self.ui.get_image("pixmaps/compressing.gif"))) + self.imagelist.append(ImageRow(image, icon)) else: - print >>sys.stderr, u"[error] %s not a supported image file" % image.fullpath + print >> sys.stderr, u"[error] %s not a supported image file" % image.fullpath self.update_table() self.thread.compress_file(delegatorlist, self.showapp, self.verbose, @@ -237,6 +238,7 @@ class StartQT4(QMainWindow): else: raise + class TriTableModel(QAbstractTableModel): def __init__(self, parent, imagelist, header, *args): @@ -278,29 +280,38 @@ class TriTableModel(QAbstractTableModel): return QVariant(self.header[col]) return QVariant() -class ImageRow: - def __init__(self, image, waitingIcon=None): - self.image=image - d={ - 'shortname': lambda i: self.statusStr() % i.shortname, - 'oldfilesizestr': lambda i: size(i.oldfilesize, system=alternative) if i.compressed else "", - 'newfilesizestr': lambda i: size(i.newfilesize, system=alternative) if i.compressed else "", - 'ratiostr': lambda i: - "%.1f%%" % (100 - (float(i.newfilesize) / i.oldfilesize * 100)) if i.compressed else "", - 'icon': lambda i: i.icon if i.compressed else waitingIcon, - 'fullpath': lambda i: i.fullpath, #only used by cli +class ImageRow: + + def __init__(self, image, waitingIcon=None): + """ Build the information visible in the table image row. """ + self.image = image + d = { + 'shortname': lambda i: self.statusStr() % i.shortname, + 'oldfilesizestr': lambda i: size(i.oldfilesize, system=alternative) + if i.compressed else "", + 'newfilesizestr': lambda i: size(i.newfilesize, system=alternative) + if i.compressed else "", + 'ratiostr': lambda i: + "%.1f%%" % (100 - (float(i.newfilesize) / i.oldfilesize * 100)) + if i.compressed else "", + 'icon': lambda i: i.icon if i.compressed else waitingIcon, + 'fullpath': lambda i: i.fullpath, #only used by cli } - for i,n in enumerate(['shortname','oldfilesizestr','newfilesizestr','ratiostr','icon']): - d[i]=d[n] + names = ['shortname', 'oldfilesizestr', 'newfilesizestr', + 'ratiostr', 'icon'] + for i, n in enumerate(names): + d[i] = d[n] self.d = d def statusStr(self): + """ Set the status message. """ if self.image.failed: return "ERROR: %s" if self.image.compressing: - return "In Progress..." + message = "Compressing %s..." + return message if not self.image.compressed and self.image.recompression: return "Queued for recompression..." if not self.image.compressed: @@ -310,7 +321,9 @@ class ImageRow: def __getitem__(self, key): return self.d[key](self.image) + class Image: + def __init__(self, fullpath): """ gather image information. """ self.valid = False @@ -327,40 +340,41 @@ class Image: def _determinetype(self): """ Determine the filetype of the file using imghdr. """ - filetype=determinetype(self.fullpath) + filetype = determinetype(self.fullpath) if filetype in ["jpeg", "png"]: - self.filetype=filetype + self.filetype = filetype else: - self.filetype=None + self.filetype = None return self.filetype def reset(self): self.failed = False self.compressed = False self.compressing = False - self.recompression= False + self.recompression = False def compress(self): """ Compress the image and return it to the thread. """ if not self.valid: - raise "Tried to compress invalid image (unsupported format or not file)" + raise "Tried to compress invalid image (unsupported format or not \ + file)" self.reset() - self.compressing=True + self.compressing = True runString = { "jpeg": u"jpegoptim -f --strip-all '%(file)s'", - "png" : u"optipng -force -o7 '%(file)s'&&advpng -z4 '%(file)s'"} + "png": u"optipng -force -o7 '%(file)s'&&advpng -z4 '%(file)s'"} try: retcode = call(runString[self.filetype] % {"file": self.fullpath}, - shell = True, stdout=PIPE) + shell=True, stdout=PIPE) except: - retcode = -1 + retcode = -1 if retcode == 0: self.newfilesize = QFile(self.fullpath).size() - self.compressed=True + self.compressed = True else: - self.failed=True - self.compressing=False - self.retcode=retcode + self.failed = True + self.compressing = False + self.retcode = retcode return self @@ -368,7 +382,7 @@ class Worker(QThread): def __init__(self, parent=None): QThread.__init__(self, parent) - self.toDisplay=Queue() + self.toDisplay = Queue() self.threadpool = ThreadPool(max_workers=cpu_count()) def __del__(self): @@ -377,8 +391,10 @@ class Worker(QThread): def compress_file(self, images, showapp, verbose, imagelist): """Start the worker thread.""" for image in images: - time.sleep(0.05) #FIXME: Workaround http://code.google.com/p/pythonthreadpool/issues/detail?id=5 - self.threadpool.add_job(image.compress, None, return_callback=self.toDisplay.put) + #FIXME:http://code.google.com/p/pythonthreadpool/issues/detail?id=5 + time.sleep(0.05) + self.threadpool.add_job(image.compress, None, + return_callback=self.toDisplay.put) self.showapp = showapp self.verbose = verbose self.imagelist = imagelist @@ -387,19 +403,20 @@ class Worker(QThread): def run(self): """Compress the given file, get data from it and call update_table.""" tp = self.threadpool - while self.showapp or not (tp._ThreadPool__active_worker_count==0 and tp._ThreadPool__jobs.empty()): + while self.showapp or not (tp._ThreadPool__active_worker_count == 0 and + tp._ThreadPool__jobs.empty()): image = self.toDisplay.get() self.emit(SIGNAL("updateUi")) if not self.showapp and self.verbose: # we work via the commandline - if image.retcode==0: - ir=ImageRow(image) + if image.retcode == 0: + ir = ImageRow(image) print("File: " + ir['fullpath'] + ", Old Size: " - + ir['oldfilesizestr'] + ", New Size: " + ir['newfilesizestr'] - + ", Ratio: " + ir['ratiostr']) + + ir['oldfilesizestr'] + ", New Size: " + + ir['newfilesizestr'] + ", Ratio: " + ir['ratiostr']) else: - print >>sys.stderr, u"[error] %s could not be compressed" % image.fullpath + print >> sys.stderr, u"[error] %s could not be compressed" % image.fullpath if __name__ == "__main__": app = QApplication(sys.argv) diff --git a/website/index.html b/website/index.html index 2bd205d..c636199 100644 --- a/website/index.html +++ b/website/index.html @@ -69,7 +69,7 @@
-

Trimage image compressor – 1.0.0b3 (beta)

+

Trimage image compressor – 1.1.0b (beta)

A cross-platform tool for losslessly optimizing PNG and JPG files.

Trimage is a cross-platform GUI and command-line interface to optimize image files via optipng, @@ -165,16 +165,12 @@

Planned features

-

Version 1.0.0 final:

+

Version 1.1.0 final:

  • Expand command line options
  • Make sure a compressed file is always smaller than the original one, or don't compress
  • General refactoring
-

Version 1.1.0

-
    -
  • Use multiprocessing instead of threading
  • -

Beyond that

  • Deletion of rows in the table view