From 6f4ed2490fbcde0b74e7b8251ad71e29b430b8ef Mon Sep 17 00:00:00 2001
From: jvoisin <julien.voisin@dustri.org>
Date: Sat, 14 Apr 2018 16:10:45 +0200
Subject: [PATCH] Thread the cleaning process
---
main.py | 32 +++++++++++++++++++++++++++++---
1 file changed, 29 insertions(+), 3 deletions(-)
diff --git a/main.py b/main.py
index 80355ae..be2508e 100755
--- a/main.py
+++ b/main.py
@@ -3,6 +3,9 @@
import os
import mimetypes
import argparse
+from threading import Thread
+import multiprocessing
+from queue import Queue
from src import parser_factory
@@ -75,6 +78,14 @@ def __get_files_recursively(files):
for _f in _files:
yield os.path.join(path, _f)
+def __do_clean_async(q):
+ while True:
+ f = q.get()
+ if f is None: # nothing more to process
+ return
+ clean_meta(f)
+ q.task_done()
+
def main():
arg_parser = create_arg_parser()
@@ -86,12 +97,27 @@ def main():
show_parsers()
return
- if args.show:
+ elif args.show:
for f in __get_files_recursively(args.files):
show_meta(f)
- else:
+ return
+
+ else: # Thread the cleaning
+ q = Queue(maxsize=0)
+ threads = list()
for f in __get_files_recursively(args.files):
- clean_meta(f)
+ q.put(f)
+
+ for _ in range(multiprocessing.cpu_count()):
+ worker = Thread(target=__do_clean_async, args=(q, ))
+ worker.start()
+ threads.append(worker)
+
+ for _ in range(multiprocessing.cpu_count()):
+ q.put(None) # stop the threads
+
+ for worker in threads:
+ worker.join()
if __name__ == '__main__':
--
GitLab