Now if option is enabled and there are multiple links being passed in at once, will make use of multiprocessing in order to speed up download and processing times. Potential race conditions in calibre add and database editing, so should test and fix that.

fanficdownload.py

@@ -11,6 +11,8 @@ from shutil import rmtree
 import socket
 from time import strftime, localtime
 
+from multiprocessing import Pool
+
 logging.getLogger("fanficfare").setLevel(logging.ERROR)
 
 class bcolors:
@@ -23,7 +25,7 @@ class bcolors:
     BOLD = '\033[1m'
     UNDERLINE = '\033[4m'
 
-def log(msg, color=None):
+def log(msg, color=None, output=True):
     if color:
         col = bcolors.HEADER
         if color == 'BLUE':
@@ -38,9 +40,14 @@ def log(msg, color=None):
             col = bcolors.BOLD
         elif color == 'UNDERLINE':
             col = bcolors.UNDERLINE
-        print '{}{}{}: \t {}{}{}'.format(bcolors.BOLD, strftime('%m/%d/%Y %H:%M:%S', localtime()), bcolors.ENDC,  col, msg, bcolors.ENDC)
+        line = '{}{}{}: \t {}{}{}'.format(bcolors.BOLD, strftime('%m/%d/%Y %H:%M:%S', localtime()), bcolors.ENDC,  col, msg, bcolors.ENDC)
+    else:
+        line = '{}{}{}: \t {}'.format(bcolors.BOLD, strftime('%m/%d/%Y %H:%M:%S', localtime()), bcolors.ENDC, msg)
+    if output:
+        print line
+        return ""
     else:
-        print '{}{}{}: \t {}'.format(bcolors.BOLD, strftime('%m/%d/%Y %H:%M:%S', localtime()), bcolors.ENDC, msg)
+        return line + "\n"
 
 def touch(fname, times=None):
     with open(fname, 'a'):
@@ -89,96 +96,60 @@ def check_regexes(output):
     if no_url.search(output):
         raise ValueError("No URL in epub to update from. Fix the metadata.")
 
-def main(user, password, server, label, inout_file, path ):
-
-    if path:
-        path = '--with-library "{}" --username calibre --password pornoboobies'.format(path)
-        try:
-            with open(devnull, 'w') as nullout:
-               call(['calibredb'], stdout=nullout, stderr=nullout)
-        except OSError as e:
-            if errno == ENOENT:
-                log("Calibredb is not installed on this system. Cannot search the calibre library or update it.", 'FAIL')
-                return
-        
-    touch(inout_file)
-
-    with open(inout_file, "r") as fp:
-        urls = set([x.replace("\n", "") for x in fp.readlines()])
-    
-    with open(inout_file, "w") as fp:
-        fp.write("")
-
-    try:
-        socket.setdefaulttimeout(55)
-        urls |= geturls.get_urls_from_imap(server, user, password, label)
-        socket.setdefaulttimeout(None)
-    except Exception as e:
-        log("Broke while getting URLs: {}".format(e), 'FAIL')
-        with open(inout_file, "w") as fp:
-            for cur in urls:
-                 fp.write("{}\n".format(cur))
-        return
-    
-    if not urls: return
-    urls = set(parse_url(x) for x in urls)
-    log("URLs to parse:", 'HEADER')
-    for url in urls:
-        log("\t{}".format(url), 'BLUE')
         
+def downloader(args):
+    url, inout_file, path, live = args
     loc = mkdtemp()
-
+    output = ""
-    for url in urls:
+    output += log("Working with url {}".format(url), 'HEADER', live)
-        log("Working with url {}".format(url), 'HEADER')
     storyId = None
     try:
         if path:
             try:
-                    res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT,stdin=PIPE, ) 
+                storyId = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT,stdin=PIPE, )
-                    storyId = res
+                output += log("\tStory is in calibre with id {}".format(storyId), 'BLUE', live)
-                    log("\tStory is in calibre with id {}".format(storyId), 'BLUE')
+                output += log("\tExporting file", 'BLUE', live)
-                    log("\tExporting file", 'BLUE')
                 res = check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format(storyId, loc, path), shell=True, stdin=PIPE, stderr=STDOUT)
                 cur = get_files(loc, ".epub", True)[0]
-                    log('\tDownloading with fanficfare, updating file "{}"'.format(cur), 'GREEN')
+                output += log('\tDownloading with fanficfare, updating file "{}"'.format(cur), 'GREEN', live)
                 moving=""
             except:
                 #story is not in calibre
                 cur = url
                 moving = 'cd "{}" && '.format(loc)
             res = check_output('cp personal.ini {}/personal.ini'.format(loc), shell=True, stderr=STDOUT, stdin=PIPE,)
-                log('\tRunning: {}fanficfare -u "{}" --update-cover'.format(moving, cur), 'BLUE')
+            output += log('\tRunning: {}fanficfare -u "{}" --update-cover'.format(moving, cur), 'BLUE', live)
             res = check_output('{}fanficfare -u "{}" --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
             check_regexes(res)
             if chapter_difference.search(res) or more_chapters.search(res):
-                    log("\tForcing download update due to:", 'WARNING')
+                output += log("\tForcing download update due to:", 'WARNING', live)
                 for line in res.split("\n"):
                     if line:
-                            log("\t\t{}".format(line), 'WARNING')
+                        output += log("\t\t{}".format(line), 'WARNING', live)
                 res = check_output('{}fanficfare -u "{}" --force --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
                 check_regexes(res)
             cur = get_files(loc, '.epub', True)[0]
 
             
             if storyId:    
-                    log("\tRemoving {} from library".format(storyId), 'BLUE')
+                output += log("\tRemoving {} from library".format(storyId), 'BLUE', live)
                 try:
                     res = check_output('calibredb remove {} {}'.format(path, storyId), shell=True,stderr=STDOUT,stdin=PIPE, )
                 except:
                     raise
             
-                log("\tAdding {} to library".format(cur), 'BLUE')
+            output += log("\tAdding {} to library".format(cur), 'BLUE', live)
             try:
                 res = check_output('calibredb add -d {} "{}"'.format(path, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
             except Exception as e:
-                    log(e)
+                output += log(e)
                 raise
             try:
                 res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=STDOUT,stdin=PIPE, )
-                    log("\tAdded {} to library with id {}".format(cur, res), 'GREEN')
+                output += log("\tAdded {} to library with id {}".format(cur, res), 'GREEN', live)
             except:
-                    log("It's been added to library, but not sure what the ID is.", 'WARNING')
+                output += log("It's been added to library, but not sure what the ID is.", 'WARNING', live)
-                    log("Added file to library with id 0", 'GREEN')
+                output += log("Added file to library with id 0", 'GREEN', live)
             remove(cur)
         else:
             res = check_output('cd "{}" && fanficfare -u "{}" --update-cover'.format(loc, url), shell=True,stderr=STDOUT,stdin=PIPE, )
@@ -186,9 +157,12 @@ def main(user, password, server, label, inout_file, path ):
             cur = get_files(loc, '.epub', True)[0]
             name = get_files(loc, '.epub', False)[0]
             rename(cur, name)
-                log("Downloaded story {} to {}".format(story_name.search(name).group(1), name), 'GREEN')
+            output += log("Downloaded story {} to {}".format(story_name.search(name).group(1), name), 'GREEN', live)
+        if not live: print output.strip()
+        rmtree(loc)
     except Exception as e:
-            log("Exception: {}".format(e), 'FAIL')
+        output += log("Exception: {}".format(e), 'FAIL', live)
+        if not live: print output.strip()
         try:
             rmtree(loc)
         except:
@@ -196,9 +170,48 @@ def main(user, password, server, label, inout_file, path ):
         loc = mkdtemp()
         with open(inout_file, "a") as fp:
             fp.write("{}\n".format(url))
-            continue
     
-    rmtree(loc)
+        
+def main(user, password, server, label, inout_file, path, live ):
+
+    if path:
+        path = '--with-library "{}" --username calibre --password pornoboobies'.format(path)
+        try:
+            with open(devnull, 'w') as nullout:
+               call(['calibredb'], stdout=nullout, stderr=nullout)
+        except OSError as e:
+            if errno == ENOENT:
+                log("Calibredb is not installed on this system. Cannot search the calibre library or update it.", 'FAIL')
+                return
+        
+    touch(inout_file)
+
+    with open(inout_file, "r") as fp:
+        urls = set([x.replace("\n", "") for x in fp.readlines()])
+    
+    with open(inout_file, "w") as fp:
+        fp.write("")
+
+    try:
+        socket.setdefaulttimeout(55)
+        urls |= geturls.get_urls_from_imap(server, user, password, label)
+        socket.setdefaulttimeout(None)
+    except Exception as e:
+        #log("Broke while getting URLs: {}".format(e), 'FAIL')
+        with open(inout_file, "w") as fp:
+            for cur in urls:
+                 fp.write("{}\n".format(cur))
+        return
+    
+    if not urls: return
+    urls = set(parse_url(x) for x in urls)
+    log("URLs to parse ({}):".format(len(urls)), 'HEADER')
+    for url in urls:
+        log("\t{}".format(url), 'BLUE')
+
+    p = Pool()
+    p.map(downloader, [[url, inout_file, path, live] for url in urls])
+
     return
 
 
@@ -219,6 +232,8 @@ if __name__ == "__main__":
     
     option_parser.add_option('-c', '--config', action='store', dest='config', help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.')
     
+    option_parser.add_option('-o', '--output', action='store_true', dest='live', help='Include this if you want all the output to be saved and posted live. Useful when multithreading.')
+    
     (options, args) = option_parser.parse_args()
     
     if options.config:
@@ -245,10 +260,13 @@ if __name__ == "__main__":
         try: options.input = updater(options.input, config.get('locations', 'input').strip())
         except: pass
         
+        try: options.live = updater(options.live, config.getboolean('output', 'live').strip())
+        except: pass
+        
     if not (options.user or options.password):
         raise ValueError("User or Password not given")
     
-    main(options.user, options.password, options.server, options.mailbox, options.input, options.library)
+    main(options.user, options.password, options.server, options.mailbox, options.input, options.library, options.live)