Updated config options and regex expressions

fanficdownload-cleaned.py

@@ -1,62 +1,68 @@
 from fanficfare import geturls
 import os
-from os import listdir, remove, rename
+from os import listdir, remove, rename, utime
 from os.path import isfile, join
-import subprocess
+from subprocess import check_output, STDOUT
-import time
 import logging
 from optparse import OptionParser
 import re
+from ConfigParser import ConfigParser
 
 logging.getLogger("fanficfare").setLevel(logging.ERROR)
 
+def touch(fname, times=None):
+    with open(fname, 'a'):
+        utime(fname, times)
 
-def main(user, password, server="imap.gmail.com", label="INBOX", path=""):
+
+ffnet = re.compile('(fanfiction.net/s/\d*)/?.*')
+neutral = re.compile('https?://(.*)')
+
+def parse_url(url):
+    if ffnet.search(url):
+        url = "www." + ffnet.search(url).group(1)
+    elif neutral.search(url):
+        url = neutral.search(url).group(1)
+    return url
+
+def main(user, password, server, label, inout_file, path="", ):
 
     if path != "":
         path = '--with-library "{}"'.format(path)
         
-    with open("thesebroke", "rb") as fp:
+    touch(inout_file)
+
+    with open(inout_file, "r") as fp:
         urls = set([x.replace("\n", "") for x in fp.readlines()])
-    with open("todo", "wb") as fp:
+        
+    with open(inout_file, "w") as fp:
         fp.write("")
         urls |= geturls.get_urls_from_imap(server, user, password, label)
                 
-    def parse_url(url):
-        url = url.replace("https://", "")
-        url = url.replace("http://", "")
-        url = url[:url.find("/", url.find("/s/") + 3)+1]
-        return url
-    
-    
         urls = set(parse_url(x) for x in urls)
         
         if len(urls) != 0: print "URLs to parse: {}".format(", ".join(urls))
 
         files = lambda x: [f for f in listdir(x) if isfile(join(x, f))]
 
-#    if len(urls) != 0:
-#        print time.strftime("%d/%m/%Y, %H:%M:%S\n\n")
-
 
         for url in urls:
             print "Working with url {}".format(url)
             try:
-            res = subprocess.check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=subprocess.STDOUT) 
+                res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT) 
                 id = res
                 print "\tStory is in calibre with id {}".format(id)
                 try:
                     print "\tExporting file"
-                res = subprocess.check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir {}'.format(id, path), shell=True)
+                    res = check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir {}'.format(id, path), shell=True)
                     onlyfiles = files(".")
                     for cur in onlyfiles:
                         if not cur.endswith(".epub"): continue
                         print '\tDownloading with fanficfare, updating file "{}"'.format(cur)
-                    res = subprocess.check_output('fanficfare -u "{}" --update-cover'.format(cur), shell=True,stderr=subprocess.STDOUT)
+                        res = check_output('fanficfare -u "{}" --update-cover'.format(cur), shell=True,stderr=STDOUT)
                         #print res
                         if "already contains" in res:
                             print "\tIssue with story, FF.net site is broken."
-                        with open("todo", "ab") as fp:
                             fp.write("{}\n".format(url))
                             remove(cur)
                             continue
@@ -65,33 +71,32 @@ def main(user, password, server="imap.gmail.com", label="INBOX", path=""):
                             continue
                         elif "more recently than Story" in res:
                             print "\tForcing download update\n"
-                        res = subprocess.check_output('fanficfare -u "{}" --force --update-cover'.format(cur), shell=True,stderr=subprocess.STDOUT)
+                            res = check_output('fanficfare -u "{}" --force --update-cover'.format(cur), shell=True,stderr=STDOUT)
                     
                         print "\tRemoving {} from library".format(id)
-                    res = subprocess.check_output('calibredb remove {} {}'.format(id, path), shell=True,stderr=subprocess.STDOUT)
+                        res = check_output('calibredb remove {} {}'.format(id, path), shell=True,stderr=STDOUT)
                         #print res
                         print "\tAdding {} to library".format(cur)
-                    res = subprocess.check_output('calibredb add "{}" {}'.format(cur, path), shell=True,stderr=subprocess.STDOUT)
+                        res = check_output('calibredb add "{}" {}'.format(cur, path), shell=True,stderr=STDOUT)
-                    res = subprocess.check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=subprocess.STDOUT)
+                        res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=STDOUT)
                         print "\tAdded {} to library with id {}".format(cur, res)
                         #print res
                         remove(cur)
                 except Exception as e:
                     print "\tSomething fucked up: {}".format(e)
                     remove(cur)
-                with open("todo", "ab") as fp:
                     fp.write("{}\n".format(url))
             except:
                 print "\tStory is not in calibre"
                 try:
-                res = subprocess.check_output('fanficfare -u "{}" --update-cover'.format(url), shell=True)
+                    res = check_output('fanficfare -u "{}" --update-cover'.format(url), shell=True)
                     #print res
                     onlyfiles = files(".")
                     for cur in onlyfiles:
                         if not cur.endswith(".epub"): continue
                         try:
                             print "\tAdding {} to library".format(cur)
-                        res = subprocess.check_output('calibredb add "{}" {}'.format(cur, path), shell=True, stderr=subprocess.STDOUT)
+                            res = check_output('calibredb add "{}" {}'.format(cur, path), shell=True, stderr=STDOUT)
                             #print res
                             remove(cur)
                         except Exception:
@@ -99,11 +104,8 @@ def main(user, password, server="imap.gmail.com", label="INBOX", path=""):
                             raise
                 except Exception as e:
                     print "\tSomething fucked up: {}".format(e)
-                with open("todo", "ab") as fp:
                     fp.write("{}\n".format(url))
 
-    remove("thesebroke")
-    rename("todo", "thesebroke")
 
 if __name__ == "__main__":
     option_parser = OptionParser(usage="usage: %prog [flags]")
@@ -114,38 +116,39 @@ if __name__ == "__main__":
     
     option_parser.add_option('-s', '--server', action='store', dest='server', default="imap.gmail.com", help='Email IMAP Server. Default is "imap.gmail.com".')
     
-    option_parser.add_option('-m', '--mailbox', action='store', dest='label', default='INBOX', help='Email Label. Default is "INBOX".')
+    option_parser.add_option('-m', '--mailbox', action='store', dest='mailbox', default='INBOX', help='Email Label. Default is "INBOX".')
     
     option_parser.add_option('-l', '--library', action='store', dest='library', default="", help="calibre library db location. If none is passed, the default is the calibre system library location. Make sure to enclose the path in quotes.")
     
-    option_parser.add_option('-c', '--config', action='store', dest-'config', help='Config file for inputs. One argument per line, format of field=value')
+    option_parser.add_option('-i', '--input', action='store', dest='input', default="./fanfiction.txt", help="Error file. Any urls that fail will be output here, and file will be read to find any urls that failed previously. If file does not exist will create. File is overwitten every time the program is run.")
+    
+    option_parser.add_option('-c', '--config', action='store', dest='config', help='Config file for inputs. Example config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank.')
     
     (options, args) = option_parser.parse_args()
     
     if options.config:
-        user = re.compile('user=(.*?)')
+        config = ConfigParser(allow_no_value=True)
-        passwrd = re.compile('password=(.*?)')
+        config.read(options.config)
-        serv = re.compile('server=(.*?)')
-        mailbox = re.compile('mailbox=(.*?)')
-        lib = re.compile('library=(.*?)')
-        with open(options.config, 'r') as configfile:
-            for line in configfile.readlines():
-                line = line.tolower().strip()
-                if user.search(line):
-                    options.user = user.search(line).group(1)
-                elif passwrd.search(line):
-                    options.password = passwrd.search(line).group(1)
-                elif serv.search(line):
-                    options.server = serv.search(line).group(1)
-                elif mailbox.search(line):
-                    options.label = mailbox.search(line).group(1)
-                elif lib.search(line):
-                    options.library = mailbox.search(line).group(1)
         
+        updater = lambda option, newval : newval if newval != "" else option
+        
+        options.user = updater(options.user, config.get('login', 'user').strip())
+        
+        options.password = updater(options.password, config.get('login', 'password').strip())
+        
+        options.server = updater(options.server, config.get('login', 'server').strip())
+        
+        options.mailbox = updater(options.mailbox, config.get('login', 'mailbox').strip())
+        
+        options.library = updater(options.library, config.get('locations', 'library').strip())
+        
+        options.input = updater(options.input, config.get('locations', 'input').strip())
+        
+    print options
     if not (options.user or options.password):
         raise ValueError("User or Password not given")
     
-    main(options.user, options.password, options.server, options.label, options.library)
+    main(options.user, options.password, options.server, options.mailbox, options.input, options.library)