Updates to calibre 3.0

Random bugfixes
Fixing of git

fanficdownload.py

@@ -8,6 +8,7 @@ import re
 from ConfigParser import ConfigParser
 from tempfile import mkdtemp
 from shutil import rmtree
+import socket
 
 logging.getLogger("fanficfare").setLevel(logging.ERROR)
 
@@ -17,7 +18,7 @@ def touch(fname, times=None):
 
 
 ffnet = re.compile('(fanfiction.net/s/\d*)/?.*')
-aooo = re.compile('(archiveofourown.org/works/\d*)/?.')
+aooo = re.compile('(archiveofourown.org/works/\d*)/?.*')
 neutral = re.compile('https?://(.*)')
 story_name = re.compile('(.*)-.*')
 
@@ -32,7 +33,7 @@ def parse_url(url):
     if ffnet.search(url):
         url = "www." + ffnet.search(url).group(1)
     elif aooo.search(url):
-        url = aooo.search(url).group(1)
+        url = "www." + aooo.search(url).group(1)
     elif neutral.search(url):
         url = neutral.search(url).group(1)
     return url
@@ -58,7 +59,7 @@ def check_regexes(output):
 
 def main(user, password, server, label, inout_file, path ):
     if path:
-        path = '--with-library "{}"'.format(path)
+        path = '--with-library "{}" --username calibre --password pornoboobies'.format(path)
         try:
             with open(devnull, 'w') as nullout:
                call(['calibredb'], stdout=nullout, stderr=nullout)
@@ -71,74 +72,97 @@ def main(user, password, server, label, inout_file, path ):
 
     with open(inout_file, "r") as fp:
         urls = set([x.replace("\n", "") for x in fp.readlines()])
-        
+    
     with open(inout_file, "w") as fp:
         fp.write("")
+
+    try:
+        socket.setdefaulttimeout(55)
         urls |= geturls.get_urls_from_imap(server, user, password, label)
+        socket.setdefaulttimeout(None)
+    except Exception as e:
+        print "Broke while getting URLs: {}".format(e)
+        with open(inout_file, "w") as fp:
+            for cur in urls:
+                 fp.write("{}\n".format(cur))
+        return
+    
+    if not urls: return
+    urls = set(parse_url(x) for x in urls)
+    
+    
+    if len(urls): print "URLs to parse: {}".format(", ".join(urls))
+
+    loc = mkdtemp()
+
+
+    
+    for url in urls:
+        print "Working with url {}".format(url)
+        storyId = None
+        try:
+            if path:
+                try:
+                    res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT,stdin=PIPE, ) 
+                    storyId = res
+                    print "\tStory is in calibre with id {}".format(storyId)
+                    print "\tExporting file"
+                    res = check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format(storyId, loc, path), shell=True, stdin=PIPE, stderr=STDOUT)
+                    cur = get_files(loc, ".epub", True)[0]
+                    print '\tDownloading with fanficfare, updating file "{}"'.format(cur)
+                    moving=""
+                except:
+                    #story is not in calibre
+                    cur = url
+                    moving = 'cd "{}" && '.format(loc)
+                res = check_output('cp personal.ini {}/personal.ini'.format(loc), shell=True, stderr=STDOUT, stdin=PIPE,)
+                print '{}fanficfare -u "{}" --update-cover'.format(moving, cur)
+                res = check_output('{}fanficfare -u "{}" --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
+                check_regexes(res)
+                if chapter_difference.search(res) or more_chapters.search(res):
+                    print "\tForcing download update due to: {}\n".format(res)
+                    res = check_output('{}fanficfare -u "{}" --force --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
+                    check_regexes(res)
+                cur = get_files(loc, '.epub', True)[0]
+
                 
-        urls = set(parse_url(x) for x in urls)
+                if storyId:    
-        
+                    print "\tRemoving {} from library".format(storyId)
-        if len(urls) != 0: print "URLs to parse: {}".format(", ".join(urls))
-
-        loc = mkdtemp()
-
-
-        
-        for url in urls:
-            print "Working with url {}".format(url)
-            storyId = None
-            try:
-                if path:
                     try:
-                        res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT,stdin=PIPE, ) 
+                        res = check_output('calibredb remove {} {}'.format(path, storyId), shell=True,stderr=STDOUT,stdin=PIPE, )
-                        storyId = res
-                        print "\tStory is in calibre with id {}".format(storyId)
-                        print "\tExporting file"
-                        res = check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format(storyId, loc, path), shell=True, stdin=PIPE, stderr=STDOUT)
-                        cur = get_files(loc, ".epub", True)[0]
-                        print '\tDownloading with fanficfare, updating file "{}"'.format(cur)
-                        moving=""
                     except:
-                        #story is not in calibre
+                        raise
-                        cur = url
+                
-                        moving = 'cd "{}" && '.format(loc)
+                print "\tAdding {} to library".format(cur)
-                    print '{}fanficfare -u "{}" --update-cover'.format(moving, cur)
+                try:
-                    res = check_output('{}fanficfare -u "{}" --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
+                    res = check_output('calibredb add -d {} "{}"'.format(path, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
-                    check_regexes(res)
+                except Exception as e:
-                    if chapter_difference.search(res) or more_chapters.search(res):
+                    print e
-                        print "\tForcing download update\n"
+                    raise
-                        res = check_output('{}fanficfare -u "{}" --force --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
+                try:
-                        check_regexes(res)
+                    res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=STDOUT,stdin=PIPE, )
-                    cur = get_files(loc, '.epub', True)[0]
+                    print "\tAdded {} to library with id {}".format(cur, res)
-
+                except:
-                    
+                    print "It's been added to library, but not sure what the ID is."
-                    if storyId:    
+                    print "Added file to library with id 0"
-                        print "\tRemoving {} from library".format(storyId)
+                remove(cur)
-                        res = check_output('calibredb remove {} {}'.format(storyId, path), shell=True,stderr=STDOUT,stdin=PIPE, )
+            else:
-                    
+                res = check_output('cd "{}" && fanficfare -u "{}" --update-cover'.format(loc, url), shell=True,stderr=STDOUT,stdin=PIPE, )
-                    print "\tAdding {} to library".format(cur)
+                check_regexes(res)
-                    res = check_output('calibredb add "{}" -d {}'.format(cur, path), shell=True,stderr=STDOUT,stdin=PIPE, )
+                cur = get_files(loc, '.epub', True)[0]
-                    try:
+                name = get_files(loc, '.epub', False)[0]
-                        res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=STDOUT,stdin=PIPE, )
+                rename(cur, name)
-                        print "\tAdded {} to library with id {}".format(cur, res)
+                print "Downloaded story {} to {}".format(story_name.search(name).group(1), name)
-                    except:
+        except Exception as e:
-                        print "It's been added to library, but not sure what the ID is."
+            print "Exception: {}".format(e)
-                    remove(cur)
+            rmtree(loc)
-                else:
+            loc = mkdtemp()
-                    res = check_output('cd "{}" && fanficfare -u "{}" --update-cover'.format(loc, url), shell=True,stderr=STDOUT,stdin=PIPE, )
+            with open(inout_file, "a") as fp:
-                    check_regexes(res)
-                    cur = get_files(loc, '.epub', True)[0]
-                    name = get_files(loc, '.epub', False)[0]
-                    rename(cur, name)
-                    print "Downloaded story {} to {}".format(story_name.search(name).group(1), name)
-            except Exception as e:
-                print "Exception: {}".format(e)
-                rmtree(loc)
-                loc = mkdtemp()
                 fp.write("{}\n".format(url))
-                continue
+            continue
  
         rmtree(loc)
+    return
 
 
 if __name__ == "__main__":

runner_notify.py

@@ -4,17 +4,20 @@ from subprocess import check_output, STDOUT
 
 from sys import platform
 from os import utime
+from os.path import join
 
-if platform == "linux" or platform == "linux2":
+'''if platform == "linux" or platform == "linux2":
     from giNotify import Notification
 elif platform == "win32":
-    from ballonNotify import Notification
+    from ballonNotify import Notification'''
+from notifications import Notification
 
 from optparse import OptionParser
 from ConfigParser import ConfigParser
+import sqlite3
+
 
 def enable_notifications(options):
-    notary_options = []
     if options.pushbullet:
         pb = Pushbullet(options.pushbullet)
         if options.pbdevice:
@@ -25,14 +28,13 @@ def enable_notifications(options):
                 pass
         temp_note = Notification()
         temp_note.send_notification = pb.push_note
-        
+        yield temp_note
-        notary_options.append(temp_note)
         
     if options.notify:
         notary = Notification()
-        notary_options.append(notary)
+        yield notary
         
-    return notary_options
+    
     
 def touch(fname, times=None):
     with open(fname, 'a'):
@@ -40,17 +42,28 @@ def touch(fname, times=None):
     
     
 def main(options):
-
+    try:
-    res = check_output("python fanficdownload.py -c config.ini", shell=True,stderr=STDOUT)
+        res = check_output("python fanficdownload.py -c config.ini", shell=True,stderr=STDOUT)
+    except Exception as e:
+        print e
+        res = None
+    if not res: return
+    if res: print res
     buf = StringIO(res)
     regex = re.compile("Added (?:.*/)?(.*)-.* to library with id \d*")
+    searcher = regex.search
+    stripper = False
     for line in buf.readlines():
-        r = regex.search(line)
+        r = searcher(line)
         if r:
             story = r.group(1).strip()
+            stripper = True
             for notify in enable_notifications(options):
                 notify.send_notification("New Fanfiction Download", story)
-    if res != "": print res
+    if stripper and options.tag:
+        with sqlite3.connect(join(options.library_path, "metadata.db")) as conn:
+            c = conn.cursor()
+            c.execute("delete from books_tags_link where id in (select id from books_tags_link where tag in (select id from tags where name like '%Last Update%'));")
     return
 
 if __name__ == "__main__":
@@ -59,9 +72,14 @@ if __name__ == "__main__":
     option_parser.add_option('-d', '--device', action='store', dest='pbdevice', help='If you wish to only send to a certian pushbullet device, put the device name here. If the device name is invalid, will just send to all pushbullets associated with the acc')
     option_parser.add_option('-n', '--notify', action='store_true', dest='notify', help='Enable if you want to use system notifications. Only for Win/Linux.')
     option_parser.add_option('-c', '--config', action='store', dest='config', help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.')
+    option_parser.add_option('-t', '--tag', action='store_true', dest='tag', help='Strip Last Updated tags from calibredb. Requires library to be passed in.')
+    option_parser.add_option('-l', '--library', action='store', dest='library', help='Path to calibre library. If you are connecting to a calibre webserver then this should be the url.')
+    option_parser.add_option('-a', '--library-path', action='store', dest='library_path', help='Path location of library. Will be equal to library if nothing is passed in.')
     
     (options, args) = option_parser.parse_args()
     
+    if options.library and not options.library_path: options.library_path = options.library
+    
     if options.config:
         touch(options.config)
         config = ConfigParser(allow_no_value=True)
@@ -76,9 +94,20 @@ if __name__ == "__main__":
         
         try: options.pbdevice = updater(options.pbdevice, config.get('runner', 'pbdevice'))
         except: pass
+
+        try: options.tag = updater(options.tag, config.getboolean('runner', 'tag'))
+        except: pass
+
+        try: options.library = updater(options.library, config.get('locations', 'library'))
+        except: pass
+        
+        try: options.library_path = updater(options.library, config.get('locations', 'library_path'))
+        except: pass
     
     if options.pbdevice and not options.pushbullet:
         raise ValueError("Can't use a pushbullet device without key")
+    if options.tag and not options.library:
+	raise ValueError("Can't strip tags from calibre library without a library location.")
     if options.pushbullet:
         from pushbullet import Pushbullet