From d0df07a055e71f8d9a98c54474feade73acf6ad2 Mon Sep 17 00:00:00 2001 From: joshua Date: Sun, 2 Jul 2017 22:57:37 -0400 Subject: [PATCH] Updates to calibre 3.0 Random bugfixes Fixing of git --- fanficdownload.py | 142 +++++++++++++++++++++++++++------------------- runner_notify.py | 51 +++++++++++++---- 2 files changed, 123 insertions(+), 70 deletions(-) diff --git a/fanficdownload.py b/fanficdownload.py index fb9ad93..1e98470 100644 --- a/fanficdownload.py +++ b/fanficdownload.py @@ -8,6 +8,7 @@ import re from ConfigParser import ConfigParser from tempfile import mkdtemp from shutil import rmtree +import socket logging.getLogger("fanficfare").setLevel(logging.ERROR) @@ -17,7 +18,7 @@ def touch(fname, times=None): ffnet = re.compile('(fanfiction.net/s/\d*)/?.*') -aooo = re.compile('(archiveofourown.org/works/\d*)/?.') +aooo = re.compile('(archiveofourown.org/works/\d*)/?.*') neutral = re.compile('https?://(.*)') story_name = re.compile('(.*)-.*') @@ -32,7 +33,7 @@ def parse_url(url): if ffnet.search(url): url = "www." + ffnet.search(url).group(1) elif aooo.search(url): - url = aooo.search(url).group(1) + url = "www." + aooo.search(url).group(1) elif neutral.search(url): url = neutral.search(url).group(1) return url @@ -58,7 +59,7 @@ def check_regexes(output): def main(user, password, server, label, inout_file, path ): if path: - path = '--with-library "{}"'.format(path) + path = '--with-library "{}" --username calibre --password pornoboobies'.format(path) try: with open(devnull, 'w') as nullout: call(['calibredb'], stdout=nullout, stderr=nullout) @@ -71,74 +72,97 @@ def main(user, password, server, label, inout_file, path ): with open(inout_file, "r") as fp: urls = set([x.replace("\n", "") for x in fp.readlines()]) - + with open(inout_file, "w") as fp: fp.write("") + + try: + socket.setdefaulttimeout(55) urls |= geturls.get_urls_from_imap(server, user, password, label) - - urls = set(parse_url(x) for x in urls) - - if len(urls) != 0: print "URLs to parse: {}".format(", ".join(urls)) + socket.setdefaulttimeout(None) + except Exception as e: + print "Broke while getting URLs: {}".format(e) + with open(inout_file, "w") as fp: + for cur in urls: + fp.write("{}\n".format(cur)) + return + + if not urls: return + urls = set(parse_url(x) for x in urls) + + + if len(urls): print "URLs to parse: {}".format(", ".join(urls)) - loc = mkdtemp() + loc = mkdtemp() - - for url in urls: - print "Working with url {}".format(url) - storyId = None - try: - if path: - try: - res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT,stdin=PIPE, ) - storyId = res - print "\tStory is in calibre with id {}".format(storyId) - print "\tExporting file" - res = check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format(storyId, loc, path), shell=True, stdin=PIPE, stderr=STDOUT) - cur = get_files(loc, ".epub", True)[0] - print '\tDownloading with fanficfare, updating file "{}"'.format(cur) - moving="" - except: - #story is not in calibre - cur = url - moving = 'cd "{}" && '.format(loc) - print '{}fanficfare -u "{}" --update-cover'.format(moving, cur) - res = check_output('{}fanficfare -u "{}" --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, ) + + for url in urls: + print "Working with url {}".format(url) + storyId = None + try: + if path: + try: + res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT,stdin=PIPE, ) + storyId = res + print "\tStory is in calibre with id {}".format(storyId) + print "\tExporting file" + res = check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format(storyId, loc, path), shell=True, stdin=PIPE, stderr=STDOUT) + cur = get_files(loc, ".epub", True)[0] + print '\tDownloading with fanficfare, updating file "{}"'.format(cur) + moving="" + except: + #story is not in calibre + cur = url + moving = 'cd "{}" && '.format(loc) + res = check_output('cp personal.ini {}/personal.ini'.format(loc), shell=True, stderr=STDOUT, stdin=PIPE,) + print '{}fanficfare -u "{}" --update-cover'.format(moving, cur) + res = check_output('{}fanficfare -u "{}" --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, ) + check_regexes(res) + if chapter_difference.search(res) or more_chapters.search(res): + print "\tForcing download update due to: {}\n".format(res) + res = check_output('{}fanficfare -u "{}" --force --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, ) check_regexes(res) - if chapter_difference.search(res) or more_chapters.search(res): - print "\tForcing download update\n" - res = check_output('{}fanficfare -u "{}" --force --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, ) - check_regexes(res) - cur = get_files(loc, '.epub', True)[0] - - - if storyId: - print "\tRemoving {} from library".format(storyId) - res = check_output('calibredb remove {} {}'.format(storyId, path), shell=True,stderr=STDOUT,stdin=PIPE, ) - - print "\tAdding {} to library".format(cur) - res = check_output('calibredb add "{}" -d {}'.format(cur, path), shell=True,stderr=STDOUT,stdin=PIPE, ) + cur = get_files(loc, '.epub', True)[0] + + + if storyId: + print "\tRemoving {} from library".format(storyId) try: - res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=STDOUT,stdin=PIPE, ) - print "\tAdded {} to library with id {}".format(cur, res) + res = check_output('calibredb remove {} {}'.format(path, storyId), shell=True,stderr=STDOUT,stdin=PIPE, ) except: - print "It's been added to library, but not sure what the ID is." - remove(cur) - else: - res = check_output('cd "{}" && fanficfare -u "{}" --update-cover'.format(loc, url), shell=True,stderr=STDOUT,stdin=PIPE, ) - check_regexes(res) - cur = get_files(loc, '.epub', True)[0] - name = get_files(loc, '.epub', False)[0] - rename(cur, name) - print "Downloaded story {} to {}".format(story_name.search(name).group(1), name) - except Exception as e: - print "Exception: {}".format(e) - rmtree(loc) - loc = mkdtemp() + raise + + print "\tAdding {} to library".format(cur) + try: + res = check_output('calibredb add -d {} "{}"'.format(path, cur), shell=True,stderr=STDOUT,stdin=PIPE, ) + except Exception as e: + print e + raise + try: + res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=STDOUT,stdin=PIPE, ) + print "\tAdded {} to library with id {}".format(cur, res) + except: + print "It's been added to library, but not sure what the ID is." + print "Added file to library with id 0" + remove(cur) + else: + res = check_output('cd "{}" && fanficfare -u "{}" --update-cover'.format(loc, url), shell=True,stderr=STDOUT,stdin=PIPE, ) + check_regexes(res) + cur = get_files(loc, '.epub', True)[0] + name = get_files(loc, '.epub', False)[0] + rename(cur, name) + print "Downloaded story {} to {}".format(story_name.search(name).group(1), name) + except Exception as e: + print "Exception: {}".format(e) + rmtree(loc) + loc = mkdtemp() + with open(inout_file, "a") as fp: fp.write("{}\n".format(url)) - continue + continue rmtree(loc) + return if __name__ == "__main__": diff --git a/runner_notify.py b/runner_notify.py index 4ab3aa3..463eed2 100644 --- a/runner_notify.py +++ b/runner_notify.py @@ -4,17 +4,20 @@ from subprocess import check_output, STDOUT from sys import platform from os import utime +from os.path import join -if platform == "linux" or platform == "linux2": +'''if platform == "linux" or platform == "linux2": from giNotify import Notification elif platform == "win32": - from ballonNotify import Notification + from ballonNotify import Notification''' +from notifications import Notification from optparse import OptionParser from ConfigParser import ConfigParser +import sqlite3 + def enable_notifications(options): - notary_options = [] if options.pushbullet: pb = Pushbullet(options.pushbullet) if options.pbdevice: @@ -25,14 +28,13 @@ def enable_notifications(options): pass temp_note = Notification() temp_note.send_notification = pb.push_note - - notary_options.append(temp_note) + yield temp_note if options.notify: notary = Notification() - notary_options.append(notary) + yield notary - return notary_options + def touch(fname, times=None): with open(fname, 'a'): @@ -40,17 +42,28 @@ def touch(fname, times=None): def main(options): - - res = check_output("python fanficdownload.py -c config.ini", shell=True,stderr=STDOUT) + try: + res = check_output("python fanficdownload.py -c config.ini", shell=True,stderr=STDOUT) + except Exception as e: + print e + res = None + if not res: return + if res: print res buf = StringIO(res) regex = re.compile("Added (?:.*/)?(.*)-.* to library with id \d*") + searcher = regex.search + stripper = False for line in buf.readlines(): - r = regex.search(line) + r = searcher(line) if r: story = r.group(1).strip() + stripper = True for notify in enable_notifications(options): notify.send_notification("New Fanfiction Download", story) - if res != "": print res + if stripper and options.tag: + with sqlite3.connect(join(options.library_path, "metadata.db")) as conn: + c = conn.cursor() + c.execute("delete from books_tags_link where id in (select id from books_tags_link where tag in (select id from tags where name like '%Last Update%'));") return if __name__ == "__main__": @@ -59,9 +72,14 @@ if __name__ == "__main__": option_parser.add_option('-d', '--device', action='store', dest='pbdevice', help='If you wish to only send to a certian pushbullet device, put the device name here. If the device name is invalid, will just send to all pushbullets associated with the acc') option_parser.add_option('-n', '--notify', action='store_true', dest='notify', help='Enable if you want to use system notifications. Only for Win/Linux.') option_parser.add_option('-c', '--config', action='store', dest='config', help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.') + option_parser.add_option('-t', '--tag', action='store_true', dest='tag', help='Strip Last Updated tags from calibredb. Requires library to be passed in.') + option_parser.add_option('-l', '--library', action='store', dest='library', help='Path to calibre library. If you are connecting to a calibre webserver then this should be the url.') + option_parser.add_option('-a', '--library-path', action='store', dest='library_path', help='Path location of library. Will be equal to library if nothing is passed in.') (options, args) = option_parser.parse_args() + if options.library and not options.library_path: options.library_path = options.library + if options.config: touch(options.config) config = ConfigParser(allow_no_value=True) @@ -76,9 +94,20 @@ if __name__ == "__main__": try: options.pbdevice = updater(options.pbdevice, config.get('runner', 'pbdevice')) except: pass + + try: options.tag = updater(options.tag, config.getboolean('runner', 'tag')) + except: pass + + try: options.library = updater(options.library, config.get('locations', 'library')) + except: pass + + try: options.library_path = updater(options.library, config.get('locations', 'library_path')) + except: pass if options.pbdevice and not options.pushbullet: raise ValueError("Can't use a pushbullet device without key") + if options.tag and not options.library: + raise ValueError("Can't strip tags from calibre library without a library location.") if options.pushbullet: from pushbullet import Pushbullet