You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
176 lines
7.8 KiB
Python
176 lines
7.8 KiB
Python
from fanficfare import geturls
|
|
from os import listdir, remove, rename, utime, errno, devnull
|
|
from os.path import isfile, join
|
|
from subprocess import check_output, STDOUT, call
|
|
import logging
|
|
from optparse import OptionParser
|
|
import re
|
|
from ConfigParser import ConfigParser
|
|
from tempfile import mkdtemp
|
|
from shutil import rmtree
|
|
|
|
logging.getLogger("fanficfare").setLevel(logging.ERROR)
|
|
|
|
def touch(fname, times=None):
|
|
with open(fname, 'a'):
|
|
utime(fname, times)
|
|
|
|
|
|
ffnet = re.compile('(fanfiction.net/s/\d*)/?.*')
|
|
neutral = re.compile('https?://(.*)')
|
|
story_name = re.compile('(.*)-.*')
|
|
|
|
def parse_url(url):
|
|
if ffnet.search(url):
|
|
url = "www." + ffnet.search(url).group(1)
|
|
elif neutral.search(url):
|
|
url = neutral.search(url).group(1)
|
|
return url
|
|
|
|
def get_files(mypath, filetype=None, fullpath=False):
|
|
ans = []
|
|
if filetype:
|
|
ans = [f for f in listdir(mypath) if isfile(join(mypath, f)) and f.endswith(filetype)]
|
|
else:
|
|
ans = [f for f in listdir(mypath) if isfile(join(mypath, f))]
|
|
if fullpath:
|
|
return [join(mypath, f) for f in ans]
|
|
else:
|
|
return ans
|
|
|
|
def main(user, password, server, label, inout_file, path ):
|
|
|
|
|
|
|
|
if path:
|
|
path = '--with-library "{}"'.format(path)
|
|
try:
|
|
with open(devnull, 'w') as nullout:
|
|
call(['calibredb'], stdout=nullout, stderr=nullout)
|
|
except OSError as e:
|
|
if errno == ENOENT:
|
|
print "Calibredb is not installed on this system. Cannot search the calibre library or update it."
|
|
return
|
|
|
|
touch(inout_file)
|
|
|
|
with open(inout_file, "r") as fp:
|
|
urls = set([x.replace("\n", "") for x in fp.readlines()])
|
|
|
|
with open(inout_file, "w") as fp:
|
|
fp.write("")
|
|
urls |= geturls.get_urls_from_imap(server, user, password, label)
|
|
|
|
urls = set(parse_url(x) for x in urls)
|
|
|
|
if len(urls) != 0: print "URLs to parse: {}".format(", ".join(urls))
|
|
|
|
loc = mkdtemp()
|
|
|
|
|
|
|
|
for url in urls:
|
|
print "Working with url {}".format(url)
|
|
storyId = None
|
|
try:
|
|
if path:
|
|
try:
|
|
res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT)
|
|
storyId = res
|
|
print "\tStory is in calibre with id {}".format(storyId)
|
|
print "\tExporting file"
|
|
res = check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format(storyId, loc, path), shell=True)
|
|
cur = get_files(loc, ".epub", True)[0]
|
|
print '\tDownloading with fanficfare, updating file "{}"'.format(cur)
|
|
moving=""
|
|
except:
|
|
#story is not in calibre
|
|
cur = url
|
|
moving = 'cd "{}" && '.format(loc)
|
|
|
|
res = check_output('{}fanficfare -u "{}" --update-cover'.format(moving, cur), shell=True,stderr=STDOUT)
|
|
|
|
if "already contains" in res:
|
|
raise ValueError("Issue with story, site is broken. Story likely hasn't updated on site yet.")
|
|
elif "Story does not exist" in res:
|
|
raise ValueError("Invalid URL")
|
|
elif "more recently than Story" in res:
|
|
print "\tForcing download update\n"
|
|
res = check_output('{}fanficfare -u "{}" --force --update-cover'.format(moving, cur), shell=True,stderr=STDOUT)
|
|
cur = get_files(loc, '.epub', True)[0]
|
|
|
|
|
|
if storyId:
|
|
print "\tRemoving {} from library".format(storyId)
|
|
res = check_output('calibredb remove {} {}'.format(storyId, path), shell=True,stderr=STDOUT)
|
|
|
|
print "\tAdding {} to library".format(cur)
|
|
res = check_output('calibredb add "{}" {}'.format(cur, path), shell=True,stderr=STDOUT)
|
|
res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=STDOUT)
|
|
print "\tAdded {} to library with id {}".format(cur, res)
|
|
remove(cur)
|
|
else:
|
|
res = check_output('cd "{}" && fanficfare -u "{}" --update-cover'.format(loc, url), shell=True,stderr=STDOUT)
|
|
if "Story does not exist" in res:
|
|
raise ValueError("Invalid URL")
|
|
cur = get_files(loc, '.epub', True)[0]
|
|
name = get_files(loc, '.epub', False)[0]
|
|
rename(cur, name)
|
|
print "Downloaded story {} to {}".format(story_name.search(name).group(1), name)
|
|
except Exception as e:
|
|
print "Exception: {}".format(e)
|
|
rmtree(loc)
|
|
loc = mkdtemp()
|
|
fp.write("{}\n".format(url))
|
|
continue
|
|
|
|
rmtree(loc)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
option_parser = OptionParser(usage="usage: %prog [flags]")
|
|
|
|
option_parser.add_option('-u', '--user', action='store', dest='user', help='Email Account Username. Required.')
|
|
|
|
option_parser.add_option('-p', '--password', action='store', dest='password', help='Email Account Password. Required.')
|
|
|
|
option_parser.add_option('-s', '--server', action='store', dest='server', default="imap.gmail.com", help='Email IMAP Server. Default is "imap.gmail.com".')
|
|
|
|
option_parser.add_option('-m', '--mailbox', action='store', dest='mailbox', default='INBOX', help='Email Label. Default is "INBOX".')
|
|
|
|
option_parser.add_option('-l', '--library', action='store', dest='library', help="calibre library db location. If none is passed, then this merely scrapes the email and error file for new stories and downloads them into the current directory.")
|
|
|
|
option_parser.add_option('-i', '--input', action='store', dest='input', default="./fanfiction.txt", help="Error file. Any urls that fail will be output here, and file will be read to find any urls that failed previously. If file does not exist will create. File is overwitten every time the program is run.")
|
|
|
|
option_parser.add_option('-c', '--config', action='store', dest='config', help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.')
|
|
|
|
(options, args) = option_parser.parse_args()
|
|
|
|
if options.config:
|
|
config = ConfigParser(allow_no_value=True)
|
|
config.read(options.config)
|
|
|
|
updater = lambda option, newval : newval if newval != "" else option
|
|
|
|
options.user = updater(options.user, config.get('login', 'user').strip())
|
|
|
|
options.password = updater(options.password, config.get('login', 'password').strip())
|
|
|
|
options.server = updater(options.server, config.get('login', 'server').strip())
|
|
|
|
options.mailbox = updater(options.mailbox, config.get('login', 'mailbox').strip())
|
|
|
|
options.library = updater(options.library, config.get('locations', 'library').strip())
|
|
|
|
options.input = updater(options.input, config.get('locations', 'input').strip())
|
|
|
|
if not (options.user or options.password):
|
|
raise ValueError("User or Password not given")
|
|
|
|
main(options.user, options.password, options.server, options.mailbox, options.input, options.library)
|
|
|
|
|
|
|
|
|
|
|