sam
/
AutomatedFanfic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Now if option is enabled and there are multiple links being passed in at once, will make use of multiprocessing in order to speed up download and processing times. Potential race conditions in calibre add and database editing, so should test and fix that.

Browse Source
master
MrTyton 7 years ago
parent 56b067d12f
commit 0615c1dd21
1 changed files with 97 additions and 79 deletions
Show Stats Download Patch File Download Diff File

140
fanficdownload.py
Unescape Escape View File

@ -11,6 +11,8 @@ from shutil import rmtree
import socket
from time import strftime, localtime
from multiprocessing import Pool
logging.getLogger("fanficfare").setLevel(logging.ERROR)
class bcolors:
@ -23,7 +25,7 @@ class bcolors:
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
def log(msg, color=None):
def log(msg, color=None, output=True):
if color:
col = bcolors.HEADER
if color == 'BLUE':
@ -38,9 +40,14 @@ def log(msg, color=None):
col = bcolors.BOLD
elif color == 'UNDERLINE':
col = bcolors.UNDERLINE
print '{}{}{}: \t {}{}{}'.format(bcolors.BOLD, strftime('%m/%d/%Y %H:%M:%S', localtime()), bcolors.ENDC, col, msg, bcolors.ENDC)
line = '{}{}{}: \t {}{}{}'.format(bcolors.BOLD, strftime('%m/%d/%Y %H:%M:%S', localtime()), bcolors.ENDC, col, msg, bcolors.ENDC)
else:
line = '{}{}{}: \t {}'.format(bcolors.BOLD, strftime('%m/%d/%Y %H:%M:%S', localtime()), bcolors.ENDC, msg)
if output:
print line
return ""
else:
print '{}{}{}: \t {}'.format(bcolors.BOLD, strftime('%m/%d/%Y %H:%M:%S', localtime()), bcolors.ENDC, msg)
return line + "\n"
def touch(fname, times=None):
with open(fname, 'a'):
@ -89,96 +96,60 @@ def check_regexes(output):
if no_url.search(output):
raise ValueError("No URL in epub to update from. Fix the metadata.")
def main(user, password, server, label, inout_file, path ):
if path:
path = '--with-library "{}" --username calibre --password pornoboobies'.format(path)
try:
with open(devnull, 'w') as nullout:
call(['calibredb'], stdout=nullout, stderr=nullout)
except OSError as e:
if errno == ENOENT:
log("Calibredb is not installed on this system. Cannot search the calibre library or update it.", 'FAIL')
return
touch(inout_file)
with open(inout_file, "r") as fp:
urls = set([x.replace("\n", "") for x in fp.readlines()])
with open(inout_file, "w") as fp:
fp.write("")
try:
socket.setdefaulttimeout(55)
urls |= geturls.get_urls_from_imap(server, user, password, label)
socket.setdefaulttimeout(None)
except Exception as e:
log("Broke while getting URLs: {}".format(e), 'FAIL')
with open(inout_file, "w") as fp:
for cur in urls:
fp.write("{}\n".format(cur))
return
if not urls: return
urls = set(parse_url(x) for x in urls)
log("URLs to parse:", 'HEADER')
for url in urls:
log("\t{}".format(url), 'BLUE')
def downloader(args):
url, inout_file, path, live = args
loc = mkdtemp()
for url in urls:
log("Working with url {}".format(url), 'HEADER')
output = ""
output += log("Working with url {}".format(url), 'HEADER', live)
storyId = None
try:
if path:
try:
res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT,stdin=PIPE, )
storyId = res
log("\tStory is in calibre with id {}".format(storyId), 'BLUE')
log("\tExporting file", 'BLUE')
storyId = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT,stdin=PIPE, )
output += log("\tStory is in calibre with id {}".format(storyId), 'BLUE', live)
output += log("\tExporting file", 'BLUE', live)
res = check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format(storyId, loc, path), shell=True, stdin=PIPE, stderr=STDOUT)
cur = get_files(loc, ".epub", True)[0]
log('\tDownloading with fanficfare, updating file "{}"'.format(cur), 'GREEN')
output += log('\tDownloading with fanficfare, updating file "{}"'.format(cur), 'GREEN', live)
moving=""
except:
#story is not in calibre
cur = url
moving = 'cd "{}" && '.format(loc)
res = check_output('cp personal.ini {}/personal.ini'.format(loc), shell=True, stderr=STDOUT, stdin=PIPE,)
log('\tRunning: {}fanficfare -u "{}" --update-cover'.format(moving, cur), 'BLUE')
output += log('\tRunning: {}fanficfare -u "{}" --update-cover'.format(moving, cur), 'BLUE', live)
res = check_output('{}fanficfare -u "{}" --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
check_regexes(res)
if chapter_difference.search(res) or more_chapters.search(res):
log("\tForcing download update due to:", 'WARNING')
output += log("\tForcing download update due to:", 'WARNING', live)
for line in res.split("\n"):
if line:
log("\t\t{}".format(line), 'WARNING')
output += log("\t\t{}".format(line), 'WARNING', live)
res = check_output('{}fanficfare -u "{}" --force --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
check_regexes(res)
cur = get_files(loc, '.epub', True)[0]
if storyId:
log("\tRemoving {} from library".format(storyId), 'BLUE')
output += log("\tRemoving {} from library".format(storyId), 'BLUE', live)
try:
res = check_output('calibredb remove {} {}'.format(path, storyId), shell=True,stderr=STDOUT,stdin=PIPE, )
except:
raise
log("\tAdding {} to library".format(cur), 'BLUE')
output += log("\tAdding {} to library".format(cur), 'BLUE', live)
try:
res = check_output('calibredb add -d {} "{}"'.format(path, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
except Exception as e:
log(e)
output += log(e)
raise
try:
res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=STDOUT,stdin=PIPE, )
log("\tAdded {} to library with id {}".format(cur, res), 'GREEN')
output += log("\tAdded {} to library with id {}".format(cur, res), 'GREEN', live)
except:
log("It's been added to library, but not sure what the ID is.", 'WARNING')
log("Added file to library with id 0", 'GREEN')
output += log("It's been added to library, but not sure what the ID is.", 'WARNING', live)
output += log("Added file to library with id 0", 'GREEN', live)
remove(cur)
else:
res = check_output('cd "{}" && fanficfare -u "{}" --update-cover'.format(loc, url), shell=True,stderr=STDOUT,stdin=PIPE, )
@ -186,9 +157,12 @@ def main(user, password, server, label, inout_file, path ):
cur = get_files(loc, '.epub', True)[0]
name = get_files(loc, '.epub', False)[0]
rename(cur, name)
log("Downloaded story {} to {}".format(story_name.search(name).group(1), name), 'GREEN')
output += log("Downloaded story {} to {}".format(story_name.search(name).group(1), name), 'GREEN', live)
if not live: print output.strip()
rmtree(loc)
except Exception as e:
log("Exception: {}".format(e), 'FAIL')
output += log("Exception: {}".format(e), 'FAIL', live)
if not live: print output.strip()
try:
rmtree(loc)
except:
@ -196,9 +170,48 @@ def main(user, password, server, label, inout_file, path ):
loc = mkdtemp()
with open(inout_file, "a") as fp:
fp.write("{}\n".format(url))
continue
rmtree(loc)
def main(user, password, server, label, inout_file, path, live ):
if path:
path = '--with-library "{}" --username calibre --password pornoboobies'.format(path)
try:
with open(devnull, 'w') as nullout:
call(['calibredb'], stdout=nullout, stderr=nullout)
except OSError as e:
if errno == ENOENT:
log("Calibredb is not installed on this system. Cannot search the calibre library or update it.", 'FAIL')
return
touch(inout_file)
with open(inout_file, "r") as fp:
urls = set([x.replace("\n", "") for x in fp.readlines()])
with open(inout_file, "w") as fp:
fp.write("")
try:
socket.setdefaulttimeout(55)
urls |= geturls.get_urls_from_imap(server, user, password, label)
socket.setdefaulttimeout(None)
except Exception as e:
#log("Broke while getting URLs: {}".format(e), 'FAIL')
with open(inout_file, "w") as fp:
for cur in urls:
fp.write("{}\n".format(cur))
return
if not urls: return
urls = set(parse_url(x) for x in urls)
log("URLs to parse ({}):".format(len(urls)), 'HEADER')
for url in urls:
log("\t{}".format(url), 'BLUE')
p = Pool()
p.map(downloader, [[url, inout_file, path, live] for url in urls])
return
@ -219,6 +232,8 @@ if __name__ == "__main__":
option_parser.add_option('-c', '--config', action='store', dest='config', help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.')
option_parser.add_option('-o', '--output', action='store_true', dest='live', help='Include this if you want all the output to be saved and posted live. Useful when multithreading.')
(options, args) = option_parser.parse_args()
if options.config:
@ -245,10 +260,13 @@ if __name__ == "__main__":
try: options.input = updater(options.input, config.get('locations', 'input').strip())
except: pass
try: options.live = updater(options.live, config.getboolean('output', 'live').strip())
except: pass
if not (options.user or options.password):
raise ValueError("User or Password not given")
main(options.user, options.password, options.server, options.mailbox, options.input, options.library)
main(options.user, options.password, options.server, options.mailbox, options.input, options.library, options.live)

Write Preview
Loading…
Cancel
Save