sam
/
AutomatedFanfic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

pep8 formatting.

Browse Source
master
MrTyton 7 years ago
parent cc14dd210c
commit 04ab385457
2 changed files with 350 additions and 146 deletions
Show Stats Download Patch File Download Diff File

346
fanficdownload.py
Unescape Escape View File

@ -1,7 +1,7 @@
from fanficfare import geturls
from os import listdir, remove, rename, utime, errno, devnull
from os.path import isfile, join
from subprocess import check_output, STDOUT, call,PIPE
from subprocess import check_output, STDOUT, call, PIPE
import logging
from optparse import OptionParser
import re
@ -10,11 +10,13 @@ from tempfile import mkdtemp
from shutil import rmtree
import socket
from time import strftime, localtime
from errno import ENOENT
from multiprocessing import Pool
logging.getLogger("fanficfare").setLevel(logging.ERROR)
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
@ -25,6 +27,7 @@ class bcolors:
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
def log(msg, color=None, output=True):
if color:
col = bcolors.HEADER
@ -40,20 +43,33 @@ def log(msg, color=None, output=True):
col = bcolors.BOLD
elif color == 'UNDERLINE':
col = bcolors.UNDERLINE
line = '{}{}{}: \t {}{}{}'.format(bcolors.BOLD, strftime('%m/%d/%Y %H:%M:%S', localtime()), bcolors.ENDC, col, msg, bcolors.ENDC)
line = '{}{}{}: \t {}{}{}'.format(
bcolors.BOLD,
strftime(
'%m/%d/%Y %H:%M:%S',
localtime()),
bcolors.ENDC,
col,
msg,
bcolors.ENDC)
else:
line = '{}{}{}: \t {}'.format(bcolors.BOLD, strftime('%m/%d/%Y %H:%M:%S', localtime()), bcolors.ENDC, msg)
line = '{}{}{}: \t {}'.format(
bcolors.BOLD,
strftime(
'%m/%d/%Y %H:%M:%S',
localtime()),
bcolors.ENDC,
msg)
if output:
print line
return ""
else:
return line + "\n"
def touch(fname, times=None):
with open(fname, 'a'):
utime(fname, times)
ffnet = re.compile('(fanfiction.net/s/\d*)/?.*')
@ -62,10 +78,13 @@ neutral = re.compile('https?://(.*)')
story_name = re.compile('(.*)-.*')
equal_chapters = re.compile('.* already contains \d* chapters.')
chapter_difference = re.compile('.* contains \d* chapters, more than source: \d*.')
bad_chapters = re.compile(".* doesn't contain any recognizable chapters, probably from a different source. Not updating.")
chapter_difference = re.compile(
'.* contains \d* chapters, more than source: \d*.')
bad_chapters = re.compile(
".* doesn't contain any recognizable chapters, probably from a different source. Not updating.")
no_url = re.compile('No story URL found in epub to update.')
more_chapters = re.compile(".*File\(.*\.epub\) Updated\(.*\) more recently than Story\(.*\) - Skipping")
more_chapters = re.compile(
".*File\(.*\.epub\) Updated\(.*\) more recently than Story\(.*\) - Skipping")
def parse_url(url):
@ -76,27 +95,32 @@ def parse_url(url):
elif neutral.search(url):
url = neutral.search(url).group(1)
return url
def get_files(mypath, filetype=None, fullpath=False):
ans = []
if filetype:
ans = [f for f in listdir(mypath) if isfile(join(mypath, f)) and f.endswith(filetype)]
ans = [f for f in listdir(mypath) if isfile(
join(mypath, f)) and f.endswith(filetype)]
else:
ans = [f for f in listdir(mypath) if isfile(join(mypath, f))]
if fullpath:
return [join(mypath, f) for f in ans]
else:
return ans
def check_regexes(output):
if equal_chapters.search(output):
raise ValueError("Issue with story, site is broken. Story likely hasn't updated on site yet.")
raise ValueError(
"Issue with story, site is broken. Story likely hasn't updated on site yet.")
if bad_chapters.search(output):
raise ValueError("Something is messed up with the site or the epub. No chapters found.")
raise ValueError(
"Something is messed up with the site or the epub. No chapters found.")
if no_url.search(output):
raise ValueError("No URL in epub to update from. Fix the metadata.")
def downloader(args):
url, inout_file, path, live = args
loc = mkdtemp()
@ -106,90 +130,133 @@ def downloader(args):
try:
if path:
try:
storyId = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT,stdin=PIPE, )
storyId = check_output(
'calibredb search "Identifiers:{}" {}'.format(
url, path), shell=True, stderr=STDOUT, stdin=PIPE, )
output += log("\tStory is in calibre with id {}".format(storyId), 'BLUE', live)
output += log("\tExporting file", 'BLUE', live)
res = check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format(storyId, loc, path), shell=True, stdin=PIPE, stderr=STDOUT)
res = check_output(
'calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format(
storyId, loc, path), shell=True, stdin=PIPE, stderr=STDOUT)
cur = get_files(loc, ".epub", True)[0]
output += log('\tDownloading with fanficfare, updating file "{}"'.format(cur), 'GREEN', live)
moving=""
except:
#story is not in calibre
output += log(
'\tDownloading with fanficfare, updating file "{}"'.format(cur),
'GREEN',
live)
moving = ""
except BaseException:
# story is not in calibre
cur = url
moving = 'cd "{}" && '.format(loc)
res = check_output('cp personal.ini {}/personal.ini'.format(loc), shell=True, stderr=STDOUT, stdin=PIPE,)
output += log('\tRunning: {}fanficfare -u "{}" --update-cover'.format(moving, cur), 'BLUE', live)
res = check_output('{}fanficfare -u "{}" --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
res = check_output(
'cp personal.ini {}/personal.ini'.format(loc),
shell=True,
stderr=STDOUT,
stdin=PIPE,
)
output += log('\tRunning: {}fanficfare -u "{}" --update-cover'.format(
moving, cur), 'BLUE', live)
res = check_output('{}fanficfare -u "{}" --update-cover'.format(
moving, cur), shell=True, stderr=STDOUT, stdin=PIPE)
check_regexes(res)
if chapter_difference.search(res) or more_chapters.search(res):
output += log("\tForcing download update due to:", 'WARNING', live)
output += log("\tForcing download update due to:",
'WARNING', live)
for line in res.split("\n"):
if line:
output += log("\t\t{}".format(line), 'WARNING', live)
res = check_output('{}fanficfare -u "{}" --force --update-cover'.format(moving, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
res = check_output(
'{}fanficfare -u "{}" --force --update-cover'.format(
moving, cur), shell=True, stderr=STDOUT, stdin=PIPE)
check_regexes(res)
cur = get_files(loc, '.epub', True)[0]
if storyId:
output += log("\tRemoving {} from library".format(storyId), 'BLUE', live)
if storyId:
output += log("\tRemoving {} from library".format(storyId),
'BLUE', live)
try:
res = check_output('calibredb remove {} {}'.format(path, storyId), shell=True,stderr=STDOUT,stdin=PIPE, )
except:
if not live: print output.strip()
res = check_output(
'calibredb remove {} {}'.format(
path,
storyId),
shell=True,
stderr=STDOUT,
stdin=PIPE,
)
except BaseException:
if not live:
print output.strip()
raise
output += log("\tAdding {} to library".format(cur), 'BLUE', live)
try:
res = check_output('calibredb add -d {} "{}"'.format(path, cur), shell=True,stderr=STDOUT,stdin=PIPE, )
res = check_output(
'calibredb add -d {} "{}"'.format(path, cur), shell=True, stderr=STDOUT, stdin=PIPE, )
except Exception as e:
output += log(e)
if not live: print output.strip()
if not live:
print output.strip()
raise
try:
res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=STDOUT,stdin=PIPE, )
output += log("\tAdded {} to library with id {}".format(cur, res), 'GREEN', live)
except:
output += log("It's been added to library, but not sure what the ID is.", 'WARNING', live)
res = check_output(
'calibredb search "Identifiers:{}" {}'.format(
url, path), shell=True, stderr=STDOUT, stdin=PIPE)
output += log("\tAdded {} to library with id {}".format(cur,
res), 'GREEN', live)
except BaseException:
output += log(
"It's been added to library, but not sure what the ID is.",
'WARNING',
live)
output += log("Added file to library with id 0", 'GREEN', live)
remove(cur)
else:
res = check_output('cd "{}" && fanficfare -u "{}" --update-cover'.format(loc, url), shell=True,stderr=STDOUT,stdin=PIPE, )
res = check_output(
'cd "{}" && fanficfare -u "{}" --update-cover'.format(
loc, url), shell=True, stderr=STDOUT, stdin=PIPE)
check_regexes(res)
cur = get_files(loc, '.epub', True)[0]
name = get_files(loc, '.epub', False)[0]
rename(cur, name)
output += log("Downloaded story {} to {}".format(story_name.search(name).group(1), name), 'GREEN', live)
if not live: print output.strip()
output += log(
"Downloaded story {} to {}".format(
story_name.search(name).group(1),
name),
'GREEN',
live)
if not live:
print output.strip()
rmtree(loc)
except Exception as e:
output += log("Exception: {}".format(e), 'FAIL', live)
if not live: print output.strip()
if not live:
print output.strip()
try:
rmtree(loc)
except:
except BaseException:
pass
with open(inout_file, "a") as fp:
fp.write("{}\n".format(url))
def main(user, password, server, label, inout_file, path, live ):
def main(user, password, server, label, inout_file, path, live):
if path:
path = '--with-library "{}" --username calibre --password pornoboobies'.format(path)
path = '--with-library "{}" --username calibre --password pornoboobies'.format(
path)
try:
with open(devnull, 'w') as nullout:
call(['calibredb'], stdout=nullout, stderr=nullout)
except OSError as e:
call(['calibredb'], stdout=nullout, stderr=nullout)
except OSError:
if errno == ENOENT:
log("Calibredb is not installed on this system. Cannot search the calibre library or update it.", 'FAIL')
return
touch(inout_file)
with open(inout_file, "r") as fp:
urls = set([x.replace("\n", "") for x in fp.readlines()])
with open(inout_file, "w") as fp:
fp.write("")
@ -197,14 +264,14 @@ def main(user, password, server, label, inout_file, path, live ):
socket.setdefaulttimeout(55)
urls |= geturls.get_urls_from_imap(server, user, password, label)
socket.setdefaulttimeout(None)
except Exception as e:
#log("Broke while getting URLs: {}".format(e), 'FAIL')
except BaseException:
with open(inout_file, "w") as fp:
for cur in urls:
fp.write("{}\n".format(cur))
fp.write("{}\n".format(cur))
return
if not urls:
return
if not urls: return
urls = set(parse_url(x) for x in urls)
log("URLs to parse ({}):".format(len(urls)), 'HEADER')
for url in urls:
@ -220,58 +287,131 @@ def main(user, password, server, label, inout_file, path, live ):
if __name__ == "__main__":
option_parser = OptionParser(usage="usage: %prog [flags]")
option_parser.add_option('-u', '--user', action='store', dest='user', help='Email Account Username. Required.')
option_parser.add_option('-p', '--password', action='store', dest='password', help='Email Account Password. Required.')
option_parser.add_option('-s', '--server', action='store', dest='server', default="imap.gmail.com", help='Email IMAP Server. Default is "imap.gmail.com".')
option_parser.add_option('-m', '--mailbox', action='store', dest='mailbox', default='INBOX', help='Email Label. Default is "INBOX".')
option_parser.add_option('-l', '--library', action='store', dest='library', help="calibre library db location. If none is passed, then this merely scrapes the email and error file for new stories and downloads them into the current directory.")
option_parser.add_option('-i', '--input', action='store', dest='input', default="./fanfiction.txt", help="Error file. Any urls that fail will be output here, and file will be read to find any urls that failed previously. If file does not exist will create. File is overwitten every time the program is run.")
option_parser.add_option('-c', '--config', action='store', dest='config', help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.')
option_parser.add_option('-o', '--output', action='store_true', dest='live', help='Include this if you want all the output to be saved and posted live. Useful when multithreading.')
option_parser.add_option(
'-u',
'--user',
action='store',
dest='user',
help='Email Account Username. Required.')
option_parser.add_option(
'-p',
'--password',
action='store',
dest='password',
help='Email Account Password. Required.')
option_parser.add_option(
'-s',
'--server',
action='store',
dest='server',
default="imap.gmail.com",
help='Email IMAP Server. Default is "imap.gmail.com".')
option_parser.add_option(
'-m',
'--mailbox',
action='store',
dest='mailbox',
default='INBOX',
help='Email Label. Default is "INBOX".')
option_parser.add_option(
'-l',
'--library',
action='store',
dest='library',
help="calibre library db location. If none is passed, then this merely scrapes the email and error file for new stories and downloads them into the current directory.")
option_parser.add_option(
'-i',
'--input',
action='store',
dest='input',
default="./fanfiction.txt",
help="Error file. Any urls that fail will be output here, and file will be read to find any urls that failed previously. If file does not exist will create. File is overwitten every time the program is run.")
option_parser.add_option(
'-c',
'--config',
action='store',
dest='config',
help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.')
option_parser.add_option(
'-o',
'--output',
action='store_true',
dest='live',
help='Include this if you want all the output to be saved and posted live. Useful when multithreading.')
(options, args) = option_parser.parse_args()
if options.config:
touch(options.config)
config = ConfigParser(allow_no_value=True)
config.read(options.config)
updater = lambda option, newval : newval if newval != "" else option
try: options.user = updater(options.user, config.get('login', 'user').strip())
except: pass
try: options.password = updater(options.password, config.get('login', 'password').strip())
except: pass
try: options.server = updater(options.server, config.get('login', 'server').strip())
except: pass
try: options.mailbox = updater(options.mailbox, config.get('login', 'mailbox').strip())
except: pass
try: options.library = updater(options.library, config.get('locations', 'library').strip())
except: pass
try: options.input = updater(options.input, config.get('locations', 'input').strip())
except: pass
try: options.live = updater(options.live, config.getboolean('output', 'live').strip())
except: pass
def updater(option, newval): return newval if newval != "" else option
try:
options.user = updater(
options.user, config.get(
'login', 'user').strip())
except BaseException:
pass
try:
options.password = updater(
options.password, config.get(
'login', 'password').strip())
except BaseException:
pass
try:
options.server = updater(
options.server, config.get(
'login', 'server').strip())
except BaseException:
pass
try:
options.mailbox = updater(
options.mailbox, config.get(
'login', 'mailbox').strip())
except BaseException:
pass
try:
options.library = updater(
options.library, config.get(
'locations', 'library').strip())
except BaseException:
pass
try:
options.input = updater(
options.input, config.get(
'locations', 'input').strip())
except BaseException:
pass
try:
options.live = updater(
options.live, config.getboolean(
'output', 'live').strip())
except BaseException:
pass
if not (options.user or options.password):
raise ValueError("User or Password not given")
main(options.user, options.password, options.server, options.mailbox, options.input, options.library, options.live)
main(
options.user,
options.password,
options.server,
options.mailbox,
options.input,
options.library,
options.live)

150
runner_notify.py
Unescape Escape View File

@ -2,7 +2,6 @@ from StringIO import StringIO
import re
from subprocess import check_output, STDOUT
from sys import platform
from os import utime
from os.path import join
@ -18,13 +17,13 @@ def enable_notifications(options):
fail = False
try:
pb = Pushbullet(options.pushbullet)
except:
except BaseException:
print "Problem wtih connecting to pushbullet. API Key likely invalid"
fail = True
if options.pbdevice and not fail:
try:
pb = pb.get_device(options.pbdevice)
except:
except BaseException:
print "Cannot get this device."
fail = True
pass
@ -32,21 +31,23 @@ def enable_notifications(options):
temp_note = Notification()
temp_note.send_notification = pb.push_note
yield temp_note
if options.notify:
notary = Notification()
yield notary
def touch(fname, times=None):
with open(fname, 'a'):
utime(fname, times)
def main(options):
try:
res = check_output("python fanficdownload.py -c config.ini", shell=True,stderr=STDOUT)
res = check_output(
"python fanficdownload.py -c config.ini",
shell=True,
stderr=STDOUT)
except Exception as e:
print e
res = None
@ -72,47 +73,110 @@ def main(options):
c.execute("delete from books_tags_link where id in (select id from books_tags_link where tag in (select id from tags where name like '%Last Update%'));")
return
if __name__ == "__main__":
option_parser = OptionParser(usage="usage: %prog [flags]")
option_parser.add_option('-p', '--pushbullet', action='store', dest='pushbullet', help='If you want to use pushbullet, pass in your key here.')
option_parser.add_option('-d', '--device', action='store', dest='pbdevice', help='If you wish to only send to a certian pushbullet device, put the device name here. If the device name is invalid, will just send to all pushbullets associated with the acc')
option_parser.add_option('-n', '--notify', action='store_true', dest='notify', help='Enable if you want to use system notifications. Only for Win/Linux.')
option_parser.add_option('-c', '--config', action='store', dest='config', help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.')
option_parser.add_option('-t', '--tag', action='store_true', dest='tag', help='Strip Last Updated tags from calibredb. Requires library to be passed in.')
option_parser.add_option('-l', '--library', action='store', dest='library', help='Path to calibre library. If you are connecting to a calibre webserver then this should be the url.')
option_parser.add_option('-a', '--library-path', action='store', dest='library_path', help='Path location of library. Will be equal to library if nothing is passed in.')
option_parser.add_option(
'-p',
'--pushbullet',
action='store',
dest='pushbullet',
help='If you want to use pushbullet, pass in your key here.')
option_parser.add_option(
'-d',
'--device',
action='store',
dest='pbdevice',
help='If you wish to only send to a certian pushbullet device, put the device name here. If the device name is invalid, will just send to all pushbullets associated with the acc')
option_parser.add_option(
'-n',
'--notify',
action='store_true',
dest='notify',
help='Enable if you want to use system notifications. Only for Win/Linux.')
option_parser.add_option(
'-c',
'--config',
action='store',
dest='config',
help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.')
option_parser.add_option(
'-t',
'--tag',
action='store_true',
dest='tag',
help='Strip Last Updated tags from calibredb. Requires library to be passed in.')
option_parser.add_option(
'-l',
'--library',
action='store',
dest='library',
help='Path to calibre library. If you are connecting to a calibre webserver then this should be the url.')
option_parser.add_option(
'-a',
'--library-path',
action='store',
dest='library_path',
help='Path location of library. Will be equal to library if nothing is passed in.')
(options, args) = option_parser.parse_args()
if options.library and not options.library_path: options.library_path = options.library
if options.library and not options.library_path:
options.library_path = options.library
if options.config:
touch(options.config)
config = ConfigParser(allow_no_value=True)
config.read(options.config)
updater = lambda option, newval : newval if newval != "" else option
try: options.notify = updater(options.notify, config.getboolean('runner', 'notification'))
except: pass
try: options.pushbullet = updater(options.pushbullet, config.get('runner', 'pushbullet'))
except: pass
try: options.pbdevice = updater(options.pbdevice, config.get('runner', 'pbdevice'))
except: pass
try: options.tag = updater(options.tag, config.getboolean('runner', 'tag'))
except: pass
try: options.library = updater(options.library, config.get('locations', 'library'))
except: pass
try: options.library_path = updater(options.library, config.get('locations', 'library_path'))
except: pass
def updater(option, newval): return newval if newval != "" else option
try:
options.notify = updater(
options.notify, config.getboolean(
'runner', 'notification'))
except BaseException:
pass
try:
options.pushbullet = updater(
options.pushbullet, config.get(
'runner', 'pushbullet'))
except BaseException:
pass
try:
options.pbdevice = updater(
options.pbdevice, config.get(
'runner', 'pbdevice'))
except BaseException:
pass
try:
options.tag = updater(
options.tag, config.getboolean(
'runner', 'tag'))
except BaseException:
pass
try:
options.library = updater(
options.library, config.get(
'locations', 'library'))
except BaseException:
pass
try:
options.library_path = updater(
options.library, config.get(
'locations', 'library_path'))
except BaseException:
pass
if options.pbdevice and not options.pushbullet:
raise ValueError("Can't use a pushbullet device without key")
if options.tag and not options.library:
raise ValueError("Can't strip tags from calibre library without a library location.")
raise ValueError(
"Can't strip tags from calibre library without a library location.")
main(options)

Write Preview
Loading…
Cancel
Save