sam/AutomatedFanfic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated config options and regex expressions

Browse Source
This commit is contained in:
MrTyton 2016-09-07 01:39:35 -04:00
parent da4e79b66b
commit b26de7e56e
1 changed files with 108 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

203
fanficdownload-cleaned.py
View File

@ -1,109 +1,111 @@
from fanficfare import geturls from fanficfare import geturls
import os import os
from os import listdir, remove, rename from os import listdir, remove, rename, utime
from os.path import isfile, join from os.path import isfile, join
import subprocess from subprocess import check_output, STDOUT
import time
import logging import logging
from optparse import OptionParser from optparse import OptionParser
import re import re
from ConfigParser import ConfigParser
logging.getLogger("fanficfare").setLevel(logging.ERROR) logging.getLogger("fanficfare").setLevel(logging.ERROR)
def touch(fname, times=None):
with open(fname, 'a'):
utime(fname, times)
def main(user, password, server="imap.gmail.com", label="INBOX", path=""):
ffnet = re.compile('(fanfiction.net/s/\d*)/?.*')
neutral = re.compile('https?://(.*)')
def parse_url(url):
if ffnet.search(url):
url = "www." + ffnet.search(url).group(1)
elif neutral.search(url):
url = neutral.search(url).group(1)
return url
def main(user, password, server, label, inout_file, path="", ):
if path != "": if path != "":
path = '--with-library "{}"'.format(path) path = '--with-library "{}"'.format(path)
with open("thesebroke", "rb") as fp: touch(inout_file)
with open(inout_file, "r") as fp:
urls = set([x.replace("\n", "") for x in fp.readlines()]) urls = set([x.replace("\n", "") for x in fp.readlines()])
with open("todo", "wb") as fp:
with open(inout_file, "w") as fp:
fp.write("") fp.write("")
urls |= geturls.get_urls_from_imap(server, user, password, label) urls |= geturls.get_urls_from_imap(server, user, password, label)
def parse_url(url): urls = set(parse_url(x) for x in urls)
url = url.replace("https://", "")
url = url.replace("http://", "") if len(urls) != 0: print "URLs to parse: {}".format(", ".join(urls))
url = url[:url.find("/", url.find("/s/") + 3)+1]
return url files = lambda x: [f for f in listdir(x) if isfile(join(x, f))]
urls = set(parse_url(x) for x in urls) for url in urls:
print "Working with url {}".format(url)
if len(urls) != 0: print "URLs to parse: {}".format(", ".join(urls))
files = lambda x: [f for f in listdir(x) if isfile(join(x, f))]
# if len(urls) != 0:
# print time.strftime("%d/%m/%Y, %H:%M:%S\n\n")
for url in urls:
print "Working with url {}".format(url)
try:
res = subprocess.check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=subprocess.STDOUT)
id = res
print "\tStory is in calibre with id {}".format(id)
try: try:
print "\tExporting file" res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT)
res = subprocess.check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir {}'.format(id, path), shell=True) id = res
onlyfiles = files(".") print "\tStory is in calibre with id {}".format(id)
for cur in onlyfiles: try:
if not cur.endswith(".epub"): continue print "\tExporting file"
print '\tDownloading with fanficfare, updating file "{}"'.format(cur) res = check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir {}'.format(id, path), shell=True)
res = subprocess.check_output('fanficfare -u "{}" --update-cover'.format(cur), shell=True,stderr=subprocess.STDOUT) onlyfiles = files(".")
#print res for cur in onlyfiles:
if "already contains" in res: if not cur.endswith(".epub"): continue
print "\tIssue with story, FF.net site is broken." print '\tDownloading with fanficfare, updating file "{}"'.format(cur)
with open("todo", "ab") as fp: res = check_output('fanficfare -u "{}" --update-cover'.format(cur), shell=True,stderr=STDOUT)
#print res
if "already contains" in res:
print "\tIssue with story, FF.net site is broken."
fp.write("{}\n".format(url)) fp.write("{}\n".format(url))
remove(cur) remove(cur)
continue continue
elif "Story does not exist" in res: elif "Story does not exist" in res:
print "\tInvalid URL" print "\tInvalid URL"
continue continue
elif "more recently than Story" in res: elif "more recently than Story" in res:
print "\tForcing download update\n" print "\tForcing download update\n"
res = subprocess.check_output('fanficfare -u "{}" --force --update-cover'.format(cur), shell=True,stderr=subprocess.STDOUT) res = check_output('fanficfare -u "{}" --force --update-cover'.format(cur), shell=True,stderr=STDOUT)
print "\tRemoving {} from library".format(id) print "\tRemoving {} from library".format(id)
res = subprocess.check_output('calibredb remove {} {}'.format(id, path), shell=True,stderr=subprocess.STDOUT) res = check_output('calibredb remove {} {}'.format(id, path), shell=True,stderr=STDOUT)
#print res #print res
print "\tAdding {} to library".format(cur)
res = subprocess.check_output('calibredb add "{}" {}'.format(cur, path), shell=True,stderr=subprocess.STDOUT)
res = subprocess.check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=subprocess.STDOUT)
print "\tAdded {} to library with id {}".format(cur, res)
#print res
remove(cur)
except Exception as e:
print "\tSomething fucked up: {}".format(e)
remove(cur)
with open("todo", "ab") as fp:
fp.write("{}\n".format(url))
except:
print "\tStory is not in calibre"
try:
res = subprocess.check_output('fanficfare -u "{}" --update-cover'.format(url), shell=True)
#print res
onlyfiles = files(".")
for cur in onlyfiles:
if not cur.endswith(".epub"): continue
try:
print "\tAdding {} to library".format(cur) print "\tAdding {} to library".format(cur)
res = subprocess.check_output('calibredb add "{}" {}'.format(cur, path), shell=True, stderr=subprocess.STDOUT) res = check_output('calibredb add "{}" {}'.format(cur, path), shell=True,stderr=STDOUT)
res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=STDOUT)
print "\tAdded {} to library with id {}".format(cur, res)
#print res #print res
remove(cur) remove(cur)
except Exception: except Exception as e:
remove(cur) print "\tSomething fucked up: {}".format(e)
raise remove(cur)
except Exception as e: fp.write("{}\n".format(url))
print "\tSomething fucked up: {}".format(e) except:
with open("todo", "ab") as fp: print "\tStory is not in calibre"
try:
res = check_output('fanficfare -u "{}" --update-cover'.format(url), shell=True)
#print res
onlyfiles = files(".")
for cur in onlyfiles:
if not cur.endswith(".epub"): continue
try:
print "\tAdding {} to library".format(cur)
res = check_output('calibredb add "{}" {}'.format(cur, path), shell=True, stderr=STDOUT)
#print res
remove(cur)
except Exception:
remove(cur)
raise
except Exception as e:
print "\tSomething fucked up: {}".format(e)
fp.write("{}\n".format(url)) fp.write("{}\n".format(url))
remove("thesebroke")
rename("todo", "thesebroke")
if __name__ == "__main__": if __name__ == "__main__":
option_parser = OptionParser(usage="usage: %prog [flags]") option_parser = OptionParser(usage="usage: %prog [flags]")
@ -114,38 +116,39 @@ if __name__ == "__main__":
option_parser.add_option('-s', '--server', action='store', dest='server', default="imap.gmail.com", help='Email IMAP Server. Default is "imap.gmail.com".') option_parser.add_option('-s', '--server', action='store', dest='server', default="imap.gmail.com", help='Email IMAP Server. Default is "imap.gmail.com".')
option_parser.add_option('-m', '--mailbox', action='store', dest='label', default='INBOX', help='Email Label. Default is "INBOX".') option_parser.add_option('-m', '--mailbox', action='store', dest='mailbox', default='INBOX', help='Email Label. Default is "INBOX".')
option_parser.add_option('-l', '--library', action='store', dest='library', default="", help="calibre library db location. If none is passed, the default is the calibre system library location. Make sure to enclose the path in quotes.") option_parser.add_option('-l', '--library', action='store', dest='library', default="", help="calibre library db location. If none is passed, the default is the calibre system library location. Make sure to enclose the path in quotes.")
option_parser.add_option('-c', '--config', action='store', dest-'config', help='Config file for inputs. One argument per line, format of field=value') option_parser.add_option('-i', '--input', action='store', dest='input', default="./fanfiction.txt", help="Error file. Any urls that fail will be output here, and file will be read to find any urls that failed previously. If file does not exist will create. File is overwitten every time the program is run.")
option_parser.add_option('-c', '--config', action='store', dest='config', help='Config file for inputs. Example config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank.')
(options, args) = option_parser.parse_args() (options, args) = option_parser.parse_args()
if options.config: if options.config:
user = re.compile('user=(.*?)') config = ConfigParser(allow_no_value=True)
passwrd = re.compile('password=(.*?)') config.read(options.config)
serv = re.compile('server=(.*?)')
mailbox = re.compile('mailbox=(.*?)')
lib = re.compile('library=(.*?)')
with open(options.config, 'r') as configfile:
for line in configfile.readlines():
line = line.tolower().strip()
if user.search(line):
options.user = user.search(line).group(1)
elif passwrd.search(line):
options.password = passwrd.search(line).group(1)
elif serv.search(line):
options.server = serv.search(line).group(1)
elif mailbox.search(line):
options.label = mailbox.search(line).group(1)
elif lib.search(line):
options.library = mailbox.search(line).group(1)
updater = lambda option, newval : newval if newval != "" else option
options.user = updater(options.user, config.get('login', 'user').strip())
options.password = updater(options.password, config.get('login', 'password').strip())
options.server = updater(options.server, config.get('login', 'server').strip())
options.mailbox = updater(options.mailbox, config.get('login', 'mailbox').strip())
options.library = updater(options.library, config.get('locations', 'library').strip())
options.input = updater(options.input, config.get('locations', 'input').strip())
print options
if not (options.user or options.password): if not (options.user or options.password):
raise ValueError("User or Password not given") raise ValueError("User or Password not given")
main(options.user, options.password, options.server, options.label, options.library) main(options.user, options.password, options.server, options.mailbox, options.input, options.library)