Updated config options and regex expressions
This commit is contained in:
MrTyton
parent
da4e79b66b
commit
b26de7e56e
@ -1,109 +1,111 @@
|
|||||||
from fanficfare import geturls
|
from fanficfare import geturls
|
||||||
import os
|
import os
|
||||||
from os import listdir, remove, rename
|
from os import listdir, remove, rename, utime
|
||||||
from os.path import isfile, join
|
from os.path import isfile, join
|
||||||
import subprocess
|
from subprocess import check_output, STDOUT
|
||||||
import time
|
|
||||||
import logging
|
import logging
|
||||||
from optparse import OptionParser
|
from optparse import OptionParser
|
||||||
import re
|
import re
|
||||||
|
from ConfigParser import ConfigParser
|
||||||
|
|
||||||
logging.getLogger("fanficfare").setLevel(logging.ERROR)
|
logging.getLogger("fanficfare").setLevel(logging.ERROR)
|
||||||
|
|
||||||
|
def touch(fname, times=None):
|
||||||
|
with open(fname, 'a'):
|
||||||
|
utime(fname, times)
|
||||||
|
|
||||||
def main(user, password, server="imap.gmail.com", label="INBOX", path=""):
|
|
||||||
|
ffnet = re.compile('(fanfiction.net/s/\d*)/?.*')
|
||||||
|
neutral = re.compile('https?://(.*)')
|
||||||
|
|
||||||
|
def parse_url(url):
|
||||||
|
if ffnet.search(url):
|
||||||
|
url = "www." + ffnet.search(url).group(1)
|
||||||
|
elif neutral.search(url):
|
||||||
|
url = neutral.search(url).group(1)
|
||||||
|
return url
|
||||||
|
|
||||||
|
def main(user, password, server, label, inout_file, path="", ):
|
||||||
|
|
||||||
if path != "":
|
if path != "":
|
||||||
path = '--with-library "{}"'.format(path)
|
path = '--with-library "{}"'.format(path)
|
||||||
|
|
||||||
|
touch(inout_file)
|
||||||
|
|
||||||
with open("thesebroke", "rb") as fp:
|
with open(inout_file, "r") as fp:
|
||||||
urls = set([x.replace("\n", "") for x in fp.readlines()])
|
urls = set([x.replace("\n", "") for x in fp.readlines()])
|
||||||
with open("todo", "wb") as fp:
|
|
||||||
|
with open(inout_file, "w") as fp:
|
||||||
fp.write("")
|
fp.write("")
|
||||||
urls |= geturls.get_urls_from_imap(server, user, password, label)
|
urls |= geturls.get_urls_from_imap(server, user, password, label)
|
||||||
|
|
||||||
def parse_url(url):
|
|
||||||
url = url.replace("https://", "")
|
|
||||||
url = url.replace("http://", "")
|
|
||||||
url = url[:url.find("/", url.find("/s/") + 3)+1]
|
|
||||||
return url
|
|
||||||
|
|
||||||
|
|
||||||
urls = set(parse_url(x) for x in urls)
|
|
||||||
|
|
||||||
if len(urls) != 0: print "URLs to parse: {}".format(", ".join(urls))
|
|
||||||
|
|
||||||
files = lambda x: [f for f in listdir(x) if isfile(join(x, f))]
|
|
||||||
|
|
||||||
# if len(urls) != 0:
|
|
||||||
# print time.strftime("%d/%m/%Y, %H:%M:%S\n\n")
|
|
||||||
|
|
||||||
|
|
||||||
for url in urls:
|
|
||||||
print "Working with url {}".format(url)
|
|
||||||
try:
|
|
||||||
res = subprocess.check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=subprocess.STDOUT)
|
|
||||||
id = res
|
|
||||||
print "\tStory is in calibre with id {}".format(id)
|
|
||||||
try:
|
|
||||||
print "\tExporting file"
|
|
||||||
res = subprocess.check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir {}'.format(id, path), shell=True)
|
|
||||||
onlyfiles = files(".")
|
|
||||||
for cur in onlyfiles:
|
|
||||||
if not cur.endswith(".epub"): continue
|
|
||||||
print '\tDownloading with fanficfare, updating file "{}"'.format(cur)
|
|
||||||
res = subprocess.check_output('fanficfare -u "{}" --update-cover'.format(cur), shell=True,stderr=subprocess.STDOUT)
|
|
||||||
#print res
|
|
||||||
if "already contains" in res:
|
|
||||||
print "\tIssue with story, FF.net site is broken."
|
|
||||||
with open("todo", "ab") as fp:
|
|
||||||
fp.write("{}\n".format(url))
|
|
||||||
remove(cur)
|
|
||||||
continue
|
|
||||||
elif "Story does not exist" in res:
|
|
||||||
print "\tInvalid URL"
|
|
||||||
continue
|
|
||||||
elif "more recently than Story" in res:
|
|
||||||
print "\tForcing download update\n"
|
|
||||||
res = subprocess.check_output('fanficfare -u "{}" --force --update-cover'.format(cur), shell=True,stderr=subprocess.STDOUT)
|
|
||||||
|
|
||||||
print "\tRemoving {} from library".format(id)
|
urls = set(parse_url(x) for x in urls)
|
||||||
res = subprocess.check_output('calibredb remove {} {}'.format(id, path), shell=True,stderr=subprocess.STDOUT)
|
|
||||||
#print res
|
if len(urls) != 0: print "URLs to parse: {}".format(", ".join(urls))
|
||||||
print "\tAdding {} to library".format(cur)
|
|
||||||
res = subprocess.check_output('calibredb add "{}" {}'.format(cur, path), shell=True,stderr=subprocess.STDOUT)
|
files = lambda x: [f for f in listdir(x) if isfile(join(x, f))]
|
||||||
res = subprocess.check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=subprocess.STDOUT)
|
|
||||||
print "\tAdded {} to library with id {}".format(cur, res)
|
|
||||||
#print res
|
for url in urls:
|
||||||
remove(cur)
|
print "Working with url {}".format(url)
|
||||||
except Exception as e:
|
|
||||||
print "\tSomething fucked up: {}".format(e)
|
|
||||||
remove(cur)
|
|
||||||
with open("todo", "ab") as fp:
|
|
||||||
fp.write("{}\n".format(url))
|
|
||||||
except:
|
|
||||||
print "\tStory is not in calibre"
|
|
||||||
try:
|
try:
|
||||||
res = subprocess.check_output('fanficfare -u "{}" --update-cover'.format(url), shell=True)
|
res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True,stderr=STDOUT)
|
||||||
#print res
|
id = res
|
||||||
onlyfiles = files(".")
|
print "\tStory is in calibre with id {}".format(id)
|
||||||
for cur in onlyfiles:
|
try:
|
||||||
if not cur.endswith(".epub"): continue
|
print "\tExporting file"
|
||||||
try:
|
res = check_output('calibredb export {} --dont-save-cover --dont-write-opf --single-dir {}'.format(id, path), shell=True)
|
||||||
|
onlyfiles = files(".")
|
||||||
|
for cur in onlyfiles:
|
||||||
|
if not cur.endswith(".epub"): continue
|
||||||
|
print '\tDownloading with fanficfare, updating file "{}"'.format(cur)
|
||||||
|
res = check_output('fanficfare -u "{}" --update-cover'.format(cur), shell=True,stderr=STDOUT)
|
||||||
|
#print res
|
||||||
|
if "already contains" in res:
|
||||||
|
print "\tIssue with story, FF.net site is broken."
|
||||||
|
fp.write("{}\n".format(url))
|
||||||
|
remove(cur)
|
||||||
|
continue
|
||||||
|
elif "Story does not exist" in res:
|
||||||
|
print "\tInvalid URL"
|
||||||
|
continue
|
||||||
|
elif "more recently than Story" in res:
|
||||||
|
print "\tForcing download update\n"
|
||||||
|
res = check_output('fanficfare -u "{}" --force --update-cover'.format(cur), shell=True,stderr=STDOUT)
|
||||||
|
|
||||||
|
print "\tRemoving {} from library".format(id)
|
||||||
|
res = check_output('calibredb remove {} {}'.format(id, path), shell=True,stderr=STDOUT)
|
||||||
|
#print res
|
||||||
print "\tAdding {} to library".format(cur)
|
print "\tAdding {} to library".format(cur)
|
||||||
res = subprocess.check_output('calibredb add "{}" {}'.format(cur, path), shell=True, stderr=subprocess.STDOUT)
|
res = check_output('calibredb add "{}" {}'.format(cur, path), shell=True,stderr=STDOUT)
|
||||||
|
res = check_output('calibredb search "Identifiers:{}" {}'.format(url, path), shell=True, stderr=STDOUT)
|
||||||
|
print "\tAdded {} to library with id {}".format(cur, res)
|
||||||
#print res
|
#print res
|
||||||
remove(cur)
|
remove(cur)
|
||||||
except Exception:
|
except Exception as e:
|
||||||
remove(cur)
|
print "\tSomething fucked up: {}".format(e)
|
||||||
raise
|
remove(cur)
|
||||||
except Exception as e:
|
fp.write("{}\n".format(url))
|
||||||
print "\tSomething fucked up: {}".format(e)
|
except:
|
||||||
with open("todo", "ab") as fp:
|
print "\tStory is not in calibre"
|
||||||
|
try:
|
||||||
|
res = check_output('fanficfare -u "{}" --update-cover'.format(url), shell=True)
|
||||||
|
#print res
|
||||||
|
onlyfiles = files(".")
|
||||||
|
for cur in onlyfiles:
|
||||||
|
if not cur.endswith(".epub"): continue
|
||||||
|
try:
|
||||||
|
print "\tAdding {} to library".format(cur)
|
||||||
|
res = check_output('calibredb add "{}" {}'.format(cur, path), shell=True, stderr=STDOUT)
|
||||||
|
#print res
|
||||||
|
remove(cur)
|
||||||
|
except Exception:
|
||||||
|
remove(cur)
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
print "\tSomething fucked up: {}".format(e)
|
||||||
fp.write("{}\n".format(url))
|
fp.write("{}\n".format(url))
|
||||||
|
|
||||||
remove("thesebroke")
|
|
||||||
rename("todo", "thesebroke")
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
option_parser = OptionParser(usage="usage: %prog [flags]")
|
option_parser = OptionParser(usage="usage: %prog [flags]")
|
||||||
@ -114,38 +116,39 @@ if __name__ == "__main__":
|
|||||||
|
|
||||||
option_parser.add_option('-s', '--server', action='store', dest='server', default="imap.gmail.com", help='Email IMAP Server. Default is "imap.gmail.com".')
|
option_parser.add_option('-s', '--server', action='store', dest='server', default="imap.gmail.com", help='Email IMAP Server. Default is "imap.gmail.com".')
|
||||||
|
|
||||||
option_parser.add_option('-m', '--mailbox', action='store', dest='label', default='INBOX', help='Email Label. Default is "INBOX".')
|
option_parser.add_option('-m', '--mailbox', action='store', dest='mailbox', default='INBOX', help='Email Label. Default is "INBOX".')
|
||||||
|
|
||||||
option_parser.add_option('-l', '--library', action='store', dest='library', default="", help="calibre library db location. If none is passed, the default is the calibre system library location. Make sure to enclose the path in quotes.")
|
option_parser.add_option('-l', '--library', action='store', dest='library', default="", help="calibre library db location. If none is passed, the default is the calibre system library location. Make sure to enclose the path in quotes.")
|
||||||
|
|
||||||
option_parser.add_option('-c', '--config', action='store', dest-'config', help='Config file for inputs. One argument per line, format of field=value')
|
option_parser.add_option('-i', '--input', action='store', dest='input', default="./fanfiction.txt", help="Error file. Any urls that fail will be output here, and file will be read to find any urls that failed previously. If file does not exist will create. File is overwitten every time the program is run.")
|
||||||
|
|
||||||
|
option_parser.add_option('-c', '--config', action='store', dest='config', help='Config file for inputs. Example config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank.')
|
||||||
|
|
||||||
(options, args) = option_parser.parse_args()
|
(options, args) = option_parser.parse_args()
|
||||||
|
|
||||||
if options.config:
|
if options.config:
|
||||||
user = re.compile('user=(.*?)')
|
config = ConfigParser(allow_no_value=True)
|
||||||
passwrd = re.compile('password=(.*?)')
|
config.read(options.config)
|
||||||
serv = re.compile('server=(.*?)')
|
|
||||||
mailbox = re.compile('mailbox=(.*?)')
|
updater = lambda option, newval : newval if newval != "" else option
|
||||||
lib = re.compile('library=(.*?)')
|
|
||||||
with open(options.config, 'r') as configfile:
|
options.user = updater(options.user, config.get('login', 'user').strip())
|
||||||
for line in configfile.readlines():
|
|
||||||
line = line.tolower().strip()
|
options.password = updater(options.password, config.get('login', 'password').strip())
|
||||||
if user.search(line):
|
|
||||||
options.user = user.search(line).group(1)
|
options.server = updater(options.server, config.get('login', 'server').strip())
|
||||||
elif passwrd.search(line):
|
|
||||||
options.password = passwrd.search(line).group(1)
|
options.mailbox = updater(options.mailbox, config.get('login', 'mailbox').strip())
|
||||||
elif serv.search(line):
|
|
||||||
options.server = serv.search(line).group(1)
|
options.library = updater(options.library, config.get('locations', 'library').strip())
|
||||||
elif mailbox.search(line):
|
|
||||||
options.label = mailbox.search(line).group(1)
|
options.input = updater(options.input, config.get('locations', 'input').strip())
|
||||||
elif lib.search(line):
|
|
||||||
options.library = mailbox.search(line).group(1)
|
print options
|
||||||
|
|
||||||
if not (options.user or options.password):
|
if not (options.user or options.password):
|
||||||
raise ValueError("User or Password not given")
|
raise ValueError("User or Password not given")
|
||||||
|
|
||||||
main(options.user, options.password, options.server, options.label, options.library)
|
main(options.user, options.password, options.server, options.mailbox, options.input, options.library)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user