You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
417 lines
14 KiB
Python
417 lines
14 KiB
Python
8 years ago
|
from fanficfare import geturls
|
||
3 years ago
|
from os import listdir, remove, rename, utime, devnull
|
||
8 years ago
|
from os.path import isfile, join
|
||
7 years ago
|
from subprocess import check_output, STDOUT, call, PIPE
|
||
8 years ago
|
import logging
|
||
|
from optparse import OptionParser
|
||
8 years ago
|
import re
|
||
4 years ago
|
from configparser import ConfigParser
|
||
8 years ago
|
from tempfile import mkdtemp
|
||
3 years ago
|
from shutil import rmtree, copyfile
|
||
7 years ago
|
import socket
|
||
7 years ago
|
from time import strftime, localtime
|
||
3 years ago
|
import os
|
||
3 years ago
|
import errno
|
||
8 years ago
|
|
||
7 years ago
|
from multiprocessing import Pool
|
||
|
|
||
8 years ago
|
logging.getLogger("fanficfare").setLevel(logging.ERROR)
|
||
|
|
||
7 years ago
|
|
||
7 years ago
|
class bcolors:
|
||
|
HEADER = '\033[95m'
|
||
|
OKBLUE = '\033[94m'
|
||
|
OKGREEN = '\033[92m'
|
||
|
WARNING = '\033[93m'
|
||
|
FAIL = '\033[91m'
|
||
|
ENDC = '\033[0m'
|
||
|
BOLD = '\033[1m'
|
||
|
UNDERLINE = '\033[4m'
|
||
|
|
||
7 years ago
|
|
||
7 years ago
|
def log(msg, color=None, output=True):
|
||
7 years ago
|
if color:
|
||
|
col = bcolors.HEADER
|
||
|
if color == 'BLUE':
|
||
|
col = bcolors.OKBLUE
|
||
|
elif color == 'GREEN':
|
||
|
col = bcolors.OKGREEN
|
||
|
elif color == 'WARNING':
|
||
|
col = bcolors.WARNING
|
||
|
elif color == 'FAIL':
|
||
|
col = bcolors.FAIL
|
||
|
elif color == 'BOLD':
|
||
|
col = bcolors.BOLD
|
||
|
elif color == 'UNDERLINE':
|
||
|
col = bcolors.UNDERLINE
|
||
7 years ago
|
line = '{}{}{}: \t {}{}{}'.format(
|
||
|
bcolors.BOLD,
|
||
|
strftime(
|
||
|
'%m/%d/%Y %H:%M:%S',
|
||
|
localtime()),
|
||
|
bcolors.ENDC,
|
||
|
col,
|
||
|
msg,
|
||
|
bcolors.ENDC)
|
||
7 years ago
|
else:
|
||
7 years ago
|
line = '{}{}{}: \t {}'.format(
|
||
|
bcolors.BOLD,
|
||
|
strftime(
|
||
|
'%m/%d/%Y %H:%M:%S',
|
||
|
localtime()),
|
||
|
bcolors.ENDC,
|
||
|
msg)
|
||
7 years ago
|
if output:
|
||
4 years ago
|
print(line)
|
||
7 years ago
|
return ""
|
||
7 years ago
|
else:
|
||
7 years ago
|
return line + "\n"
|
||
7 years ago
|
|
||
7 years ago
|
|
||
8 years ago
|
def touch(fname, times=None):
|
||
|
with open(fname, 'a'):
|
||
|
utime(fname, times)
|
||
8 years ago
|
|
||
8 years ago
|
|
||
3 years ago
|
url_parsers = [(re.compile('(fanfiction.net/s/\d*/?).*'), "www."), #ffnet
|
||
4 years ago
|
(re.compile('(archiveofourown.org/works/\d*)/?.*'), ""), #ao3
|
||
|
(re.compile('(fictionpress.com/s/\d*)/?.*'), ""), #fictionpress
|
||
|
(re.compile('(royalroad.com/fiction/\d*)/?.*'), ""), #royalroad
|
||
|
(re.compile('https?://(.*)'), "")] #other sites
|
||
8 years ago
|
story_name = re.compile('(.*)-.*')
|
||
8 years ago
|
|
||
8 years ago
|
equal_chapters = re.compile('.* already contains \d* chapters.')
|
||
7 years ago
|
chapter_difference = re.compile(
|
||
|
'.* contains \d* chapters, more than source: \d*.')
|
||
|
bad_chapters = re.compile(
|
||
|
".* doesn't contain any recognizable chapters, probably from a different source. Not updating.")
|
||
8 years ago
|
no_url = re.compile('No story URL found in epub to update.')
|
||
7 years ago
|
more_chapters = re.compile(
|
||
|
".*File\(.*\.epub\) Updated\(.*\) more recently than Story\(.*\) - Skipping")
|
||
8 years ago
|
|
||
|
|
||
8 years ago
|
def parse_url(url):
|
||
4 years ago
|
for cur_parser, cur_prefix in url_parsers:
|
||
|
if cur_parser.search(url):
|
||
|
url = cur_prefix + cur_parser.search(url).group(1)
|
||
|
return url
|
||
8 years ago
|
return url
|
||
7 years ago
|
|
||
|
|
||
8 years ago
|
def get_files(mypath, filetype=None, fullpath=False):
|
||
|
ans = []
|
||
|
if filetype:
|
||
7 years ago
|
ans = [f for f in listdir(mypath) if isfile(
|
||
|
join(mypath, f)) and f.endswith(filetype)]
|
||
8 years ago
|
else:
|
||
|
ans = [f for f in listdir(mypath) if isfile(join(mypath, f))]
|
||
|
if fullpath:
|
||
|
return [join(mypath, f) for f in ans]
|
||
|
else:
|
||
|
return ans
|
||
7 years ago
|
|
||
|
|
||
8 years ago
|
def check_regexes(output):
|
||
|
if equal_chapters.search(output):
|
||
7 years ago
|
raise ValueError(
|
||
|
"Issue with story, site is broken. Story likely hasn't updated on site yet.")
|
||
8 years ago
|
if bad_chapters.search(output):
|
||
7 years ago
|
raise ValueError(
|
||
|
"Something is messed up with the site or the epub. No chapters found.")
|
||
8 years ago
|
if no_url.search(output):
|
||
|
raise ValueError("No URL in epub to update from. Fix the metadata.")
|
||
8 years ago
|
|
||
7 years ago
|
|
||
7 years ago
|
def downloader(args):
|
||
|
url, inout_file, path, live = args
|
||
|
loc = mkdtemp()
|
||
|
output = ""
|
||
|
output += log("Working with url {}".format(url), 'HEADER', live)
|
||
|
storyId = None
|
||
|
try:
|
||
|
if path:
|
||
|
try:
|
||
7 years ago
|
storyId = check_output(
|
||
|
'calibredb search "Identifiers:{}" {}'.format(
|
||
4 years ago
|
url, path), shell=True, stderr=STDOUT, stdin=PIPE, ).decode('utf-8')
|
||
7 years ago
|
output += log("\tStory is in calibre with id {}".format(storyId), 'BLUE', live)
|
||
|
output += log("\tExporting file", 'BLUE', live)
|
||
7 years ago
|
res = check_output(
|
||
|
'calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format(
|
||
4 years ago
|
storyId, loc, path), shell=True, stdin=PIPE, stderr=STDOUT).decode('utf-8')
|
||
7 years ago
|
cur = get_files(loc, ".epub", True)[0]
|
||
7 years ago
|
output += log(
|
||
|
'\tDownloading with fanficfare, updating file "{}"'.format(cur),
|
||
|
'GREEN',
|
||
|
live)
|
||
|
moving = ""
|
||
|
except BaseException:
|
||
|
# story is not in calibre
|
||
7 years ago
|
cur = url
|
||
|
moving = 'cd "{}" && '.format(loc)
|
||
3 years ago
|
copyfile("/config/personal.ini", "{}/personal.ini".format(loc))
|
||
|
copyfile("/config/defaults.ini", "{}/defaults.ini".format(moving))
|
||
|
output += log('\tRunning: {}python3.9 -m fanficfare.cli -u "{}" --update-cover --non-interactive'.format(
|
||
7 years ago
|
moving, cur), 'BLUE', live)
|
||
3 years ago
|
res = check_output('{}python3.9 -m fanficfare.cli -u "{}" --update-cover --non-interactive --config={}/personal.ini'.format(
|
||
|
moving, cur, loc), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8')
|
||
7 years ago
|
check_regexes(res)
|
||
|
if chapter_difference.search(res) or more_chapters.search(res):
|
||
7 years ago
|
output += log("\tForcing download update due to:",
|
||
|
'WARNING', live)
|
||
7 years ago
|
for line in res.split("\n"):
|
||
|
if line:
|
||
|
output += log("\t\t{}".format(line), 'WARNING', live)
|
||
7 years ago
|
res = check_output(
|
||
3 years ago
|
'{}python3.9 -m fanficfare.cli -u "{}" --force --update-cover --non-interactive --config={}/personal.ini'.format(
|
||
|
moving, cur, loc), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8')
|
||
7 years ago
|
check_regexes(res)
|
||
|
cur = get_files(loc, '.epub', True)[0]
|
||
|
|
||
7 years ago
|
if storyId:
|
||
|
output += log("\tRemoving {} from library".format(storyId),
|
||
|
'BLUE', live)
|
||
7 years ago
|
try:
|
||
7 years ago
|
res = check_output(
|
||
|
'calibredb remove {} {}'.format(
|
||
|
path,
|
||
|
storyId),
|
||
|
shell=True,
|
||
|
stderr=STDOUT,
|
||
|
stdin=PIPE,
|
||
4 years ago
|
).decode('utf-8')
|
||
7 years ago
|
except BaseException:
|
||
|
if not live:
|
||
4 years ago
|
print(output.strip())
|
||
7 years ago
|
raise
|
||
7 years ago
|
|
||
7 years ago
|
output += log("\tAdding {} to library".format(cur), 'BLUE', live)
|
||
|
try:
|
||
7 years ago
|
res = check_output(
|
||
4 years ago
|
'calibredb add -d {} "{}"'.format(path, cur), shell=True, stderr=STDOUT, stdin=PIPE, ).decode('utf-8')
|
||
7 years ago
|
except Exception as e:
|
||
|
output += log(e)
|
||
7 years ago
|
if not live:
|
||
4 years ago
|
print(output.strip())
|
||
7 years ago
|
raise
|
||
|
try:
|
||
7 years ago
|
res = check_output(
|
||
|
'calibredb search "Identifiers:{}" {}'.format(
|
||
4 years ago
|
url, path), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8')
|
||
7 years ago
|
output += log("\tAdded {} to library with id {}".format(cur,
|
||
|
res), 'GREEN', live)
|
||
|
except BaseException:
|
||
|
output += log(
|
||
|
"It's been added to library, but not sure what the ID is.",
|
||
|
'WARNING',
|
||
|
live)
|
||
6 years ago
|
output += log("Added /Story-file to library with id 0", 'GREEN', live)
|
||
7 years ago
|
remove(cur)
|
||
|
else:
|
||
7 years ago
|
res = check_output(
|
||
|
'cd "{}" && fanficfare -u "{}" --update-cover'.format(
|
||
4 years ago
|
loc, url), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8')
|
||
7 years ago
|
check_regexes(res)
|
||
|
cur = get_files(loc, '.epub', True)[0]
|
||
|
name = get_files(loc, '.epub', False)[0]
|
||
|
rename(cur, name)
|
||
7 years ago
|
output += log(
|
||
|
"Downloaded story {} to {}".format(
|
||
|
story_name.search(name).group(1),
|
||
|
name),
|
||
|
'GREEN',
|
||
|
live)
|
||
|
if not live:
|
||
4 years ago
|
print(output.strip())
|
||
7 years ago
|
rmtree(loc)
|
||
|
except Exception as e:
|
||
|
output += log("Exception: {}".format(e), 'FAIL', live)
|
||
7 years ago
|
if not live:
|
||
4 years ago
|
print(output.strip())
|
||
7 years ago
|
try:
|
||
|
rmtree(loc)
|
||
7 years ago
|
except BaseException:
|
||
7 years ago
|
pass
|
||
|
with open(inout_file, "a") as fp:
|
||
|
fp.write("{}\n".format(url))
|
||
7 years ago
|
|
||
|
|
||
|
def main(user, password, server, label, inout_file, path, live):
|
||
7 years ago
|
|
||
8 years ago
|
if path:
|
||
7 years ago
|
path = '--with-library "{}" --username calibre --password pornoboobies'.format(
|
||
|
path)
|
||
8 years ago
|
try:
|
||
|
with open(devnull, 'w') as nullout:
|
||
7 years ago
|
call(['calibredb'], stdout=nullout, stderr=nullout)
|
||
3 years ago
|
except OSError as e:
|
||
|
if e.errno == errno.ENOENT:
|
||
7 years ago
|
log("Calibredb is not installed on this system. Cannot search the calibre library or update it.", 'FAIL')
|
||
8 years ago
|
return
|
||
7 years ago
|
|
||
8 years ago
|
touch(inout_file)
|
||
8 years ago
|
|
||
8 years ago
|
with open(inout_file, "r") as fp:
|
||
8 years ago
|
urls = set([x.replace("\n", "") for x in fp.readlines()])
|
||
7 years ago
|
|
||
8 years ago
|
with open(inout_file, "w") as fp:
|
||
8 years ago
|
fp.write("")
|
||
7 years ago
|
|
||
|
try:
|
||
|
socket.setdefaulttimeout(55)
|
||
8 years ago
|
urls |= geturls.get_urls_from_imap(server, user, password, label)
|
||
7 years ago
|
socket.setdefaulttimeout(None)
|
||
7 years ago
|
except BaseException:
|
||
7 years ago
|
with open(inout_file, "w") as fp:
|
||
|
for cur in urls:
|
||
7 years ago
|
fp.write("{}\n".format(cur))
|
||
|
return
|
||
|
|
||
|
if not urls:
|
||
7 years ago
|
return
|
||
|
urls = set(parse_url(x) for x in urls)
|
||
7 years ago
|
log("URLs to parse ({}):".format(len(urls)), 'HEADER')
|
||
7 years ago
|
for url in urls:
|
||
7 years ago
|
log("\t{}".format(url), 'BLUE')
|
||
7 years ago
|
if len(urls) == 1:
|
||
7 years ago
|
downloader([list(urls)[0], inout_file, path, True])
|
||
7 years ago
|
else:
|
||
3 years ago
|
for url in urls:
|
||
|
downloader([url, inout_file, path, True])
|
||
|
with open(inout_file, "r") as fp:
|
||
|
urls = set([x.replace("\n", "") for x in fp.readlines()])
|
||
|
with open(inout_file, "w") as fp:
|
||
|
fp.writelines(["{}\n".format(x) for x in urls])
|
||
7 years ago
|
return
|
||
8 years ago
|
|
||
|
|
||
|
if __name__ == "__main__":
|
||
|
option_parser = OptionParser(usage="usage: %prog [flags]")
|
||
7 years ago
|
|
||
|
option_parser.add_option(
|
||
|
'-u',
|
||
|
'--user',
|
||
|
action='store',
|
||
|
dest='user',
|
||
|
help='Email Account Username. Required.')
|
||
|
|
||
|
option_parser.add_option(
|
||
|
'-p',
|
||
|
'--password',
|
||
|
action='store',
|
||
|
dest='password',
|
||
|
help='Email Account Password. Required.')
|
||
|
|
||
|
option_parser.add_option(
|
||
|
'-s',
|
||
|
'--server',
|
||
|
action='store',
|
||
|
dest='server',
|
||
|
default="imap.gmail.com",
|
||
|
help='Email IMAP Server. Default is "imap.gmail.com".')
|
||
|
|
||
|
option_parser.add_option(
|
||
|
'-m',
|
||
|
'--mailbox',
|
||
|
action='store',
|
||
|
dest='mailbox',
|
||
|
default='INBOX',
|
||
|
help='Email Label. Default is "INBOX".')
|
||
|
|
||
|
option_parser.add_option(
|
||
|
'-l',
|
||
|
'--library',
|
||
|
action='store',
|
||
|
dest='library',
|
||
|
help="calibre library db location. If none is passed, then this merely scrapes the email and error file for new stories and downloads them into the current directory.")
|
||
|
|
||
|
option_parser.add_option(
|
||
|
'-i',
|
||
|
'--input',
|
||
|
action='store',
|
||
|
dest='input',
|
||
|
default="./fanfiction.txt",
|
||
|
help="Error file. Any urls that fail will be output here, and file will be read to find any urls that failed previously. If file does not exist will create. File is overwitten every time the program is run.")
|
||
|
|
||
|
option_parser.add_option(
|
||
|
'-c',
|
||
|
'--config',
|
||
|
action='store',
|
||
|
dest='config',
|
||
|
help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.')
|
||
|
|
||
|
option_parser.add_option(
|
||
|
'-o',
|
||
|
'--output',
|
||
|
action='store_true',
|
||
|
dest='live',
|
||
|
help='Include this if you want all the output to be saved and posted live. Useful when multithreading.')
|
||
|
|
||
8 years ago
|
(options, args) = option_parser.parse_args()
|
||
7 years ago
|
|
||
8 years ago
|
if options.config:
|
||
8 years ago
|
touch(options.config)
|
||
8 years ago
|
config = ConfigParser(allow_no_value=True)
|
||
|
config.read(options.config)
|
||
7 years ago
|
|
||
|
def updater(option, newval): return newval if newval != "" else option
|
||
|
try:
|
||
|
options.user = updater(
|
||
|
options.user, config.get(
|
||
|
'login', 'user').strip())
|
||
|
except BaseException:
|
||
|
pass
|
||
|
|
||
|
try:
|
||
|
options.password = updater(
|
||
|
options.password, config.get(
|
||
|
'login', 'password').strip())
|
||
|
except BaseException:
|
||
|
pass
|
||
|
|
||
|
try:
|
||
|
options.server = updater(
|
||
|
options.server, config.get(
|
||
|
'login', 'server').strip())
|
||
|
except BaseException:
|
||
|
pass
|
||
|
|
||
|
try:
|
||
|
options.mailbox = updater(
|
||
|
options.mailbox, config.get(
|
||
|
'login', 'mailbox').strip())
|
||
|
except BaseException:
|
||
|
pass
|
||
|
|
||
|
try:
|
||
|
options.library = updater(
|
||
|
options.library, config.get(
|
||
|
'locations', 'library').strip())
|
||
|
except BaseException:
|
||
|
pass
|
||
|
|
||
|
try:
|
||
|
options.input = updater(
|
||
|
options.input, config.get(
|
||
|
'locations', 'input').strip())
|
||
|
except BaseException:
|
||
|
pass
|
||
|
|
||
|
try:
|
||
|
options.live = updater(
|
||
|
options.live, config.getboolean(
|
||
|
'output', 'live').strip())
|
||
|
except BaseException:
|
||
|
pass
|
||
|
|
||
8 years ago
|
if not (options.user or options.password):
|
||
8 years ago
|
raise ValueError("User or Password not given")
|
||
7 years ago
|
main(
|
||
|
options.user,
|
||
|
options.password,
|
||
|
options.server,
|
||
|
options.mailbox,
|
||
|
options.input,
|
||
|
options.library,
|
||
|
options.live)
|