add files
This commit is contained in:
parent
7880ef118e
commit
10982f7172
80
Dockerfile
Normal file
80
Dockerfile
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
FROM python:3.9-slim
|
||||||
|
|
||||||
|
# set version label
|
||||||
|
ARG VERSION
|
||||||
|
ARG CALIBRE_RELEASE
|
||||||
|
ARG FFF_RELEASE
|
||||||
|
LABEL build_version="FFDL-Auto version:- ${VERSION} Calibre: ${CALIBRE_RELEASE} FFF: ${FFF_RELEASE}"
|
||||||
|
|
||||||
|
ENV PUID="911" \
|
||||||
|
PGID="911"
|
||||||
|
|
||||||
|
RUN set -x && \
|
||||||
|
apt-get update && \
|
||||||
|
apt-get install -y --upgrade \
|
||||||
|
bash \
|
||||||
|
ca-certificates \
|
||||||
|
gcc \
|
||||||
|
wget \
|
||||||
|
xdg-utils \
|
||||||
|
curl \
|
||||||
|
dbus \
|
||||||
|
jq \
|
||||||
|
python3
|
||||||
|
|
||||||
|
RUN set -x && \
|
||||||
|
addgroup --gid "$PGID" abc && \
|
||||||
|
adduser \
|
||||||
|
--gecos "" \
|
||||||
|
--disabled-password \
|
||||||
|
--no-create-home \
|
||||||
|
--uid "$PUID" \
|
||||||
|
--ingroup abc \
|
||||||
|
--shell /bin/bash \
|
||||||
|
abc
|
||||||
|
|
||||||
|
RUN echo "**** install calibre ****" && \
|
||||||
|
set -x && \
|
||||||
|
apt-get install -y calibre && \
|
||||||
|
dbus-uuidgen > /etc/machine-id
|
||||||
|
|
||||||
|
|
||||||
|
RUN echo "**** s6 omsta;; ****" && \
|
||||||
|
set -ex && \
|
||||||
|
ARCH=`uname -m` && \
|
||||||
|
if [ "$ARCH" = "x86_64" ]; then \
|
||||||
|
s6_package="s6-overlay-amd64.tar.gz" ; \
|
||||||
|
elif [ "$ARCH" = "aarch64" ]; then \
|
||||||
|
s6_package="s6-overlay-aarch64.tar.gz" ; \
|
||||||
|
else \
|
||||||
|
echo "unknown arch: ${ARCH}" && \
|
||||||
|
exit 1 ; \
|
||||||
|
fi && \
|
||||||
|
wget -P /tmp/ https://github.com/just-containers/s6-overlay/releases/download/v2.2.0.3/${s6_package} && \
|
||||||
|
tar -xzf /tmp/${s6_package} -C /
|
||||||
|
|
||||||
|
RUN echo *** Install Packages *** && \
|
||||||
|
set -x && \
|
||||||
|
if [ -z ${FFF_RELEASE} ]; then \
|
||||||
|
echo "FFF Using Default Release"; \
|
||||||
|
python3 -m pip --no-cache-dir install FanFicFare; \
|
||||||
|
else \
|
||||||
|
echo "FF Using ${FFF_RELEASE} Release"; \
|
||||||
|
python3 -m pip --no-cache-dir install --extra-index-url https://testpypi.python.org/pypi FanFicFare==${FFF_RELEASE}; \
|
||||||
|
fi && \
|
||||||
|
python3 -m pip --no-cache-dir install pushbullet.py pillow && \
|
||||||
|
ln -s /opt/calibre/calibredb /bin/calibredb
|
||||||
|
|
||||||
|
RUN echo "**** cleanup ****" && \
|
||||||
|
rm -rf \
|
||||||
|
/tmp/* \
|
||||||
|
/var/lib/apt/lists/* \
|
||||||
|
/var/tmp/*
|
||||||
|
|
||||||
|
COPY root/ /
|
||||||
|
|
||||||
|
VOLUME /config
|
||||||
|
|
||||||
|
WORKDIR /config
|
||||||
|
|
||||||
|
ENTRYPOINT ["/init"]
|
12
README.md
12
README.md
@ -0,0 +1,12 @@
|
|||||||
|
# AutomatedFanfic
|
||||||
|
Automated Fanfiction Download using FanficFare CLI
|
||||||
|
|
||||||
|
This is a docker image to run the Automated FFF CLI, with pushbullet integration.
|
||||||
|
|
||||||
|
[Dockerhub Link](https://hub.docker.com/r/mrtyton/automated-ffdl)
|
||||||
|
|
||||||
|
## How to Install
|
||||||
|
|
||||||
|
1. Install the docker image with `docker pull mrtyton/automated-ffdl`
|
||||||
|
2. Map the `/config` volume to someplace on your drive.
|
||||||
|
3. After running the image once, it will have copied over default configs. Fill them out and everything should start working.
|
1
release-versions/calibre.txt
Normal file
1
release-versions/calibre.txt
Normal file
@ -0,0 +1 @@
|
|||||||
|
6.12.0
|
1
release-versions/fff.txt
Normal file
1
release-versions/fff.txt
Normal file
@ -0,0 +1 @@
|
|||||||
|
4.20.6
|
1
release-versions/latest.txt
Normal file
1
release-versions/latest.txt
Normal file
@ -0,0 +1 @@
|
|||||||
|
2023.02.26
|
447
root/app/fanficdownload.py
Normal file
447
root/app/fanficdownload.py
Normal file
@ -0,0 +1,447 @@
|
|||||||
|
from fanficfare import geturls
|
||||||
|
from os import listdir, remove, rename, utime, devnull
|
||||||
|
from os.path import isfile, join
|
||||||
|
from subprocess import check_output, STDOUT, call, PIPE
|
||||||
|
import logging
|
||||||
|
from optparse import OptionParser
|
||||||
|
import re
|
||||||
|
from configparser import ConfigParser
|
||||||
|
from tempfile import mkdtemp
|
||||||
|
from shutil import rmtree, copyfile
|
||||||
|
import socket
|
||||||
|
from time import strftime, localtime
|
||||||
|
import os
|
||||||
|
import errno
|
||||||
|
|
||||||
|
from multiprocessing import Pool
|
||||||
|
|
||||||
|
logging.getLogger("fanficfare").setLevel(logging.ERROR)
|
||||||
|
|
||||||
|
|
||||||
|
class bcolors:
|
||||||
|
HEADER = '\033[95m'
|
||||||
|
OKBLUE = '\033[94m'
|
||||||
|
OKGREEN = '\033[92m'
|
||||||
|
WARNING = '\033[93m'
|
||||||
|
FAIL = '\033[91m'
|
||||||
|
ENDC = '\033[0m'
|
||||||
|
BOLD = '\033[1m'
|
||||||
|
UNDERLINE = '\033[4m'
|
||||||
|
|
||||||
|
|
||||||
|
def log(msg, color=None, output=True):
|
||||||
|
if color:
|
||||||
|
col = bcolors.HEADER
|
||||||
|
if color == 'BLUE':
|
||||||
|
col = bcolors.OKBLUE
|
||||||
|
elif color == 'GREEN':
|
||||||
|
col = bcolors.OKGREEN
|
||||||
|
elif color == 'WARNING':
|
||||||
|
col = bcolors.WARNING
|
||||||
|
elif color == 'FAIL':
|
||||||
|
col = bcolors.FAIL
|
||||||
|
elif color == 'BOLD':
|
||||||
|
col = bcolors.BOLD
|
||||||
|
elif color == 'UNDERLINE':
|
||||||
|
col = bcolors.UNDERLINE
|
||||||
|
line = '{}{}{}: \t {}{}{}'.format(
|
||||||
|
bcolors.BOLD,
|
||||||
|
strftime(
|
||||||
|
'%m/%d/%Y %H:%M:%S',
|
||||||
|
localtime()),
|
||||||
|
bcolors.ENDC,
|
||||||
|
col,
|
||||||
|
msg,
|
||||||
|
bcolors.ENDC)
|
||||||
|
else:
|
||||||
|
line = '{}{}{}: \t {}'.format(
|
||||||
|
bcolors.BOLD,
|
||||||
|
strftime(
|
||||||
|
'%m/%d/%Y %H:%M:%S',
|
||||||
|
localtime()),
|
||||||
|
bcolors.ENDC,
|
||||||
|
msg)
|
||||||
|
if output:
|
||||||
|
print(line)
|
||||||
|
return ""
|
||||||
|
else:
|
||||||
|
return line + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def touch(fname, times=None):
|
||||||
|
with open(fname, 'a'):
|
||||||
|
utime(fname, times)
|
||||||
|
|
||||||
|
|
||||||
|
url_parsers = [(re.compile('(fanfiction.net/s/\d*/?).*'), "www."), #ffnet
|
||||||
|
(re.compile('(archiveofourown.org/works/\d*)/?.*'), ""), #ao3
|
||||||
|
(re.compile('(fictionpress.com/s/\d*)/?.*'), ""), #fictionpress
|
||||||
|
(re.compile('(royalroad.com/fiction/\d*)/?.*'), ""), #royalroad
|
||||||
|
(re.compile('https?://(.*)'), "")] #other sites
|
||||||
|
story_name = re.compile('(.*)-.*')
|
||||||
|
|
||||||
|
equal_chapters = re.compile('.* already contains \d* chapters.')
|
||||||
|
chapter_difference = re.compile(
|
||||||
|
'.* contains \d* chapters, more than source: \d*.')
|
||||||
|
bad_chapters = re.compile(
|
||||||
|
".* doesn't contain any recognizable chapters, probably from a different source. Not updating.")
|
||||||
|
no_url = re.compile('No story URL found in epub to update.')
|
||||||
|
more_chapters = re.compile(
|
||||||
|
".*File\(.*\.epub\) Updated\(.*\) more recently than Story\(.*\) - Skipping")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_url(url):
|
||||||
|
for cur_parser, cur_prefix in url_parsers:
|
||||||
|
if cur_parser.search(url):
|
||||||
|
url = cur_prefix + cur_parser.search(url).group(1)
|
||||||
|
return url
|
||||||
|
return url
|
||||||
|
|
||||||
|
|
||||||
|
def get_files(mypath, filetype=None, fullpath=False):
|
||||||
|
ans = []
|
||||||
|
if filetype:
|
||||||
|
ans = [f for f in listdir(mypath) if isfile(
|
||||||
|
join(mypath, f)) and f.endswith(filetype)]
|
||||||
|
else:
|
||||||
|
ans = [f for f in listdir(mypath) if isfile(join(mypath, f))]
|
||||||
|
if fullpath:
|
||||||
|
return [join(mypath, f) for f in ans]
|
||||||
|
else:
|
||||||
|
return ans
|
||||||
|
|
||||||
|
|
||||||
|
def check_regexes(output):
|
||||||
|
if equal_chapters.search(output):
|
||||||
|
raise ValueError(
|
||||||
|
"Issue with story, site is broken. Story likely hasn't updated on site yet.")
|
||||||
|
if bad_chapters.search(output):
|
||||||
|
raise ValueError(
|
||||||
|
"Something is messed up with the site or the epub. No chapters found.")
|
||||||
|
if no_url.search(output):
|
||||||
|
raise ValueError("No URL in epub to update from. Fix the metadata.")
|
||||||
|
|
||||||
|
|
||||||
|
def downloader(args):
|
||||||
|
url, inout_file, path, live = args
|
||||||
|
loc = mkdtemp()
|
||||||
|
output = ""
|
||||||
|
output += log("Working with url {}".format(url), 'HEADER', live)
|
||||||
|
storyId = None
|
||||||
|
try:
|
||||||
|
if path:
|
||||||
|
try:
|
||||||
|
storyId = check_output(
|
||||||
|
'calibredb search "Identifiers:{}" {}'.format(
|
||||||
|
url, path), shell=True, stderr=STDOUT, stdin=PIPE, ).decode('utf-8')
|
||||||
|
output += log("\tStory is in calibre with id {}".format(storyId), 'BLUE', live)
|
||||||
|
output += log("\tExporting file", 'BLUE', live)
|
||||||
|
res = check_output(
|
||||||
|
'calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format(
|
||||||
|
storyId, loc, path), shell=True, stdin=PIPE, stderr=STDOUT).decode('utf-8')
|
||||||
|
cur = get_files(loc, ".epub", True)[0]
|
||||||
|
output += log(
|
||||||
|
'\tDownloading with fanficfare, updating file "{}"'.format(cur),
|
||||||
|
'GREEN',
|
||||||
|
live)
|
||||||
|
moving = ""
|
||||||
|
except BaseException:
|
||||||
|
# story is not in calibre
|
||||||
|
output += log("\tStory is not in Calibre", 'WARNING', live)
|
||||||
|
cur = url
|
||||||
|
moving = 'cd "{}" && '.format(loc)
|
||||||
|
copyfile("/config/personal.ini", "{}/personal.ini".format(loc))
|
||||||
|
copyfile("/config/defaults.ini", "{}/defaults.ini".format(loc))
|
||||||
|
output += log('\tRunning: {}python3.9 -m fanficfare.cli -u "{}" --update-cover --non-interactive'.format(
|
||||||
|
moving, cur), 'BLUE', live)
|
||||||
|
res = check_output('{}python3.9 -m fanficfare.cli -u "{}" --update-cover --non-interactive --config={}/personal.ini'.format(
|
||||||
|
moving, cur, loc), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8')
|
||||||
|
check_regexes(res)
|
||||||
|
if chapter_difference.search(res) or more_chapters.search(res):
|
||||||
|
output += log("\tForcing download update due to:",
|
||||||
|
'WARNING', live)
|
||||||
|
for line in res.split("\n"):
|
||||||
|
if line:
|
||||||
|
output += log("\t\t{}".format(line), 'WARNING', live)
|
||||||
|
res = check_output(
|
||||||
|
'{}python3.9 -m fanficfare.cli -u "{}" --force --update-cover --non-interactive --config={}/personal.ini'.format(
|
||||||
|
moving, cur, loc), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8')
|
||||||
|
check_regexes(res)
|
||||||
|
cur = get_files(loc, '.epub', True)[0]
|
||||||
|
|
||||||
|
if storyId:
|
||||||
|
output += log("\tRemoving {} from library".format(storyId),
|
||||||
|
'BLUE', live)
|
||||||
|
try:
|
||||||
|
res = check_output(
|
||||||
|
'calibredb remove {} {}'.format(
|
||||||
|
path,
|
||||||
|
storyId),
|
||||||
|
shell=True,
|
||||||
|
stderr=STDOUT,
|
||||||
|
stdin=PIPE,
|
||||||
|
).decode('utf-8')
|
||||||
|
except BaseException:
|
||||||
|
if not live:
|
||||||
|
print(output.strip())
|
||||||
|
raise
|
||||||
|
|
||||||
|
output += log("\tAdding {} to library".format(cur), 'BLUE', live)
|
||||||
|
try:
|
||||||
|
res = check_output(
|
||||||
|
'calibredb add -d {} "{}"'.format(path, cur), shell=True, stderr=STDOUT, stdin=PIPE, ).decode('utf-8')
|
||||||
|
except Exception as e:
|
||||||
|
output += log(e)
|
||||||
|
if not live:
|
||||||
|
print(output.strip())
|
||||||
|
raise
|
||||||
|
try:
|
||||||
|
res = check_output(
|
||||||
|
'calibredb search "Identifiers:{}" {}'.format(
|
||||||
|
url, path), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8')
|
||||||
|
output += log("\tAdded {} to library with id {}".format(cur,
|
||||||
|
res), 'GREEN', live)
|
||||||
|
except BaseException:
|
||||||
|
output += log(
|
||||||
|
"It's been added to library, but not sure what the ID is.",
|
||||||
|
'WARNING',
|
||||||
|
live)
|
||||||
|
output += log("Added /Story-file to library with id 0", 'GREEN', live)
|
||||||
|
remove(cur)
|
||||||
|
else:
|
||||||
|
res = check_output(
|
||||||
|
'cd "{}" && fanficfare -u "{}" --update-cover'.format(
|
||||||
|
loc, url), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8')
|
||||||
|
check_regexes(res)
|
||||||
|
cur = get_files(loc, '.epub', True)[0]
|
||||||
|
name = get_files(loc, '.epub', False)[0]
|
||||||
|
rename(cur, name)
|
||||||
|
output += log(
|
||||||
|
"Downloaded story {} to {}".format(
|
||||||
|
story_name.search(name).group(1),
|
||||||
|
name),
|
||||||
|
'GREEN',
|
||||||
|
live)
|
||||||
|
if not live:
|
||||||
|
print(output.strip())
|
||||||
|
rmtree(loc)
|
||||||
|
except Exception as e:
|
||||||
|
output += log("Exception: {}".format(e), 'FAIL', live)
|
||||||
|
if not live:
|
||||||
|
print(output.strip())
|
||||||
|
try:
|
||||||
|
rmtree(loc)
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
with open(inout_file, "a") as fp:
|
||||||
|
fp.write("{}\n".format(url))
|
||||||
|
|
||||||
|
|
||||||
|
def main(user, password, server, label, inout_file, path, lib_user, lib_password, live):
|
||||||
|
|
||||||
|
if path:
|
||||||
|
path = '--with-library "{}" --username {} --password {}'.format(
|
||||||
|
path, lib_user, lib_password)
|
||||||
|
try:
|
||||||
|
with open(devnull, 'w') as nullout:
|
||||||
|
call(['calibredb'], stdout=nullout, stderr=nullout)
|
||||||
|
except OSError as e:
|
||||||
|
if e.errno == errno.ENOENT:
|
||||||
|
log("Calibredb is not installed on this system. Cannot search the calibre library or update it.", 'FAIL')
|
||||||
|
return
|
||||||
|
|
||||||
|
touch(inout_file)
|
||||||
|
|
||||||
|
with open(inout_file, "r") as fp:
|
||||||
|
urls = set([x.replace("\n", "") for x in fp.readlines()])
|
||||||
|
|
||||||
|
with open(inout_file, "w") as fp:
|
||||||
|
fp.write("")
|
||||||
|
|
||||||
|
try:
|
||||||
|
socket.setdefaulttimeout(55)
|
||||||
|
urls |= geturls.get_urls_from_imap(server, user, password, label)
|
||||||
|
socket.setdefaulttimeout(None)
|
||||||
|
except BaseException:
|
||||||
|
with open(inout_file, "w") as fp:
|
||||||
|
for cur in urls:
|
||||||
|
fp.write("{}\n".format(cur))
|
||||||
|
return
|
||||||
|
|
||||||
|
if not urls:
|
||||||
|
return
|
||||||
|
urls = set(parse_url(x) for x in urls)
|
||||||
|
log("URLs to parse ({}):".format(len(urls)), 'HEADER')
|
||||||
|
for url in urls:
|
||||||
|
log("\t{}".format(url), 'BLUE')
|
||||||
|
if len(urls) == 1:
|
||||||
|
downloader([list(urls)[0], inout_file, path, True])
|
||||||
|
else:
|
||||||
|
for url in urls:
|
||||||
|
downloader([url, inout_file, path, True])
|
||||||
|
with open(inout_file, "r") as fp:
|
||||||
|
urls = set([x.replace("\n", "") for x in fp.readlines()])
|
||||||
|
with open(inout_file, "w") as fp:
|
||||||
|
fp.writelines(["{}\n".format(x) for x in urls])
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
option_parser = OptionParser(usage="usage: %prog [flags]")
|
||||||
|
|
||||||
|
option_parser.add_option(
|
||||||
|
'-u',
|
||||||
|
'--user',
|
||||||
|
action='store',
|
||||||
|
dest='user',
|
||||||
|
help='Email Account Username. Required.')
|
||||||
|
|
||||||
|
option_parser.add_option(
|
||||||
|
'-p',
|
||||||
|
'--password',
|
||||||
|
action='store',
|
||||||
|
dest='password',
|
||||||
|
help='Email Account Password. Required.')
|
||||||
|
|
||||||
|
option_parser.add_option(
|
||||||
|
'-s',
|
||||||
|
'--server',
|
||||||
|
action='store',
|
||||||
|
dest='server',
|
||||||
|
default="imap.gmail.com",
|
||||||
|
help='Email IMAP Server. Default is "imap.gmail.com".')
|
||||||
|
|
||||||
|
option_parser.add_option(
|
||||||
|
'-m',
|
||||||
|
'--mailbox',
|
||||||
|
action='store',
|
||||||
|
dest='mailbox',
|
||||||
|
default='INBOX',
|
||||||
|
help='Email Label. Default is "INBOX".')
|
||||||
|
|
||||||
|
option_parser.add_option(
|
||||||
|
'-l',
|
||||||
|
'--library',
|
||||||
|
action='store',
|
||||||
|
dest='library',
|
||||||
|
help="calibre library db location. If none is passed, then this merely scrapes the email and error file for new stories and downloads them into the current directory.")
|
||||||
|
|
||||||
|
option_parser.add_option(
|
||||||
|
'-i',
|
||||||
|
'--input',
|
||||||
|
action='store',
|
||||||
|
dest='input',
|
||||||
|
default="./fanfiction.txt",
|
||||||
|
help="Error file. Any urls that fail will be output here, and file will be read to find any urls that failed previously. If file does not exist will create. File is overwitten every time the program is run.")
|
||||||
|
|
||||||
|
option_parser.add_option(
|
||||||
|
'-c',
|
||||||
|
'--config',
|
||||||
|
action='store',
|
||||||
|
dest='config',
|
||||||
|
help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.')
|
||||||
|
|
||||||
|
option_parser.add_option(
|
||||||
|
'-o',
|
||||||
|
'--output',
|
||||||
|
action='store_true',
|
||||||
|
dest='live',
|
||||||
|
help='Include this if you want all the output to be saved and posted live. Useful when multithreading.')
|
||||||
|
|
||||||
|
option_parser.add_option(
|
||||||
|
'-q',
|
||||||
|
'--libuser',
|
||||||
|
action='store',
|
||||||
|
dest='libuser',
|
||||||
|
help='Calibre User. Required.')
|
||||||
|
|
||||||
|
option_parser.add_option(
|
||||||
|
'-w',
|
||||||
|
'--libpassword',
|
||||||
|
action='store',
|
||||||
|
dest='libpassword',
|
||||||
|
help='Calibre Password. Required.')
|
||||||
|
|
||||||
|
(options, args) = option_parser.parse_args()
|
||||||
|
|
||||||
|
if options.config:
|
||||||
|
touch(options.config)
|
||||||
|
config = ConfigParser(allow_no_value=True)
|
||||||
|
config.read(options.config)
|
||||||
|
|
||||||
|
def updater(option, newval): return newval if newval != "" else option
|
||||||
|
try:
|
||||||
|
options.user = updater(
|
||||||
|
options.user, config.get(
|
||||||
|
'login', 'user').strip())
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.password = updater(
|
||||||
|
options.password, config.get(
|
||||||
|
'login', 'password').strip())
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.libuser = updater(
|
||||||
|
options.libuser, config.get(
|
||||||
|
'login', 'libuser').strip())
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.libpassword = updater(
|
||||||
|
options.libpassword, config.get(
|
||||||
|
'login', 'libpassword').strip())
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.server = updater(
|
||||||
|
options.server, config.get(
|
||||||
|
'login', 'server').strip())
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.mailbox = updater(
|
||||||
|
options.mailbox, config.get(
|
||||||
|
'login', 'mailbox').strip())
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.library = updater(
|
||||||
|
options.library, config.get(
|
||||||
|
'locations', 'library').strip())
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.input = updater(
|
||||||
|
options.input, config.get(
|
||||||
|
'locations', 'input').strip())
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.live = updater(
|
||||||
|
options.live, config.getboolean(
|
||||||
|
'output', 'live').strip())
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not (options.user or options.password):
|
||||||
|
raise ValueError("User or Password not given")
|
||||||
|
main(
|
||||||
|
options.user,
|
||||||
|
options.password,
|
||||||
|
options.server,
|
||||||
|
options.mailbox,
|
||||||
|
options.input,
|
||||||
|
options.library,
|
||||||
|
options.libuser,
|
||||||
|
options.libpassword,
|
||||||
|
options.live)
|
8
root/app/notifications.py
Normal file
8
root/app/notifications.py
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
class Notification():
|
||||||
|
def __init__(self):
|
||||||
|
|
||||||
|
self.name="Fanfiction"
|
||||||
|
|
||||||
|
def send_notification(self, title, text):
|
||||||
|
return
|
||||||
|
|
5
root/app/run.sh
Normal file
5
root/app/run.sh
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
python3 /app/runner_notify.py -c /config/config.ini
|
||||||
|
sleep 60
|
||||||
|
|
171
root/app/runner_notify.py
Normal file
171
root/app/runner_notify.py
Normal file
@ -0,0 +1,171 @@
|
|||||||
|
from io import StringIO
|
||||||
|
import re
|
||||||
|
from subprocess import check_output, STDOUT
|
||||||
|
from time import sleep
|
||||||
|
import ntpath
|
||||||
|
|
||||||
|
from os import utime
|
||||||
|
from os.path import join
|
||||||
|
|
||||||
|
from notifications import Notification
|
||||||
|
from pushbullet import Pushbullet
|
||||||
|
|
||||||
|
from optparse import OptionParser
|
||||||
|
from configparser import ConfigParser
|
||||||
|
|
||||||
|
def enable_notifications(options):
|
||||||
|
if options.pushbullet:
|
||||||
|
fail = False
|
||||||
|
try:
|
||||||
|
pb = Pushbullet(options.pushbullet)
|
||||||
|
except BaseException:
|
||||||
|
print("Problem wtih connecting to pushbullet. API Key likely invalid")
|
||||||
|
fail = True
|
||||||
|
if options.pbdevice and not fail:
|
||||||
|
try:
|
||||||
|
pb = pb.get_device(options.pbdevice)
|
||||||
|
except BaseException:
|
||||||
|
print("Cannot get this device.")
|
||||||
|
fail = True
|
||||||
|
pass
|
||||||
|
if not fail:
|
||||||
|
temp_note = Notification()
|
||||||
|
temp_note.send_notification = pb.push_note
|
||||||
|
yield temp_note
|
||||||
|
|
||||||
|
|
||||||
|
def touch(fname, times=None):
|
||||||
|
with open(fname, 'a'):
|
||||||
|
utime(fname, times)
|
||||||
|
|
||||||
|
|
||||||
|
def main(options):
|
||||||
|
try:
|
||||||
|
res = check_output(
|
||||||
|
f"python3.9 /app/fanficdownload.py -c {options.config}",
|
||||||
|
shell=True,
|
||||||
|
stderr=STDOUT)
|
||||||
|
except Exception as e:
|
||||||
|
print(e)
|
||||||
|
res = None
|
||||||
|
if not res:
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
res = res.decode('utf-8')
|
||||||
|
print(res)
|
||||||
|
buf = StringIO(res)
|
||||||
|
regex = re.compile("Added (?:.*/)?(.*)-.* to library with id \d*")
|
||||||
|
searcher = regex.search
|
||||||
|
stripper = False
|
||||||
|
for line in buf.readlines():
|
||||||
|
r = searcher(line)
|
||||||
|
if r:
|
||||||
|
story = ntpath.basename(r.group(1).strip())
|
||||||
|
stripper = True
|
||||||
|
for notify in enable_notifications(options):
|
||||||
|
notify.send_notification("New Fanfiction Download", story)
|
||||||
|
if stripper and options.tag:
|
||||||
|
import sqlite3
|
||||||
|
with sqlite3.connect(join(options.library_path, "metadata.db")) as conn:
|
||||||
|
c = conn.cursor()
|
||||||
|
c.execute("delete from books_tags_link where id in (select id from books_tags_link where tag in (select id from tags where name like '%Last Update%'));")
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
option_parser = OptionParser(usage="usage: %prog [flags]")
|
||||||
|
option_parser.add_option(
|
||||||
|
'-p',
|
||||||
|
'--pushbullet',
|
||||||
|
action='store',
|
||||||
|
dest='pushbullet',
|
||||||
|
help='If you want to use pushbullet, pass in your key here.')
|
||||||
|
option_parser.add_option(
|
||||||
|
'-d',
|
||||||
|
'--device',
|
||||||
|
action='store',
|
||||||
|
dest='pbdevice',
|
||||||
|
help='If you wish to only send to a certian pushbullet device, put the device name here. If the device name is invalid, will just send to all pushbullets associated with the acc')
|
||||||
|
option_parser.add_option(
|
||||||
|
'-n',
|
||||||
|
'--notify',
|
||||||
|
action='store_true',
|
||||||
|
dest='notify',
|
||||||
|
help='Enable if you want to use system notifications. Only for Win/Linux.')
|
||||||
|
option_parser.add_option(
|
||||||
|
'-c',
|
||||||
|
'--config',
|
||||||
|
action='store',
|
||||||
|
dest='config',
|
||||||
|
help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.')
|
||||||
|
option_parser.add_option(
|
||||||
|
'-t',
|
||||||
|
'--tag',
|
||||||
|
action='store_true',
|
||||||
|
dest='tag',
|
||||||
|
help='Strip Last Updated tags from calibredb. Requires library to be passed in.')
|
||||||
|
option_parser.add_option(
|
||||||
|
'-l',
|
||||||
|
'--library',
|
||||||
|
action='store',
|
||||||
|
dest='library',
|
||||||
|
help='Path to calibre library. If you are connecting to a calibre webserver then this should be the url.')
|
||||||
|
option_parser.add_option(
|
||||||
|
'-a',
|
||||||
|
'--library-path',
|
||||||
|
action='store',
|
||||||
|
dest='library_path',
|
||||||
|
help='Path location of library. Will be equal to library if nothing is passed in.')
|
||||||
|
|
||||||
|
(options, args) = option_parser.parse_args()
|
||||||
|
|
||||||
|
if options.library and not options.library_path:
|
||||||
|
options.library_path = options.library
|
||||||
|
|
||||||
|
if options.config:
|
||||||
|
config = ConfigParser(allow_no_value=True)
|
||||||
|
config.read(options.config)
|
||||||
|
|
||||||
|
def updater(option, newval): return newval if newval != "" else option
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.pushbullet = updater(
|
||||||
|
options.pushbullet, config.get(
|
||||||
|
'runner', 'pushbullet'))
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.pbdevice = updater(
|
||||||
|
options.pbdevice, config.get(
|
||||||
|
'runner', 'pbdevice'))
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.tag = updater(
|
||||||
|
options.tag, config.getboolean(
|
||||||
|
'runner', 'tag'))
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.library = updater(
|
||||||
|
options.library, config.get(
|
||||||
|
'locations', 'library'))
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
options.library_path = updater(
|
||||||
|
options.library, config.get(
|
||||||
|
'locations', 'library_path'))
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if options.pbdevice and not options.pushbullet:
|
||||||
|
raise ValueError("Can't use a pushbullet device without key")
|
||||||
|
if options.tag and not options.library:
|
||||||
|
raise ValueError(
|
||||||
|
"Can't strip tags from calibre library without a library location.")
|
||||||
|
main(options)
|
18
root/config.default/config.ini
Normal file
18
root/config.default/config.ini
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
[login]
|
||||||
|
user=
|
||||||
|
password=
|
||||||
|
server=
|
||||||
|
mailbox=
|
||||||
|
libuser=
|
||||||
|
libpassword=
|
||||||
|
|
||||||
|
[locations]
|
||||||
|
library=
|
||||||
|
input=/config/fanfiction_file
|
||||||
|
|
||||||
|
[runner]
|
||||||
|
pushbullet=False
|
||||||
|
pbdevice=
|
||||||
|
|
||||||
|
[output]
|
||||||
|
live=False
|
2372
root/config.default/defaults.ini
Normal file
2372
root/config.default/defaults.ini
Normal file
File diff suppressed because it is too large
Load Diff
0
root/config.default/fanfiction_file
Normal file
0
root/config.default/fanfiction_file
Normal file
140
root/config.default/personal.ini
Normal file
140
root/config.default/personal.ini
Normal file
@ -0,0 +1,140 @@
|
|||||||
|
## This is an example of what your personal configuration might look
|
||||||
|
## like. Uncomment options by removing the '#' in front of them.
|
||||||
|
|
||||||
|
[defaults]
|
||||||
|
## [defaults] section applies to all formats and sites but may be
|
||||||
|
## overridden at several levels. Example:
|
||||||
|
|
||||||
|
## [defaults]
|
||||||
|
## titlepage_entries: category,genre, status
|
||||||
|
## [www.whofic.com]
|
||||||
|
## # overrides defaults.
|
||||||
|
## titlepage_entries: category,genre, status,dateUpdated,rating
|
||||||
|
## [epub]
|
||||||
|
## # overrides defaults & site section
|
||||||
|
## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
|
||||||
|
## [www.whofic.com:epub]
|
||||||
|
## # overrides defaults, site section & format section
|
||||||
|
## titlepage_entries: category,genre, status,datePublished
|
||||||
|
## [overrides]
|
||||||
|
## # overrides all other sections
|
||||||
|
## titlepage_entries: category
|
||||||
|
|
||||||
|
## Some sites also require the user to confirm they are adult for
|
||||||
|
## adult content. Uncomment by removing '#' in front of is_adult.
|
||||||
|
is_adult:true
|
||||||
|
|
||||||
|
## Don't like the numbers at the start of chapter titles on some
|
||||||
|
## sites? You can use strip_chapter_numbers to strip them off. Just
|
||||||
|
## want to make them all look the same? Strip them off, then add them
|
||||||
|
## back on with add_chapter_numbers. Don't like the way it strips
|
||||||
|
## numbers or adds them back? See chapter_title_strip_pattern and
|
||||||
|
## chapter_title_add_pattern.
|
||||||
|
#strip_chapter_numbers:true
|
||||||
|
#add_chapter_numbers:true
|
||||||
|
|
||||||
|
## Add this to genre if there's more than one category.
|
||||||
|
#add_genre_when_multi_category: Crossover
|
||||||
|
|
||||||
|
[epub]
|
||||||
|
## include images from img tags in the body and summary of stories.
|
||||||
|
## Images will be converted to jpg for size if possible. Images work
|
||||||
|
## in epub format only. To get mobi or other format with images,
|
||||||
|
## download as epub and use Calibre to convert.
|
||||||
|
include_images:true
|
||||||
|
|
||||||
|
## If not set, the summary will have all html stripped for safety.
|
||||||
|
## Both this and include_images must be true to get images in the
|
||||||
|
## summary.
|
||||||
|
#keep_summary_html:true
|
||||||
|
|
||||||
|
## If set, the first image found will be made the cover image. If
|
||||||
|
## keep_summary_html is true, any images in summary will be before any
|
||||||
|
## in chapters.
|
||||||
|
make_firstimage_cover:true
|
||||||
|
|
||||||
|
## Resize images down to width, height, preserving aspect ratio.
|
||||||
|
## Nook size, with margin.
|
||||||
|
#image_max_size: 580, 725
|
||||||
|
|
||||||
|
## Change image to grayscale, if graphics library allows, to save
|
||||||
|
## space.
|
||||||
|
#grayscale_images: false
|
||||||
|
|
||||||
|
|
||||||
|
## Most common, I expect will be using this to save username/passwords
|
||||||
|
## for different sites. Here are a few examples. See defaults.ini
|
||||||
|
## for the full list.
|
||||||
|
|
||||||
|
[www.twilighted.net]
|
||||||
|
#username:YourPenname
|
||||||
|
#password:YourPassword
|
||||||
|
## default is false
|
||||||
|
#collect_series: true
|
||||||
|
|
||||||
|
[ficwad.com]
|
||||||
|
#username:YourUsername
|
||||||
|
#password:YourPassword
|
||||||
|
|
||||||
|
[www.adastrafanfic.com]
|
||||||
|
## Some sites do not require a login, but do require the user to
|
||||||
|
## confirm they are adult for adult content.
|
||||||
|
#is_adult:true
|
||||||
|
|
||||||
|
[www.fictionalley.org]
|
||||||
|
#is_adult:true
|
||||||
|
|
||||||
|
[www.harrypotterfanfiction.com]
|
||||||
|
#is_adult:true
|
||||||
|
|
||||||
|
[www.fimfiction.net]
|
||||||
|
#is_adult:true
|
||||||
|
#fail_on_password: false
|
||||||
|
|
||||||
|
[www.tthfanfic.org]
|
||||||
|
#is_adult:true
|
||||||
|
## tth is a little unusual--it doesn't require user/pass, but the site
|
||||||
|
## keeps track of which chapters you've read and won't send another
|
||||||
|
## update until it thinks you're up to date. This way, on download,
|
||||||
|
## it thinks you're up to date.
|
||||||
|
#username:YourName
|
||||||
|
#password:yourpassword
|
||||||
|
|
||||||
|
[www.fanfiction.net]
|
||||||
|
check_next_chapter:true
|
||||||
|
skip_author_cover:false
|
||||||
|
user_agent:Mozilla/5.0
|
||||||
|
continue_on_chapter_error:false
|
||||||
|
# for use with the workaround
|
||||||
|
#use_nsapa_proxy:true
|
||||||
|
#use_cloudscraper:false
|
||||||
|
use_flaresolverr_proxy:true
|
||||||
|
# option settings, these are the defaults:
|
||||||
|
flaresolverr_proxy_address:localhost
|
||||||
|
flaresolverr_proxy_port:8191
|
||||||
|
flaresolverr_proxy_protocol:http
|
||||||
|
|
||||||
|
## This option if uncommented, will put a box around the spoiler
|
||||||
|
## blocks with the original spoiler button text as a label using
|
||||||
|
## fieldset and legend HTML tags. For a simple box, see the
|
||||||
|
## add_to_output_css example for [base_xenforoforum:epub].
|
||||||
|
#legend_spoilers:true
|
||||||
|
|
||||||
|
|
||||||
|
## This section will override anything in the system defaults or other
|
||||||
|
## sections here.
|
||||||
|
[overrides]
|
||||||
|
## default varies by site. Set true here to force all sites to
|
||||||
|
## collect series.
|
||||||
|
#collect_series: true
|
||||||
|
|
||||||
|
# Change oneshot to ['Completed'] to ['Completed','Oneshot'] only when
|
||||||
|
# numChapters is exactly 1.
|
||||||
|
# with ',' instead of '\,' it would be ['Completed'] to
|
||||||
|
# ['Completed,Oneshot']--one string instead of two.
|
||||||
|
add_to_replace_metadata:
|
||||||
|
oneshot=>Completed=>Completed\,Oneshot&&numChapters=>^1$
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
25
root/etc/cont-init.d/05-default-confs
Normal file
25
root/etc/cont-init.d/05-default-confs
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/with-contenv sh
|
||||||
|
|
||||||
|
if ! [ -f "/config/config.ini" ]
|
||||||
|
then
|
||||||
|
echo "[default-confs] restoring default 'config.ini'."
|
||||||
|
cp /config.default/config.ini /config/
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! [ -f "/config/defaults.ini" ]
|
||||||
|
then
|
||||||
|
echo "[default-confs] restoring default 'defaults.ini'."
|
||||||
|
cp /config.default/defaults.ini /config/
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! [ -f "/config/personal.ini" ]
|
||||||
|
then
|
||||||
|
echo "[default-confs] restoring default 'personal.ini'."
|
||||||
|
cp /config.default/personal.ini /config/
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! [ -f "/config/fanfiction_file" ]
|
||||||
|
then
|
||||||
|
echo "[default-confs] restoring default 'fanfiction_file'."
|
||||||
|
cp /config.default/fanfiction_file /config/
|
||||||
|
fi
|
10
root/etc/cont-init.d/90-user-permissions
Normal file
10
root/etc/cont-init.d/90-user-permissions
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/with-contenv bash
|
||||||
|
|
||||||
|
groupmod -o -g "$PGID" abc
|
||||||
|
usermod -o -u "$PUID" abc
|
||||||
|
|
||||||
|
chown -R abc:abc /app
|
||||||
|
chown -R abc:abc /config
|
||||||
|
chown -R abc:abc /root
|
||||||
|
|
||||||
|
chmod +x /app/run.sh
|
11
root/etc/cont-init.d/99-init.d-finish
Normal file
11
root/etc/cont-init.d/99-init.d-finish
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
#!/usr/bin/with-contenv sh
|
||||||
|
|
||||||
|
echo "
|
||||||
|
UID/GID
|
||||||
|
---------------------------------------
|
||||||
|
User uid: $(id -u abc)
|
||||||
|
User gid: $(id -g abc)
|
||||||
|
---------------------------------------
|
||||||
|
|
||||||
|
Image Made By: MrTyton
|
||||||
|
"
|
4
root/etc/services.d/ffdl/run
Normal file
4
root/etc/services.d/ffdl/run
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
#!/usr/bin/with-contenv bash
|
||||||
|
|
||||||
|
sleep 30s
|
||||||
|
exec s6-setuidgid abc /app/run.sh
|
Loading…
Reference in New Issue
Block a user