@ -1,62 +1,68 @@
from fanficfare import geturls
from fanficfare import geturls
import os
import os
from os import listdir , remove , rename
from os import listdir , remove , rename , utime
from os . path import isfile , join
from os . path import isfile , join
import subprocess
from subprocess import check_output , STDOUT
import time
import logging
import logging
from optparse import OptionParser
from optparse import OptionParser
import re
import re
from ConfigParser import ConfigParser
logging . getLogger ( " fanficfare " ) . setLevel ( logging . ERROR )
logging . getLogger ( " fanficfare " ) . setLevel ( logging . ERROR )
def touch ( fname , times = None ) :
with open ( fname , ' a ' ) :
utime ( fname , times )
def main ( user , password , server = " imap.gmail.com " , label = " INBOX " , path = " " ) :
ffnet = re . compile ( ' (fanfiction.net/s/ \ d*)/?.* ' )
neutral = re . compile ( ' https?://(.*) ' )
def parse_url ( url ) :
if ffnet . search ( url ) :
url = " www. " + ffnet . search ( url ) . group ( 1 )
elif neutral . search ( url ) :
url = neutral . search ( url ) . group ( 1 )
return url
def main ( user , password , server , label , inout_file , path = " " , ) :
if path != " " :
if path != " " :
path = ' --with-library " {} " ' . format ( path )
path = ' --with-library " {} " ' . format ( path )
with open ( " thesebroke " , " rb " ) as fp :
touch ( inout_file )
with open ( inout_file , " r " ) as fp :
urls = set ( [ x . replace ( " \n " , " " ) for x in fp . readlines ( ) ] )
urls = set ( [ x . replace ( " \n " , " " ) for x in fp . readlines ( ) ] )
with open ( " todo " , " wb " ) as fp :
with open ( inout_file , " w " ) as fp :
fp . write ( " " )
fp . write ( " " )
urls | = geturls . get_urls_from_imap ( server , user , password , label )
urls | = geturls . get_urls_from_imap ( server , user , password , label )
def parse_url ( url ) :
url = url . replace ( " https:// " , " " )
url = url . replace ( " http:// " , " " )
url = url [ : url . find ( " / " , url . find ( " /s/ " ) + 3 ) + 1 ]
return url
urls = set ( parse_url ( x ) for x in urls )
urls = set ( parse_url ( x ) for x in urls )
if len ( urls ) != 0 : print " URLs to parse: {} " . format ( " , " . join ( urls ) )
if len ( urls ) != 0 : print " URLs to parse: {} " . format ( " , " . join ( urls ) )
files = lambda x : [ f for f in listdir ( x ) if isfile ( join ( x , f ) ) ]
files = lambda x : [ f for f in listdir ( x ) if isfile ( join ( x , f ) ) ]
# if len(urls) != 0:
# print time.strftime("%d/%m/%Y, %H:%M:%S\n\n")
for url in urls :
for url in urls :
print " Working with url {} " . format ( url )
print " Working with url {} " . format ( url )
try :
try :
res = subprocess . check_output ( ' calibredb search " Identifiers: {} " {} ' . format ( url , path ) , shell = True , stderr = subprocess . STDOUT )
res = check_output ( ' calibredb search " Identifiers: {} " {} ' . format ( url , path ) , shell = True , stderr = STDOUT )
id = res
id = res
print " \t Story is in calibre with id {} " . format ( id )
print " \t Story is in calibre with id {} " . format ( id )
try :
try :
print " \t Exporting file "
print " \t Exporting file "
res = subprocess . check_output ( ' calibredb export {} --dont-save-cover --dont-write-opf --single-dir {} ' . format ( id , path ) , shell = True )
res = check_output ( ' calibredb export {} --dont-save-cover --dont-write-opf --single-dir {} ' . format ( id , path ) , shell = True )
onlyfiles = files ( " . " )
onlyfiles = files ( " . " )
for cur in onlyfiles :
for cur in onlyfiles :
if not cur . endswith ( " .epub " ) : continue
if not cur . endswith ( " .epub " ) : continue
print ' \t Downloading with fanficfare, updating file " {} " ' . format ( cur )
print ' \t Downloading with fanficfare, updating file " {} " ' . format ( cur )
res = subprocess . check_output ( ' fanficfare -u " {} " --update-cover ' . format ( cur ) , shell = True , stderr = subprocess . STDOUT )
res = check_output ( ' fanficfare -u " {} " --update-cover ' . format ( cur ) , shell = True , stderr = STDOUT )
#print res
#print res
if " already contains " in res :
if " already contains " in res :
print " \t Issue with story, FF.net site is broken. "
print " \t Issue with story, FF.net site is broken. "
with open ( " todo " , " ab " ) as fp :
fp . write ( " {} \n " . format ( url ) )
fp . write ( " {} \n " . format ( url ) )
remove ( cur )
remove ( cur )
continue
continue
@ -65,33 +71,32 @@ def main(user, password, server="imap.gmail.com", label="INBOX", path=""):
continue
continue
elif " more recently than Story " in res :
elif " more recently than Story " in res :
print " \t Forcing download update \n "
print " \t Forcing download update \n "
res = subprocess . check_output ( ' fanficfare -u " {} " --force --update-cover ' . format ( cur ) , shell = True , stderr = subprocess . STDOUT )
res = check_output ( ' fanficfare -u " {} " --force --update-cover ' . format ( cur ) , shell = True , stderr = STDOUT )
print " \t Removing {} from library " . format ( id )
print " \t Removing {} from library " . format ( id )
res = subprocess . check_output ( ' calibredb remove {} {} ' . format ( id , path ) , shell = True , stderr = subprocess . STDOUT )
res = check_output ( ' calibredb remove {} {} ' . format ( id , path ) , shell = True , stderr = STDOUT )
#print res
#print res
print " \t Adding {} to library " . format ( cur )
print " \t Adding {} to library " . format ( cur )
res = subprocess . check_output ( ' calibredb add " {} " {} ' . format ( cur , path ) , shell = True , stderr = subprocess . STDOUT )
res = check_output ( ' calibredb add " {} " {} ' . format ( cur , path ) , shell = True , stderr = STDOUT )
res = subprocess . check_output ( ' calibredb search " Identifiers: {} " {} ' . format ( url , path ) , shell = True , stderr = subprocess . STDOUT )
res = check_output ( ' calibredb search " Identifiers: {} " {} ' . format ( url , path ) , shell = True , stderr = STDOUT )
print " \t Added {} to library with id {} " . format ( cur , res )
print " \t Added {} to library with id {} " . format ( cur , res )
#print res
#print res
remove ( cur )
remove ( cur )
except Exception as e :
except Exception as e :
print " \t Something fucked up: {} " . format ( e )
print " \t Something fucked up: {} " . format ( e )
remove ( cur )
remove ( cur )
with open ( " todo " , " ab " ) as fp :
fp . write ( " {} \n " . format ( url ) )
fp . write ( " {} \n " . format ( url ) )
except :
except :
print " \t Story is not in calibre "
print " \t Story is not in calibre "
try :
try :
res = subprocess . check_output ( ' fanficfare -u " {} " --update-cover ' . format ( url ) , shell = True )
res = check_output ( ' fanficfare -u " {} " --update-cover ' . format ( url ) , shell = True )
#print res
#print res
onlyfiles = files ( " . " )
onlyfiles = files ( " . " )
for cur in onlyfiles :
for cur in onlyfiles :
if not cur . endswith ( " .epub " ) : continue
if not cur . endswith ( " .epub " ) : continue
try :
try :
print " \t Adding {} to library " . format ( cur )
print " \t Adding {} to library " . format ( cur )
res = subprocess . check_output ( ' calibredb add " {} " {} ' . format ( cur , path ) , shell = True , stderr = subprocess . STDOUT )
res = check_output ( ' calibredb add " {} " {} ' . format ( cur , path ) , shell = True , stderr = STDOUT )
#print res
#print res
remove ( cur )
remove ( cur )
except Exception :
except Exception :
@ -99,11 +104,8 @@ def main(user, password, server="imap.gmail.com", label="INBOX", path=""):
raise
raise
except Exception as e :
except Exception as e :
print " \t Something fucked up: {} " . format ( e )
print " \t Something fucked up: {} " . format ( e )
with open ( " todo " , " ab " ) as fp :
fp . write ( " {} \n " . format ( url ) )
fp . write ( " {} \n " . format ( url ) )
remove ( " thesebroke " )
rename ( " todo " , " thesebroke " )
if __name__ == " __main__ " :
if __name__ == " __main__ " :
option_parser = OptionParser ( usage = " usage: % prog [flags] " )
option_parser = OptionParser ( usage = " usage: % prog [flags] " )
@ -114,38 +116,39 @@ if __name__ == "__main__":
option_parser . add_option ( ' -s ' , ' --server ' , action = ' store ' , dest = ' server ' , default = " imap.gmail.com " , help = ' Email IMAP Server. Default is " imap.gmail.com " . ' )
option_parser . add_option ( ' -s ' , ' --server ' , action = ' store ' , dest = ' server ' , default = " imap.gmail.com " , help = ' Email IMAP Server. Default is " imap.gmail.com " . ' )
option_parser . add_option ( ' -m ' , ' --mailbox ' , action = ' store ' , dest = ' label ' , default = ' INBOX ' , help = ' Email Label. Default is " INBOX " . ' )
option_parser . add_option ( ' -m ' , ' --mailbox ' , action = ' store ' , dest = ' mailbox ' , default = ' INBOX ' , help = ' Email Label. Default is " INBOX " . ' )
option_parser . add_option ( ' -l ' , ' --library ' , action = ' store ' , dest = ' library ' , default = " " , help = " calibre library db location. If none is passed, the default is the calibre system library location. Make sure to enclose the path in quotes. " )
option_parser . add_option ( ' -l ' , ' --library ' , action = ' store ' , dest = ' library ' , default = " " , help = " calibre library db location. If none is passed, the default is the calibre system library location. Make sure to enclose the path in quotes. " )
option_parser . add_option ( ' -c ' , ' --config ' , action = ' store ' , dest - ' config ' , help = ' Config file for inputs. One argument per line, format of field=value ' )
option_parser . add_option ( ' -i ' , ' --input ' , action = ' store ' , dest = ' input ' , default = " ./fanfiction.txt " , help = " Error file. Any urls that fail will be output here, and file will be read to find any urls that failed previously. If file does not exist will create. File is overwitten every time the program is run. " )
option_parser . add_option ( ' -c ' , ' --config ' , action = ' store ' , dest = ' config ' , help = ' Config file for inputs. Example config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. ' )
( options , args ) = option_parser . parse_args ( )
( options , args ) = option_parser . parse_args ( )
if options . config :
if options . config :
user = re . compile ( ' user=(.*?) ' )
config = ConfigParser ( allow_no_value = True )
passwrd = re . compile ( ' password=(.*?) ' )
config . read ( options . config )
serv = re . compile ( ' server=(.*?) ' )
mailbox = re . compile ( ' mailbox=(.*?) ' )
updater = lambda option , newval : newval if newval != " " else option
lib = re . compile ( ' library=(.*?) ' )
with open ( options . config , ' r ' ) as configfile :
options . user = updater ( options . user , config . get ( ' login ' , ' user ' ) . strip ( ) )
for line in configfile . readlines ( ) :
line = line . tolower ( ) . strip ( )
options . password = updater ( options . password , config . get ( ' login ' , ' password ' ) . strip ( ) )
if user . search ( line ) :
options . user = user . search ( line ) . group ( 1 )
options . server = updater ( options . server , config . get ( ' login ' , ' server ' ) . strip ( ) )
elif passwrd . search ( line ) :
options . password = passwrd . search ( line ) . group ( 1 )
options . mailbox = updater ( options . mailbox , config . get ( ' login ' , ' mailbox ' ) . strip ( ) )
elif serv . search ( line ) :
options . server = serv . search ( line ) . group ( 1 )
options . library = updater ( options . library , config . get ( ' locations ' , ' library ' ) . strip ( ) )
elif mailbox . search ( line ) :
options . label = mailbox . search ( line ) . group ( 1 )
options . input = updater ( options . input , config . get ( ' locations ' , ' input ' ) . strip ( ) )
elif lib . search ( line ) :
options . library = mailbox . search ( line ) . group ( 1 )
print options
if not ( options . user or options . password ) :
if not ( options . user or options . password ) :
raise ValueError ( " User or Password not given " )
raise ValueError ( " User or Password not given " )
main ( options . user , options . password , options . server , options . label , options . library )
main ( options . user , options . password , options . server , options . mailbox, options . input , options . library )