6 from future.builtins import str as text
13 from urllib.parse import urlparse
15 from urlparse import urlparse
17 from nhentai.logger import logger
18 from nhentai.parser import request
19 from nhentai.utils import Singleton
21 requests.packages.urllib3.disable_warnings()
22 semaphore = multiprocessing.Semaphore(1)
25 class NHentaiImageNotExistException(Exception):
29 class Downloader(Singleton):
31 def __init__(self, path='', size=5, timeout=30, delay=0):
34 self.timeout = timeout
37 def download_(self, url, folder='', filename='', retried=0):
39 time.sleep(self.delay)
40 logger.info('Starting to download {0} ...'.format(url))
41 filename = filename if filename else os.path.basename(urlparse(url).path)
42 base_filename, extension = os.path.splitext(filename)
44 if os.path.exists(os.path.join(folder, base_filename.zfill(3) + extension)):
45 logger.warning('File: {0} exists, ignoring'.format(os.path.join(folder, base_filename.zfill(3) +
50 with open(os.path.join(folder, base_filename.zfill(3) + extension), "wb") as f:
54 response = request('get', url, stream=True, timeout=self.timeout)
55 if response.status_code != 200:
56 raise NHentaiImageNotExistException
58 except NHentaiImageNotExistException as e:
61 except Exception as e:
64 logger.critical(str(e))
70 length = response.headers.get('content-length')
72 f.write(response.content)
74 for chunk in response.iter_content(2048):
77 except (requests.HTTPError, requests.Timeout) as e:
79 logger.warning('Warning: {0}, retrying({1}) ...'.format(str(e), retried))
80 return 0, self.download_(url=url, folder=folder, filename=filename, retried=retried+1)
84 except NHentaiImageNotExistException as e:
85 os.remove(os.path.join(folder, base_filename.zfill(3) + extension))
88 except Exception as e:
90 traceback.print_stack()
91 logger.critical(str(e))
94 except KeyboardInterrupt:
99 def _download_callback(self, result):
100 result, data = result
102 logger.warning('fatal errors occurred, ignored')
105 logger.warning('url {} return status code 404'.format(data))
107 logger.warning('Ctrl-C pressed, exiting sub processes ...')
109 # workers wont be run, just pass
112 logger.log(15, '{0} downloaded successfully'.format(data))
114 def download(self, queue, folder=''):
115 if not isinstance(folder, text):
119 folder = os.path.join(self.path, folder)
121 if not os.path.exists(folder):
122 logger.warn('Path \'{0}\' does not exist, creating.'.format(folder))
125 except EnvironmentError as e:
126 logger.critical('{0}'.format(str(e)))
129 logger.warn('Path \'{0}\' already exist.'.format(folder))
131 queue = [(self, url, folder) for url in queue]
133 pool = multiprocessing.Pool(self.size, init_worker)
134 [pool.apply_async(download_wrapper, args=item) for item in queue]
140 def download_wrapper(obj, url, folder=''):
141 if sys.platform == 'darwin' or semaphore.get_value():
142 return Downloader.download_(obj, url=url, folder=folder)
148 signal.signal(signal.SIGINT, subprocess_signal)
151 def subprocess_signal(signal, frame):
152 if semaphore.acquire(timeout=1):
153 logger.warning('Ctrl-C pressed, exiting sub processes ...')
155 raise KeyboardInterrupt