X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=nhentai%2Futils.py;h=835463785a18240fe875911b14f7a4687a4a48e2;hb=ea356a1ca24b99e3229a6c547a77a79221c201a9;hp=16f8bdf84a542cc0ab15b51eb9eb22758fcccb72;hpb=c1a82635bdce40e5800d4f0498bb30cad7aa268d;p=nhentai.git diff --git a/nhentai/utils.py b/nhentai/utils.py index 16f8bdf..8354637 100644 --- a/nhentai/utils.py +++ b/nhentai/utils.py @@ -1,34 +1,41 @@ # coding: utf-8 -from __future__ import unicode_literals, print_function import sys import re import os -import string import zipfile import shutil import requests +import sqlite3 from nhentai import constant from nhentai.logger import logger -from nhentai.serializer import serialize, set_js_database +from nhentai.serializer import serialize_json, serialize_comic_xml, set_js_database def request(method, url, **kwargs): session = requests.Session() session.headers.update({ 'Referer': constant.LOGIN_URL, - 'User-Agent': 'nhentai command line client (https://github.com/RicterZ/nhentai)', - 'Cookie': constant.COOKIE + 'User-Agent': constant.CONFIG['useragent'], + 'Cookie': constant.CONFIG['cookie'] }) - return getattr(session, method)(url, proxies=constant.PROXY, verify=False, **kwargs) + + if not kwargs.get('proxies', None): + kwargs['proxies'] = constant.CONFIG['proxy'] + + return getattr(session, method)(url, verify=False, **kwargs) def check_cookie(): - response = request('get', constant.BASE_URL).text - username = re.findall('"/users/\d+/(.*?)"', response) + response = request('get', constant.BASE_URL) + if response.status_code == 503 and 'cf-browser-verification' in response.text: + logger.error('Blocked by Cloudflare captcha, please set your cookie and useragent') + exit(-1) + + username = re.findall('"/users/\d+/(.*?)"', response.text) if not username: - logger.error('Cannot get your username, please check your cookie or use `nhentai --cookie` to set your cookie') + logger.warning('Cannot get your username, please check your cookie or use `nhentai --cookie` to set your cookie') else: logger.info('Login successfully! Your username: {}'.format(username[0])) @@ -63,7 +70,7 @@ def readfile(path): return file.read() -def generate_html(output_dir='.', doujinshi_obj=None): +def generate_html(output_dir='.', doujinshi_obj=None, template='default'): image_html = '' if doujinshi_obj is not None: @@ -71,6 +78,13 @@ def generate_html(output_dir='.', doujinshi_obj=None): else: doujinshi_dir = '.' + if not os.path.exists(doujinshi_dir): + logger.warning('Path \'{0}\' does not exist, creating.'.format(doujinshi_dir)) + try: + os.makedirs(doujinshi_dir) + except EnvironmentError as e: + logger.critical('{0}'.format(str(e))) + file_list = os.listdir(doujinshi_dir) file_list.sort() @@ -78,14 +92,14 @@ def generate_html(output_dir='.', doujinshi_obj=None): if not os.path.splitext(image)[1] in ('.jpg', '.png'): continue - image_html += '\n'\ + image_html += '\n' \ .format(image) - html = readfile('viewer/index.html') - css = readfile('viewer/styles.css') - js = readfile('viewer/scripts.js') + html = readfile('viewer/{}/index.html'.format(template)) + css = readfile('viewer/{}/styles.css'.format(template)) + js = readfile('viewer/{}/scripts.js'.format(template)) if doujinshi_obj is not None: - serialize(doujinshi_obj, doujinshi_dir) + serialize_json(doujinshi_obj, doujinshi_dir) name = doujinshi_obj.name if sys.version_info < (3, 0): name = doujinshi_obj.name.encode('utf-8') @@ -101,9 +115,9 @@ def generate_html(output_dir='.', doujinshi_obj=None): with open(os.path.join(doujinshi_dir, 'index.html'), 'wb') as f: f.write(data.encode('utf-8')) - logger.log(15, 'HTML Viewer has been write to \'{0}\''.format(os.path.join(doujinshi_dir, 'index.html'))) + logger.log(15, 'HTML Viewer has been written to \'{0}\''.format(os.path.join(doujinshi_dir, 'index.html'))) except Exception as e: - logger.warning('Writen HTML Viewer failed ({})'.format(str(e))) + logger.warning('Writing HTML Viewer failed ({})'.format(str(e))) def generate_main_html(output_dir='./'): @@ -149,7 +163,7 @@ def generate_main_html(output_dir='./'): image_html += element.format(FOLDER=folder, IMAGE=image, TITLE=title) if image_html == '': - logger.warning('None index.html found, --gen-main paused.') + logger.warning('No index.html found, --gen-main paused.') return try: data = main.format(STYLES=css, SCRIPTS=js, PICTURE=image_html) @@ -159,17 +173,19 @@ def generate_main_html(output_dir='./'): else: with open('./main.html', 'wb') as f: f.write(data.encode('utf-8')) - shutil.copy(os.path.dirname(__file__)+'/viewer/logo.png', './') + shutil.copy(os.path.dirname(__file__) + '/viewer/logo.png', './') set_js_database() logger.log( - 15, 'Main Viewer has been write to \'{0}main.html\''.format(output_dir)) + 15, 'Main Viewer has been written to \'{0}main.html\''.format(output_dir)) except Exception as e: - logger.warning('Writen Main Viewer failed ({})'.format(str(e))) + logger.warning('Writing Main Viewer failed ({})'.format(str(e))) -def generate_cbz(output_dir='.', doujinshi_obj=None, rm_origin_dir=False): +def generate_cbz(output_dir='.', doujinshi_obj=None, rm_origin_dir=False, write_comic_info=True): if doujinshi_obj is not None: doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename) + if write_comic_info: + serialize_comic_xml(doujinshi_obj, doujinshi_dir) cbz_filename = os.path.join(os.path.join(doujinshi_dir, '..'), '{}.cbz'.format(doujinshi_obj.filename)) else: cbz_filename = './doujinshi.cbz' @@ -187,30 +203,148 @@ def generate_cbz(output_dir='.', doujinshi_obj=None, rm_origin_dir=False): if rm_origin_dir: shutil.rmtree(doujinshi_dir, ignore_errors=True) - logger.log(15, 'Comic Book CBZ file has been write to \'{0}\''.format(doujinshi_dir)) + logger.log(15, 'Comic Book CBZ file has been written to \'{0}\''.format(doujinshi_dir)) + + +def generate_pdf(output_dir='.', doujinshi_obj=None, rm_origin_dir=False): + try: + import img2pdf + + """Write images to a PDF file using img2pdf.""" + if doujinshi_obj is not None: + doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename) + pdf_filename = os.path.join( + os.path.join(doujinshi_dir, '..'), + '{}.pdf'.format(doujinshi_obj.filename) + ) + else: + pdf_filename = './doujinshi.pdf' + doujinshi_dir = '.' + + file_list = os.listdir(doujinshi_dir) + file_list.sort() + + logger.info('Writing PDF file to path: {}'.format(pdf_filename)) + with open(pdf_filename, 'wb') as pdf_f: + full_path_list = ( + [os.path.join(doujinshi_dir, image) for image in file_list] + ) + pdf_f.write(img2pdf.convert(full_path_list)) + + if rm_origin_dir: + shutil.rmtree(doujinshi_dir, ignore_errors=True) + + logger.log(15, 'PDF file has been written to \'{0}\''.format(doujinshi_dir)) + + except ImportError: + logger.error("Please install img2pdf package by using pip.") + + +def unicode_truncate(s, length, encoding='utf-8'): + """https://stackoverflow.com/questions/1809531/truncating-unicode-so-it-fits-a-maximum-size-when-encoded-for-wire-transfer + """ + encoded = s.encode(encoding)[:length] + return encoded.decode(encoding, 'ignore') def format_filename(s): - """Take a string and return a valid filename constructed from the string. -Uses a whitelist approach: any characters not present in valid_chars are -removed. Also spaces are replaced with underscores. - -Note: this method may produce invalid filenames such as ``, `.` or `..` -When I use this method I prepend a date string like '2009_01_15_19_46_32_' -and append a file extension like '.txt', so I avoid the potential of using -an invalid filename. - -""" - valid_chars = "-_.()[] %s%s" % (string.ascii_letters, string.digits) - filename = ''.join(c for c in s if c in valid_chars) + """ + It used to be a whitelist approach allowed only alphabet and a part of symbols. + but most doujinshi's names include Japanese 2-byte characters and these was rejected. + so it is using blacklist approach now. + if filename include forbidden characters (\'/:,;*?"<>|) ,it replace space character(' '). + """ + # maybe you can use `--format` to select a suitable filename + ban_chars = '\\\'/:,;*?"<>|\t' + filename = s.translate(str.maketrans(ban_chars, ' ' * len(ban_chars))).strip() + filename = ' '.join(filename.split()) + + while filename.endswith('.'): + filename = filename[:-1] + if len(filename) > 100: - filename = filename[:100] + '...]' + filename = filename[:100] + u'…' # Remove [] from filename - filename = filename.replace('[]', '') + filename = filename.replace('[]', '').strip() return filename def signal_handler(signal, frame): logger.error('Ctrl-C signal received. Stopping...') exit(1) + + +def paging(page_string): + # 1,3-5,14 -> [1, 3, 4, 5, 14] + if not page_string: + return [] + + page_list = [] + for i in page_string.split(','): + if '-' in i: + start, end = i.split('-') + if not (start.isdigit() and end.isdigit()): + raise Exception('Invalid page number') + page_list.extend(list(range(int(start), int(end) + 1))) + else: + if not i.isdigit(): + raise Exception('Invalid page number') + page_list.append(int(i)) + + return page_list + + +def generate_metadata_file(output_dir, table, doujinshi_obj=None): + logger.info('Writing Metadata Info') + + if doujinshi_obj is not None: + doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename) + else: + doujinshi_dir = '.' + + logger.info(doujinshi_dir) + + f = open(os.path.join(doujinshi_dir, 'info.txt'), 'w', encoding='utf-8') + + fields = ['TITLE', 'ORIGINAL TITLE', 'AUTHOR', 'ARTIST', 'CIRCLE', 'SCANLATOR', + 'TRANSLATOR', 'PUBLISHER', 'DESCRIPTION', 'STATUS', 'CHAPTERS', 'PAGES', + 'TAGS', 'TYPE', 'LANGUAGE', 'RELEASED', 'READING DIRECTION', 'CHARACTERS', + 'SERIES', 'PARODY', 'URL'] + special_fields = ['PARODY', 'TITLE', 'ORIGINAL TITLE', 'CHARACTERS', 'AUTHOR', + 'LANGUAGE', 'TAGS', 'URL', 'PAGES'] + + for i in range(len(fields)): + f.write('{}: '.format(fields[i])) + if fields[i] in special_fields: + f.write(str(table[special_fields.index(fields[i])][1])) + f.write('\n') + + f.close() + + +class DB(object): + conn = None + cur = None + + def __enter__(self): + self.conn = sqlite3.connect(constant.NHENTAI_HISTORY) + self.cur = self.conn.cursor() + self.cur.execute('CREATE TABLE IF NOT EXISTS download_history (id text)') + self.conn.commit() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.conn.close() + + def clean_all(self): + self.cur.execute('DELETE FROM download_history WHERE 1') + self.conn.commit() + + def add_one(self, data): + self.cur.execute('INSERT INTO download_history VALUES (?)', [data]) + self.conn.commit() + + def get_all(self): + data = self.cur.execute('SELECT id FROM download_history') + return [i[0] for i in data]