X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=nhentai%2Fparser.py;h=c61f8f71515bdf0d3a0fc2dd1b567d6b92bb9341;hb=f157ac32469231a01823fd14b17870fb4e6b44e8;hp=ae628d607c6d64b5596f4b5653d0b12ff616eec4;hpb=f5b7d89fb043da0c6271247b31dd1f98ee8c07b2;p=nhentai.git diff --git a/nhentai/parser.py b/nhentai/parser.py index ae628d6..c61f8f7 100644 --- a/nhentai/parser.py +++ b/nhentai/parser.py @@ -1,34 +1,18 @@ # coding: utf-8 from __future__ import unicode_literals, print_function +import sys import os import re -import threadpool -import requests import time from bs4 import BeautifulSoup from tabulate import tabulate import nhentai.constant as constant +from nhentai.utils import request from nhentai.logger import logger -session = requests.Session() -session.headers.update({ - 'Referer': constant.LOGIN_URL, - 'User-Agent': 'nhentai command line client (https://github.com/RicterZ/nhentai)', -}) - - -def request(method, url, **kwargs): - global session - if not hasattr(session, method): - raise AttributeError('\'requests.Session\' object has no attribute \'{0}\''.format(method)) - - session.headers.update({'Cookie': constant.COOKIE}) - return getattr(session, method)(url, proxies=constant.PROXY, verify=False, **kwargs) - - def _get_csrf_token(content): html = BeautifulSoup(content, 'html.parser') csrf_token_elem = html.find('input', attrs={'name': 'csrfmiddlewaretoken'}) @@ -66,7 +50,22 @@ def login(username, password): exit(2) -def favorites_parser(): +def _get_title_and_id(response): + result = [] + html = BeautifulSoup(response, 'html.parser') + doujinshi_search_result = html.find_all('div', attrs={'class': 'gallery'}) + for doujinshi in doujinshi_search_result: + doujinshi_container = doujinshi.find('div', attrs={'class': 'caption'}) + title = doujinshi_container.text.strip() + title = title if len(title) < 85 else title[:82] + '...' + id_ = re.search('/g/(\d+)/', doujinshi.a['href']).group(1) + result.append({'id': id_, 'title': title}) + + return result + + +def favorites_parser(page=None): + result = [] html = BeautifulSoup(request('get', constant.FAV_URL).content, 'html.parser') count = html.find('span', attrs={'class': 'count'}) if not count: @@ -79,30 +78,31 @@ def favorites_parser(): return [] pages = int(count / 25) - if pages: - pages += 1 if count % (25 * pages) else 0 + if page: + page_range_list = page else: - pages = 1 + if pages: + pages += 1 if count % (25 * pages) else 0 + else: + pages = 1 - logger.info('You have %d favorites in %d pages.' % (count, pages)) + logger.info('You have %d favorites in %d pages.' % (count, pages)) - if os.getenv('DEBUG'): - pages = 1 + if os.getenv('DEBUG'): + pages = 1 - ret = [] - doujinshi_id = re.compile('data-id="([\d]+)"') + page_range_list = range(1, pages + 1) - for page in range(1, pages + 1): + for page in page_range_list: try: logger.info('Getting doujinshi ids of page %d' % page) - resp = request('get', constant.FAV_URL + '?page=%d' % page).text - ids = doujinshi_id.findall(resp) - ret.extend(ids) + resp = request('get', constant.FAV_URL + '?page=%d' % page).content + result.extend(_get_title_and_id(resp)) except Exception as e: logger.error('Error: %s, continue', str(e)) - return ret + return result def doujinshi_parser(id_): @@ -117,7 +117,7 @@ def doujinshi_parser(id_): try: response = request('get', url) - if response.status_code in (200, ): + if response.status_code in (200,): response = response.content else: logger.debug('Slow down and retry ({}) ...'.format(id_)) @@ -125,8 +125,8 @@ def doujinshi_parser(id_): return doujinshi_parser(str(id_)) except Exception as e: - logger.critical(str(e)) - raise SystemExit + logger.warn('Error: {}, ignored'.format(str(e))) + return None html = BeautifulSoup(response, 'html.parser') doujinshi_info = html.find('div', attrs={'id': 'info'}) @@ -138,7 +138,7 @@ def doujinshi_parser(id_): doujinshi['subtitle'] = subtitle.text if subtitle else '' doujinshi_cover = html.find('div', attrs={'id': 'cover'}) - img_id = re.search('/galleries/([\d]+)/cover\.(jpg|png)$', doujinshi_cover.a.img.attrs['data-src']) + img_id = re.search('/galleries/([\d]+)/cover\.(jpg|png|gif)$', doujinshi_cover.a.img.attrs['data-src']) ext = [] for i in html.find_all('div', attrs={'class': 'thumb-container'}): @@ -152,51 +152,89 @@ def doujinshi_parser(id_): doujinshi['img_id'] = img_id.group(1) doujinshi['ext'] = ext - pages = 0 - for _ in doujinshi_info.find_all('div', class_=''): - pages = re.search('([\d]+) pages', _.text) - if pages: - pages = pages.group(1) - break + for _ in doujinshi_info.find_all('div', class_='tag-container field-name'): + if re.search('Pages:', _.text): + pages = _.find('span', class_='name').string doujinshi['pages'] = int(pages) # gain information of the doujinshi information_fields = doujinshi_info.find_all('div', attrs={'class': 'field-name'}) - needed_fields = ['Characters', 'Artists', 'Languages', 'Tags'] + needed_fields = ['Characters', 'Artists', 'Languages', 'Tags', 'Parodies', 'Groups', 'Categories'] for field in information_fields: field_name = field.contents[0].strip().strip(':') if field_name in needed_fields: - data = [sub_field.contents[0].strip() for sub_field in + data = [sub_field.find('span', attrs={'class': 'name'}).contents[0].strip() for sub_field in field.find_all('a', attrs={'class': 'tag'})] doujinshi[field_name.lower()] = ', '.join(data) + time_field = doujinshi_info.find('time') + if time_field.has_attr('datetime'): + doujinshi['date'] = time_field['datetime'] return doujinshi -def search_parser(keyword, page): +def old_search_parser(keyword, sorting='date', page=1): logger.debug('Searching doujinshis of keyword {0}'.format(keyword)) - result = [] - try: - response = request('get', url=constant.SEARCH_URL, params={'q': keyword, 'page': page}).content - except requests.ConnectionError as e: - logger.critical(e) - logger.warn('If you are in China, please configure the proxy to fu*k GFW.') - raise SystemExit + response = request('get', url=constant.SEARCH_URL, params={'q': keyword, 'page': page, 'sort': sorting}).content - html = BeautifulSoup(response, 'html.parser') - doujinshi_search_result = html.find_all('div', attrs={'class': 'gallery'}) - for doujinshi in doujinshi_search_result: - doujinshi_container = doujinshi.find('div', attrs={'class': 'caption'}) - title = doujinshi_container.text.strip() - title = title if len(title) < 85 else title[:82] + '...' - id_ = re.search('/g/(\d+)/', doujinshi.a['href']).group(1) - result.append({'id': id_, 'title': title}) + result = _get_title_and_id(response) if not result: logger.warn('Not found anything of keyword {}'.format(keyword)) return result +def print_doujinshi(doujinshi_list): + if not doujinshi_list: + return + doujinshi_list = [(i['id'], i['title']) for i in doujinshi_list] + headers = ['id', 'doujinshi'] + logger.info('Search Result || Found %i doujinshis \n' % doujinshi_list.__len__() + + tabulate(tabular_data=doujinshi_list, headers=headers, tablefmt='rst')) + + +def search_parser(keyword, sorting, page, is_page_all=False): + # keyword = '+'.join([i.strip().replace(' ', '-').lower() for i in keyword.split(',')]) + result = [] + if not page: + page = [1] + + if is_page_all: + url = request('get', url=constant.SEARCH_URL, params={'query': keyword}).url + init_response = request('get', url.replace('%2B', '+')).json() + page = range(1, init_response['num_pages']+1) + + for p in page: + i = 0 + if is_page_all: + total = '/{0}'.format(page[-1]) + + logger.info('Searching doujinshis using keywords "{0}" on page {1}{2}'.format(keyword, p, total)) + while i < 3: + try: + url = request('get', url=constant.SEARCH_URL, params={'query': keyword, + 'page': p, 'sort': sorting}).url + response = request('get', url.replace('%2B', '+')).json() + except Exception as e: + logger.critical(str(e)) + + break + + if 'result' not in response: + logger.warn('No result in response in page {}'.format(p)) + break + + for row in response['result']: + title = row['title']['english'] + title = title[:85] + '..' if len(title) > 85 else title + result.append({'id': row['id'], 'title': title}) + + if not result: + logger.warn('No results for keywords {}'.format(keyword)) + + return result + + def __api_suspended_doujinshi_parser(id_): if not isinstance(id_, (int,)) and (isinstance(id_, (str,)) and not id_.isdigit()): raise Exception('Doujinshi id({0}) is not valid'.format(id_)) @@ -221,11 +259,11 @@ def __api_suspended_doujinshi_parser(id_): doujinshi['name'] = response['title']['english'] doujinshi['subtitle'] = response['title']['japanese'] doujinshi['img_id'] = response['media_id'] - doujinshi['ext'] = ''.join(map(lambda s: s['t'], response['images']['pages'])) + doujinshi['ext'] = ''.join([i['t'] for i in response['images']['pages']]) doujinshi['pages'] = len(response['images']['pages']) # gain information of the doujinshi - needed_fields = ['character', 'artist', 'language', 'tag'] + needed_fields = ['character', 'artist', 'language', 'tag', 'parody', 'group', 'category'] for tag in response['tags']: tag_type = tag['type'] if tag_type in needed_fields: @@ -244,94 +282,5 @@ def __api_suspended_doujinshi_parser(id_): return doujinshi -def __api_suspended_search_parser(keyword, page): - logger.debug('Searching doujinshis using keywords {0}'.format(keyword)) - result = [] - i = 0 - while i < 5: - try: - response = request('get', url=constant.SEARCH_URL, params={'query': keyword, 'page': page}).json() - except Exception as e: - i += 1 - if not i < 5: - logger.critical(str(e)) - logger.warn('If you are in China, please configure the proxy to fu*k GFW.') - exit(1) - continue - break - - if 'result' not in response: - raise Exception('No result in response') - - for row in response['result']: - title = row['title']['english'] - title = title[:85] + '..' if len(title) > 85 else title - result.append({'id': row['id'], 'title': title}) - - if not result: - logger.warn('No results for keywords {}'.format(keyword)) - - return result - - -def print_doujinshi(doujinshi_list): - if not doujinshi_list: - return - doujinshi_list = [(i['id'], i['title']) for i in doujinshi_list] - headers = ['id', 'doujinshi'] - logger.info('Search Result\n' + - tabulate(tabular_data=doujinshi_list, headers=headers, tablefmt='rst')) - - -def __api_suspended_tag_parser(tag_id, max_page=1): - logger.info('Searching for doujinshi with tag id {0}'.format(tag_id)) - result = [] - response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json() - page = max_page if max_page <= response['num_pages'] else int(response['num_pages']) - - for i in range(1, page + 1): - logger.info('Getting page {} ...'.format(i)) - - if page != 1: - response = request('get', url=constant.TAG_API_URL, - params={'sort': 'popular', 'tag_id': tag_id}).json() - for row in response['result']: - title = row['title']['english'] - title = title[:85] + '..' if len(title) > 85 else title - result.append({'id': row['id'], 'title': title}) - - if not result: - logger.warn('No results for tag id {}'.format(tag_id)) - - return result - - -def tag_parser(tag_name, max_page=1): - result = [] - tag_name = tag_name.lower() - tag_name = tag_name.replace(' ', '-') - - for p in range(1, max_page + 1): - logger.debug('Fetching page {0} for doujinshi with tag \'{1}\''.format(p, tag_name)) - response = request('get', url='%s/%s?page=%d' % (constant.TAG_URL, tag_name, p)).content - - html = BeautifulSoup(response, 'html.parser') - doujinshi_items = html.find_all('div', attrs={'class': 'gallery'}) - if not doujinshi_items: - logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name)) - return - - for i in doujinshi_items: - doujinshi_id = i.a.attrs['href'].strip('/g') - doujinshi_title = i.a.text.strip() - doujinshi_title = doujinshi_title if len(doujinshi_title) < 85 else doujinshi_title[:82] + '...' - result.append({'title': doujinshi_title, 'id': doujinshi_id}) - - if not result: - logger.warn('No results for tag \'{}\''.format(tag_name)) - - return result - - if __name__ == '__main__': print(doujinshi_parser("32271"))