project is now Py3 and Py2 compatible

[nhentai.git] / nhentai / parser.py

diff --git a/nhentai/parser.py b/nhentai/parser.py
index 0d8630320754598c633f215f61aecab50e68548b..f8977540bde2fd2c54ba9e35bdd81ace4f3fc9b2 100644 (file)
--- a/nhentai/parser.py
+++ b/nhentai/parser.py
@@ -1,77 +1,105 @@
+# coding: utf-8
 from __future__ import print_function
-import sys
+
+from bs4 import BeautifulSoup
 import re
 import requests
-from bs4 import BeautifulSoup
-from constant import DETAIL_URL, SEARCH_URL
-from logger import logger
 from tabulate import tabulate
 
+import nhentai.constant as constant
+from nhentai.logger import logger
+
+
+def request(method, url, **kwargs):
+    if not hasattr(requests, method):
+        raise AttributeError('\'requests\' object has no attribute \'{0}\''.format(method))
+
+    return requests.__dict__[method](url, proxies=constant.PROXY, **kwargs)
+
 
-def dojinshi_parser(id):
-    if not isinstance(id, (int, )) and (isinstance(id, (str, )) and not id.isdigit()):
-        raise Exception('Dojinshi id(%s) is not valid' % str(id))
-    id = int(id)
-    logger.debug('Fetching dojinshi information of id %d' % id)
-    dojinshi = dict()
-    dojinshi['id'] = id
-    url = '%s/%d/' % (DETAIL_URL, id)
+def doujinshi_parser(id_):
+    if not isinstance(id_, (int,)) and (isinstance(id_, (str,)) and not id_.isdigit()):
+        raise Exception('Doujinshi id({0}) is not valid'.format(id_))
+
+    id_ = int(id_)
+    logger.log(15, 'Fetching doujinshi information of id {0}'.format(id_))
+    doujinshi = dict()
+    doujinshi['id'] = id_
+    url = '{0}/{1}/'.format(constant.DETAIL_URL, id_)
 
     try:
-        response = requests.get(url).content
+        response = request('get', url).content
     except Exception as e:
-        logger.critical('%s%s' % tuple(e.message))
-        sys.exit()
+        logger.critical(str(e))
+        exit(1)
 
     html = BeautifulSoup(response)
-    dojinshi_info = html.find('div', attrs={'id': 'info'})
+    doujinshi_info = html.find('div', attrs={'id': 'info'})
 
-    title = dojinshi_info.find('h1').text
-    subtitle = dojinshi_info.find('h2')
+    title = doujinshi_info.find('h1').text
+    subtitle = doujinshi_info.find('h2')
 
-    dojinshi['name'] = title
-    dojinshi['subtitle'] = subtitle.text if subtitle else ''
+    doujinshi['name'] = title
+    doujinshi['subtitle'] = subtitle.text if subtitle else ''
 
-    dojinshi_cover = html.find('div', attrs={'id': 'cover'})
-    img_id = re.search('/galleries/([\d]+)/cover\.(jpg|png)$', dojinshi_cover.a.img['src'])
+    doujinshi_cover = html.find('div', attrs={'id': 'cover'})
+    img_id = re.search('/galleries/([\d]+)/cover\.(jpg|png)$', doujinshi_cover.a.img['src'])
     if not img_id:
         logger.critical('Tried yo get image id failed')
-        sys.exit()
-    dojinshi['img_id'] = img_id.group(1)
-    dojinshi['ext'] = img_id.group(2)
+        exit(1)
+
+    doujinshi['img_id'] = img_id.group(1)
+    doujinshi['ext'] = img_id.group(2)
 
     pages = 0
-    for _ in dojinshi_info.find_all('div', class_=''):
+    for _ in doujinshi_info.find_all('div', class_=''):
         pages = re.search('([\d]+) pages', _.text)
         if pages:
             pages = pages.group(1)
             break
-    dojinshi['pages'] = int(pages)
-    return dojinshi
+    doujinshi['pages'] = int(pages)
+
+    # gain information of the doujinshi
+    information_fields = doujinshi_info.find_all('div', attrs={'class': 'field-name'})
+    needed_fields = ['Characters', 'Artists', 'Language', 'Tags']
+    for field in information_fields:
+        field_name = field.contents[0].strip().strip(':')
+        if field_name in needed_fields:
+            data = [sub_field.contents[0].strip() for sub_field in
+                    field.find_all('a', attrs={'class': 'tag'})]
+            doujinshi[field_name.lower()] = ', '.join(data)
+
+    return doujinshi
 
 
 def search_parser(keyword, page):
-    logger.debug('Searching dojinshis of keyword %s' % keyword)
+    logger.debug('Searching doujinshis of keyword {0}'.format(keyword))
     result = []
-    response = requests.get(SEARCH_URL, params={'q': keyword, 'page': page}).content
+    try:
+        response = request('get', url=constant.SEARCH_URL, params={'q': keyword, 'page': page}).content
+    except requests.ConnectionError as e:
+        logger.critical(e)
+        logger.warn('If you are in China, please configure the proxy to fu*k GFW.')
+        exit(1)
+
     html = BeautifulSoup(response)
-    dojinshi_search_result = html.find_all('div', attrs={'class': 'gallery'})
-    for dojinshi in dojinshi_search_result:
-        dojinshi_container = dojinshi.find('div', attrs={'class': 'caption'})
-        title = dojinshi_container.text.strip()
+    doujinshi_search_result = html.find_all('div', attrs={'class': 'gallery'})
+    for doujinshi in doujinshi_search_result:
+        doujinshi_container = doujinshi.find('div', attrs={'class': 'caption'})
+        title = doujinshi_container.text.strip()
         title = (title[:85] + '..') if len(title) > 85 else title
-        id_ = re.search('/g/(\d+)/', dojinshi.a['href']).group(1)
+        id_ = re.search('/g/(\d+)/', doujinshi.a['href']).group(1)
         result.append({'id': id_, 'title': title})
     return result
 
 
-def print_dojinshi(dojinshi_list):
-    if not dojinshi_list:
+def print_doujinshi(doujinshi_list):
+    if not doujinshi_list:
         return
-    dojinshi_list = [i.values() for i in dojinshi_list]
-    headers = ['id', 'dojinshi']
+    doujinshi_list = [i.values() for i in doujinshi_list]
+    headers = ['id', 'doujinshi']
     logger.info('Search Result\n' +
-                tabulate(tabular_data=dojinshi_list, headers=headers, tablefmt='rst'))
+                tabulate(tabular_data=doujinshi_list, headers=headers, tablefmt='rst'))
 
 if __name__ == '__main__':
-    print(dojinshi_parser("32271"))
+    print(doujinshi_parser("32271"))