Merge pull request #96 from symant233/dev

[nhentai.git] / nhentai / parser.py

diff --git a/nhentai/parser.py b/nhentai/parser.py
index cb53cc8894ac906f22d1d7c27cbd6b109f16dac4..3c01af164e1733de6e5b2538b44c6c3f28c1f40d 100644 (file)
--- a/nhentai/parser.py
+++ b/nhentai/parser.py
@@ -121,8 +121,8 @@ def doujinshi_parser(id_):
             return doujinshi_parser(str(id_))
 
     except Exception as e:
-        logger.critical(str(e))
-        raise SystemExit
+        logger.warn('Error: {}, ignored'.format(str(e)))
+        return None
 
     html = BeautifulSoup(response, 'html.parser')
     doujinshi_info = html.find('div', attrs={'id': 'info'})
@@ -134,7 +134,7 @@ def doujinshi_parser(id_):
     doujinshi['subtitle'] = subtitle.text if subtitle else ''
 
     doujinshi_cover = html.find('div', attrs={'id': 'cover'})
-    img_id = re.search('/galleries/([\d]+)/cover\.(jpg|png)$', doujinshi_cover.a.img.attrs['data-src'])
+    img_id = re.search('/galleries/([\d]+)/cover\.(jpg|png|gif)$', doujinshi_cover.a.img.attrs['data-src'])
 
     ext = []
     for i in html.find_all('div', attrs={'class': 'thumb-container'}):
@@ -158,7 +158,7 @@ def doujinshi_parser(id_):
 
     # gain information of the doujinshi
     information_fields = doujinshi_info.find_all('div', attrs={'class': 'field-name'})
-    needed_fields = ['Characters', 'Artists', 'Languages', 'Tags']
+    needed_fields = ['Characters', 'Artists', 'Languages', 'Tags', 'Parodies', 'Groups', 'Categories']
     for field in information_fields:
         field_name = field.contents[0].strip().strip(':')
         if field_name in needed_fields:
@@ -166,17 +166,15 @@ def doujinshi_parser(id_):
                     field.find_all('a', attrs={'class': 'tag'})]
             doujinshi[field_name.lower()] = ', '.join(data)
 
+    time_field = doujinshi_info.find('time')
+    if time_field.has_attr('datetime'):
+        doujinshi['date'] = time_field['datetime']
     return doujinshi
 
 
-def search_parser(keyword, page):
+def search_parser(keyword, sorting='date', page=1):
     logger.debug('Searching doujinshis of keyword {0}'.format(keyword))
-    try:
-        response = request('get', url=constant.SEARCH_URL, params={'q': keyword, 'page': page}).content
-    except requests.ConnectionError as e:
-        logger.critical(e)
-        logger.warn('If you are in China, please configure the proxy to fu*k GFW.')
-        raise SystemExit
+    response = request('get', url=constant.SEARCH_URL, params={'q': keyword, 'page': page, 'sort': sorting}).content
 
     result = _get_title_and_id(response)
     if not result:
@@ -194,16 +192,28 @@ def print_doujinshi(doujinshi_list):
                 tabulate(tabular_data=doujinshi_list, headers=headers, tablefmt='rst'))
 
 
-def tag_parser(tag_name, max_page=1):
+def tag_parser(tag_name, sorting='date', max_page=1, index=0):
     result = []
     tag_name = tag_name.lower()
-    tag_name = tag_name.replace(' ', '-')
+    if ',' in tag_name:
+        tag_name = [i.strip().replace(' ', '-') for i in tag_name.split(',')]
+    else:
+        tag_name = tag_name.strip().replace(' ', '-')
+    if sorting == 'date':
+        sorting = ''
 
     for p in range(1, max_page + 1):
-        logger.debug('Fetching page {0} for doujinshi with tag \'{1}\''.format(p, tag_name))
-        response = request('get', url='%s/%s/?page=%d' % (constant.TAG_URL, tag_name, p)).content
+        if isinstance(tag_name, str):
+            logger.debug('Fetching page {0} for doujinshi with tag \'{1}\''.format(p, tag_name))
+            response = request('get', url='%s/%s/%s?page=%d' % (constant.TAG_URL[index], tag_name, sorting, p)).content
+            result += _get_title_and_id(response)
+        else:
+            for i in tag_name:
+                logger.debug('Fetching page {0} for doujinshi with tag \'{1}\''.format(p, i))
+                response = request('get',
+                                   url='%s/%s/%s?page=%d' % (constant.TAG_URL[index], i, sorting, p)).content
+                result += _get_title_and_id(response)
 
-        result += _get_title_and_id(response)
         if not result:
             logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
             return
@@ -214,13 +224,13 @@ def tag_parser(tag_name, max_page=1):
     return result
 
 
-def __api_suspended_search_parser(keyword, page):
+def __api_suspended_search_parser(keyword, sorting, page):
     logger.debug('Searching doujinshis using keywords {0}'.format(keyword))
     result = []
     i = 0
     while i < 5:
         try:
-            response = request('get', url=constant.SEARCH_URL, params={'query': keyword, 'page': page}).json()
+            response = request('get', url=constant.SEARCH_URL, params={'query': keyword, 'page': page, 'sort': sorting}).json()
         except Exception as e:
             i += 1
             if not i < 5:
@@ -244,10 +254,10 @@ def __api_suspended_search_parser(keyword, page):
     return result
 
 
-def __api_suspended_tag_parser(tag_id, max_page=1):
+def __api_suspended_tag_parser(tag_id, sorting, max_page=1):
     logger.info('Searching for doujinshi with tag id {0}'.format(tag_id))
     result = []
-    response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json()
+    response = request('get', url=constant.TAG_API_URL, params={'sort': sorting, 'tag_id': tag_id}).json()
     page = max_page if max_page <= response['num_pages'] else int(response['num_pages'])
 
     for i in range(1, page + 1):
@@ -255,7 +265,7 @@ def __api_suspended_tag_parser(tag_id, max_page=1):
 
         if page != 1:
             response = request('get', url=constant.TAG_API_URL,
-                               params={'sort': 'popular', 'tag_id': tag_id}).json()
+                               params={'sort': sorting, 'tag_id': tag_id}).json()
     for row in response['result']:
         title = row['title']['english']
         title = title[:85] + '..' if len(title) > 85 else title
@@ -291,11 +301,11 @@ def __api_suspended_doujinshi_parser(id_):
     doujinshi['name'] = response['title']['english']
     doujinshi['subtitle'] = response['title']['japanese']
     doujinshi['img_id'] = response['media_id']
-    doujinshi['ext'] = ''.join(map(lambda s: s['t'], response['images']['pages']))
+    doujinshi['ext'] = ''.join([i['t'] for i in response['images']['pages']])
     doujinshi['pages'] = len(response['images']['pages'])
 
     # gain information of the doujinshi
-    needed_fields = ['character', 'artist', 'language', 'tag']
+    needed_fields = ['character', 'artist', 'language', 'tag', 'parody', 'group', 'category']
     for tag in response['tags']:
         tag_type = tag['type']
         if tag_type in needed_fields: