pjht/buku
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Detect page charset from content-type field

Browse Source
This commit is contained in:
Arun Prakash Jana 2018-08-01 05:53:28 +05:30
parent 88c4ccbccb
commit 69281444fd
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
buku.py
View File

@ -18,6 +18,7 @@
# along with Buku. If not, see <http://www.gnu.org/licenses/>. # along with Buku. If not, see <http://www.gnu.org/licenses/>.
import argparse import argparse
import cgi
import collections import collections
import html.parser as HTMLParser import html.parser as HTMLParser
import json import json
@ -2948,9 +2949,14 @@ def get_page_title(resp):
""" """
parser = BukuHTMLParser() parser = BukuHTMLParser()
charset = 'utf-8'
try: try:
parser.feed(resp.data.decode('utf-8')) if 'content-type' in resp.headers:
_, params = cgi.parse_header(resp.headers['content-type'])
if 'charset' in params:
charset = params['charset']
parser.feed(resp.data.decode(charset))
except Exception as e: except Exception as e:
# Suppress Exception due to intentional self.reset() in BHTMLParser # Suppress Exception due to intentional self.reset() in BHTMLParser
if (logger.isEnabledFor(logging.DEBUG) and str(e) != 'we should not get here!'): if (logger.isEnabledFor(logging.DEBUG) and str(e) != 'we should not get here!'):