diff --git a/sickbeard/db.py b/sickbeard/db.py index e1045711e7f09f2380978610b9f325579df46675..a5f8701f7c2ee99e2667ed77d9a6ed0b3f6fe843 100644 --- a/sickbeard/db.py +++ b/sickbeard/db.py @@ -229,19 +229,23 @@ class DBConnection(object): def _unicode_text_factory(self, x): try: + # Already unicode, empty string, or ascii x = unicode(x) except Exception: try: - x = unicode(x, sickbeard.SYS_ENCODING) + # most common encoding from web + x = unicode(x, 'utf-8') except Exception: try: - x = unicode(x, 'utf-8') + # most common from web if utf-8 fails + x = unicode(x, 'latin-1') except Exception: try: - x = unicode(x, 'latin-1') + # try system encoding before trusting chardet + x = unicode(x, sickbeard.SYS_ENCODING) except Exception: try: - # Chardet can be wrong, so try it before ignoring + # Chardet can be wrong, so try it last before ignoring x = unicode(x, chardet.detect(x).get('encoding')) except Exception: x = unicode(x, sickbeard.SYS_ENCODING, errors="ignore") diff --git a/sickbeard/encodingKludge.py b/sickbeard/encodingKludge.py index 04a2ec27d5dc990febda5d48730784d2600ef4bc..622cfc8aba371da49777df10a1bad3f9eddafda9 100644 --- a/sickbeard/encodingKludge.py +++ b/sickbeard/encodingKludge.py @@ -26,16 +26,19 @@ def _toUnicode(x): x = unicode(x) except Exception: try: - x = unicode(x, sickbeard.SYS_ENCODING) + x = unicode(x, 'utf-8') except Exception: try: - x = unicode(x, 'utf-8') + x = unicode(x, 'latin-1') except Exception: try: - x = unicode(x, 'latin-1') + x = unicode(x, sickbeard.SYS_ENCODING) except Exception: - # Chardet can be wrong, so try it last - x = unicode(x, chardet.detect(x).get('encoding')) + try: + # Chardet can be wrong, so try it last + x = unicode(x, chardet.detect(x).get('encoding')) + except Exception: + x = unicode(x, sickbeard.SYS_ENCODING, 'replace') return x def ss(x): diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index e08d1da049095362e0ed81a43189dd1f3624741c..ffa29e2d5a36b9177b853dd01c189e7b8fbdc548 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -96,13 +96,13 @@ class html_entities(CheetahFilter): filtered = unicode(val) except Exception: try: - filtered = unicode(val, sickbeard.SYS_ENCODING) + filtered = unicode(val, 'utf-8') except Exception: try: - filtered = unicode(val, 'utf-8') + filtered = unicode(val, 'latin-1') except Exception: try: - filtered = unicode(val, 'latin-1') + filtered = unicode(val, sickbeard.SYS_ENCODING) except Exception: logger.log(u'Unable to decode using %s, utf-8, or latin-1. Falling back to chardet!' % sickbeard.SYS_ENCODING, logger.ERROR)