diff --git a/gui/slick/images/network/rdi.png b/gui/slick/images/network/rdi.png
new file mode 100644
index 0000000000000000000000000000000000000000..43602d994d238ea863229b7d210c5ce1d9267e68
Binary files /dev/null and b/gui/slick/images/network/rdi.png differ
diff --git a/gui/slick/views/inc_defs.mako b/gui/slick/views/inc_defs.mako
index 8a339ba4fd52032c923267639f50abbb7c88f540..eaa54116836d691c2ff9eecc16cf25c501a2f502 100644
--- a/gui/slick/views/inc_defs.mako
+++ b/gui/slick/views/inc_defs.mako
@@ -3,13 +3,6 @@
from sickbeard.common import Quality, qualityPresets, qualityPresetStrings
%>
<%def name="renderQualityPill(quality, showTitle=False, overrideClass=None)"><%
- iQuality = quality & 0xFFFF
- pQuality = quality >> 16
-
- # If initial and preferred qualities are the same, show pill as initial quality
- if iQuality == pQuality:
- quality = iQuality
-
# Build a string of quality names to use as title attribute
if showTitle:
iQuality, pQuality = Quality.splitQuality(quality)
@@ -29,6 +22,13 @@
else:
title = ""
+ iQuality = quality & 0xFFFF
+ pQuality = quality >> 16
+
+ # If initial and preferred qualities are the same, show pill as initial quality
+ if iQuality == pQuality:
+ quality = iQuality
+
if quality in qualityPresets:
cssClass = qualityPresetStrings[quality]
qualityString = qualityPresetStrings[quality]
diff --git a/lib/subliminal/__init__.py b/lib/subliminal/__init__.py
index 943bb1d13c60d9d8e8a0ebb5355308e352a96fd2..68de539bffcb3e3ecac2062f79addbe0c58e2474 100644
--- a/lib/subliminal/__init__.py
+++ b/lib/subliminal/__init__.py
@@ -1,20 +1,18 @@
# -*- coding: utf-8 -*-
__title__ = 'subliminal'
-__version__ = '0.8.0-dev'
+__version__ = '1.0.dev0'
__author__ = 'Antoine Bertin'
__license__ = 'MIT'
-__copyright__ = 'Copyright 2013 Antoine Bertin'
+__copyright__ = 'Copyright 2015, Antoine Bertin'
import logging
-from .api import list_subtitles, download_subtitles, download_best_subtitles, save_subtitles
-from .cache import MutexLock, region as cache_region
+
+from .api import (ProviderPool, check_video, provider_manager, download_best_subtitles, download_subtitles,
+ list_subtitles, save_subtitles)
+from .cache import region
from .exceptions import Error, ProviderError
-from .providers import Provider, ProviderPool, provider_manager
+from .providers import Provider
from .subtitle import Subtitle
-from .video import VIDEO_EXTENSIONS, SUBTITLE_EXTENSIONS, Video, Episode, Movie, scan_videos, scan_video
-
-class NullHandler(logging.Handler):
- def emit(self, record):
- pass
+from .video import SUBTITLE_EXTENSIONS, VIDEO_EXTENSIONS, Episode, Movie, Video, scan_video, scan_videos
-logging.getLogger(__name__).addHandler(NullHandler())
+logging.getLogger(__name__).addHandler(logging.NullHandler())
diff --git a/lib/subliminal/api.py b/lib/subliminal/api.py
index 47d6a2cb93cc7ce638c873f8bca1f1af81ce5d19..92d7b4b7f24ab01f2716a237ef71714a67991b08 100644
--- a/lib/subliminal/api.py
+++ b/lib/subliminal/api.py
@@ -1,140 +1,447 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
-import collections
+from collections import defaultdict
import io
import logging
import operator
import os.path
-import babelfish
-from .providers import ProviderPool
-from .subtitle import get_subtitle_path
+import socket
+from babelfish import Language
+from pkg_resources import EntryPoint
+import requests
+from stevedore import ExtensionManager
+from stevedore.dispatch import DispatchExtensionManager
+
+from .subtitle import compute_score, get_subtitle_path
logger = logging.getLogger(__name__)
-def list_subtitles(videos, languages, providers=None, provider_configs=None):
- """List subtitles for `videos` with the given `languages` using the specified `providers`
+class InternalExtensionManager(ExtensionManager):
+ """Add support for internal entry points to the :class:`~stevedore.extension.Extensionmanager`
+
+ Internal entry points are useful for libraries that ship their own plugins but still keep the entry point open.
+
+ All other parameters are passed onwards to the :class:`~stevedore.extension.Extensionmanager` constructor.
+
+ :param internal_entry_points: the internal providers
+ :type internal_entry_points: list of :class:`~pkg_resources.EntryPoint`
+
+ """
+ def __init__(self, namespace, internal_entry_points, **kwargs):
+ self.internal_entry_points = list(internal_entry_points)
+ super(InternalExtensionManager, self).__init__(namespace, **kwargs)
+
+ def _find_entry_points(self, namespace):
+ return self.internal_entry_points + super(InternalExtensionManager, self)._find_entry_points(namespace)
+
+
+provider_manager = InternalExtensionManager('subliminal.providers', [EntryPoint.parse(ep) for ep in (
+ 'addic7ed = subliminal.providers.addic7ed:Addic7edProvider',
+ 'opensubtitles = subliminal.providers.opensubtitles:OpenSubtitlesProvider',
+ 'podnapisi = subliminal.providers.podnapisi:PodnapisiProvider',
+ 'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider',
+ 'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider'
+)])
+
+
+class ProviderPool(object):
+ """A pool of providers with the same API as a single :class:`~subliminal.providers.Provider`.
+
+ It has a few extra features:
+
+ * Lazy loads providers when needed and supports the :keyword:`with` statement to :meth:`terminate`
+ the providers on exit.
+ * Automatically discard providers on failure.
+
+ :param providers: name of providers to use, if not all.
+ :type providers: list
+ :param dict provider_configs: provider configuration as keyword arguments per provider name to pass when
+ instanciating the :class:`~subliminal.providers.Provider`.
+
+ """
+ def __init__(self, providers=None, provider_configs=None):
+ #: Name of providers to use
+ self.providers = providers or provider_manager.names()
+
+ #: Provider configuration
+ self.provider_configs = provider_configs or {}
+
+ #: Initialized providers
+ self.initialized_providers = {}
+
+ #: Discarded providers
+ self.discarded_providers = set()
+
+ #: Dedicated :data:`provider_manager` as :class:`~stevedore.dispatch.DispatchExtensionManager`
+ self.manager = DispatchExtensionManager(provider_manager.namespace, lambda e: e.name in self.providers)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.terminate()
+
+ def __getitem__(self, name):
+ if name not in self.initialized_providers:
+ logger.info('Initializing provider %s', name)
+ provider = self.manager[name].plugin(**self.provider_configs.get(name, {}))
+ provider.initialize()
+ self.initialized_providers[name] = provider
+
+ return self.initialized_providers[name]
+
+ def __delitem__(self, name):
+ if name not in self.initialized_providers:
+ raise KeyError(name)
+
+ try:
+ logger.info('Terminating provider %s', name)
+ self.initialized_providers[name].terminate()
+ except (requests.Timeout, socket.timeout):
+ logger.error('Provider %r timed out, improperly terminated', name)
+ except:
+ logger.exception('Provider %r terminated unexpectedly', name)
+
+ del self.initialized_providers[name]
+
+ def __iter__(self):
+ return iter(self.initialized_providers)
+
+ def list_subtitles(self, video, languages):
+ """List subtitles.
+
+ :param video: video to list subtitles for.
+ :type video: :class:`~subliminal.video.Video`
+ :param languages: languages to search for.
+ :type languages: set of :class:`~babelfish.language.Language`
+ :return: found subtitles.
+ :rtype: list of :class:`~subliminal.subtitle.Subtitle`
+
+ """
+ subtitles = []
+
+ for name in self.providers:
+ # check discarded providers
+ if name in self.discarded_providers:
+ logger.debug('Skipping discarded provider %r', name)
+ continue
+
+ # check video validity
+ if not self.manager[name].plugin.check(video):
+ logger.info('Skipping provider %r: not a valid video', name)
+ continue
+
+ # check supported languages
+ provider_languages = self.manager[name].plugin.languages & languages
+ if not provider_languages:
+ logger.info('Skipping provider %r: no language to search for', name)
+ continue
+
+ # list subtitles
+ logger.info('Listing subtitles with provider %r and languages %r', name, provider_languages)
+ try:
+ provider_subtitles = self[name].list_subtitles(video, provider_languages)
+ except (requests.Timeout, socket.timeout):
+ logger.error('Provider %r timed out, discarding it', name)
+ self.discarded_providers.add(name)
+ continue
+ except:
+ logger.exception('Unexpected error in provider %r, discarding it', name)
+ self.discarded_providers.add(name)
+ continue
+ subtitles.extend(provider_subtitles)
+
+ return subtitles
+
+ def download_subtitle(self, subtitle):
+ """Download `subtitle`'s :attr:`~subliminal.subtitle.Subtitle.content`.
+
+ :param subtitle: subtitle to download.
+ :type subtitle: :class:`~subliminal.subtitle.Subtitle`
+ :return: `True` if the subtitle has been successfully downloaded, `False` otherwise.
+ :rtype: bool
+
+ """
+ # check discarded providers
+ if subtitle.provider_name in self.discarded_providers:
+ logger.warning('Provider %r is discarded', subtitle.provider_name)
+ return False
+
+ logger.info('Downloading subtitle %r', subtitle)
+ try:
+ self[subtitle.provider_name].download_subtitle(subtitle)
+ except (requests.Timeout, socket.timeout):
+ logger.error('Provider %r timed out, discarding it', subtitle.provider_name)
+ self.discarded_providers.add(subtitle.provider_name)
+ return False
+ except:
+ logger.exception('Unexpected error in provider %r, discarding it', subtitle.provider_name)
+ self.discarded_providers.add(subtitle.provider_name)
+ return False
+
+ # check subtitle validity
+ if not subtitle.is_valid():
+ logger.error('Invalid subtitle')
+ return False
+
+ return True
+
+ def download_best_subtitles(self, subtitles, video, languages, min_score=0, hearing_impaired=False, only_one=False,
+ scores=None):
+ """Download the best matching subtitles.
+
+ :param subtitles: the subtitles to use.
+ :type subtitles: list of :class:`~subliminal.subtitle.Subtitle`
+ :param video: video to download subtitles for.
+ :type video: :class:`~subliminal.video.Video`
+ :param languages: languages to download.
+ :type languages: set of :class:`~babelfish.language.Language`
+ :param int min_score: minimum score for a subtitle to be downloaded.
+ :param bool hearing_impaired: hearing impaired preference.
+ :param bool only_one: download only one subtitle, not one per language.
+ :param dict scores: scores to use, if `None`, the :attr:`~subliminal.video.Video.scores` from the video are
+ used.
+ :return: downloaded subtitles.
+ :rtype: list of :class:`~subliminal.subtitle.Subtitle`
+
+ """
+ # sort subtitles by score
+ scored_subtitles = sorted([(s, compute_score(s.get_matches(video, hearing_impaired=hearing_impaired), video,
+ scores=scores))
+ for s in subtitles], key=operator.itemgetter(1), reverse=True)
- :param videos: videos to list subtitles for
+ # download best subtitles, falling back on the next on error
+ downloaded_subtitles = []
+ for subtitle, score in scored_subtitles:
+ # check score
+ if score < min_score:
+ logger.info('Score %d is below min_score (%d)', (score, min_score))
+ break
+
+ # check downloaded languages
+ if subtitle.language in set(s.language for s in downloaded_subtitles):
+ logger.debug('Skipping subtitle: %r already downloaded', subtitle.language)
+ continue
+
+ # download
+ logger.info('Downloading subtitle %r with score %d', subtitle, score)
+ if self.download_subtitle(subtitle):
+ downloaded_subtitles.append(subtitle)
+
+ # stop when all languages are downloaded
+ if set(s.language for s in downloaded_subtitles) == languages:
+ logger.debug('All languages downloaded')
+ break
+
+ # stop if only one subtitle is requested
+ if only_one:
+ logger.debug('Only one subtitle downloaded')
+ break
+
+ return downloaded_subtitles
+
+ def terminate(self):
+ """Terminate all the :attr:`initialized_providers`."""
+ logger.debug('Terminating initialized providers')
+ for name in list(self.initialized_providers):
+ del self[name]
+
+
+def check_video(video, languages=None, age=None, undefined=False):
+ """Perform some checks on the `video`.
+
+ All the checks are optional. Return `False` if any of this check fails:
+
+ * `languages` already exist in `video`'s :attr:`~subliminal.video.Video.subtitle_languages`.
+ * `video` is older than `age`.
+ * `video` has an `undefined` language in :attr:`~subliminal.video.Video.subtitle_languages`.
+
+ :param video: video to check.
+ :type video: :class:`~subliminal.video.Video`
+ :param languages: desired languages.
+ :type languages: set of :class:`~babelfish.language.Language`
+ :param datetime.timedelta age: maximum age of the video.
+ :param bool undefined: fail on existing undefined language.
+ :return: `True` if the video passes the checks, `False` otherwise.
+ :rtype: bool
+
+ """
+ # language test
+ if languages and not (languages - video.subtitle_languages):
+ logger.debug('All languages %r exist', languages)
+ return False
+
+ # age test
+ if age and video.age > age:
+ logger.debug('Video is older than %r', age)
+ return False
+
+ # undefined test
+ if undefined and Language('und') in video.subtitle_languages:
+ logger.debug('Undefined language found')
+ return False
+
+ return True
+
+
+def list_subtitles(videos, languages, **kwargs):
+ """List subtitles.
+
+ The `videos` must pass the `languages` check of :func:`check_video`.
+
+ All other parameters are passed onwards to the :class:`ProviderPool` constructor.
+
+ :param videos: videos to list subtitles for.
:type videos: set of :class:`~subliminal.video.Video`
- :param languages: languages of subtitles to search for
- :type languages: set of :class:`babelfish.Language`
- :param providers: providers to use, if not all
- :type providers: list of string or None
- :param provider_configs: configuration for providers
- :type provider_configs: dict of provider name => provider constructor kwargs or None
- :return: found subtitles
- :rtype: dict of :class:`~subliminal.video.Video` => [:class:`~subliminal.subtitle.Subtitle`]
+ :param languages: languages to search for.
+ :type languages: set of :class:`~babelfish.language.Language`
+ :param providers: name of providers to use, if not all.
+ :return: found subtitles per video.
+ :rtype: dict of :class:`~subliminal.video.Video` to list of :class:`~subliminal.subtitle.Subtitle`
"""
- subtitles = collections.defaultdict(list)
- with ProviderPool(providers, provider_configs) as pp:
- for video in videos:
+ listed_subtitles = defaultdict(list)
+
+ # check videos
+ checked_videos = []
+ for video in videos:
+ if not check_video(video, languages=languages):
+ logger.info('Skipping video %r', video)
+ continue
+ checked_videos.append(video)
+
+ # return immediatly if no video passed the checks
+ if not checked_videos:
+ return listed_subtitles
+
+ # list subtitles
+ with ProviderPool(**kwargs) as pool:
+ for video in checked_videos:
logger.info('Listing subtitles for %r', video)
- video_subtitles = pp.list_subtitles(video, languages)
- logger.info('Found %d subtitles total', len(video_subtitles))
- subtitles[video].extend(video_subtitles)
- return subtitles
+ subtitles = pool.list_subtitles(video, languages - video.subtitle_languages)
+ listed_subtitles[video].extend(subtitles)
+ logger.info('Found %d subtitle(s)', len(subtitles))
+
+ return listed_subtitles
+
+def download_subtitles(subtitles, **kwargs):
+ """Download :attr:`~subliminal.subtitle.Subtitle.content` of `subtitles`.
-def download_subtitles(subtitles, provider_configs=None):
- """Download subtitles
+ All other parameters are passed onwards to the :class:`ProviderPool` constructor.
- :param subtitles: subtitles to download
+ :param subtitles: subtitles to download.
:type subtitles: list of :class:`~subliminal.subtitle.Subtitle`
- :param provider_configs: configuration for providers
- :type provider_configs: dict of provider name => provider constructor kwargs or None
+ :param dict provider_configs: provider configuration as keyword arguments per provider name to pass when.
+ instanciating the :class:`~subliminal.providers.Provider`.
"""
- with ProviderPool(provider_configs=provider_configs) as pp:
+ with ProviderPool(**kwargs) as pool:
for subtitle in subtitles:
logger.info('Downloading subtitle %r', subtitle)
- pp.download_subtitle(subtitle)
+ pool.download_subtitle(subtitle)
-def download_best_subtitles(videos, languages, providers=None, provider_configs=None, min_score=0,
- hearing_impaired=False, single=False):
- """Download the best subtitles for `videos` with the given `languages` using the specified `providers`
+def download_best_subtitles(videos, languages, min_score=0, hearing_impaired=False, only_one=False, scores=None,
+ **kwargs):
+ """List and download the best matching subtitles.
- :param videos: videos to download subtitles for
+ The `videos` must pass the `languages` and `undefined` (`only_one`) checks of :func:`check_video`.
+
+ All other parameters are passed onwards to the :class:`ProviderPool` constructor.
+
+ :param videos: videos to download subtitles for.
:type videos: set of :class:`~subliminal.video.Video`
- :param languages: languages of subtitles to download
- :type languages: set of :class:`babelfish.Language`
- :param providers: providers to use for the search, if not all
- :type providers: list of string or None
- :param provider_configs: configuration for providers
- :type provider_configs: dict of provider name => provider constructor kwargs or None
- :param int min_score: minimum score for subtitles to download
- :param bool hearing_impaired: download hearing impaired subtitles
- :param bool single: do not download for videos with an undetermined subtitle language detected
+ :param languages: languages to download.
+ :type languages: set of :class:`~babelfish.language.Language`
+ :param providers: name of providers to use, if not all.
+ :param int min_score: minimum score for a subtitle to be downloaded.
+ :param bool hearing_impaired: hearing impaired preference.
+ :param bool only_one: download only one subtitle, not one per language.
+ :param dict scores: scores to use, if `None`, the :attr:`~subliminal.video.Video.scores` from the video are used.
+ :return: downloaded subtitles per video.
+ :rtype: dict of :class:`~subliminal.video.Video` to list of :class:`~subliminal.subtitle.Subtitle`
"""
- downloaded_subtitles = collections.defaultdict(list)
- with ProviderPool(providers, provider_configs) as pp:
- for video in videos:
- # filter
- if single and babelfish.Language('und') in video.subtitle_languages:
- logger.debug('Skipping video %r: undetermined language found')
- continue
+ downloaded_subtitles = defaultdict(list)
- # list
- logger.info('Listing subtitles for %r', video)
- video_subtitles = pp.list_subtitles(video, languages)
- logger.info('Found %d subtitles total', len(video_subtitles))
+ # check videos
+ checked_videos = []
+ for video in videos:
+ if not check_video(video, languages=languages, undefined=only_one):
+ logger.info('Skipping video %r')
+ continue
+ checked_videos.append(video)
+
+ # return immediatly if no video passed the checks
+ if not checked_videos:
+ return downloaded_subtitles
+
+ # download best subtitles
+ with ProviderPool(**kwargs) as pool:
+ for video in checked_videos:
+ logger.info('Downloading best subtitles for %r', video)
+ subtitles = pool.download_best_subtitles(pool.list_subtitles(video, languages - video.subtitle_languages),
+ video, languages, min_score=min_score,
+ hearing_impaired=hearing_impaired, only_one=only_one,
+ scores=scores)
+ logger.info('Downloaded %d subtitle(s)', len(subtitles))
+ downloaded_subtitles[video].extend(subtitles)
- # download
- downloaded_languages = set()
- for subtitle, score in sorted([(s, s.compute_score(video)) for s in video_subtitles],
- key=operator.itemgetter(1), reverse=True):
- if score < min_score:
- logger.info('No subtitle with score >= %d', min_score)
- break
- if subtitle.hearing_impaired != hearing_impaired:
- logger.debug('Skipping subtitle: hearing impaired != %r', hearing_impaired)
- continue
- if subtitle.language in downloaded_languages:
- logger.debug('Skipping subtitle: %r already downloaded', subtitle.language)
- continue
- logger.info('Downloading subtitle %r with score %d', subtitle, score)
- if pp.download_subtitle(subtitle):
- downloaded_languages.add(subtitle.language)
- downloaded_subtitles[video].append(subtitle)
- if single or downloaded_languages == languages:
- logger.debug('All languages downloaded')
- break
return downloaded_subtitles
-def save_subtitles(subtitles, single=False, directory=None, encoding=None):
- """Save subtitles on disk next to the video or in a specific folder if `folder_path` is specified
+def save_subtitles(video, subtitles, single=False, directory=None, encoding=None):
+ """Save subtitles on filesystem.
+
+ Subtitles are saved in the order of the list. If a subtitle with a language has already been saved, other subtitles
+ with the same language are silently ignored.
+
+ The extension used is `.lang.srt` by default or `.srt` is `single` is `True`, with `lang` being the IETF code for
+ the :attr:`~subliminal.subtitle.Subtitle.language` of the subtitle.
- :param bool single: download with .srt extension if ``True``, add language identifier otherwise
- :param directory: path to directory where to save the subtitles, if any
- :type directory: string or None
- :param encoding: encoding for the subtitles or ``None`` to use the original encoding
- :type encoding: string or None
+ :param video: video of the subtitles.
+ :type video: :class:`~subliminal.video.Video`
+ :param subtitles: subtitles to save.
+ :type subtitles: list of :class:`~subliminal.subtitle.Subtitle`
+ :param bool single: save a single subtitle, default is to save one subtitle per language.
+ :param str directory: path to directory where to save the subtitles, default is next to the video.
+ :param str encoding: encoding in which to save the subtitles, default is to keep original encoding.
+ :return: the saved subtitles
+ :rtype: list of :class:`~subliminal.subtitle.Subtitle`
"""
- for video, video_subtitles in subtitles.items():
- saved_languages = set()
- for video_subtitle in video_subtitles:
- if video_subtitle.content is None:
- logger.debug('Skipping subtitle %r: no content', video_subtitle)
- continue
- if video_subtitle.language in saved_languages:
- logger.debug('Skipping subtitle %r: language already saved', video_subtitle)
- continue
- subtitle_path = get_subtitle_path(video.name, None if single else video_subtitle.language)
- if directory is not None:
- subtitle_path = os.path.join(directory, os.path.split(subtitle_path)[1])
- logger.info('Saving %r to %r', video_subtitle, subtitle_path)
- if encoding is None:
- with io.open(subtitle_path, 'wb') as f:
- f.write(video_subtitle.content)
- else:
- with io.open(subtitle_path, 'w', encoding=encoding) as f:
- f.write(video_subtitle.text)
- saved_languages.add(video_subtitle.language)
- if single:
- break
+ saved_subtitles = []
+ for subtitle in subtitles:
+ # check content
+ if subtitle.content is None:
+ logger.error('Skipping subtitle %r: no content', subtitle)
+ continue
+
+ # check language
+ if subtitle.language in set(s.language for s in saved_subtitles):
+ logger.debug('Skipping subtitle %r: language already saved', subtitle)
+ continue
+
+ # create subtitle path
+ subtitle_path = get_subtitle_path(video.name, None if single else subtitle.language)
+ if directory is not None:
+ subtitle_path = os.path.join(directory, os.path.split(subtitle_path)[1])
+
+ # save content as is or in the specified encoding
+ logger.info('Saving %r to %r', subtitle, subtitle_path)
+ if encoding is None:
+ with io.open(subtitle_path, 'wb') as f:
+ f.write(subtitle.content)
+ else:
+ with io.open(subtitle_path, 'w', encoding=encoding) as f:
+ f.write(subtitle.text)
+ saved_subtitles.append(subtitle)
+
+ # check single
+ if single:
+ break
+
+ return saved_subtitles
diff --git a/lib/subliminal/cache.py b/lib/subliminal/cache.py
index 1cc1fe182b6e588b2204139d3f73f3d124735252..79fd753325ce333e08e45a257106141f8121f669 100644
--- a/lib/subliminal/cache.py
+++ b/lib/subliminal/cache.py
@@ -1,62 +1,17 @@
# -*- coding: utf-8 -*-
import datetime
-import inspect
-from dogpile.cache import make_region # @UnresolvedImport
-from dogpile.cache.backends.file import AbstractFileLock # @UnresolvedImport
-from dogpile.cache.compat import string_type # @UnresolvedImport
-from dogpile.core.readwrite_lock import ReadWriteMutex # @UnresolvedImport
+
+from dogpile.cache import make_region
#: Subliminal's cache version
CACHE_VERSION = 1
-EXPIRE_SECONDS = datetime.timedelta(weeks=3)
-
#: Expiration time for show caching
-SHOW_EXPIRATION_TIME = EXPIRE_SECONDS.days * 1440 + EXPIRE_SECONDS.seconds
+SHOW_EXPIRATION_TIME = datetime.timedelta(weeks=3).total_seconds()
#: Expiration time for episode caching
-EPISODE_EXPIRATION_TIME = EXPIRE_SECONDS.days * 1440 + EXPIRE_SECONDS.seconds
-
-
-def subliminal_key_generator(namespace, fn, to_str=string_type):
- """Add a :data:`CACHE_VERSION` to dogpile.cache's default function_key_generator"""
- if namespace is None:
- namespace = '%d:%s:%s' % (CACHE_VERSION, fn.__module__, fn.__name__)
- else:
- namespace = '%d:%s:%s|%s' % (CACHE_VERSION, fn.__module__, fn.__name__, namespace)
-
- args = inspect.getargspec(fn)
- has_self = args[0] and args[0][0] in ('self', 'cls')
-
- def generate_key(*args, **kw):
- if kw:
- raise ValueError('Keyword arguments not supported')
- if has_self:
- args = args[1:]
- return namespace + '|' + ' '.join(map(to_str, args))
- return generate_key
-
-
-class MutexLock(AbstractFileLock):
- """:class:`MutexLock` is a thread-based rw lock based on :class:`dogpile.core.ReadWriteMutex`"""
- def __init__(self, filename):
- self.mutex = ReadWriteMutex()
-
- def acquire_read_lock(self, wait):
- ret = self.mutex.acquire_read_lock(wait)
- return wait or ret
-
- def acquire_write_lock(self, wait):
- ret = self.mutex.acquire_write_lock(wait)
- return wait or ret
-
- def release_read_lock(self):
- return self.mutex.release_read_lock()
-
- def release_write_lock(self):
- return self.mutex.release_write_lock()
+EPISODE_EXPIRATION_TIME = datetime.timedelta(days=3).total_seconds()
-#: The dogpile.cache region
-region = make_region(function_key_generator=subliminal_key_generator)
+region = make_region()
diff --git a/lib/subliminal/cli.py b/lib/subliminal/cli.py
index cabcdfc8bd872a81498ddc9d44572f5964136fd4..c1e690b0d6cb7fd0c90448595edc1cc2bb4e6ca4 100644
--- a/lib/subliminal/cli.py
+++ b/lib/subliminal/cli.py
@@ -1,197 +1,272 @@
# -*- coding: utf-8 -*-
-from __future__ import unicode_literals, print_function
-import argparse
-import datetime
+"""
+Subliminal uses `click <http://click.pocoo.org>`_ to provide a powerful :abbr:`CLI (command-line interface)`.
+
+"""
+from __future__ import unicode_literals
+from collections import defaultdict
+from datetime import timedelta
import logging
import os
import re
-import sys
-import babelfish
-import xdg.BaseDirectory
-from subliminal import (__version__, cache_region, MutexLock, provider_manager, Video, Episode, Movie, scan_videos,
- download_best_subtitles, save_subtitles)
-try:
- import colorlog
-except ImportError:
- colorlog = None
-
-
-DEFAULT_CACHE_FILE = os.path.join(xdg.BaseDirectory.save_cache_path('subliminal'), 'cli.dbm')
-
-
-def subliminal():
- parser = argparse.ArgumentParser(prog='subliminal', description='Subtitles, faster than your thoughts',
- epilog='Suggestions and bug reports are greatly appreciated: '
- 'https://github.com/Diaoul/subliminal/issues', add_help=False)
-
- # required arguments
- required_arguments_group = parser.add_argument_group('required arguments')
- required_arguments_group.add_argument('paths', nargs='+', metavar='PATH', help='path to video file or folder')
- required_arguments_group.add_argument('-l', '--languages', nargs='+', required=True, metavar='LANGUAGE',
- help='wanted languages as IETF codes e.g. fr, pt-BR, sr-Cyrl ')
-
- # configuration
- configuration_group = parser.add_argument_group('configuration')
- configuration_group.add_argument('-s', '--single', action='store_true',
- help='download without language code in subtitle\'s filename i.e. .srt only')
- configuration_group.add_argument('-c', '--cache-file', default=DEFAULT_CACHE_FILE,
- help='cache file (default: %(default)s)')
-
- # filtering
- filtering_group = parser.add_argument_group('filtering')
- filtering_group.add_argument('-p', '--providers', nargs='+', metavar='PROVIDER',
- help='providers to use (%s)' % ', '.join(provider_manager.available_providers))
- filtering_group.add_argument('-m', '--min-score', type=int, default=0,
- help='minimum score for subtitles (0-%d for episodes, 0-%d for movies)'
- % (Episode.scores['hash'], Movie.scores['hash']))
- filtering_group.add_argument('-a', '--age', help='download subtitles for videos newer than AGE e.g. 12h, 1w2d')
- filtering_group.add_argument('-h', '--hearing-impaired', action='store_true',
- help='download hearing impaired subtitles')
- filtering_group.add_argument('-f', '--force', action='store_true',
- help='force subtitle download for videos with existing subtitles')
-
- # addic7ed
- addic7ed_group = parser.add_argument_group('addic7ed')
- addic7ed_group.add_argument('--addic7ed-username', metavar='USERNAME', help='username for addic7ed provider')
- addic7ed_group.add_argument('--addic7ed-password', metavar='PASSWORD', help='password for addic7ed provider')
-
- # output
- output_group = parser.add_argument_group('output')
- output_group.add_argument('-d', '--directory',
- help='save subtitles in the given directory rather than next to the video')
- output_group.add_argument('-e', '--encoding', default=None,
- help='encoding to convert the subtitle to (default: no conversion)')
- output_exclusive_group = output_group.add_mutually_exclusive_group()
- output_exclusive_group.add_argument('-q', '--quiet', action='store_true', help='disable output')
- output_exclusive_group.add_argument('-v', '--verbose', action='store_true', help='verbose output')
- output_group.add_argument('--log-file', help='log into a file instead of stdout')
- output_group.add_argument('--color', action='store_true', help='add color to console output (requires colorlog)')
-
- # troubleshooting
- troubleshooting_group = parser.add_argument_group('troubleshooting')
- troubleshooting_group.add_argument('--debug', action='store_true', help='debug output')
- troubleshooting_group.add_argument('--version', action='version', version=__version__)
- troubleshooting_group.add_argument('--help', action='help', help='show this help message and exit')
-
- # parse args
- args = parser.parse_args()
-
- # parse paths
- try:
- args.paths = [os.path.abspath(os.path.expanduser(p.decode('utf-8') if isinstance(p, bytes) else p))
- for p in args.paths]
- except UnicodeDecodeError:
- parser.error('argument paths: encodings is not utf-8: %r' % args.paths)
-
- # parse languages
- try:
- args.languages = {babelfish.Language.fromietf(l) for l in args.languages}
- except babelfish.Error:
- parser.error('argument -l/--languages: codes are not IETF: %r' % args.languages)
-
- # parse age
- if args.age is not None:
- match = re.match(r'^(?:(?P<weeks>\d+?)w)?(?:(?P<days>\d+?)d)?(?:(?P<hours>\d+?)h)?$', args.age)
+
+from babelfish import Error as BabelfishError, Language
+import click
+from dogpile.cache.backends.file import AbstractFileLock
+from dogpile.core import ReadWriteMutex
+
+from subliminal import (Episode, Movie, ProviderPool, Video, __version__, check_video, provider_manager, region,
+ save_subtitles, scan_video, scan_videos)
+from subliminal.subtitle import compute_score
+
+
+class MutexLock(AbstractFileLock):
+ """:class:`MutexLock` is a thread-based rw lock based on :class:`dogpile.core.ReadWriteMutex`."""
+ def __init__(self, filename):
+ self.mutex = ReadWriteMutex()
+
+ def acquire_read_lock(self, wait):
+ ret = self.mutex.acquire_read_lock(wait)
+ return wait or ret
+
+ def acquire_write_lock(self, wait):
+ ret = self.mutex.acquire_write_lock(wait)
+ return wait or ret
+
+ def release_read_lock(self):
+ return self.mutex.release_read_lock()
+
+ def release_write_lock(self):
+ return self.mutex.release_write_lock()
+
+
+class LanguageParamType(click.ParamType):
+ """:class:`~click.ParamType` for languages that returns a :class:`~babelfish.language.Language`"""
+ name = 'language'
+
+ def convert(self, value, param, ctx):
+ try:
+ return Language.fromietf(value)
+ except BabelfishError:
+ self.fail('%s is not a valid language' % value)
+
+LANGUAGE = LanguageParamType()
+
+
+class AgeParamType(click.ParamType):
+ """:class:`~click.ParamType` for age strings that returns a :class:`~datetime.timedelta`
+
+ An age string is in the form `number + identifier` with possible identifiers:
+
+ * ``w`` for weeks
+ * ``d`` for days
+ * ``h`` for hours
+
+ The form can be specified multiple times but only with that idenfier ordering. For example:
+
+ * ``1w2d4h`` for 1 week, 2 days and 4 hours
+ * ``2w`` for 2 weeks
+ * ``3w6h`` for 3 weeks and 6 hours
+
+ """
+ name = 'age'
+
+ def convert(self, value, param, ctx):
+ match = re.match(r'^(?:(?P<weeks>\d+?)w)?(?:(?P<days>\d+?)d)?(?:(?P<hours>\d+?)h)?$', value)
if not match:
- parser.error('argument -a/--age: invalid age: %r' % args.age)
- args.age = datetime.timedelta(**{k: int(v) for k, v in match.groupdict(0).items()})
-
- # parse cache-file
- args.cache_file = os.path.abspath(os.path.expanduser(args.cache_file))
- if not os.path.exists(os.path.split(args.cache_file)[0]):
- parser.error('argument -c/--cache-file: directory %r for cache file does not exist'
- % os.path.split(args.cache_file)[0])
-
- # parse provider configs
- provider_configs = {}
- if (args.addic7ed_username is not None and args.addic7ed_password is None
- or args.addic7ed_username is None and args.addic7ed_password is not None):
- parser.error('argument --addic7ed-username/--addic7ed-password: both arguments are required or none')
- if args.addic7ed_username is not None and args.addic7ed_password is not None:
- provider_configs['addic7ed'] = {'username': args.addic7ed_username, 'password': args.addic7ed_password}
-
- # parse color
- if args.color and colorlog is None:
- parser.error('argument --color: colorlog required')
-
- # setup output
- if args.log_file is None:
+ self.fail('%s is not a valid age' % value)
+
+ return timedelta(**{k: int(v) for k, v in match.groupdict(0).items()})
+
+AGE = AgeParamType()
+
+PROVIDER = click.Choice(sorted(provider_manager.names()))
+
+subliminal_cache = 'subliminal.dbm'
+
+
+@click.group(context_settings={'max_content_width': 100}, epilog='Suggestions and bug reports are greatly appreciated: '
+ 'https://github.com/Diaoul/subliminal/')
+@click.option('--addic7ed', type=click.STRING, nargs=2, metavar='USERNAME PASSWORD', help='Addic7ed configuration.')
+@click.option('--cache-dir', type=click.Path(writable=True, resolve_path=True, file_okay=False),
+ default=click.get_app_dir('subliminal'), show_default=True, expose_value=True,
+ help='Path to the cache directory.')
+@click.option('--debug', is_flag=True, help='Print useful information for debugging subliminal and for reporting bugs.')
+@click.version_option(__version__)
+@click.pass_context
+def subliminal(ctx, addic7ed, cache_dir, debug):
+ """Subtitles, faster than your thoughts."""
+ # create cache directory
+ os.makedirs(cache_dir, exist_ok=True)
+
+ # configure cache
+ region.configure('dogpile.cache.dbm', expiration_time=timedelta(days=30),
+ arguments={'filename': os.path.join(cache_dir, subliminal_cache), 'lock_factory': MutexLock})
+
+ # configure logging
+ if debug:
handler = logging.StreamHandler()
- else:
- handler = logging.FileHandler(args.log_file, encoding='utf-8')
- if args.debug:
- if args.color:
- if args.log_file is None:
- log_format = '%(log_color)s%(levelname)-8s%(reset)s [%(blue)s%(name)s-%(funcName)s:%(lineno)d%(reset)s] %(message)s'
- else:
- log_format = '%(purple)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s [%(blue)s%(name)s-%(funcName)s:%(lineno)d%(reset)s] %(message)s'
- handler.setFormatter(colorlog.ColoredFormatter(log_format,
- log_colors=dict(colorlog.default_log_colors.items() + [('DEBUG', 'cyan')])))
- else:
- if args.log_file is None:
- log_format = '%(levelname)-8s [%(name)s-%(funcName)s:%(lineno)d] %(message)s'
- else:
- log_format = '%(asctime)s %(levelname)-8s [%(name)s-%(funcName)s:%(lineno)d] %(message)s'
- handler.setFormatter(logging.Formatter(log_format))
- logging.getLogger().addHandler(handler)
- logging.getLogger().setLevel(logging.DEBUG)
- elif args.verbose:
- if args.color:
- if args.log_file is None:
- log_format = '%(log_color)s%(levelname)-8s%(reset)s [%(blue)s%(name)s%(reset)s] %(message)s'
- else:
- log_format = '%(purple)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s [%(blue)s%(name)s%(reset)s] %(message)s'
- handler.setFormatter(colorlog.ColoredFormatter(log_format))
- else:
- log_format = '%(levelname)-8s [%(name)s] %(message)s'
- if args.log_file is not None:
- log_format = '%(asctime)s ' + log_format
- handler.setFormatter(logging.Formatter(log_format))
+ handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
logging.getLogger('subliminal').addHandler(handler)
- logging.getLogger('subliminal').setLevel(logging.INFO)
- elif not args.quiet:
- if args.color:
- if args.log_file is None:
- log_format = '[%(log_color)s%(levelname)s%(reset)s] %(message)s'
- else:
- log_format = '%(purple)s%(asctime)s%(reset)s [%(log_color)s%(levelname)s%(reset)s] %(message)s'
- handler.setFormatter(colorlog.ColoredFormatter(log_format))
- else:
- if args.log_file is None:
- log_format = '%(levelname)s: %(message)s'
- else:
- log_format = '%(asctime)s %(levelname)s: %(message)s'
- handler.setFormatter(logging.Formatter(log_format))
- logging.getLogger('subliminal.api').addHandler(handler)
- logging.getLogger('subliminal.api').setLevel(logging.INFO)
+ logging.getLogger('subliminal').setLevel(logging.DEBUG)
- # configure cache
- cache_region.configure('dogpile.cache.dbm', expiration_time=datetime.timedelta(days=30), # @UndefinedVariable
- arguments={'filename': args.cache_file, 'lock_factory': MutexLock})
+ # provider configs
+ ctx.obj = {'provider_configs': {}}
+ if addic7ed:
+ ctx.obj['provider_configs']['addic7ed'] = {'username': addic7ed[0], 'password': addic7ed[1]}
+
+
+@subliminal.command()
+@click.option('--clear-subliminal', is_flag=True, help='Clear subliminal\'s cache. Use this ONLY if your cache is '
+ 'corrupted or if you experience issues.')
+@click.pass_context
+def cache(ctx, clear_subliminal):
+ """Cache management."""
+ if clear_subliminal:
+ os.remove(os.path.join(ctx.parent.params['cache_dir'], subliminal_cache))
+ click.echo('Subliminal\'s cache cleared.')
+ else:
+ click.echo('Nothing done.')
+
+
+@subliminal.command()
+@click.option('-l', '--language', type=LANGUAGE, required=True, multiple=True, help='Language as IETF code, '
+ 'e.g. en, pt-BR (can be used multiple times).')
+@click.option('-p', '--provider', type=PROVIDER, multiple=True, help='Provider to use (can be used multiple times).')
+@click.option('-a', '--age', type=AGE, help='Filter videos newer than AGE, e.g. 12h, 1w2d.')
+@click.option('-d', '--directory', type=click.STRING, metavar='DIR', help='Directory where to save subtitles, '
+ 'default is next to the video file.')
+@click.option('-e', '--encoding', type=click.STRING, metavar='ENC', help='Subtitle file encoding, default is to '
+ 'preserve original encoding.')
+@click.option('-s', '--single', is_flag=True, default=False, help='Save subtitle without language code in the file '
+ 'name, i.e. use .srt extension.')
+@click.option('-f', '--force', is_flag=True, default=False, help='Force download even if a subtitle already exist.')
+@click.option('-hi', '--hearing-impaired', is_flag=True, default=False, help='Prefer hearing impaired subtitles.')
+@click.option('-m', '--min-score', type=click.IntRange(0, 100), default=0, help='Minimum score for a subtitle '
+ 'to be downloaded.')
+@click.option('-v', '--verbose', count=True, help='Increase verbosity.')
+@click.argument('path', type=click.Path(), required=True, nargs=-1)
+@click.pass_obj
+def download(obj, provider, language, age, directory, encoding, single, force, hearing_impaired, min_score, verbose,
+ path):
+ """Download best subtitles.
+
+ PATH can be an directory containing videos, a video file path or a video file name. It can be used multiple times.
+
+ If an existing subtitle is detected (external or embedded) in the correct language, the download is skipped for
+ the associated video.
+
+ """
+ # process parameters
+ language = set(language)
# scan videos
- videos = scan_videos([p for p in args.paths if os.path.exists(p)], subtitles=not args.force,
- embedded_subtitles=not args.force, age=args.age)
+ videos = []
+ ignored_videos = []
+ with click.progressbar(path, label='Collecting videos',
+ item_show_func=lambda p: str(p) if p is not None else '') as bar:
+ for p in bar:
+ # non-existing
+ if not os.path.exists(p):
+ videos.append(Video.fromname(p))
+ continue
+
+ # directories
+ if os.path.isdir(p):
+ for video in scan_videos(p, subtitles=not force, embedded_subtitles=not force):
+ if check_video(video, languages=language, age=age, undefined=single):
+ videos.append(video)
+ else:
+ ignored_videos.append(video)
+ continue
+
+ # other inputs
+ video = scan_video(p, subtitles=not force, embedded_subtitles=not force)
+ if check_video(video, languages=language, age=age, undefined=single):
+ videos.append(video)
+ else:
+ ignored_videos.append(video)
+
+ # output ignored videos
+ if verbose > 1:
+ for video in ignored_videos:
+ click.secho('%s ignored - subtitles: %s / age: %d day%s ' % (
+ os.path.split(video.name)[1],
+ ', '.join(str(s) for s in video.subtitle_languages) or 'none',
+ video.age.days,
+ 's' if video.age.days > 1 else ''
+ ), fg='yellow')
- # guess videos
- videos.extend([Video.fromname(p) for p in args.paths if not os.path.exists(p)])
+ # report collected videos
+ click.echo('%s video%s collected / %s video%s ignored' % (click.style(str(len(videos)), bold=True),
+ 's' if len(videos) > 1 else '',
+ click.style(str(len(ignored_videos)), bold=True),
+ 's' if len(ignored_videos) > 1 else ''))
+
+ # exit if no video collected
+ if not videos:
+ return
# download best subtitles
- subtitles = download_best_subtitles(videos, args.languages, providers=args.providers,
- provider_configs=provider_configs, min_score=args.min_score,
- hearing_impaired=args.hearing_impaired, single=args.single)
+ downloaded_subtitles = defaultdict(list)
+ with ProviderPool(providers=provider, provider_configs=obj['provider_configs']) as pool:
+ with click.progressbar(videos, label='Downloading subtitles',
+ item_show_func=lambda v: os.path.split(v.name)[1] if v is not None else '') as bar:
+ for v in bar:
+ subtitles = pool.download_best_subtitles(pool.list_subtitles(v, language - v.subtitle_languages),
+ v, language, min_score=v.scores['hash'] * min_score / 100,
+ hearing_impaired=hearing_impaired, only_one=single)
+ downloaded_subtitles[v] = subtitles
# save subtitles
- save_subtitles(subtitles, single=args.single, directory=args.directory, encoding=args.encoding)
-
- # result output
- if not subtitles:
- if not args.quiet:
- print('No subtitles downloaded', file=sys.stderr)
- exit(1)
- if not args.quiet:
- subtitles_count = sum([len(s) for s in subtitles.values()])
- if subtitles_count == 1:
- print('%d subtitle downloaded' % subtitles_count)
- else:
- print('%d subtitles downloaded' % subtitles_count)
+ total_subtitles = 0
+ for v, subtitles in downloaded_subtitles.items():
+ saved_subtitles = save_subtitles(v, subtitles, single=single, directory=directory, encoding=encoding)
+ total_subtitles += len(saved_subtitles)
+
+ if verbose > 0:
+ click.echo('%s subtitle%s downloaded for %s' % (click.style(str(len(saved_subtitles)), bold=True),
+ 's' if len(saved_subtitles) > 1 else '',
+ os.path.split(v.name)[1]))
+
+ if verbose > 1:
+ for s in saved_subtitles:
+ matches = s.get_matches(v, hearing_impaired=hearing_impaired)
+ score = compute_score(matches, v)
+
+ # score color
+ score_color = None
+ if isinstance(v, Movie):
+ if score < v.scores['title']:
+ score_color = 'red'
+ elif score < v.scores['title'] + v.scores['year'] + v.scores['release_group']:
+ score_color = 'yellow'
+ else:
+ score_color = 'green'
+ elif isinstance(v, Episode):
+ if score < v.scores['series'] + v.scores['season'] + v.scores['episode']:
+ score_color = 'red'
+ elif score < (v.scores['series'] + v.scores['season'] + v.scores['episode'] +
+ v.scores['release_group']):
+ score_color = 'yellow'
+ else:
+ score_color = 'green'
+
+ # scale score from 0 to 100 taking out preferences
+ scaled_score = score
+ if s.hearing_impaired == hearing_impaired:
+ scaled_score -= v.scores['hearing_impaired']
+ scaled_score *= 100 / v.scores['hash']
+
+ # echo some nice colored output
+ click.echo(' - [{score}] - {language} subtitle from {provider_name} (match on {matches})'.format(
+ score=click.style('{:5.1f}'.format(scaled_score), fg=score_color, bold=score >= v.scores['hash']),
+ language=s.language.name if s.language.country is None else '%s (%s)' % (s.language.name,
+ s.language.country.name),
+ provider_name=s.provider_name,
+ matches=', '.join(sorted(matches, key=v.scores.get, reverse=True))
+ ))
+
+ if verbose == 0:
+ click.echo('Downloaded %s subtitle%s' % (click.style(str(total_subtitles), bold=True),
+ 's' if total_subtitles > 1 else ''))
diff --git a/lib/subliminal/converters/addic7ed.py b/lib/subliminal/converters/addic7ed.py
index 0e862931ddd760dc633d0b59fa0e8f7f0451da72..571f8ee11f41427a5c4bc863e19fa9c2768e6beb 100644
--- a/lib/subliminal/converters/addic7ed.py
+++ b/lib/subliminal/converters/addic7ed.py
@@ -23,9 +23,11 @@ class Addic7edConverter(LanguageReverseConverter):
return self.to_addic7ed[(alpha3, country)]
if (alpha3,) in self.to_addic7ed:
return self.to_addic7ed[(alpha3,)]
+
return self.name_converter.convert(alpha3, country, script)
def reverse(self, addic7ed):
if addic7ed in self.from_addic7ed:
return self.from_addic7ed[addic7ed]
+
return self.name_converter.reverse(addic7ed)
diff --git a/lib/subliminal/converters/tvsubtitles.py b/lib/subliminal/converters/tvsubtitles.py
index e9b7e74fd8af4e0f772e4cd47539be1cc321b268..9507df268f7e6d8470df51a3150c9499786e6b44 100644
--- a/lib/subliminal/converters/tvsubtitles.py
+++ b/lib/subliminal/converters/tvsubtitles.py
@@ -8,7 +8,7 @@ class TVsubtitlesConverter(LanguageReverseConverter):
self.alpha2_converter = language_converters['alpha2']
self.from_tvsubtitles = {'br': ('por', 'BR'), 'ua': ('ukr',), 'gr': ('ell',), 'cn': ('zho',), 'jp': ('jpn',),
'cz': ('ces',)}
- self.to_tvsubtitles = {v: k for k, v in self.from_tvsubtitles}
+ self.to_tvsubtitles = {v: k for k, v in self.from_tvsubtitles.items()}
self.codes = self.alpha2_converter.codes | set(self.from_tvsubtitles.keys())
def convert(self, alpha3, country=None, script=None):
@@ -16,9 +16,11 @@ class TVsubtitlesConverter(LanguageReverseConverter):
return self.to_tvsubtitles[(alpha3, country)]
if (alpha3,) in self.to_tvsubtitles:
return self.to_tvsubtitles[(alpha3,)]
+
return self.alpha2_converter.convert(alpha3, country, script)
def reverse(self, tvsubtitles):
if tvsubtitles in self.from_tvsubtitles:
return self.from_tvsubtitles[tvsubtitles]
+
return self.alpha2_converter.reverse(tvsubtitles)
diff --git a/lib/subliminal/exceptions.py b/lib/subliminal/exceptions.py
index be954800a055c7e7bb1c6348dfdd0be7d5899b21..e1ac1c67b04318141f5a4ef9ed840c81f4fb2fe2 100644
--- a/lib/subliminal/exceptions.py
+++ b/lib/subliminal/exceptions.py
@@ -3,20 +3,25 @@ from __future__ import unicode_literals
class Error(Exception):
- """Base class for exceptions in subliminal"""
+ """Base class for exceptions in subliminal."""
+ pass
class ProviderError(Error):
- """Exception raised by providers"""
+ """Exception raised by providers."""
+ pass
class ConfigurationError(ProviderError):
- """Exception raised by providers when badly configured"""
+ """Exception raised by providers when badly configured."""
+ pass
class AuthenticationError(ProviderError):
- """Exception raised by providers when authentication failed"""
+ """Exception raised by providers when authentication failed."""
+ pass
class DownloadLimitExceeded(ProviderError):
- """Exception raised by providers when download limit is exceeded"""
+ """Exception raised by providers when download limit is exceeded."""
+ pass
diff --git a/lib/subliminal/providers/__init__.py b/lib/subliminal/providers/__init__.py
index fc0cb9556171513efe5c2d53597cea5f5117303c..4181c8b0bf1cb76d6a7f848eff569c53408682bd 100644
--- a/lib/subliminal/providers/__init__.py
+++ b/lib/subliminal/providers/__init__.py
@@ -1,27 +1,77 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
-import contextlib
import logging
-import socket
-import babelfish
-from pkg_resources import iter_entry_points, EntryPoint
-import requests
-from ..video import Episode, Movie
+from bs4 import BeautifulSoup, FeatureNotFound
+from six.moves.xmlrpc_client import SafeTransport
+
+from ..video import Episode, Movie
logger = logging.getLogger(__name__)
+def get_version(version):
+ """Put the `version` in the major.minor form.
+
+ :param str version: the full version.
+ :return: the major.minor form of the `version`.
+ :rtype: str
+
+ """
+ return '.'.join(version.split('.')[:2])
+
+
+class TimeoutSafeTransport(SafeTransport):
+ """Timeout support for ``xmlrpc.client.SafeTransport``."""
+ def __init__(self, timeout, *args, **kwargs):
+ SafeTransport.__init__(self, *args, **kwargs)
+ self.timeout = timeout
+
+ def make_connection(self, host):
+ c = SafeTransport.make_connection(self, host)
+ c.timeout = self.timeout
+
+ return c
+
+
+class ParserBeautifulSoup(BeautifulSoup):
+ """A ``bs4.BeautifulSoup`` that picks the first parser available in `parsers`.
+
+ :param markup: markup for the ``bs4.BeautifulSoup``.
+ :param list parsers: parser names, in order of preference
+
+ """
+ def __init__(self, markup, parsers, **kwargs):
+ # reject features
+ if set(parsers).intersection({'fast', 'permissive', 'strict', 'xml', 'html', 'html5'}):
+ raise ValueError('Features not allowed, only parser names')
+
+ # reject some kwargs
+ if 'features' in kwargs:
+ raise ValueError('Cannot use features kwarg')
+ if 'builder' in kwargs:
+ raise ValueError('Cannot use builder kwarg')
+
+ # pick the first parser available
+ for parser in parsers:
+ try:
+ super(ParserBeautifulSoup, self).__init__(markup, parser, **kwargs)
+ return
+ except FeatureNotFound:
+ pass
+
+ raise FeatureNotFound
+
+
class Provider(object):
- """Base class for providers
+ """Base class for providers.
- If any configuration is possible for the provider, like credentials, it must take place during instantiation
+ If any configuration is possible for the provider, like credentials, it must take place during instantiation.
- :param \*\*kwargs: configuration
- :raise: :class:`~subliminal.exceptions.ProviderConfigurationError` if there is a configuration error
+ :raise: :class:`~subliminal.exceptions.ConfigurationError` if there is a configuration error
"""
- #: Supported BabelFish languages
+ #: Supported set of :class:`~babelfish.language.Language`
languages = set()
#: Supported video types
@@ -30,53 +80,46 @@ class Provider(object):
#: Required hash, if any
required_hash = None
- def __init__(self, **kwargs):
- pass
-
def __enter__(self):
self.initialize()
return self
- def __exit__(self, type, value, traceback): # @ReservedAssignment
+ def __exit__(self, exc_type, exc_value, traceback):
self.terminate()
def initialize(self):
- """Initialize the provider
+ """Initialize the provider.
Must be called when starting to work with the provider. This is the place for network initialization
or login operations.
- .. note:
- This is called automatically if you use the :keyword:`with` statement
-
-
- :raise: :class:`~subliminal.exceptions.ProviderNotAvailable` if the provider is unavailable
+ .. note::
+ This is called automatically when entering the :keyword:`with` statement
"""
- pass
+ raise NotImplementedError
def terminate(self):
- """Terminate the provider
+ """Terminate the provider.
Must be called when done with the provider. This is the place for network shutdown or logout operations.
- .. note:
- This is called automatically if you use the :keyword:`with` statement
+ .. note::
+ This is called automatically when exiting the :keyword:`with` statement
- :raise: :class:`~subliminal.exceptions.ProviderNotAvailable` if the provider is unavailable
"""
- pass
+ raise NotImplementedError
@classmethod
def check(cls, video):
- """Check if the `video` can be processed
+ """Check if the `video` can be processed.
- The video is considered invalid if not an instance of :attr:`video_types` or if the :attr:`required_hash` is
- not present in :attr:`~subliminal.video.Video`'s `hashes` attribute.
+ The `video` is considered invalid if not an instance of :attr:`video_types` or if the :attr:`required_hash` is
+ not present in :attr:`~subliminal.video.Video.hashes` attribute of the `video`.
- :param video: the video to check
+ :param video: the video to check.
:type video: :class:`~subliminal.video.Video`
- :return: `True` if the `video` and `languages` are valid, `False` otherwise
+ :return: `True` if the `video` is valid, `False` otherwise.
:rtype: bool
"""
@@ -84,255 +127,47 @@ class Provider(object):
return False
if cls.required_hash is not None and cls.required_hash not in video.hashes:
return False
+
return True
- def query(self, languages, *args, **kwargs):
- """Query the provider for subtitles
+ def query(self, *args, **kwargs):
+ """Query the provider for subtitles.
- This method arguments match as much as possible the actual parameters for querying the provider
+ Arguments should match as much as possible the actual parameters for querying the provider
- :param languages: languages to search for
- :type languages: set of :class:`babelfish.Language`
- :param \*args: other required arguments
- :param \*\*kwargs: other optional arguments
- :return: the subtitles
+ :return: found subtitles.
:rtype: list of :class:`~subliminal.subtitle.Subtitle`
- :raise: :class:`~subliminal.exceptions.ProviderNotAvailable` if the provider is unavailable
- :raise: :class:`~subliminal.exceptions.ProviderError` if something unexpected occured
+ :raise: :class:`~subliminal.exceptions.ProviderError`
"""
raise NotImplementedError
def list_subtitles(self, video, languages):
- """List subtitles for the `video` with the given `languages`
+ """List subtitles for the `video` with the given `languages`.
- This is a proxy for the :meth:`query` method. The parameters passed to the :meth:`query` method may
- vary depending on the amount of information available in the `video`
+ This will call the :meth:`query` method internally. The parameters passed to the :meth:`query` method may
+ vary depending on the amount of information available in the `video`.
- :param video: video to list subtitles for
+ :param video: video to list subtitles for.
:type video: :class:`~subliminal.video.Video`
- :param languages: languages to search for
- :type languages: set of :class:`babelfish.Language`
- :return: the subtitles
+ :param languages: languages to search for.
+ :type languages: set of :class:`~babelfish.language.Language`
+ :return: found subtitles.
:rtype: list of :class:`~subliminal.subtitle.Subtitle`
- :raise: :class:`~subliminal.exceptions.ProviderNotAvailable` if the provider is unavailable
- :raise: :class:`~subliminal.exceptions.ProviderError` if something unexpected occured
+ :raise: :class:`~subliminal.exceptions.ProviderError`
"""
raise NotImplementedError
def download_subtitle(self, subtitle):
- """Download the `subtitle` an fill its :attr:`~subliminal.subtitle.Subtitle.content` attribute with
- subtitle's text
+ """Download `subtitle`'s :attr:`~subliminal.subtitle.Subtitle.content`.
- :param subtitle: subtitle to download
+ :param subtitle: subtitle to download.
:type subtitle: :class:`~subliminal.subtitle.Subtitle`
- :raise: :class:`~subliminal.exceptions.ProviderNotAvailable` if the provider is unavailable
- :raise: :class:`~subliminal.exceptions.ProviderError` if something unexpected occured
+ :raise: :class:`~subliminal.exceptions.ProviderError`
"""
raise NotImplementedError
def __repr__(self):
return '<%s [%r]>' % (self.__class__.__name__, self.video_types)
-
-
-class ProviderManager(object):
- """Manager for providers behaving like a dict with lazy loading
-
- Loading is done in this order:
-
- * Entry point providers
- * Registered providers
-
- .. attribute:: entry_point
-
- The entry point where to look for providers
-
- """
- entry_point = 'subliminal.providers'
-
- def __init__(self):
- #: Registered providers with entry point syntax
- self.registered_providers = ['addic7ed = subliminal.providers.addic7ed:Addic7edProvider',
- 'opensubtitles = subliminal.providers.opensubtitles:OpenSubtitlesProvider',
- 'podnapisi = subliminal.providers.podnapisi:PodnapisiProvider',
- 'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider',
- 'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider']
-
- #: Loaded providers
- self.providers = {}
-
- @property
- def available_providers(self):
- """Available providers"""
- available_providers = set(self.providers.keys())
- available_providers.update([ep.name for ep in iter_entry_points(self.entry_point)])
- available_providers.update([EntryPoint.parse(c).name for c in self.registered_providers])
- return available_providers
-
- def __getitem__(self, name):
- """Get a provider, lazy loading it if necessary"""
- if name in self.providers:
- return self.providers[name]
- for ep in iter_entry_points(self.entry_point):
- if ep.name == name:
- self.providers[ep.name] = ep.load()
- return self.providers[ep.name]
- for ep in (EntryPoint.parse(c) for c in self.registered_providers):
- if ep.name == name:
- self.providers[ep.name] = ep.load(require=False)
- return self.providers[ep.name]
- raise KeyError(name)
-
- def __setitem__(self, name, provider):
- """Load a provider"""
- self.providers[name] = provider
-
- def __delitem__(self, name):
- """Unload a provider"""
- del self.providers[name]
-
- def __iter__(self):
- """Iterator over loaded providers"""
- return iter(self.providers)
-
- def register(self, entry_point):
- """Register a provider
-
- :param string entry_point: provider to register (entry point syntax)
- :raise: ValueError if already registered
-
- """
- if entry_point in self.registered_providers:
- raise ValueError('Entry point \'%s\' already registered' % entry_point)
- entry_point_name = EntryPoint.parse(entry_point).name
- if entry_point_name in self.available_providers:
- raise ValueError('An entry point with name \'%s\' already registered' % entry_point_name)
- self.registered_providers.insert(0, entry_point)
-
- def unregister(self, entry_point):
- """Unregister a provider
-
- :param string entry_point: provider to unregister (entry point syntax)
-
- """
- self.registered_providers.remove(entry_point)
-
- def __contains__(self, name):
- return name in self.providers
-
-provider_manager = ProviderManager()
-
-
-class ProviderPool(object):
- """A pool of providers with the same API as a single :class:`Provider`
-
- The :class:`ProviderPool` supports the ``with`` statement to :meth:`terminate` the providers
-
- :param providers: providers to use, if not all
- :type providers: list of string or None
- :param provider_configs: configuration for providers
- :type provider_configs: dict of provider name => provider constructor kwargs or None
-
- """
- def __init__(self, providers=None, provider_configs=None):
- self.provider_configs = provider_configs or {}
- self.providers = dict([(p, provider_manager[p]) for p in (providers or provider_manager.available_providers)])
- self.initialized_providers = {}
- self.discarded_providers = set()
-
- def __enter__(self):
- return self
-
- def __exit__(self, type, value, traceback): # @ReservedAssignment
- self.terminate()
-
- def get_initialized_provider(self, name):
- """Get a :class:`Provider` by name, initializing it if necessary
-
- :param string name: name of the provider
- :return: the initialized provider
- :rtype: :class:`Provider`
-
- """
- if name in self.initialized_providers:
- return self.initialized_providers[name]
- provider = self.providers[name](**self.provider_configs.get(name, {}))
- provider.initialize()
- self.initialized_providers[name] = provider
- return provider
-
- def list_subtitles(self, video, languages):
- """List subtitles for `video` with the given `languages`
-
- :param video: video to list subtitles for
- :type video: :class:`~subliminal.video.Video`
- :param languages: languages of subtitles to search for
- :type languages: set of :class:`babelfish.Language`
- :return: found subtitles
- :rtype: list of :class:`~subliminal.subtitle.Subtitle`
-
- """
- subtitles = []
- for provider_name, provider_class in self.providers.items():
- if not provider_class.check(video):
- logger.info('Skipping provider %r: not a valid video', provider_name)
- continue
- provider_languages = provider_class.languages & languages - video.subtitle_languages
- if not provider_languages:
- logger.info('Skipping provider %r: no language to search for', provider_name)
- continue
- if provider_name in self.discarded_providers:
- logger.debug('Skipping discarded provider %r', provider_name)
- continue
- try:
- provider = self.get_initialized_provider(provider_name)
- logger.info('Listing subtitles with provider %r and languages %r', provider_name, provider_languages)
- provider_subtitles = provider.list_subtitles(video, provider_languages)
- logger.info('Found %d subtitles', len(provider_subtitles))
- subtitles.extend(provider_subtitles)
- except (requests.exceptions.Timeout, socket.timeout):
- logger.warning('Provider %r timed out, discarding it', provider_name)
- self.discarded_providers.add(provider_name)
- except:
- logger.exception('Unexpected error in provider %r, discarding it', provider_name)
- self.discarded_providers.add(provider_name)
- return subtitles
-
- def download_subtitle(self, subtitle):
- """Download a subtitle
-
- :param subtitle: subtitle to download
- :type subtitle: :class:`~subliminal.subtitle.Subtitle`
- :return: ``True`` if the subtitle has been successfully downloaded, ``False`` otherwise
- :rtype: bool
-
- """
- if subtitle.provider_name in self.discarded_providers:
- logger.debug('Discarded provider %r', subtitle.provider_name)
- return False
- try:
- provider = self.get_initialized_provider(subtitle.provider_name)
- provider.download_subtitle(subtitle)
- if not subtitle.is_valid:
- logger.warning('Invalid subtitle')
- return False
- return True
- except (requests.exceptions.Timeout, socket.timeout):
- logger.warning('Provider %r timed out, discarding it', subtitle.provider_name)
- self.discarded_providers.add(subtitle.provider_name)
- except:
- logger.exception('Unexpected error in provider %r, discarding it', subtitle.provider_name)
- self.discarded_providers.add(subtitle.provider_name)
- return False
-
- def terminate(self):
- """Terminate all the initialized providers"""
- for (provider_name, provider) in self.initialized_providers.items():
- try:
- provider.terminate()
- except (requests.exceptions.Timeout, socket.timeout):
- logger.warning('Provider %r timed out, unable to terminate', provider_name)
- except:
- logger.exception('Unexpected error in provider %r', provider_name)
diff --git a/lib/subliminal/providers/addic7ed.py b/lib/subliminal/providers/addic7ed.py
index 2a7f4bd53a71f2342ba7ace0517c2c8d485bc9cd..b0a86c2bfe91070b767fc0626cc097e4c8fbe0cc 100644
--- a/lib/subliminal/providers/addic7ed.py
+++ b/lib/subliminal/providers/addic7ed.py
@@ -1,26 +1,29 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
-import babelfish
-import bs4
-import requests
-from . import Provider
+import re
+
+from babelfish import Language, language_converters
+from requests import Session
+
+from . import ParserBeautifulSoup, Provider, get_version
from .. import __version__
-from ..cache import region, SHOW_EXPIRATION_TIME
-from ..exceptions import ConfigurationError, AuthenticationError, DownloadLimitExceeded, ProviderError
-from ..subtitle import Subtitle, fix_line_endings, compute_guess_properties_matches
+from ..cache import SHOW_EXPIRATION_TIME, region
+from ..exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded
+from ..subtitle import Subtitle, fix_line_ending, guess_matches, guess_properties
from ..video import Episode
-
logger = logging.getLogger(__name__)
-babelfish.language_converters.register('addic7ed = subliminal.converters.addic7ed:Addic7edConverter')
+language_converters.register('addic7ed = subliminal.converters.addic7ed:Addic7edConverter')
+
+series_year_re = re.compile('^(?P<series>[ \w]+)(?: \((?P<year>\d{4})\))?$')
class Addic7edSubtitle(Subtitle):
provider_name = 'addic7ed'
- def __init__(self, language, series, season, episode, title, year, version, hearing_impaired, download_link,
- page_link):
+ def __init__(self, language, hearing_impaired, page_link, series, season, episode, title, year, version,
+ download_link):
super(Addic7edSubtitle, self).__init__(language, hearing_impaired, page_link)
self.series = series
self.season = season
@@ -30,10 +33,15 @@ class Addic7edSubtitle(Subtitle):
self.version = version
self.download_link = download_link
- def compute_matches(self, video):
- matches = set()
+ @property
+ def id(self):
+ return self.download_link
+
+ def get_matches(self, video, hearing_impaired=False):
+ matches = super(Addic7edSubtitle, self).get_matches(video, hearing_impaired=hearing_impaired)
+
# series
- if video.series and self.series == video.series:
+ if video.series and self.series.lower() == video.series.lower():
matches.add('series')
# season
if video.season and self.season == video.season:
@@ -45,152 +53,205 @@ class Addic7edSubtitle(Subtitle):
if video.title and self.title.lower() == video.title.lower():
matches.add('title')
# year
- if self.year == video.year:
+ if video.year == self.year:
matches.add('year')
# release_group
if video.release_group and self.version and video.release_group.lower() in self.version.lower():
matches.add('release_group')
- """
# resolution
if video.resolution and self.version and video.resolution in self.version.lower():
matches.add('resolution')
# format
- if video.format and self.version and video.format in self.version.lower:
+ if video.format and self.version and video.format.lower() in self.version.lower():
matches.add('format')
- """
- # we don't have the complete filename, so we need to guess the matches separately
- # guess resolution (screenSize in guessit)
- matches |= compute_guess_properties_matches(video, self.version, 'screenSize')
- # guess format
- matches |= compute_guess_properties_matches(video, self.version, 'format')
- # guess video codec
- matches |= compute_guess_properties_matches(video, self.version, 'videoCodec')
+ # other properties
+ matches |= guess_matches(video, guess_properties(self.version), partial=True)
+
return matches
class Addic7edProvider(Provider):
- languages = {babelfish.Language('por', 'BR')} | {babelfish.Language(l)
- for l in ['ara', 'aze', 'ben', 'bos', 'bul', 'cat', 'ces', 'dan', 'deu', 'ell', 'eng', 'eus', 'fas',
- 'fin', 'fra', 'glg', 'heb', 'hrv', 'hun', 'hye', 'ind', 'ita', 'jpn', 'kor', 'mkd', 'msa',
- 'nld', 'nor', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'spa', 'sqi', 'srp', 'swe', 'tha',
- 'tur', 'ukr', 'vie', 'zho']}
+ languages = {Language('por', 'BR')} | {Language(l) for l in [
+ 'ara', 'aze', 'ben', 'bos', 'bul', 'cat', 'ces', 'dan', 'deu', 'ell', 'eng', 'eus', 'fas', 'fin', 'fra', 'glg',
+ 'heb', 'hrv', 'hun', 'hye', 'ind', 'ita', 'jpn', 'kor', 'mkd', 'msa', 'nld', 'nor', 'pol', 'por', 'ron', 'rus',
+ 'slk', 'slv', 'spa', 'sqi', 'srp', 'swe', 'tha', 'tur', 'ukr', 'vie', 'zho'
+ ]}
video_types = (Episode,)
- server = 'http://www.addic7ed.com'
+ server_url = 'http://www.addic7ed.com/'
def __init__(self, username=None, password=None):
if username is not None and password is None or username is None and password is not None:
raise ConfigurationError('Username and password must be specified')
+
self.username = username
self.password = password
self.logged_in = False
def initialize(self):
- self.session = requests.Session()
- self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__.split('-')[0]}
+ self.session = Session()
+ self.session.headers = {'User-Agent': 'Subliminal/%s' % get_version(__version__)}
+
# login
if self.username is not None and self.password is not None:
- logger.debug('Logging in')
+ logger.info('Logging in')
data = {'username': self.username, 'password': self.password, 'Submit': 'Log in'}
- r = self.session.post(self.server + '/dologin.php', data, timeout=10, allow_redirects=False)
- if r.status_code == 302:
- logger.info('Logged in')
- self.logged_in = True
- else:
+ r = self.session.post(self.server_url + 'dologin.php', data, allow_redirects=False, timeout=10)
+
+ if r.status_code != 302:
raise AuthenticationError(self.username)
+ logger.debug('Logged in')
+ self.logged_in = True
+
def terminate(self):
# logout
if self.logged_in:
- r = self.session.get(self.server + '/logout.php', timeout=10)
- logger.info('Logged out')
- if r.status_code != 200:
- raise ProviderError('Request failed with status code %d' % r.status_code)
- self.session.close()
-
- def get(self, url, params=None):
- """Make a GET request on `url` with the given parameters
+ logger.info('Logging out')
+ r = self.session.get(self.server_url + 'logout.php', timeout=10)
+ r.raise_for_status()
+ logger.debug('Logged out')
+ self.logged_in = False
- :param string url: part of the URL to reach with the leading slash
- :param params: params of the request
- :return: the response
- :rtype: :class:`bs4.BeautifulSoup`
-
- """
- r = self.session.get(self.server + url, params=params, timeout=10)
- if r.status_code != 200:
- raise ProviderError('Request failed with status code %d' % r.status_code)
- return bs4.BeautifulSoup(r.content, ['permissive'])
+ self.session.close()
@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
- def get_show_ids(self):
- """Load the shows page with default series to show ids mapping
+ def _get_show_ids(self):
+ """Get the ``dict`` of show ids per series by querying the `shows.php` page.
- :return: series to show ids
+ :return: show id per series, lower case and without quotes.
:rtype: dict
"""
- soup = self.get('/shows.php')
+ # get the show page
+ logger.info('Getting show ids')
+ r = self.session.get(self.server_url + 'shows.php', timeout=10)
+ r.raise_for_status()
+ soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+ # populate the show ids
show_ids = {}
- for html_show in soup.select('td.version > h3 > a[href^="/show/"]'):
- show_ids[html_show.string.lower()] = int(html_show['href'][6:])
+ for show in soup.select('td.version > h3 > a[href^="/show/"]'):
+ show_ids[show.text.lower().replace('\'', '')] = int(show['href'][6:])
+ logger.debug('Found %d show ids', len(show_ids))
+
return show_ids
@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
- def find_show_id(self, series, year=None):
- """Find the show id from the `series` with optional `year`
-
- Use this only if the show id cannot be found with :meth:`get_show_ids`
+ def _search_show_id(self, series, year=None):
+ """Search the show id from the `series` and `year`.
- :param string series: series of the episode in lowercase
- :param year: year of the series, if any
+ :param string series: series of the episode.
+ :param year: year of the series, if any.
:type year: int or None
- :return: the show id, if any
+ :return: the show id, if found.
:rtype: int or None
"""
- series_year = series
- if year is not None:
- series_year += ' (%d)' % year
+ # build the params
+ series_year = '%s (%d)' % (series, year) if year is not None else series
params = {'search': series_year, 'Submit': 'Search'}
- logger.debug('Searching series %r', params)
- suggested_shows = self.get('/search.php', params).select('span.titulo > a[href^="/show/"]')
- if not suggested_shows:
- logger.info('Series %r not found', series_year)
+
+ # make the search
+ logger.info('Searching show ids with %r', params)
+ r = self.session.get(self.server_url + 'search.php', params=params, timeout=10)
+ r.raise_for_status()
+ soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+ # get the suggestion
+ suggestion = soup.select('span.titulo > a[href^="/show/"]')
+ if not suggestion:
+ logger.warning('Show id not found: no suggestion')
return None
- return int(suggested_shows[0]['href'][6:])
+ if not suggestion[0].i.text.lower() == series_year.lower():
+ logger.warning('Show id not found: suggestion does not match')
+ return None
+ show_id = int(suggestion[0]['href'][6:])
+ logger.debug('Found show id %d', show_id)
+
+ return show_id
+
+ def get_show_id(self, series, year=None, country_code=None):
+ """Get the best matching show id for `series`, `year` and `country_code`.
+
+ First search in the result of :meth:`_get_show_ids` and fallback on a search with :meth:`_search_show_id`
+
+ :param str series: series of the episode.
+ :param year: year of the series, if any.
+ :type year: int or None
+ :param country_code: country code of the series, if any.
+ :type country_code: str or None
+ :return: the show id, if found.
+ :rtype: int or None
- def query(self, series, season, year=None):
- show_ids = self.get_show_ids()
+ """
+ series_clean = series.lower().replace('\'', '')
+ show_ids = self._get_show_ids()
show_id = None
- if year is not None: # search with the year
- series_year = '%s (%d)' % (series.lower(), year)
- if series_year in show_ids:
- show_id = show_ids[series_year]
- else:
- show_id = self.find_show_id(series.lower(), year)
- if show_id is None: # search without the year
- year = None
- if series.lower() in show_ids:
- show_id = show_ids[series.lower()]
- else:
- show_id = self.find_show_id(series.lower())
+
+ # attempt with country
+ if not show_id and country_code:
+ logger.debug('Getting show id with country')
+ show_id = show_ids.get('%s (%s)' % (series_clean, country_code.lower()))
+
+ # attempt with year
+ if not show_id and year:
+ logger.debug('Getting show id with year')
+ show_id = show_ids.get('%s (%d)' % (series_clean, year))
+
+ # attempt clean
+ if not show_id:
+ logger.debug('Getting show id')
+ show_id = show_ids.get(series_clean)
+
+ # search as last resort
+ if not show_id:
+ logger.warning('Series not found in show ids')
+ show_id = self._search_show_id(series)
+
+ return show_id
+
+ def query(self, series, season, year=None, country=None):
+ # get the show id
+ show_id = self.get_show_id(series, year, country)
if show_id is None:
+ logger.error('No show id found for %r (%r)', series, {'year': year, 'country': country})
return []
- params = {'show_id': show_id, 'season': season}
- logger.debug('Searching subtitles %r', params)
- link = '/show/{show_id}&season={season}'.format(**params)
- soup = self.get(link)
+
+ # get the page of the season of the show
+ logger.info('Getting the page of show id %d, season %d', show_id, season)
+ r = self.session.get(self.server_url + 'show/%d' % show_id, params={'season': season}, timeout=10)
+ r.raise_for_status()
+ soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+ # loop over subtitle rows
+ match = series_year_re.match(soup.select('#header font')[0].text.strip()[:-10])
+ series = match.group('series')
+ year = int(match.group('year')) if match.group('year') else None
subtitles = []
- for row in soup('tr', class_='epeven completed'):
+ for row in soup.select('tr.epeven'):
cells = row('td')
- if cells[5].string != 'Completed':
- continue
- if not cells[3].string:
+
+ # ignore incomplete subtitles
+ status = cells[5].text
+ if status != 'Completed':
+ logger.debug('Ignoring subtitle with status %s', status)
continue
- subtitles.append(Addic7edSubtitle(babelfish.Language.fromaddic7ed(cells[3].string), series, season,
- int(cells[1].string), cells[2].string, year, cells[4].string,
- bool(cells[6].string), cells[9].a['href'],
- self.server + cells[2].a['href']))
+
+ # read the item
+ language = Language.fromaddic7ed(cells[3].text)
+ hearing_impaired = bool(cells[6].text)
+ page_link = self.server_url + cells[2].a['href'][1:]
+ season = int(cells[0].text)
+ episode = int(cells[1].text)
+ title = cells[2].text
+ version = cells[4].text
+ download_link = cells[9].a['href'][1:]
+
+ subtitle = Addic7edSubtitle(language, hearing_impaired, page_link, series, season, episode, title, year,
+ version, download_link)
+ logger.debug('Found subtitle %r', subtitle)
+ subtitles.append(subtitle)
+
return subtitles
def list_subtitles(self, video, languages):
@@ -198,9 +259,14 @@ class Addic7edProvider(Provider):
if s.language in languages and s.episode == video.episode]
def download_subtitle(self, subtitle):
- r = self.session.get(self.server + subtitle.download_link, timeout=10, headers={'Referer': subtitle.page_link})
- if r.status_code != 200:
- raise ProviderError('Request failed with status code %d' % r.status_code)
+ # download the subtitle
+ logger.info('Downloading subtitle %r', subtitle)
+ r = self.session.get(self.server_url + subtitle.download_link, headers={'Referer': subtitle.page_link},
+ timeout=10)
+ r.raise_for_status()
+
+ # detect download limit exceeded
if r.headers['Content-Type'] == 'text/html':
raise DownloadLimitExceeded
- subtitle.content = fix_line_endings(r.content)
+
+ subtitle.content = fix_line_ending(r.content)
diff --git a/lib/subliminal/providers/opensubtitles.py b/lib/subliminal/providers/opensubtitles.py
index f763837d5c192ee857e6224de42bf4a00928f6b9..64962b04545403970de3c57ce3feb5bff995943c 100644
--- a/lib/subliminal/providers/opensubtitles.py
+++ b/lib/subliminal/providers/opensubtitles.py
@@ -5,16 +5,17 @@ import logging
import os
import re
import zlib
-import babelfish
-import guessit
-from . import Provider
+
+from babelfish import Language, language_converters
+from guessit import guess_episode_info, guess_movie_info
+from six.moves.xmlrpc_client import ServerProxy
+
+from . import Provider, TimeoutSafeTransport, get_version
from .. import __version__
-from ..compat import ServerProxy, TimeoutTransport
-from ..exceptions import ProviderError, AuthenticationError, DownloadLimitExceeded
-from ..subtitle import Subtitle, fix_line_endings, compute_guess_matches
+from ..exceptions import AuthenticationError, DownloadLimitExceeded, ProviderError
+from ..subtitle import Subtitle, fix_line_ending, guess_matches
from ..video import Episode, Movie
-
logger = logging.getLogger(__name__)
@@ -22,10 +23,10 @@ class OpenSubtitlesSubtitle(Subtitle):
provider_name = 'opensubtitles'
series_re = re.compile('^"(?P<series_name>.*)" (?P<series_title>.*)$')
- def __init__(self, language, hearing_impaired, id, matched_by, movie_kind, hash, movie_name, movie_release_name, # @ReservedAssignment
- movie_year, movie_imdb_id, series_season, series_episode, page_link):
+ def __init__(self, language, hearing_impaired, page_link, subtitle_id, matched_by, movie_kind, hash, movie_name,
+ movie_release_name, movie_year, movie_imdb_id, series_season, series_episode):
super(OpenSubtitlesSubtitle, self).__init__(language, hearing_impaired, page_link)
- self.id = id
+ self.subtitle_id = subtitle_id
self.matched_by = matched_by
self.movie_kind = movie_kind
self.hash = hash
@@ -36,6 +37,10 @@ class OpenSubtitlesSubtitle(Subtitle):
self.series_season = series_season
self.series_episode = series_episode
+ @property
+ def id(self):
+ return str(self.subtitle_id)
+
@property
def series_name(self):
return self.series_re.match(self.movie_name).group('series_name')
@@ -44,8 +49,9 @@ class OpenSubtitlesSubtitle(Subtitle):
def series_title(self):
return self.series_re.match(self.movie_name).group('series_title')
- def compute_matches(self, video):
- matches = set()
+ def get_matches(self, video, hearing_impaired=False):
+ matches = super(OpenSubtitlesSubtitle, self).get_matches(video, hearing_impaired=hearing_impaired)
+
# episode
if isinstance(video, Episode) and self.movie_kind == 'episode':
# series
@@ -57,133 +63,174 @@ class OpenSubtitlesSubtitle(Subtitle):
# episode
if video.episode and self.series_episode == video.episode:
matches.add('episode')
+ # title
+ if video.title and self.series_title.lower() == video.title.lower():
+ matches.add('title')
# guess
- logger.info('Trying to guess movie relese name: %r ', self.movie_release_name)
- matches |= compute_guess_matches(video, guessit.guess_episode_info(self.movie_release_name + '.mkv'))
+ matches |= guess_matches(video, guess_episode_info(self.movie_release_name + '.mkv'))
# movie
elif isinstance(video, Movie) and self.movie_kind == 'movie':
+ # title
+ if video.title and self.movie_name.lower() == video.title.lower():
+ matches.add('title')
# year
if video.year and self.movie_year == video.year:
matches.add('year')
# guess
- matches |= compute_guess_matches(video, guessit.guess_movie_info(self.movie_release_name + '.mkv'))
+ matches |= guess_matches(video, guess_movie_info(self.movie_release_name + '.mkv'))
else:
- logger.info('%r is not a valid movie_kind for %r', self.movie_kind, video)
+ logger.info('%r is not a valid movie_kind', self.movie_kind)
return matches
+
# hash
if 'opensubtitles' in video.hashes and self.hash == video.hashes['opensubtitles']:
matches.add('hash')
# imdb_id
if video.imdb_id and self.movie_imdb_id == video.imdb_id:
matches.add('imdb_id')
- # title
- if video.title and self.movie_name.lower() == video.title.lower():
- matches.add('title')
+
return matches
class OpenSubtitlesProvider(Provider):
- languages = {babelfish.Language.fromopensubtitles(l) for l in babelfish.language_converters['opensubtitles'].codes}
+ languages = {Language.fromopensubtitles(l) for l in language_converters['opensubtitles'].codes}
def __init__(self):
- self.server = ServerProxy('http://api.opensubtitles.org/xml-rpc', transport=TimeoutTransport(10))
+ self.server = ServerProxy('https://api.opensubtitles.org/xml-rpc', TimeoutSafeTransport(10))
self.token = None
def initialize(self):
- response = checked(self.server.LogIn('', '', 'eng', 'subliminal v%s' % __version__.split('-')[0]))
+ logger.info('Logging in')
+ response = checked(self.server.LogIn('', '', 'eng', 'subliminal v%s' % get_version(__version__)))
self.token = response['token']
+ logger.debug('Logged in with token %r', self.token)
def terminate(self):
+ logger.info('Logging out')
checked(self.server.LogOut(self.token))
self.server.close()
+ logger.debug('Logged out')
def no_operation(self):
+ logger.debug('No operation')
checked(self.server.NoOperation(self.token))
- def query(self, languages, hash=None, size=None, imdb_id=None, query=None, season=None, episode=None): # @ReservedAssignment
- searches = []
+ def query(self, languages, hash=None, size=None, imdb_id=None, query=None, season=None, episode=None):
+ # fill the search criteria
+ criteria = []
if hash and size:
- searches.append({'moviehash': hash, 'moviebytesize': str(size)})
+ criteria.append({'moviehash': hash, 'moviebytesize': str(size)})
if imdb_id:
- searches.append({'imdbid': imdb_id})
+ criteria.append({'imdbid': imdb_id})
if query and season and episode:
- searches.append({'query': query, 'season': season, 'episode': episode})
+ criteria.append({'query': query, 'season': season, 'episode': episode})
elif query:
- searches.append({'query': query})
- if not searches:
- raise ValueError('One or more parameter missing')
- for search in searches:
- search['sublanguageid'] = ','.join(l.opensubtitles for l in languages)
- logger.debug('Searching subtitles %r', searches)
- response = checked(self.server.SearchSubtitles(self.token, searches))
+ criteria.append({'query': query})
+ if not criteria:
+ raise ValueError('Not enough information')
+
+ # add the language
+ for criterion in criteria:
+ criterion['sublanguageid'] = ','.join(sorted(l.opensubtitles for l in languages))
+
+ # query the server
+ logger.info('Searching subtitles %r', criteria)
+ response = checked(self.server.SearchSubtitles(self.token, criteria))
+ subtitles = []
+
+ # exit if no data
if not response['data']:
- logger.debug('No subtitle found')
- return []
- return [OpenSubtitlesSubtitle(babelfish.Language.fromopensubtitles(r['SubLanguageID']),
- bool(int(r['SubHearingImpaired'])), r['IDSubtitleFile'], r['MatchedBy'],
- r['MovieKind'], r['MovieHash'], r['MovieName'], r['MovieReleaseName'],
- int(r['MovieYear']) if r['MovieYear'] else None, int(r['IDMovieImdb']),
- int(r['SeriesSeason']) if r['SeriesSeason'] else None,
- int(r['SeriesEpisode']) if r['SeriesEpisode'] else None, r['SubtitlesLink'])
- for r in response['data']]
+ logger.info('No subtitles found')
+ return subtitles
+
+ # loop over subtitle items
+ for subtitle_item in response['data']:
+ # read the item
+ language = Language.fromopensubtitles(subtitle_item['SubLanguageID'])
+ hearing_impaired = bool(int(subtitle_item['SubHearingImpaired']))
+ page_link = subtitle_item['SubtitlesLink']
+ subtitle_id = int(subtitle_item['IDSubtitleFile'])
+ matched_by = subtitle_item['MatchedBy']
+ movie_kind = subtitle_item['MovieKind']
+ hash = subtitle_item['MovieHash']
+ movie_name = subtitle_item['MovieName']
+ movie_release_name = subtitle_item['MovieReleaseName']
+ movie_year = int(subtitle_item['MovieYear']) if subtitle_item['MovieYear'] else None
+ movie_imdb_id = int(subtitle_item['IDMovieImdb'])
+ series_season = int(subtitle_item['SeriesSeason']) if subtitle_item['SeriesSeason'] else None
+ series_episode = int(subtitle_item['SeriesEpisode']) if subtitle_item['SeriesEpisode'] else None
+
+ subtitle = OpenSubtitlesSubtitle(language, hearing_impaired, page_link, subtitle_id, matched_by, movie_kind,
+ hash, movie_name, movie_release_name, movie_year, movie_imdb_id,
+ series_season, series_episode)
+ logger.debug('Found subtitle %r', subtitle)
+ subtitles.append(subtitle)
+
+ return subtitles
def list_subtitles(self, video, languages):
- query = None
- season = None
- episode = None
- if ('opensubtitles' not in video.hashes or not video.size) and not video.imdb_id:
- query = video.name.split(os.sep)[-1]
+ query = season = episode = None
if isinstance(video, Episode):
query = video.series
season = video.season
episode = video.episode
+ elif ('opensubtitles' not in video.hashes or not video.size) and not video.imdb_id:
+ query = video.name.split(os.sep)[-1]
+
return self.query(languages, hash=video.hashes.get('opensubtitles'), size=video.size, imdb_id=video.imdb_id,
query=query, season=season, episode=episode)
def download_subtitle(self, subtitle):
- response = checked(self.server.DownloadSubtitles(self.token, [subtitle.id]))
- if not response['data']:
- raise ProviderError('Nothing to download')
- subtitle.content = fix_line_endings(zlib.decompress(base64.b64decode(response['data'][0]['data']), 47))
+ logger.info('Downloading subtitle %r', subtitle)
+ response = checked(self.server.DownloadSubtitles(self.token, [str(subtitle.subtitle_id)]))
+ subtitle.content = fix_line_ending(zlib.decompress(base64.b64decode(response['data'][0]['data']), 47))
class OpenSubtitlesError(ProviderError):
- """Base class for non-generic :class:`OpenSubtitlesProvider` exceptions"""
+ """Base class for non-generic :class:`OpenSubtitlesProvider` exceptions."""
+ pass
class Unauthorized(OpenSubtitlesError, AuthenticationError):
- """Exception raised when status is '401 Unauthorized'"""
+ """Exception raised when status is '401 Unauthorized'."""
+ pass
class NoSession(OpenSubtitlesError, AuthenticationError):
- """Exception raised when status is '406 No session'"""
+ """Exception raised when status is '406 No session'."""
+ pass
class DownloadLimitReached(OpenSubtitlesError, DownloadLimitExceeded):
- """Exception raised when status is '407 Download limit reached'"""
+ """Exception raised when status is '407 Download limit reached'."""
+ pass
class InvalidImdbid(OpenSubtitlesError):
- """Exception raised when status is '413 Invalid ImdbID'"""
+ """Exception raised when status is '413 Invalid ImdbID'."""
+ pass
class UnknownUserAgent(OpenSubtitlesError, AuthenticationError):
- """Exception raised when status is '414 Unknown User Agent'"""
+ """Exception raised when status is '414 Unknown User Agent'."""
+ pass
class DisabledUserAgent(OpenSubtitlesError, AuthenticationError):
- """Exception raised when status is '415 Disabled user agent'"""
+ """Exception raised when status is '415 Disabled user agent'."""
+ pass
class ServiceUnavailable(OpenSubtitlesError):
- """Exception raised when status is '503 Service Unavailable'"""
+ """Exception raised when status is '503 Service Unavailable'."""
+ pass
def checked(response):
- """Check a response status before returning it
+ """Check a response status before returning it.
- :param response: a response from a XMLRPC call to OpenSubtitles
- :return: the response
+ :param response: a response from a XMLRPC call to OpenSubtitles.
+ :return: the response.
:raise: :class:`OpenSubtitlesError`
"""
@@ -204,4 +251,5 @@ def checked(response):
raise ServiceUnavailable
if status_code != 200:
raise OpenSubtitlesError(response['status'])
+
return response
diff --git a/lib/subliminal/providers/podnapisi.py b/lib/subliminal/providers/podnapisi.py
index 328cee0731b53ae77fa6c340268016a13bf708e5..c83145a44a4132a91d4d48b6c38b5dedf46c322c 100644
--- a/lib/subliminal/providers/podnapisi.py
+++ b/lib/subliminal/providers/podnapisi.py
@@ -3,44 +3,52 @@ from __future__ import unicode_literals
import io
import logging
import re
-import xml.etree.ElementTree
-import zipfile
-import babelfish
-import bs4
-import guessit
-import requests
-from . import Provider
+
+from babelfish import Language, language_converters
+from guessit import guess_episode_info, guess_movie_info
+try:
+ from lxml import etree
+except ImportError:
+ try:
+ import xml.etree.cElementTree as etree
+ except ImportError:
+ import xml.etree.ElementTree as etree
+from requests import Session
+from zipfile import ZipFile
+
+from . import Provider, get_version
from .. import __version__
from ..exceptions import ProviderError
-from ..subtitle import Subtitle, fix_line_endings, compute_guess_matches
+from ..subtitle import Subtitle, fix_line_ending, guess_matches
from ..video import Episode, Movie
-
logger = logging.getLogger(__name__)
-babelfish.language_converters.register('podnapisi = subliminal.converters.podnapisi:PodnapisiConverter')
class PodnapisiSubtitle(Subtitle):
provider_name = 'podnapisi'
- def __init__(self, language, id, releases, hearing_impaired, page_link, series=None, season=None, episode=None, # @ReservedAssignment
- title=None, year=None):
+ def __init__(self, language, hearing_impaired, page_link, pid, releases, title, season=None, episode=None,
+ year=None):
super(PodnapisiSubtitle, self).__init__(language, hearing_impaired, page_link)
- self.id = id
+ self.pid = pid
self.releases = releases
- self.hearing_impaired = hearing_impaired
- self.series = series
+ self.title = title
self.season = season
self.episode = episode
- self.title = title
self.year = year
- def compute_matches(self, video):
- matches = set()
+ @property
+ def id(self):
+ return self.pid
+
+ def get_matches(self, video, hearing_impaired=False):
+ matches = super(PodnapisiSubtitle, self).get_matches(video, hearing_impaired=hearing_impaired)
+
# episode
if isinstance(video, Episode):
# series
- if video.series and self.series.lower() == video.series.lower():
+ if video.series and self.title.lower() == video.series.lower():
matches.add('series')
# season
if video.season and self.season == video.season:
@@ -50,7 +58,7 @@ class PodnapisiSubtitle(Subtitle):
matches.add('episode')
# guess
for release in self.releases:
- matches |= compute_guess_matches(video, guessit.guess_episode_info(release + '.mkv'))
+ matches |= guess_matches(video, guess_episode_info(release + '.mkv'))
# movie
elif isinstance(video, Movie):
# title
@@ -58,94 +66,108 @@ class PodnapisiSubtitle(Subtitle):
matches.add('title')
# guess
for release in self.releases:
- matches |= compute_guess_matches(video, guessit.guess_movie_info(release + '.mkv'))
+ matches |= guess_matches(video, guess_movie_info(release + '.mkv'))
# year
- if self.year == video.year:
+ if video.year and self.year == video.year:
matches.add('year')
+
return matches
class PodnapisiProvider(Provider):
- languages = {babelfish.Language.frompodnapisi(l) for l in babelfish.language_converters['podnapisi'].codes}
+ languages = ({Language('por', 'BR'), Language('srp', script='Latn')} |
+ {Language.fromalpha2(l) for l in language_converters['alpha2'].codes})
video_types = (Episode, Movie)
- search_url = 'http://simple.podnapisi.net/ppodnapisi/search'
- download_url_suffix = '/download'
+ server_url = 'http://podnapisi.net/subtitles/'
def initialize(self):
- self.session = requests.Session()
- self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__.split('-')[0]}
+ self.session = Session()
+ self.session.headers = {'User-Agent': 'Subliminal/%s' % get_version(__version__)}
def terminate(self):
self.session.close()
- def get(self, url, params=None, is_xml=True):
- """Make a GET request on `url` with the given parameters
-
- :param string url: the URL to reach
- :param dict params: params of the request
- :param bool is_xml: whether the response content is XML or not
- :return: the response
- :rtype: :class:`xml.etree.ElementTree.Element` or :class:`bs4.BeautifulSoup`
-
- """
- r = self.session.get(url, params=params, timeout=10)
- if r.status_code != 200:
- raise ProviderError('Request failed with status code %d' % r.status_code)
- if is_xml:
- return xml.etree.ElementTree.fromstring(r.content)
- else:
- return bs4.BeautifulSoup(r.content, ['permissive'])
-
- def query(self, language, series=None, season=None, episode=None, title=None, year=None):
- params = {'sXML': 1, 'sJ': language.podnapisi}
- if series and season and episode:
- params['sK'] = series
+ def query(self, language, keyword, season=None, episode=None, year=None):
+ # set parameters, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164#p212652
+ params = {'sXML': 1, 'sL': str(language), 'sK': keyword}
+ is_episode = False
+ if season and episode:
+ is_episode = True
params['sTS'] = season
params['sTE'] = episode
- elif title:
- params['sK'] = title
- else:
- raise ValueError('Missing parameters series and season and episode or title')
if year:
params['sY'] = year
- logger.debug('Searching episode %r', params)
+
+ # loop over paginated results
+ logger.info('Searching subtitles %r', params)
subtitles = []
+ pids = set()
while True:
- root = self.get(self.search_url, params)
- if not int(root.find('pagination/results').text):
- logger.debug('No subtitle found')
+ # query the server
+ xml = etree.fromstring(self.session.get(self.server_url + 'search/old', params=params, timeout=10).content)
+
+ # exit if no results
+ if not int(xml.find('pagination/results').text):
+ logger.debug('No subtitles found')
break
- if series and season and episode:
- subtitles.extend([PodnapisiSubtitle(language, int(s.find('id').text),
- s.find('release').text.split() if s.find('release').text else [],
- 'n' in (s.find('flags').text or ''), s.find('url').text,
- series=series, season=season, episode=episode,
- year=s.find('year').text)
- for s in root.findall('subtitle')])
- elif title:
- subtitles.extend([PodnapisiSubtitle(language, int(s.find('id').text),
- s.find('release').text.split() if s.find('release').text else [],
- 'n' in (s.find('flags').text or ''), s.find('url').text,
- title=title, year=s.find('year').text)
- for s in root.findall('subtitle')])
- if int(root.find('pagination/current').text) >= int(root.find('pagination/count').text):
+
+ # loop over subtitles
+ for subtitle_xml in xml.findall('subtitle'):
+ # read xml elements
+ language = Language.fromietf(subtitle_xml.find('language').text)
+ hearing_impaired = 'n' in (subtitle_xml.find('flags').text or '')
+ page_link = subtitle_xml.find('url').text
+ pid = subtitle_xml.find('pid').text
+ releases = []
+ if subtitle_xml.find('release').text:
+ for release in subtitle_xml.find('release').text.split():
+ releases.append(re.sub(r'\.+$', '', release)) # remove trailing dots
+ title = subtitle_xml.find('title').text
+ season = int(subtitle_xml.find('tvSeason').text)
+ episode = int(subtitle_xml.find('tvEpisode').text)
+ year = int(subtitle_xml.find('year').text)
+
+ if is_episode:
+ subtitle = PodnapisiSubtitle(language, hearing_impaired, page_link, pid, releases, title,
+ season=season, episode=episode, year=year)
+ else:
+ subtitle = PodnapisiSubtitle(language, hearing_impaired, page_link, pid, releases, title,
+ year=year)
+
+ # ignore duplicates, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164&start=10#p213321
+ if pid in pids:
+ continue
+
+ logger.debug('Found subtitle %r', subtitle)
+ subtitles.append(subtitle)
+ pids.add(pid)
+
+ # stop on last page
+ if int(xml.find('pagination/current').text) >= int(xml.find('pagination/count').text):
break
- params['page'] = int(root.find('pagination/current').text) + 1
+
+ # increment current page
+ params['page'] = int(xml.find('pagination/current').text) + 1
+ logger.debug('Getting page %d', params['page'])
+
return subtitles
def list_subtitles(self, video, languages):
if isinstance(video, Episode):
- return [s for l in languages for s in self.query(l, series=video.series, season=video.season,
+ return [s for l in languages for s in self.query(l, video.series, season=video.season,
episode=video.episode, year=video.year)]
elif isinstance(video, Movie):
- return [s for l in languages for s in self.query(l, title=video.title, year=video.year)]
+ return [s for l in languages for s in self.query(l, video.title, year=video.year)]
def download_subtitle(self, subtitle):
- download_url = subtitle.page_link + self.download_url_suffix
- r = self.session.get(download_url, timeout=10)
- if r.status_code != 200:
- raise ProviderError('Request failed with status code %d' % r.status_code)
- with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
+ # download as a zip
+ logger.info('Downloading subtitle %r')
+ r = self.session.get(self.server_url + subtitle.pid + '/download', params={'container': 'zip'}, timeout=10)
+ r.raise_for_status()
+
+ # open the zip
+ with ZipFile(io.BytesIO(r.content)) as zf:
if len(zf.namelist()) > 1:
raise ProviderError('More than one file to unzip')
- subtitle.content = fix_line_endings(zf.read(zf.namelist()[0]))
+
+ subtitle.content = fix_line_ending(zf.read(zf.namelist()[0]))
diff --git a/lib/subliminal/providers/thesubdb.py b/lib/subliminal/providers/thesubdb.py
index 446231736e87fb1895639142aab54591f3d8f0c8..0cc890e9425599c2cede0bf0fb7e4517a6d76486 100644
--- a/lib/subliminal/providers/thesubdb.py
+++ b/lib/subliminal/providers/thesubdb.py
@@ -1,12 +1,13 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
-import babelfish
-import requests
-from . import Provider
+
+from babelfish import Language
+from requests import Session
+
+from . import Provider, get_version
from .. import __version__
-from ..exceptions import ProviderError
-from ..subtitle import Subtitle, fix_line_endings
+from ..subtitle import Subtitle, fix_line_ending
logger = logging.getLogger(__name__)
@@ -15,58 +16,67 @@ logger = logging.getLogger(__name__)
class TheSubDBSubtitle(Subtitle):
provider_name = 'thesubdb'
- def __init__(self, language, hash): # @ReservedAssignment
+ def __init__(self, language, hash):
super(TheSubDBSubtitle, self).__init__(language)
self.hash = hash
- def compute_matches(self, video):
- matches = set()
+ @property
+ def id(self):
+ return self.hash
+
+ def get_matches(self, video, hearing_impaired=False):
+ matches = super(TheSubDBSubtitle, self).get_matches(video, hearing_impaired=hearing_impaired)
+
# hash
if 'thesubdb' in video.hashes and video.hashes['thesubdb'] == self.hash:
matches.add('hash')
+
return matches
class TheSubDBProvider(Provider):
- languages = {babelfish.Language.fromalpha2(l) for l in ['en', 'es', 'fr', 'it', 'nl', 'pl', 'pt', 'ro', 'sv', 'tr']}
+ languages = {Language.fromalpha2(l) for l in ['en', 'es', 'fr', 'it', 'nl', 'pl', 'pt', 'ro', 'sv', 'tr']}
required_hash = 'thesubdb'
+ server_url = 'http://api.thesubdb.com/'
def initialize(self):
- self.session = requests.Session()
+ self.session = Session()
self.session.headers = {'User-Agent': 'SubDB/1.0 (subliminal/%s; https://github.com/Diaoul/subliminal)' %
- __version__.split('-')[0]}
+ get_version(__version__)}
def terminate(self):
self.session.close()
- def get(self, params):
- """Make a GET request on the server with the given parameters
-
- :param params: params of the request
- :return: the response
- :rtype: :class:`requests.Response`
-
- """
- return self.session.get('http://api.thesubdb.com', params=params, timeout=10)
-
- def query(self, hash): # @ReservedAssignment
+ def query(self, hash):
+ # make the query
params = {'action': 'search', 'hash': hash}
- logger.debug('Searching subtitles %r', params)
- r = self.get(params)
+ logger.info('Searching subtitles %r', params)
+ r = self.session.get(self.server_url, params=params, timeout=10)
+
+ # handle subtitles not found and errors
if r.status_code == 404:
- logger.debug('No subtitle found')
+ logger.debug('No subtitles found')
return []
- elif r.status_code != 200:
- raise ProviderError('Request failed with status code %d' % r.status_code)
- return [TheSubDBSubtitle(language, hash) for language in
- {babelfish.Language.fromalpha2(l) for l in r.content.decode('utf-8').split(',')}]
+ r.raise_for_status()
+
+ # loop over languages
+ subtitles = []
+ for language_code in r.text.split(','):
+ language = Language.fromalpha2(language_code)
+
+ subtitle = TheSubDBSubtitle(language, hash)
+ logger.info('Found subtitle %r', subtitle)
+ subtitles.append(subtitle)
+
+ return subtitles
def list_subtitles(self, video, languages):
return [s for s in self.query(video.hashes['thesubdb']) if s.language in languages]
def download_subtitle(self, subtitle):
+ logger.info('Downloading subtitle %r')
params = {'action': 'download', 'hash': subtitle.hash, 'language': subtitle.language.alpha2}
- r = self.get(params)
- if r.status_code != 200:
- raise ProviderError('Request failed with status code %d' % r.status_code)
- subtitle.content = fix_line_endings(r.content)
+ r = self.session.get(self.server_url, params=params, timeout=10)
+ r.raise_for_status()
+
+ subtitle.content = fix_line_ending(r.content)
diff --git a/lib/subliminal/providers/tvsubtitles.py b/lib/subliminal/providers/tvsubtitles.py
index 3f21928b0d94eb37d7b5860b6998418c1eb2878e..5e60bc9e4af04b085af49d288416c3b2c8ffceca 100644
--- a/lib/subliminal/providers/tvsubtitles.py
+++ b/lib/subliminal/providers/tvsubtitles.py
@@ -3,39 +3,47 @@ from __future__ import unicode_literals
import io
import logging
import re
-import zipfile
-import babelfish
-import bs4
-import requests
-from . import Provider
+from zipfile import ZipFile
+
+from babelfish import Language, language_converters
+from requests import Session
+
+from . import ParserBeautifulSoup, Provider, get_version
from .. import __version__
-from ..cache import region, SHOW_EXPIRATION_TIME, EPISODE_EXPIRATION_TIME
+from ..cache import EPISODE_EXPIRATION_TIME, SHOW_EXPIRATION_TIME, region
from ..exceptions import ProviderError
-from ..subtitle import Subtitle, fix_line_endings, compute_guess_properties_matches
+from ..subtitle import Subtitle, fix_line_ending, guess_matches, guess_properties
from ..video import Episode
-
logger = logging.getLogger(__name__)
-babelfish.language_converters.register('tvsubtitles = subliminal.converters.tvsubtitles:TVsubtitlesConverter')
+language_converters.register('tvsubtitles = subliminal.converters.tvsubtitles:TVsubtitlesConverter')
+
+link_re = re.compile('^(?P<series>.+?)(?: \(?\d{4}\)?| \((?:US|UK)\))? \((?P<first_year>\d{4})-\d{4}\)$')
+episode_id_re = re.compile('^episode-\d+\.html$')
class TVsubtitlesSubtitle(Subtitle):
provider_name = 'tvsubtitles'
- def __init__(self, language, series, season, episode, year, id, rip, release, page_link): # @ReservedAssignment
+ def __init__(self, language, page_link, subtitle_id, series, season, episode, year, rip, release):
super(TVsubtitlesSubtitle, self).__init__(language, page_link=page_link)
+ self.subtitle_id = subtitle_id
self.series = series
self.season = season
self.episode = episode
self.year = year
- self.id = id
self.rip = rip
self.release = release
- def compute_matches(self, video):
- matches = set()
+ @property
+ def id(self):
+ return str(self.subtitle_id)
+
+ def get_matches(self, video, hearing_impaired=False):
+ matches = super(TVsubtitlesSubtitle, self).get_matches(video, hearing_impaired=hearing_impaired)
+
# series
- if video.series and self.series == video.series:
+ if video.series and self.series.lower() == video.series.lower():
matches.add('series')
# season
if video.season and self.season == video.season:
@@ -49,143 +57,147 @@ class TVsubtitlesSubtitle(Subtitle):
# release_group
if video.release_group and self.release and video.release_group.lower() in self.release.lower():
matches.add('release_group')
- """
- # video_codec
- if video.video_codec and self.release and (video.video_codec in self.release.lower()
- or video.video_codec == 'h264' and 'x264' in self.release.lower()):
- matches.add('video_codec')
- # resolution
- if video.resolution and self.rip and video.resolution in self.rip.lower():
- matches.add('resolution')
- # format
- if video.format and self.rip and video.format in self.rip.lower():
- matches.add('format')
- """
- # we don't have the complete filename, so we need to guess the matches separately
- # guess video_codec (videoCodec in guessit)
- matches |= compute_guess_properties_matches(video, self.release, 'videoCodec')
- # guess resolution (screenSize in guessit)
- matches |= compute_guess_properties_matches(video, self.rip, 'screenSize')
- # guess format
- matches |= compute_guess_properties_matches(video, self.rip, 'format')
+ # other properties
+ if self.release:
+ matches |= guess_matches(video, guess_properties(self.release), partial=True)
+ if self.rip:
+ matches |= guess_matches(video, guess_properties(self.rip), partial=True)
+
return matches
class TVsubtitlesProvider(Provider):
- languages = {babelfish.Language('por', 'BR')} | {babelfish.Language(l)
- for l in ['ara', 'bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'fin', 'fra', 'hun', 'ita', 'jpn', 'kor',
- 'nld', 'pol', 'por', 'ron', 'rus', 'spa', 'swe', 'tur', 'ukr', 'zho']}
+ languages = {Language('por', 'BR')} | {Language(l) for l in [
+ 'ara', 'bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'fin', 'fra', 'hun', 'ita', 'jpn', 'kor', 'nld', 'pol', 'por',
+ 'ron', 'rus', 'spa', 'swe', 'tur', 'ukr', 'zho'
+ ]}
video_types = (Episode,)
- server = 'http://www.tvsubtitles.net'
- episode_id_re = re.compile('^episode-\d+\.html$')
- subtitle_re = re.compile('^\/subtitle-\d+\.html$')
- link_re = re.compile('^(?P<series>[A-Za-z0-9 \'.]+).*\((?P<first_year>\d{4})-\d{4}\)$')
+ server_url = 'http://www.tvsubtitles.net/'
def initialize(self):
- self.session = requests.Session()
- self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__.split('-')[0]}
+ self.session = Session()
+ self.session.headers = {'User-Agent': 'Subliminal/%s' % get_version(__version__)}
def terminate(self):
self.session.close()
- def request(self, url, params=None, data=None, method='GET'):
- """Make a `method` request on `url` with the given parameters
-
- :param string url: part of the URL to reach with the leading slash
- :param dict params: params of the request
- :param dict data: data of the request
- :param string method: method of the request
- :return: the response
- :rtype: :class:`bs4.BeautifulSoup`
-
- """
- r = self.session.request(method, self.server + url, params=params, data=data, timeout=10)
- if r.status_code != 200:
- raise ProviderError('Request failed with status code %d' % r.status_code)
- return bs4.BeautifulSoup(r.content, ['permissive'])
-
@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
- def find_show_id(self, series, year=None):
- """Find the show id from the `series` with optional `year`
+ def search_show_id(self, series, year=None):
+ """Search the show id from the `series` and `year`.
- :param string series: series of the episode in lowercase
- :param year: year of the series, if any
+ :param string series: series of the episode.
+ :param year: year of the series, if any.
:type year: int or None
- :return: the show id, if any
+ :return: the show id, if any.
:rtype: int or None
"""
- data = {'q': series}
- logger.debug('Searching series %r', data)
- soup = self.request('/search.php', data=data, method='POST')
- links = soup.select('div.left li div a[href^="/tvshow-"]')
- if not links:
- logger.info('Series %r not found', series)
- return None
- matched_links = [link for link in links if self.link_re.match(link.string)]
- for link in matched_links: # first pass with exact match on series
- match = self.link_re.match(link.string)
- if match.group('series').lower().replace('.', ' ').strip() == series:
- if year is not None and int(match.group('first_year')) != year:
- continue
- return int(link['href'][8:-5])
- for link in matched_links: # less selective second pass
- match = self.link_re.match(link.string)
- if match.group('series').lower().replace('.', ' ').strip().startswith(series):
+ # make the search
+ logger.info('Searching show id for %r', series)
+ r = self.session.post(self.server_url + 'search.php', data={'q': series}, timeout=10)
+ r.raise_for_status()
+
+ # get the series out of the suggestions
+ soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+ show_id = None
+ for suggestion in soup.select('div.left li div a[href^="/tvshow-"]'):
+ match = link_re.match(suggestion.text)
+ if not match:
+ logger.error('Failed to match %s', suggestion.text)
+ continue
+
+ if match.group('series').lower() == series.lower():
if year is not None and int(match.group('first_year')) != year:
+ logger.debug('Year does not match')
continue
- return int(link['href'][8:-5])
- return None
+ show_id = int(suggestion['href'][8:-5])
+ logger.debug('Found show id %d', show_id)
+ break
+
+ return show_id
@region.cache_on_arguments(expiration_time=EPISODE_EXPIRATION_TIME)
- def find_episode_ids(self, show_id, season):
- """Find episode ids from the show id and the season
+ def get_episode_ids(self, show_id, season):
+ """Get episode ids from the show id and the season.
- :param int show_id: show id
- :param int season: season of the episode
- :return: episode ids per episode number
+ :param int show_id: show id.
+ :param int season: season of the episode.
+ :return: episode ids per episode number.
:rtype: dict
"""
- params = {'show_id': show_id, 'season': season}
- logger.debug('Searching episodes %r', params)
- soup = self.request('/tvshow-{show_id}-{season}.html'.format(**params))
+ # get the page of the season of the show
+ logger.info('Getting the page of show id %d, season %d', show_id, season)
+ r = self.session.get(self.server_url + 'tvshow-%d-%d.html' % (show_id, season), timeout=10)
+ soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+ # loop over episode rows
episode_ids = {}
for row in soup.select('table#table5 tr'):
- if not row('a', href=self.episode_id_re):
+ # skip rows that do not have a link to the episode page
+ if not row('a', href=episode_id_re):
continue
+
+ # extract data from the cells
cells = row('td')
- episode_ids[int(cells[0].string.split('x')[1])] = int(cells[1].a['href'][8:-5])
+ episode = int(cells[0].text.split('x')[1])
+ episode_id = int(cells[1].a['href'][8:-5])
+ episode_ids[episode] = episode_id
+
+ if episode_ids:
+ logger.debug('Found episode ids %r', episode_ids)
+ else:
+ logger.warning('No episode ids found')
+
return episode_ids
def query(self, series, season, episode, year=None):
- show_id = self.find_show_id(series.lower(), year)
+ # search the show id
+ show_id = self.search_show_id(series, year)
if show_id is None:
+ logger.error('No show id found for %r (%r)', series, {'year': year})
return []
- episode_ids = self.find_episode_ids(show_id, season)
+
+ # get the episode ids
+ episode_ids = self.get_episode_ids(show_id, season)
if episode not in episode_ids:
- logger.info('Episode %d not found', episode)
+ logger.error('Episode %d not found', episode)
return []
- params = {'episode_id': episode_ids[episode]}
- logger.debug('Searching episode %r', params)
- link = '/episode-{episode_id}.html'.format(**params)
- soup = self.request(link)
- return [TVsubtitlesSubtitle(babelfish.Language.fromtvsubtitles(row.h5.img['src'][13:-4]), series, season,
- episode, year if year and show_id != self.find_show_id(series.lower()) else None,
- int(row['href'][10:-5]), row.find('p', title='rip').text.strip() or None,
- row.find('p', title='release').text.strip() or None,
- self.server + '/subtitle-%d.html' % int(row['href'][10:-5]))
- for row in soup('a', href=self.subtitle_re)]
+
+ # get the episode page
+ logger.info('Getting the page for episode %d', episode_ids[episode])
+ r = self.session.get(self.server_url + 'episode-%d.html' % episode_ids[episode], timeout=10)
+ soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
+
+ # loop over subtitles rows
+ subtitles = []
+ for row in soup.select('.subtitlen'):
+ # read the item
+ language = Language.fromtvsubtitles(row.h5.img['src'][13:-4])
+ subtitle_id = int(row.parent['href'][10:-5])
+ page_link = self.server_url + 'subtitle-%d.html' % subtitle_id
+ rip = row.find('p', title='rip').text.strip() or None
+ release = row.find('p', title='release').text.strip() or None
+
+ subtitle = TVsubtitlesSubtitle(language, page_link, subtitle_id, series, season, episode, year, rip,
+ release)
+ logger.info('Found subtitle %s', subtitle)
+ subtitles.append(subtitle)
+
+ return subtitles
def list_subtitles(self, video, languages):
return [s for s in self.query(video.series, video.season, video.episode, video.year) if s.language in languages]
def download_subtitle(self, subtitle):
- r = self.session.get(self.server + '/download-{subtitle_id}.html'.format(subtitle_id=subtitle.id),
- timeout=10)
- if r.status_code != 200:
- raise ProviderError('Request failed with status code %d' % r.status_code)
- with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
+ # download as a zip
+ logger.info('Downloading subtitle %r', subtitle)
+ r = self.session.get(self.server_url + 'download-%d.html' % subtitle.subtitle_id, timeout=10)
+ r.raise_for_status()
+
+ # open the zip
+ with ZipFile(io.BytesIO(r.content)) as zf:
if len(zf.namelist()) > 1:
raise ProviderError('More than one file to unzip')
- subtitle.content = fix_line_endings(zf.read(zf.namelist()[0]))
+
+ subtitle.content = fix_line_ending(zf.read(zf.namelist()[0]))
diff --git a/lib/subliminal/score.py b/lib/subliminal/score.py
index f9dcaedee5880fa8f4f7713c6505c0fc47fc3e50..831314bc28b8927c39c07fcdc4e294329fa3e89a 100755
--- a/lib/subliminal/score.py
+++ b/lib/subliminal/score.py
@@ -1,17 +1,50 @@
-#!/usr/bin/env python
# -*- coding: utf-8 -*-
-from __future__ import print_function, unicode_literals
-from sympy import Eq, symbols, solve
+"""
+This module is responsible for calculating the :attr:`~subliminal.video.Video.scores` dicts
+(:attr:`Episode.scores <subliminal.video.Episode.scores>` and :attr:`Movie.scores <subliminal.video.Movie.scores>`)
+by assigning a score to a match.
+
+.. note::
+
+ To avoid unnecessary dependency on `sympy <http://www.sympy.org/>`_ and boost subliminal's import time, the
+ resulting scores are hardcoded in their respective classes and manually updated when the set of equations change.
+
+Available matches:
+
+ * hearing_impaired
+ * format
+ * release_group
+ * resolution
+ * video_codec
+ * audio_codec
+ * imdb_id
+ * hash
+ * title
+ * year
+ * series
+ * season
+ * episode
+ * tvdb_id
+
+
+The :meth:`Subtitle.get_matches <subliminal.subtitle.Subtitle.get_matches>` method get the matches between the
+:class:`~subliminal.subtitle.Subtitle` and the :class:`~subliminal.video.Video` and
+:func:`~subliminal.subtitle.compute_score` computes the score.
+
+"""
+from __future__ import unicode_literals, print_function
+
+from sympy import Eq, solve, symbols
# Symbols
-release_group, resolution, format, video_codec, audio_codec = symbols('release_group resolution format video_codec audio_codec')
-imdb_id, hash, title, series, tvdb_id, season, episode = symbols('imdb_id hash title series tvdb_id season episode') # @ReservedAssignment
-year = symbols('year')
+hearing_impaired, format, release_group, resolution = symbols('hearing_impaired format release_group resolution')
+video_codec, audio_codec, imdb_id, hash, title, year = symbols('video_codec audio_codec imdb_id hash title year')
+series, season, episode, tvdb_id = symbols('series season episode tvdb_id')
-def get_episode_equations():
- """Get the score equations for a :class:`~subliminal.video.Episode`
+def solve_episode_equations():
+ """Solve the score equations for an :class:`~subliminal.video.Episode`.
The equations are the following:
@@ -27,31 +60,36 @@ def get_episode_equations():
10. title = season + episode
11. season = episode
12. release_group = season
- 13. audio_codec = 1
+ 13. audio_codec = 2 * hearing_impaired
+ 14. hearing_impaired = 1
- :return: the score equations for an episode
- :rtype: list of :class:`sympy.Eq`
+ :return: the result of the equations.
+ :rtype: dict
"""
- equations = []
- equations.append(Eq(hash, resolution + format + video_codec + audio_codec + series + season + episode + year + release_group))
- equations.append(Eq(series, resolution + video_codec + audio_codec + season + episode + release_group + 1))
- equations.append(Eq(series, year))
- equations.append(Eq(tvdb_id, series + year))
- equations.append(Eq(season, resolution + video_codec + audio_codec + 1))
- equations.append(Eq(imdb_id, series + season + episode + year))
- equations.append(Eq(format, video_codec + audio_codec))
- equations.append(Eq(resolution, video_codec))
- equations.append(Eq(video_codec, 2 * audio_codec))
- equations.append(Eq(title, season + episode))
- equations.append(Eq(season, episode))
- equations.append(Eq(release_group, season))
- equations.append(Eq(audio_codec, 1))
- return equations
-
-
-def get_movie_equations():
- """Get the score equations for a :class:`~subliminal.video.Movie`
+ equations = [
+ Eq(hash, resolution + format + video_codec + audio_codec + series + season + episode + year + release_group),
+ Eq(series, resolution + video_codec + audio_codec + season + episode + release_group + 1),
+ Eq(year, series),
+ Eq(tvdb_id, series + year),
+ Eq(season, resolution + video_codec + audio_codec + 1),
+ Eq(imdb_id, series + season + episode + year),
+ Eq(format, video_codec + audio_codec),
+ Eq(resolution, video_codec),
+ Eq(video_codec, 2 * audio_codec),
+ Eq(title, season + episode),
+ Eq(season, episode),
+ Eq(release_group, season),
+ Eq(audio_codec, 2 * hearing_impaired),
+ Eq(hearing_impaired, 1)
+ ]
+
+ return solve(equations, [hearing_impaired, format, release_group, resolution, video_codec, audio_codec, imdb_id,
+ hash, series, season, episode, title, year, tvdb_id])
+
+
+def solve_movie_equations():
+ """Solve the score equations for a :class:`~subliminal.video.Movie`.
The equations are the following:
@@ -63,28 +101,25 @@ def get_movie_equations():
6. title = resolution + video_codec + audio_codec + year + 1
7. release_group = resolution + video_codec + audio_codec + 1
8. year = release_group + 1
- 9. audio_codec = 1
+ 9. audio_codec = 2 * hearing_impaired
+ 10. hearing_impaired = 1
- :return: the score equations for a movie
- :rtype: list of :class:`sympy.Eq`
+ :return: the result of the equations.
+ :rtype: dict
"""
- equations = []
- equations.append(Eq(hash, resolution + format + video_codec + audio_codec + title + year + release_group))
- equations.append(Eq(imdb_id, hash))
- equations.append(Eq(resolution, video_codec))
- equations.append(Eq(video_codec, 2 * audio_codec))
- equations.append(Eq(format, video_codec + audio_codec))
- equations.append(Eq(title, resolution + video_codec + audio_codec + year + 1))
- equations.append(Eq(video_codec, 2 * audio_codec))
- equations.append(Eq(release_group, resolution + video_codec + audio_codec + 1))
- equations.append(Eq(year, release_group + 1))
- equations.append(Eq(audio_codec, 1))
- return equations
-
-
-if __name__ == '__main__':
- print(solve(get_episode_equations(), [release_group, resolution, format, video_codec, audio_codec, imdb_id,
- hash, series, tvdb_id, season, episode, title, year]))
- print(solve(get_movie_equations(), [release_group, resolution, format, video_codec, audio_codec, imdb_id,
- hash, title, year]))
+ equations = [
+ Eq(hash, resolution + format + video_codec + audio_codec + title + year + release_group),
+ Eq(imdb_id, hash),
+ Eq(resolution, video_codec),
+ Eq(video_codec, 2 * audio_codec),
+ Eq(format, video_codec + audio_codec),
+ Eq(title, resolution + video_codec + audio_codec + year + 1),
+ Eq(release_group, resolution + video_codec + audio_codec + 1),
+ Eq(year, release_group + 1),
+ Eq(audio_codec, 2 * hearing_impaired),
+ Eq(hearing_impaired, 1)
+ ]
+
+ return solve(equations, [hearing_impaired, format, release_group, resolution, video_codec, audio_codec, imdb_id,
+ hash, title, year])
diff --git a/lib/subliminal/subtitle.py b/lib/subliminal/subtitle.py
index 3522b8e9e9946712661759345b51a10b49b1c624..68f5984249df41a8d5c59a56390dd82890c3ee8d 100644
--- a/lib/subliminal/subtitle.py
+++ b/lib/subliminal/subtitle.py
@@ -1,31 +1,39 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
-import os.path
-import babelfish
+import os
+
import chardet
-import guessit.matchtree
-import guessit.transfo
+from guessit.matchtree import MatchTree
+from guessit.plugins.transformers import get_transformer
import pysrt
-from .video import Episode, Movie
+from .video import Episode, Movie
logger = logging.getLogger(__name__)
class Subtitle(object):
- """Base class for subtitle
+ """Base class for subtitle.
- :param language: language of the subtitle
- :type language: :class:`babelfish.Language`
- :param bool hearing_impaired: `True` if the subtitle is hearing impaired, `False` otherwise
- :param page_link: link to the web page from which the subtitle can be downloaded, if any
- :type page_link: string or None
+ :param language: language of the subtitle.
+ :type language: :class:`~babelfish.language.Language`
+ :param bool hearing_impaired: whether or not the subtitle is hearing impaired.
+ :param page_link: URL of the web page from which the subtitle can be downloaded.
+ :type page_link: str
"""
+ #: Name of the provider that returns that class of subtitle
+ provider_name = ''
+
def __init__(self, language, hearing_impaired=False, page_link=None):
+ #: Language of the subtitle
self.language = language
+
+ #: Whether or not the subtitle is hearing impaired
self.hearing_impaired = hearing_impaired
+
+ #: URL of the web page from which the subtitle can be downloaded
self.page_link = page_link
#: Content as bytes
@@ -35,8 +43,49 @@ class Subtitle(object):
self.encoding = None
@property
- def guessed_encoding(self):
- """Guessed encoding using the language, falling back on chardet"""
+ def id(self):
+ """Unique identifier of the subtitle."""
+ raise NotImplementedError
+
+ @property
+ def text(self):
+ """Content as string.
+
+ If :attr:`encoding` is None, the encoding is guessed with :meth:`guess_encoding`
+
+ """
+ if not self.content:
+ return
+
+ return self.content.decode(self.encoding or self.guess_encoding(), errors='replace')
+
+ def is_valid(self):
+ """Check if a :attr:`text` is a valid SubRip format.
+
+ :return: whether or not the subtitle is valid.
+ :rtype: bool
+
+ """
+ if not self.text:
+ return False
+
+ try:
+ pysrt.from_string(self.text, error_handling=pysrt.ERROR_RAISE)
+ except pysrt.Error as e:
+ if e.args[0] < 80:
+ return False
+
+ return True
+
+ def guess_encoding(self):
+ """Guess encoding using the language, falling back on chardet.
+
+ :return: the guessed encoding.
+ :rtype: str
+
+ """
+ logger.info('Guessing encoding for language %s', self.language)
+
# always try utf-8 first
encodings = ['utf-8']
@@ -62,122 +111,126 @@ class Subtitle(object):
encodings.append('latin-1')
# try to decode
+ logger.debug('Trying encodings %r', encodings)
for encoding in encodings:
try:
self.content.decode(encoding)
- return encoding
except UnicodeDecodeError:
pass
+ else:
+ logger.info('Guessed encoding %s', encoding)
+ return encoding
+
+ logger.warning('Could not guess encoding from language')
# fallback on chardet
- logger.warning('Could not decode content with encodings %r', encodings)
- return chardet.detect(self.content)['encoding']
+ encoding = chardet.detect(self.content)['encoding']
+ logger.info('Chardet found encoding %s', encoding)
- @property
- def text(self):
- """Content as string
+ return encoding
+
+ def get_matches(self, video, hearing_impaired=False):
+ """Get the matches against the `video`.
- If :attr:`encoding` is None, the encoding is guessed with :attr:`guessed_encoding`
+ :param video: the video to get the matches with.
+ :type video: :class:`~subliminal.video.Video`
+ :param bool hearing_impaired: hearing impaired preference.
+ :return: matches of the subtitle.
+ :rtype: set
"""
- if not self.content:
- return ''
- return self.content.decode(self.encoding or self.guessed_encoding, errors='replace')
+ matches = set()
- @property
- def is_valid(self):
- """Check if a subtitle text is a valid SubRip format"""
- try:
- pysrt.from_string(self.text, error_handling=pysrt.ERROR_RAISE)
- return True
- except pysrt.Error as e:
- if e.args[0] > 80:
- return True
- except:
- logger.exception('Unexpected error when validating subtitle')
- return False
+ # hearing_impaired
+ if self.hearing_impaired == hearing_impaired:
+ matches.add('hearing_impaired')
- def compute_matches(self, video):
- """Compute the matches of the subtitle against the `video`
+ return matches
- :param video: the video to compute the matches against
- :type video: :class:`~subliminal.video.Video`
- :return: matches of the subtitle
- :rtype: set
+ def __hash__(self):
+ return hash(self.provider_name + '-' + self.id)
- """
- raise NotImplementedError
+ def __repr__(self):
+ return '<%s %r [%s]>' % (self.__class__.__name__, self.id, self.language)
- def compute_score(self, video):
- """Compute the score of the subtitle against the `video`
- There are equivalent matches so that a provider can match one element or its equivalent. This is
- to give all provider a chance to have a score in the same range without hurting quality.
+def compute_score(matches, video, scores=None):
+ """Compute the score of the `matches` against the `video`.
- * Matching :class:`~subliminal.video.Video`'s `hashes` is equivalent to matching everything else
- * Matching :class:`~subliminal.video.Episode`'s `season` and `episode`
- is equivalent to matching :class:`~subliminal.video.Episode`'s `title`
- * Matching :class:`~subliminal.video.Episode`'s `tvdb_id` is equivalent to matching
- :class:`~subliminal.video.Episode`'s `series`
+ Some matches count as much as a combination of others in order to level the final score:
- :param video: the video to compute the score against
- :type video: :class:`~subliminal.video.Video`
- :return: score of the subtitle
- :rtype: int
+ * `hash` removes everything else
+ * For :class:`~subliminal.video.Episode`
- """
- score = 0
- # compute matches
- initial_matches = self.compute_matches(video)
- matches = initial_matches.copy()
- # hash is the perfect match
- if 'hash' in matches:
- score = video.scores['hash']
- else:
- # remove equivalences
- if isinstance(video, Episode):
- if 'imdb_id' in matches:
- matches -= set(('series', 'tvdb_id', 'season', 'episode', 'title', 'year'))
- if 'tvdb_id' in matches:
- matches -= set(('series', 'year'))
- if 'title' in matches:
- matches -= set(('season', 'episode'))
- # add other scores
- score += sum((video.scores[match] for match in matches))
- logger.info('Computed score %d with matches %r', score, initial_matches)
- return score
+ * `imdb_id` removes `series`, `tvdb_id`, `season`, `episode`, `title` and `year`
+ * `tvdb_id` removes `series` and `year`
+ * `title` removes `season` and `episode`
- def __repr__(self):
- return '<%s [%s]>' % (self.__class__.__name__, self.language)
+ :param video: the video to get the score with.
+ :type video: :class:`~subliminal.video.Video`
+ :param dict scores: scores to use, if `None`, the :attr:`~subliminal.video.Video.scores` from the video are used.
+ :return: score of the subtitle.
+ :rtype: int
+
+ """
+ final_matches = matches.copy()
+ scores = scores or video.scores
+
+ logger.info('Computing score for matches %r and %r', matches, video)
-def get_subtitle_path(video_path, language=None):
- """Create the subtitle path from the given `video_path` and `language`
+ # remove equivalent match combinations
+ if 'hash' in final_matches:
+ final_matches &= {'hash', 'hearing_impaired'}
+ elif isinstance(video, Episode):
+ if 'imdb_id' in final_matches:
+ final_matches -= {'series', 'tvdb_id', 'season', 'episode', 'title', 'year'}
+ if 'tvdb_id' in final_matches:
+ final_matches -= {'series', 'year'}
+ if 'title' in final_matches:
+ final_matches -= {'season', 'episode'}
- :param string video_path: path to the video
- :param language: language of the subtitle to put in the path
- :type language: :class:`babelfish.Language` or None
- :return: path of the subtitle
- :rtype: string
+ # compute score
+ logger.debug('Final matches: %r', final_matches)
+ score = sum((scores[match] for match in final_matches))
+ logger.info('Computed score %d', score)
+
+ # ensure score is capped by the best possible score (hash + preferences)
+ assert score <= scores['hash'] + scores['hearing_impaired']
+
+ return score
+
+
+def get_subtitle_path(video_path, language=None, extension='.srt'):
+ """Get the subtitle path using the `video_path` and `language`.
+
+ :param str video_path: path to the video.
+ :param language: language of the subtitle to put in the path.
+ :type language: :class:`~babelfish.language.Language`
+ :param str extension: extension of the subtitle.
+ :return: path of the subtitle.
+ :rtype: str
"""
- subtitle_path = os.path.splitext(video_path)[0]
- if language is not None:
- try:
- return subtitle_path + '.%s.%s' % (language.alpha2, 'srt')
- except babelfish.LanguageConvertError:
- return subtitle_path + '.%s.%s' % (language.alpha3, 'srt')
- return subtitle_path + '.srt'
+ subtitle_root = os.path.splitext(video_path)[0]
+
+ if language:
+ subtitle_root += '.' + str(language)
+
+ return subtitle_root + extension
+
+def guess_matches(video, guess, partial=False):
+ """Get matches between a `video` and a `guess`.
-def compute_guess_matches(video, guess):
- """Compute matches between a `video` and a `guess`
+ If a guess is `partial`, the absence information won't be counted as a match.
- :param video: the video to compute the matches on
+ :param video: the video.
:type video: :class:`~subliminal.video.Video`
- :param guess: the guess to compute the matches on
- :type guess: :class:`guessit.Guess`
- :return: matches of the `guess`
+ :param guess: the guess.
+ :type guess: dict
+ :param bool partial: whether or not the guess is partial.
+ :return: matches between the `video` and the `guess`.
:rtype: set
"""
@@ -187,13 +240,16 @@ def compute_guess_matches(video, guess):
if video.series and 'series' in guess and guess['series'].lower() == video.series.lower():
matches.add('series')
# season
- if video.season and 'seasonNumber' in guess and guess['seasonNumber'] == video.season:
+ if video.season and 'season' in guess and guess['season'] == video.season:
matches.add('season')
# episode
if video.episode and 'episodeNumber' in guess and guess['episodeNumber'] == video.episode:
matches.add('episode')
# year
- if video.year == guess.get('year'): # count "no year" as an information
+ if video.year and 'year' in guess and guess['year'] == video.year:
+ matches.add('year')
+ # count "no year" as an information
+ if not partial and video.year is None and 'year' not in guess:
matches.add('year')
elif isinstance(video, Movie):
# year
@@ -202,83 +258,44 @@ def compute_guess_matches(video, guess):
# title
if video.title and 'title' in guess and guess['title'].lower() == video.title.lower():
matches.add('title')
- # release group
+ # release_group
if video.release_group and 'releaseGroup' in guess and guess['releaseGroup'].lower() == video.release_group.lower():
matches.add('release_group')
- # screen size
+ # resolution
if video.resolution and 'screenSize' in guess and guess['screenSize'] == video.resolution:
matches.add('resolution')
# format
if video.format and 'format' in guess and guess['format'].lower() == video.format.lower():
matches.add('format')
- # video codec
+ # video_codec
if video.video_codec and 'videoCodec' in guess and guess['videoCodec'] == video.video_codec:
matches.add('video_codec')
- # audio codec
+ # audio_codec
if video.audio_codec and 'audioCodec' in guess and guess['audioCodec'] == video.audio_codec:
matches.add('audio_codec')
+
return matches
-def compute_guess_properties_matches(video, string, propertytype):
- """Compute matches between a `video` and properties of a certain property type
+def guess_properties(string):
+ """Extract properties from `string` using guessit's `guess_properties` transformer.
- :param video: the video to compute the matches on
- :type video: :class:`~subliminal.video.Video`
- :param string string: the string to check for a certain property type
- :param string propertytype: the type of property to check (as defined in guessit)
- :return: matches of a certain property type (but will only be 1 match because we are checking for 1 property type)
- :rtype: set
-
- Supported property types: result of guessit.transfo.guess_properties.GuessProperties().supported_properties()
- [u'audioProfile',
- u'videoCodec',
- u'container',
- u'format',
- u'episodeFormat',
- u'videoApi',
- u'screenSize',
- u'videoProfile',
- u'audioChannels',
- u'other',
- u'audioCodec']
+ :param str string: the string potentially containing properties.
+ :return: the guessed properties.
+ :rtype: dict
"""
- matches = set()
- # We only check for the property types relevant for us
- if propertytype == 'screenSize' and video.resolution:
- for prop in guess_properties(string, propertytype):
- if prop.lower() == video.resolution.lower():
- matches.add('resolution')
- elif propertytype == 'format' and video.format:
- for prop in guess_properties(string, propertytype):
- if prop.lower() == video.format.lower():
- matches.add('format')
- elif propertytype == 'videoCodec' and video.video_codec:
- for prop in guess_properties(string, propertytype):
- if prop.lower() == video.video_codec.lower():
- matches.add('video_codec')
- elif propertytype == 'audioCodec' and video.audio_codec:
- for prop in guess_properties(string, propertytype):
- if prop.lower() == video.audio_codec.lower():
- matches.add('audio_codec')
- return matches
-
+ mtree = MatchTree(string)
+ get_transformer('guess_properties').process(mtree)
-def guess_properties(string, propertytype):
- properties = set()
- if string:
- tree = guessit.matchtree.MatchTree(string)
- guessit.transfo.guess_properties.GuessProperties().process(tree)
- properties = set(n.guess[propertytype] for n in tree.nodes() if propertytype in n.guess)
- return properties
+ return mtree.matched()
-def fix_line_endings(content):
- """Fix line ending of `content` by changing it to \n
+def fix_line_ending(content):
+ """Fix line ending of `content` by changing it to \n.
- :param bytes content: content of the subtitle
- :return: the content with fixed line endings
+ :param bytes content: content of the subtitle.
+ :return: the content with fixed line endings.
:rtype: bytes
"""
diff --git a/lib/subliminal/video.py b/lib/subliminal/video.py
index 09e36a9ec3b371e2a05dd781335836b7c0f0f832..7e52502e3b4758a5dce3e93a08ddc6320f49d128 100644
--- a/lib/subliminal/video.py
+++ b/lib/subliminal/video.py
@@ -1,14 +1,14 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, division
-import datetime
+from datetime import datetime, timedelta
import hashlib
import logging
import os
import struct
-import babelfish
-import enzyme
-import guessit
+from babelfish import Error as BabelfishError, Language
+from enzyme import Error as EnzymeError, MKV
+from guessit import guess_episode_info, guess_file_info, guess_movie_info
logger = logging.getLogger(__name__)
@@ -26,50 +26,91 @@ SUBTITLE_EXTENSIONS = ('.srt', '.sub', '.smi', '.txt', '.ssa', '.ass', '.mpl')
class Video(object):
- """Base class for videos
-
- Represent a video, existing or not, with various properties that defines it.
- Each property has an associated score based on equations that are described in
- subclasses.
-
- :param string name: name or path of the video
- :param string format: format of the video (HDTV, WEB-DL, ...)
- :param string release_group: release group of the video
- :param string resolution: screen size of the video stream (480p, 720p, 1080p or 1080i)
- :param string video_codec: codec of the video stream
- :param string audio_codec: codec of the main audio stream
- :param int imdb_id: IMDb id of the video
- :param dict hashes: hashes of the video file by provider names
- :param int size: byte size of the video file
+ """Base class for videos.
+
+ Represent a video, existing or not. Attributes have an associated score based on equations defined in
+ :mod:`~subliminal.score`.
+
+ :param str name: name or path of the video.
+ :param str format: format of the video (HDTV, WEB-DL, BluRay, ...).
+ :param str release_group: release group of the video.
+ :param str resolution: resolution of the video stream (480p, 720p, 1080p or 1080i).
+ :param str video_codec: codec of the video stream.
+ :param str audio_codec: codec of the main audio stream.
+ :param int imdb_id: IMDb id of the video.
+ :param dict hashes: hashes of the video file by provider names.
+ :param int size: size of the video file in bytes.
:param set subtitle_languages: existing subtitle languages
"""
+ #: Score by match property
scores = {}
def __init__(self, name, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None,
imdb_id=None, hashes=None, size=None, subtitle_languages=None):
+ #: Name or path of the video
self.name = name
+
+ #: Format of the video (HDTV, WEB-DL, BluRay, ...)
self.format = format
+
+ #: Release group of the video
self.release_group = release_group
+
+ #: Resolution of the video stream (480p, 720p, 1080p or 1080i)
self.resolution = resolution
+
+ #: Codec of the video stream
self.video_codec = video_codec
+
+ #: Codec of the main audio stream
self.audio_codec = audio_codec
+
+ #: IMDb id of the video
self.imdb_id = imdb_id
+
+ #: Hashes of the video file by provider names
self.hashes = hashes or {}
+
+ #: Size of the video file in bytes
self.size = size
+
+ #: Existing subtitle languages
self.subtitle_languages = subtitle_languages or set()
+ @property
+ def exists(self):
+ """Test whether the video exists."""
+ return os.path.exists(self.name)
+
+ @property
+ def age(self):
+ """Age of the video."""
+ if self.exists:
+ return datetime.utcnow() - datetime.utcfromtimestamp(os.path.getmtime(self.name))
+
+ return timedelta()
+
@classmethod
def fromguess(cls, name, guess):
+ """Create an :class:`Episode` or a :class:`Movie` with the given `name` based on the `guess`.
+
+ :param str name: name of the video.
+ :param dict guess: guessed data, like a :class:`~guessit.guess.Guess` instance.
+ :raise: :class:`ValueError` if the `type` of the `guess` is invalid
+
+ """
if guess['type'] == 'episode':
return Episode.fromguess(name, guess)
+
if guess['type'] == 'movie':
return Movie.fromguess(name, guess)
+
raise ValueError('The guess must be an episode or a movie guess')
@classmethod
def fromname(cls, name):
- return cls.fromguess(os.path.split(name)[1], guessit.guess_file_info(name))
+ return cls.fromguess(name, guess_file_info(name))
def __repr__(self):
return '<%s [%r]>' % (self.__class__.__name__, self.name)
@@ -79,39 +120,54 @@ class Video(object):
class Episode(Video):
- """Episode :class:`Video`
+ """Episode :class:`Video`.
- Scores are defined by a set of equations, see :func:`~subliminal.score.get_episode_equations`
+ Scores are defined by a set of equations, see :func:`~subliminal.score.solve_episode_equations`
- :param string series: series of the episode
- :param int season: season number of the episode
- :param int episode: episode number of the episode
- :param string title: title of the episode
- :param int year: year of series
- :param int tvdb_id: TheTVDB id of the episode
+ :param str series: series of the episode.
+ :param int season: season number of the episode.
+ :param int episode: episode number of the episode.
+ :param str title: title of the episode.
+ :param int year: year of series.
+ :param int tvdb_id: TVDB id of the episode
"""
- scores = {'format': 3, 'video_codec': 2, 'tvdb_id': 48, 'title': 12, 'imdb_id': 60, 'audio_codec': 1, 'year': 24,
- 'resolution': 2, 'season': 6, 'release_group': 6, 'series': 24, 'episode': 6, 'hash': 74}
-
- def __init__(self, name, series, season, episode, format=None, release_group=None, resolution=None, video_codec=None,
- audio_codec=None, imdb_id=None, hashes=None, size=None, subtitle_languages=None, title=None,
- year=None, tvdb_id=None):
- super(Episode, self).__init__(name, format, release_group, resolution, video_codec, audio_codec, imdb_id, hashes,
- size, subtitle_languages)
+ #: Score by match property
+ scores = {'hash': 137, 'imdb_id': 110, 'tvdb_id': 88, 'series': 44, 'year': 44, 'title': 22, 'season': 11,
+ 'episode': 11, 'release_group': 11, 'format': 6, 'video_codec': 4, 'resolution': 4, 'audio_codec': 2,
+ 'hearing_impaired': 1}
+
+ def __init__(self, name, series, season, episode, format=None, release_group=None, resolution=None,
+ video_codec=None, audio_codec=None, imdb_id=None, hashes=None, size=None, subtitle_languages=None,
+ title=None, year=None, tvdb_id=None):
+ super(Episode, self).__init__(name, format, release_group, resolution, video_codec, audio_codec, imdb_id,
+ hashes, size, subtitle_languages)
+ #: Series of the episode
self.series = series
+
+ #: Season number of the episode
self.season = season
+
+ #: Episode number of the episode
self.episode = episode
+
+ #: Title of the episode
self.title = title
+
+ #: Year of series
self.year = year
+
+ #: TVDB id of the episode
self.tvdb_id = tvdb_id
@classmethod
def fromguess(cls, name, guess):
if guess['type'] != 'episode':
raise ValueError('The guess must be an episode guess')
+
if 'series' not in guess or 'season' not in guess or 'episodeNumber' not in guess:
raise ValueError('Insufficient data to process the guess')
+
return cls(name, guess['series'], guess['season'], guess['episodeNumber'], format=guess.get('format'),
release_group=guess.get('releaseGroup'), resolution=guess.get('screenSize'),
video_codec=guess.get('videoCodec'), audio_codec=guess.get('audioCodec'),
@@ -119,88 +175,128 @@ class Episode(Video):
@classmethod
def fromname(cls, name):
- return cls.fromguess(os.path.split(name)[1], guessit.guess_episode_info(name))
+ return cls.fromguess(name, guess_episode_info(name))
def __repr__(self):
if self.year is None:
return '<%s [%r, %dx%d]>' % (self.__class__.__name__, self.series, self.season, self.episode)
+
return '<%s [%r, %d, %dx%d]>' % (self.__class__.__name__, self.series, self.year, self.season, self.episode)
class Movie(Video):
- """Movie :class:`Video`
+ """Movie :class:`Video`.
- Scores are defined by a set of equations, see :func:`~subliminal.score.get_movie_equations`
+ Scores are defined by a set of equations, see :func:`~subliminal.score.solve_movie_equations`
- :param string title: title of the movie
+ :param str title: title of the movie.
:param int year: year of the movie
"""
- scores = {'format': 3, 'video_codec': 2, 'title': 13, 'imdb_id': 34, 'audio_codec': 1, 'year': 7, 'resolution': 2,
- 'release_group': 6, 'hash': 34}
+ #: Score by match property
+ scores = {'hash': 62, 'imdb_id': 62, 'title': 23, 'year': 12, 'release_group': 11, 'format': 6, 'video_codec': 4,
+ 'resolution': 4, 'audio_codec': 2, 'hearing_impaired': 1}
- def __init__(self, name, title, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None,
- imdb_id=None, hashes=None, size=None, subtitle_languages=None, year=None):
+ def __init__(self, name, title, format=None, release_group=None, resolution=None, video_codec=None,
+ audio_codec=None, imdb_id=None, hashes=None, size=None, subtitle_languages=None, year=None):
super(Movie, self).__init__(name, format, release_group, resolution, video_codec, audio_codec, imdb_id, hashes,
size, subtitle_languages)
+ #: Title of the movie
self.title = title
+
+ #: Year of the movie
self.year = year
@classmethod
def fromguess(cls, name, guess):
if guess['type'] != 'movie':
raise ValueError('The guess must be a movie guess')
+
if 'title' not in guess:
raise ValueError('Insufficient data to process the guess')
+
return cls(name, guess['title'], format=guess.get('format'), release_group=guess.get('releaseGroup'),
resolution=guess.get('screenSize'), video_codec=guess.get('videoCodec'),
- audio_codec=guess.get('audioCodec'),year=guess.get('year'))
+ audio_codec=guess.get('audioCodec'), year=guess.get('year'))
@classmethod
def fromname(cls, name):
- return cls.fromguess(os.path.split(name)[1], guessit.guess_movie_info(name))
+ return cls.fromguess(name, guess_movie_info(name))
def __repr__(self):
if self.year is None:
return '<%s [%r]>' % (self.__class__.__name__, self.title)
+
return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year)
-def scan_subtitle_languages(path):
- """Search for subtitles with alpha2 extension from a video `path` and return their language
+def search_external_subtitles(path):
+ """Search for external subtitles from a video `path` and their associated language.
- :param string path: path to the video
- :return: found subtitle languages
- :rtype: set
+ :param str path: path to the video.
+ :return: found subtitles with their languages.
+ :rtype: dict
"""
- language_extensions = tuple('.' + c for c in babelfish.language_converters['alpha2'].codes)
dirpath, filename = os.path.split(path)
- subtitles = set()
+ dirpath = dirpath or '.'
+ fileroot, fileext = os.path.splitext(filename)
+ subtitles = {}
for p in os.listdir(dirpath):
- if not isinstance(p, bytes) and p.startswith(os.path.splitext(filename)[0]) and p.endswith(SUBTITLE_EXTENSIONS):
- if os.path.splitext(p)[0].endswith(language_extensions):
- subtitles.add(babelfish.Language.fromalpha2(os.path.splitext(p)[0][-2:]))
- else:
- subtitles.add(babelfish.Language('und'))
+ # skip badly encoded filenames
+ if isinstance(p, bytes): # pragma: no cover
+ logger.error('Skipping badly encoded filename %r in %r', p.decode('utf-8', errors='replace'), dirpath)
+ continue
+
+ # keep only valid subtitle filenames
+ if not p.startswith(fileroot) or not p.endswith(SUBTITLE_EXTENSIONS):
+ continue
+
+ # extract the potential language code
+ language_code = p[len(fileroot):-len(os.path.splitext(p)[1])].replace(fileext, '').replace('_', '-')[1:]
+
+ # default language is undefined
+ language = Language('und')
+
+ # attempt to parse
+ if language_code:
+ try:
+ language = Language.fromietf(language_code)
+ except ValueError:
+ logger.error('Cannot parse language code %r', language_code)
+
+ subtitles[p] = language
+
logger.debug('Found subtitles %r', subtitles)
+
return subtitles
def scan_video(path, subtitles=True, embedded_subtitles=True):
- """Scan a video and its subtitle languages from a video `path`
+ """Scan a video and its subtitle languages from a video `path`.
- :param string path: absolute path to the video
- :param bool subtitles: scan for subtitles with the same name
- :param bool embedded_subtitles: scan for embedded subtitles
- :return: the scanned video
+ :param str path: existing path to the video.
+ :param bool subtitles: scan for subtitles with the same name.
+ :param bool embedded_subtitles: scan for embedded subtitles.
+ :return: the scanned video.
:rtype: :class:`Video`
- :raise: ValueError if cannot guess enough information from the path
"""
+ # check for non-existing path
+ if not os.path.exists(path):
+ raise ValueError('Path does not exist')
+
+ # check video extension
+ if not path.endswith(VIDEO_EXTENSIONS):
+ raise ValueError('%s is not a valid video extension' % os.path.splitext(path)[1])
+
dirpath, filename = os.path.split(path)
logger.info('Scanning video %r in %r', filename, dirpath)
- video = Video.fromguess(path, guessit.guess_file_info(path))
+
+ # guess
+ video = Video.fromguess(path, guess_file_info(path))
+
+ # size and hashes
video.size = os.path.getsize(path)
if video.size > 10485760:
logger.debug('Size is %d', video.size)
@@ -209,23 +305,29 @@ def scan_video(path, subtitles=True, embedded_subtitles=True):
logger.debug('Computed hashes %r', video.hashes)
else:
logger.warning('Size is lower than 10MB: hashes not computed')
+
+ # external subtitles
if subtitles:
- video.subtitle_languages |= scan_subtitle_languages(path)
- # enzyme
+ video.subtitle_languages |= set(search_external_subtitles(path).values())
+
+ # video metadata with enzyme
try:
if filename.endswith('.mkv'):
with open(path, 'rb') as f:
- mkv = enzyme.MKV(f)
+ mkv = MKV(f)
+
+ # main video track
if mkv.video_tracks:
video_track = mkv.video_tracks[0]
+
# resolution
if video_track.height in (480, 720, 1080):
if video_track.interlaced:
video.resolution = '%di' % video_track.height
- logger.debug('Found resolution %s with enzyme', video.resolution)
else:
video.resolution = '%dp' % video_track.height
- logger.debug('Found resolution %s with enzyme', video.resolution)
+ logger.debug('Found resolution %s with enzyme', video.resolution)
+
# video codec
if video_track.codec_id == 'V_MPEG4/ISO/AVC':
video.video_codec = 'h264'
@@ -238,6 +340,8 @@ def scan_video(path, subtitles=True, embedded_subtitles=True):
logger.debug('Found video_codec %s with enzyme', video.video_codec)
else:
logger.warning('MKV has no video track')
+
+ # main audio track
if mkv.audio_tracks:
audio_track = mkv.audio_tracks[0]
# audio codec
@@ -252,124 +356,119 @@ def scan_video(path, subtitles=True, embedded_subtitles=True):
logger.debug('Found audio_codec %s with enzyme', video.audio_codec)
else:
logger.warning('MKV has no audio track')
+
+ # subtitle tracks
if mkv.subtitle_tracks:
- # embedded subtitles
if embedded_subtitles:
embedded_subtitle_languages = set()
for st in mkv.subtitle_tracks:
if st.language:
try:
- embedded_subtitle_languages.add(babelfish.Language.fromalpha3b(st.language))
- except babelfish.Error:
+ embedded_subtitle_languages.add(Language.fromalpha3b(st.language))
+ except BabelfishError:
logger.error('Embedded subtitle track language %r is not a valid language', st.language)
- embedded_subtitle_languages.add(babelfish.Language('und'))
+ embedded_subtitle_languages.add(Language('und'))
elif st.name:
try:
- embedded_subtitle_languages.add(babelfish.Language.fromname(st.name))
- except babelfish.Error:
+ embedded_subtitle_languages.add(Language.fromname(st.name))
+ except BabelfishError:
logger.debug('Embedded subtitle track name %r is not a valid language', st.name)
- embedded_subtitle_languages.add(babelfish.Language('und'))
+ embedded_subtitle_languages.add(Language('und'))
else:
- embedded_subtitle_languages.add(babelfish.Language('und'))
+ embedded_subtitle_languages.add(Language('und'))
logger.debug('Found embedded subtitle %r with enzyme', embedded_subtitle_languages)
video.subtitle_languages |= embedded_subtitle_languages
else:
logger.debug('MKV has no subtitle track')
- except enzyme.Error:
+
+ except EnzymeError:
logger.exception('Parsing video metadata with enzyme failed')
+
return video
-def scan_videos(paths, subtitles=True, embedded_subtitles=True, age=None):
- """Scan `paths` for videos and their subtitle languages
+def scan_videos(path, subtitles=True, embedded_subtitles=True):
+ """Scan `path` for videos and their subtitles.
- :params paths: absolute paths to scan for videos
- :type paths: list of string
- :param bool subtitles: scan for subtitles with the same name
- :param bool embedded_subtitles: scan for embedded subtitles
- :param age: age of the video, if any
- :type age: datetime.timedelta or None
- :return: the scanned videos
+ :params path: existing directory path to scan.
+ :type path: str
+ :param bool subtitles: scan for subtitles with the same name.
+ :param bool embedded_subtitles: scan for embedded subtitles.
+ :return: the scanned videos.
:rtype: list of :class:`Video`
"""
+ # check for non-existing path
+ if not os.path.exists(path):
+ raise ValueError('Path does not exist')
+
+ # check for non-directory path
+ if not os.path.isdir(path):
+ raise ValueError('Path is not a directory')
+
+ # walk the path
videos = []
- # scan files
- for filepath in [p for p in paths if os.path.isfile(p)]:
- if age is not None:
- try:
- video_age = datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(filepath))
- except ValueError:
- logger.exception('Error while getting video age, skipping it')
+ for dirpath, dirnames, filenames in os.walk(path):
+ # skip badly encoded directory names
+ if isinstance(dirpath, bytes): # pragma: no cover
+ logger.error('Skipping badly encoded directory %r', dirpath.decode('utf-8', errors='replace'))
+ continue
+
+ logger.debug('Walking directory %s', dirpath)
+
+ # remove badly encoded and hidden dirnames
+ for dirname in list(dirnames):
+ if isinstance(dirname, bytes): # pragma: no cover
+ logger.error('Skipping badly encoded dirname %r in %r', dirname.decode('utf-8', errors='replace'),
+ dirpath)
+ dirnames.remove(dirname)
+ elif dirname.startswith('.'):
+ logger.debug('Skipping hidden dirname %r in %r', dirname, dirpath)
+ dirnames.remove(dirname)
+
+ # scan for videos
+ for filename in filenames:
+ # skip badly encoded filenames
+ if isinstance(filename, bytes): # pragma: no cover
+ logger.error('Skipping badly encoded filename %r in %r', filename.decode('utf-8', errors='replace'),
+ dirpath)
continue
- if video_age > age:
- logger.info('Skipping video %r: older than %r', filepath, age)
+
+ # filter on videos
+ if not filename.endswith(VIDEO_EXTENSIONS):
continue
- try:
- videos.append(scan_video(filepath, subtitles, embedded_subtitles))
- except ValueError as e:
- logger.error('Skipping video: %s', e)
- continue
- # scan directories
- for path in [p for p in paths if os.path.isdir(p)]:
- logger.info('Scanning directory %r', path)
- for dirpath, dirnames, filenames in os.walk(path):
- # skip badly encoded directories
- if isinstance(dirpath, bytes):
- logger.error('Skipping badly encoded directory %r', dirpath.decode('utf-8', errors='replace'))
+
+ # skip hidden files
+ if filename.startswith('.'):
+ logger.debug('Skipping hidden filename %r in %r', filename, dirpath)
+ continue
+
+ # reconstruct the file path
+ filepath = os.path.join(dirpath, filename)
+
+ # skip links
+ if os.path.islink(filepath):
+ logger.debug('Skipping link %r in %r', filename, dirpath)
+ continue
+
+ # scan video
+ try:
+ video = scan_video(filepath, subtitles=subtitles, embedded_subtitles=embedded_subtitles)
+ except ValueError: # pragma: no cover
+ logger.exception('Error scanning video')
continue
- # skip badly encoded and hidden sub directories
- for dirname in list(dirnames):
- if isinstance(dirname, bytes):
- logger.error('Skipping badly encoded dirname %r in %r', dirname.decode('utf-8', errors='replace'),
- dirpath)
- dirnames.remove(dirname)
- elif dirname.startswith('.'):
- logger.debug('Skipping hidden dirname %r in %r', dirname, dirpath)
- dirnames.remove(dirname)
- # scan for videos
- for filename in filenames:
- # skip badly encoded files
- if isinstance(filename, bytes):
- logger.error('Skipping badly encoded filename %r in %r', filename.decode('utf-8', errors='replace'),
- dirpath)
- continue
- # filter videos
- if not filename.endswith(VIDEO_EXTENSIONS):
- continue
- # skip hidden files
- if filename.startswith('.'):
- logger.debug('Skipping hidden filename %r in %r', filename, dirpath)
- continue
- filepath = os.path.join(dirpath, filename)
- # skip links
- if os.path.islink(filepath):
- logger.debug('Skipping link %r in %r', filename, dirpath)
- continue
- if age is not None:
- try:
- video_age = datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(filepath))
- except ValueError:
- logger.exception('Error while getting video age, skipping it')
- continue
- if video_age > age:
- logger.info('Skipping video %r: older than %r', filepath, age)
- continue
- try:
- video = scan_video(filepath, subtitles, embedded_subtitles)
- except ValueError as e:
- logger.error('Skipping video: %s', e)
- continue
- videos.append(video)
+
+ videos.append(video)
+
return videos
def hash_opensubtitles(video_path):
- """Compute a hash using OpenSubtitles' algorithm
+ """Compute a hash using OpenSubtitles' algorithm.
- :param string video_path: path of the video
- :return: the hash
- :rtype: string
+ :param str video_path: path of the video.
+ :return: the hash.
+ :rtype: str
"""
bytesize = struct.calcsize(b'<q')
@@ -377,35 +476,37 @@ def hash_opensubtitles(video_path):
filesize = os.path.getsize(video_path)
filehash = filesize
if filesize < 65536 * 2:
- return None
+ return
for _ in range(65536 // bytesize):
filebuffer = f.read(bytesize)
(l_value,) = struct.unpack(b'<q', filebuffer)
filehash += l_value
- filehash = filehash & 0xFFFFFFFFFFFFFFFF # to remain as 64bit number
+ filehash &= 0xFFFFFFFFFFFFFFFF # to remain as 64bit number
f.seek(max(0, filesize - 65536), 0)
for _ in range(65536 // bytesize):
filebuffer = f.read(bytesize)
(l_value,) = struct.unpack(b'<q', filebuffer)
filehash += l_value
- filehash = filehash & 0xFFFFFFFFFFFFFFFF
+ filehash &= 0xFFFFFFFFFFFFFFFF
returnedhash = '%016x' % filehash
+
return returnedhash
def hash_thesubdb(video_path):
- """Compute a hash using TheSubDB's algorithm
+ """Compute a hash using TheSubDB's algorithm.
- :param string video_path: path of the video
- :return: the hash
- :rtype: string
+ :param str video_path: path of the video.
+ :return: the hash.
+ :rtype: str
"""
readsize = 64 * 1024
if os.path.getsize(video_path) < readsize:
- return None
+ return
with open(video_path, 'rb') as f:
data = f.read(readsize)
f.seek(-readsize, os.SEEK_END)
data += f.read(readsize)
+
return hashlib.md5(data).hexdigest()
diff --git a/lib/tvdb_api/tvdb_api.py b/lib/tvdb_api/tvdb_api.py
index bf0180303a1a92e991c1f9c153929a06beea91f4..4800ff10eef7d7371b50d0bb0d5204c5ee740278 100644
--- a/lib/tvdb_api/tvdb_api.py
+++ b/lib/tvdb_api/tvdb_api.py
@@ -178,9 +178,9 @@ class Show(dict):
Search terms are converted to lower case (unicode) strings.
# Examples
-
+
These examples assume t is an instance of Tvdb():
-
+
>>> t = Tvdb()
>>>
@@ -296,7 +296,7 @@ class Episode(dict):
"""Search episode data for term, if it matches, return the Episode (self).
The key parameter can be used to limit the search to a specific element,
for example, episodename.
-
+
This primarily for use use by Show.search and Season.search. See
Show.search for further information on search
@@ -422,7 +422,7 @@ class Tvdb:
By default, Tvdb will only search in the language specified using
the language option. When this is True, it will search for the
show in and language
-
+
apikey (str/unicode):
Override the default thetvdb.com API key. By default it will use
tvdb_api's own key (fine for small scripts), but you can use your
@@ -617,12 +617,12 @@ class Tvdb:
zipdata = StringIO.StringIO()
zipdata.write(resp.content)
myzipfile = zipfile.ZipFile(zipdata)
- return xmltodict.parse(myzipfile.read('%s.xml' % language), postprocessor=process)
+ return xmltodict.parse(myzipfile.read('%s.xml' % language).replace(' ',' '), postprocessor=process)
except zipfile.BadZipfile:
raise tvdb_error("Bad zip file received from thetvdb.com, could not read it")
else:
try:
- return xmltodict.parse(resp.content.decode('utf-8'), postprocessor=process)
+ return xmltodict.parse(resp.content.decode('utf-8').replace(' ',' '), postprocessor=process)
except:
return dict([(u'data', None)])
@@ -671,7 +671,8 @@ class Tvdb:
- Replaces & with &
- Trailing whitespace
"""
- data = data.replace(u"&", u"&")
+
+ data = unicode(data).replace(u"&", u"&")
data = data.strip()
return data
diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py
index 6b44cd198f98d5fe863435b1b4912b5134d2abd3..22ea83e07d40830d421c45a4115a701ad8ab856e 100644
--- a/sickbeard/helpers.py
+++ b/sickbeard/helpers.py
@@ -41,13 +41,12 @@ import errno
import ast
import operator
import platform
-from contextlib import closing
-
import sickbeard
-
import adba
import requests
import certifi
+from contextlib import closing
+from socket import timeout as SocketTimeout
try:
@@ -562,8 +561,7 @@ def hardlinkFile(srcFile, destFile):
ek(link, srcFile, destFile)
fixSetGroupID(destFile)
except Exception as e:
- logger.log(u"Failed to create hardlink of " + srcFile + " at " + destFile + ": " + ex(e) + ". Copying instead",
- logger.ERROR)
+ logger.log(u"Failed to create hardlink of %s at %s. Error: %r. Copying instead" % (srcFile, destFile, ex(e)),logger.ERROR)
copyFile(srcFile, destFile)
@@ -595,8 +593,8 @@ def moveAndSymlinkFile(srcFile, destFile):
ek(shutil.move, srcFile, destFile)
fixSetGroupID(destFile)
ek(symlink, destFile, srcFile)
- except Exception:
- logger.log(u"Failed to create symlink of " + srcFile + " at " + destFile + ". Copying instead", logger.ERROR)
+ except Exception as e:
+ logger.log(u"Failed to create symlink of %s at %s. Error: %r. Copying instead" % (srcFile, destFile, ex(e)),logger.ERROR)
copyFile(srcFile, destFile)
@@ -615,7 +613,7 @@ def make_dirs(path):
logger.log(u"Folder %s didn't exist, creating it" % path, logger.DEBUG)
ek(os.makedirs, path)
except (OSError, IOError) as e:
- logger.log(u"Failed creating %s : %s" % (path, ex(e)), logger.ERROR)
+ logger.log(u"Failed creating %s : %r" % (path, ex(e)), logger.ERROR)
return False
# not Windows, create all missing folders and set permissions
@@ -639,7 +637,7 @@ def make_dirs(path):
# do the library update for synoindex
notifiers.synoindex_notifier.addFolder(sofar)
except (OSError, IOError) as e:
- logger.log(u"Failed creating %s : %s" % (sofar, ex(e)), logger.ERROR)
+ logger.log(u"Failed creating %s : %r" % (sofar, ex(e)), logger.ERROR)
return False
return True
@@ -683,7 +681,7 @@ def rename_ep_file(cur_path, new_path, old_path_length=0):
logger.log(u"Renaming file from %s to %s" % (cur_path, new_path))
ek(shutil.move, cur_path, new_path)
except (OSError, IOError) as e:
- logger.log(u"Failed renaming %s to %s : %s" % (cur_path, new_path, ex(e)), logger.ERROR)
+ logger.log(u"Failed renaming %s to %s : %r" % (cur_path, new_path, ex(e)), logger.ERROR)
return False
# clean up any old folders that are empty
@@ -719,7 +717,7 @@ def delete_empty_folders(check_empty_dir, keep_dir=None):
# do the library update for synoindex
notifiers.synoindex_notifier.deleteFolder(check_empty_dir)
except OSError as e:
- logger.log(u"Unable to delete " + check_empty_dir + ": " + repr(e) + " / " + str(e), logger.WARNING)
+ logger.log(u"Unable to delete %s. Error: %r" % (check_empty_dir, repr(e)), logger.WARNING)
break
check_empty_dir = ek(os.path.dirname, check_empty_dir)
else:
@@ -865,14 +863,9 @@ def get_absolute_number_from_season_and_episode(show, season, episode):
if len(sqlResults) == 1:
absolute_number = int(sqlResults[0]["absolute_number"])
- logger.log(
- "Found absolute_number:" + str(absolute_number) + " by " + str(season) + "x" + str(episode),
- logger.DEBUG)
+ logger.log("Found absolute number %s for show %s S%02dE%02d" % (absolute_number, show.name, season, episode), logger.DEBUG)
else:
- logger.log(
- "No entries for absolute number in show: " + show.name + " found using " + str(season) + "x" + str(
- episode),
- logger.DEBUG)
+ logger.log("No entries for absolute number for show %s S%02dE%02d" % (show.name, season, episode), logger.DEBUG)
return absolute_number
@@ -1017,7 +1010,7 @@ def backupVersionedFile(old_file, version):
logger.log(u"Backup done", logger.DEBUG)
break
except Exception as e:
- logger.log(u"Error while trying to back up %s to %s : %s" % (old_file, new_file, ex(e)), logger.WARNING)
+ logger.log(u"Error while trying to back up %s to %s : %r" % (old_file, new_file, ex(e)), logger.WARNING)
numTries += 1
time.sleep(1)
logger.log(u"Trying again.", logger.DEBUG)
@@ -1048,35 +1041,33 @@ def restoreVersionedFile(backup_file, version):
return False
try:
- logger.log(
- u"Trying to backup " + new_file + " to " + new_file + "." + "r" + str(version) + " before restoring backup",
+ logger.log(u"Trying to backup %s to %s.r%s before restoring backup" % (new_file, new_file, version),
logger.DEBUG)
+
shutil.move(new_file, new_file + '.' + 'r' + str(version))
except Exception as e:
- logger.log(
- u"Error while trying to backup DB file " + restore_file + " before proceeding with restore: " + ex(e),
+ logger.log(u"Error while trying to backup DB file %s before proceeding with restore: %r" % (restore_file, ex(e)),
logger.WARNING)
return False
while not ek(os.path.isfile, new_file):
if not ek(os.path.isfile, restore_file):
- logger.log(u"Not restoring, " + restore_file + " doesn't exist", logger.DEBUG)
+ logger.log(u"Not restoring, %s doesn't exist" % restore_file, logger.DEBUG)
break
try:
- logger.log(u"Trying to restore " + restore_file + " to " + new_file, logger.DEBUG)
+ logger.log(u"Trying to restore file %s to %s" % (restore_file, new_file), logger.DEBUG)
shutil.copy(restore_file, new_file)
logger.log(u"Restore done", logger.DEBUG)
break
except Exception as e:
- logger.log(u"Error while trying to restore " + restore_file + ": " + ex(e), logger.WARNING)
+ logger.log(u"Error while trying to restore file %s. Error: %r" % (restore_file, ex(e)), logger.WARNING)
numTries += 1
time.sleep(1)
- logger.log(u"Trying again.", logger.DEBUG)
+ logger.log(u"Trying again. Attempt #: %s" % numTries, logger.DEBUG)
if numTries >= 10:
- logger.log(u"Unable to restore " + restore_file + " to " + new_file + " please do it manually.",
- logger.ERROR)
+ logger.log(u"Unable to restore file %s to %s" % (restore_file, new_file), logger.WARNING)
return False
return True
@@ -1247,7 +1238,7 @@ def get_show(name, tryIndexers=False, trySceneExceptions=False):
if showObj and not fromCache:
sickbeard.name_cache.addNameToCache(name, showObj.indexerid)
except Exception as e:
- logger.log(u"Error when attempting to find show: " + name + " in SickRage: " + str(e), logger.DEBUG)
+ logger.log(u"Error when attempting to find show: %s in SickRage. Error: %r " % (name, repr(e)), logger.DEBUG)
return showObj
@@ -1318,11 +1309,11 @@ def set_up_anidb_connection():
return False
if not sickbeard.ADBA_CONNECTION:
- anidb_logger = lambda x: logger.log("ANIDB: " + str(x), logger.DEBUG)
+ anidb_logger = lambda x: logger.log("anidb: %s " % x, logger.DEBUG)
try:
sickbeard.ADBA_CONNECTION = adba.Connection(keepAlive=True, log=anidb_logger)
except Exception as e:
- logger.log(u"anidb exception msg: " + str(e))
+ logger.log(u"anidb exception msg: %r " % repr(e), logger.WARNING)
return False
try:
@@ -1331,7 +1322,7 @@ def set_up_anidb_connection():
else:
return True
except Exception as e:
- logger.log(u"anidb exception msg: " + str(e))
+ logger.log(u"anidb exception msg: %r " % repr(e), logger.WARNING)
return False
return sickbeard.ADBA_CONNECTION.authed()
@@ -1352,7 +1343,7 @@ def makeZip(fileList, archive):
a.close()
return True
except Exception as e:
- logger.log(u"Zip creation error: " + str(e), logger.ERROR)
+ logger.log(u"Zip creation error: %r " % repr(e), logger.ERROR)
return False
@@ -1384,7 +1375,7 @@ def extractZip(archive, targetDir):
zip_file.close()
return True
except Exception as e:
- logger.log(u"Zip extraction error: " + str(e), logger.ERROR)
+ logger.log(u"Zip extraction error: %r " % repr(e), logger.ERROR)
return False
@@ -1405,7 +1396,7 @@ def backupConfigZip(fileList, archive, arcname=None):
a.close()
return True
except Exception as e:
- logger.log(u"Zip creation error: " + str(e), logger.ERROR)
+ logger.log(u"Zip creation error: %r " % repr(e), logger.ERROR)
return False
@@ -1434,7 +1425,7 @@ def restoreConfigZip(archive, targetDir):
zip_file.close()
return True
except Exception as e:
- logger.log(u"Zip extraction error: " + str(e), logger.ERROR)
+ logger.log(u"Zip extraction error: %r" % ex(e), logger.ERROR)
shutil.rmtree(targetDir)
return False
@@ -1512,11 +1503,11 @@ def touchFile(fname, atime=None):
return True
except Exception as e:
if e.errno == errno.ENOSYS:
- logger.log(u"File air date stamping not available on your OS", logger.DEBUG)
+ logger.log(u"File air date stamping not available on your OS. Please disable setting", logger.DEBUG)
elif e.errno == errno.EACCES:
logger.log(u"File air date stamping failed(Permission denied). Check permissions for file: %s" % fname, logger.ERROR)
else:
- logger.log(u"File air date stamping failed. The error is: %s." % ex(e), logger.ERROR)
+ logger.log(u"File air date stamping failed. The error is: %r" % ex(e), logger.ERROR)
return False
@@ -1620,8 +1611,8 @@ def getURL(url, post_data=None, params={}, headers={}, timeout=30, session=None,
resp = session.get(url, timeout=timeout, allow_redirects=True, verify=session.verify)
if not resp.ok:
- logger.log(u"Requested getURL " + url + " returned status code is " + str(
- resp.status_code) + ': ' + codeDescription(resp.status_code), logger.DEBUG)
+ logger.log(u"Requested getURL %s returned status code is %s: %s" % (url, resp.status_code, codeDescription(resp.status_code)),
+ logger.DEBUG)
return None
if proxyGlypeProxySSLwarning is not None:
@@ -1629,25 +1620,28 @@ def getURL(url, post_data=None, params={}, headers={}, timeout=30, session=None,
resp = session.get(proxyGlypeProxySSLwarning, timeout=timeout, allow_redirects=True, verify=session.verify)
if not resp.ok:
- logger.log(u"GlypeProxySSLwarning: Requested getURL " + url + " returned status code is " + str(
- resp.status_code) + ': ' + codeDescription(resp.status_code), logger.DEBUG)
+ logger.log(u"GlypeProxySSLwarning: Requested getURL %s returned status code is %s: %s" % (url, resp.status_code, codeDescription(resp.status_code)),
+ logger.DEBUG)
return None
+ except SocketTimeout:
+ logger.log(u"Connection timed out (sockets) accessing getURL %s Error: %r" % (url, ex(e)), logger.WARNING)
+ return None
except requests.exceptions.HTTPError as e:
- logger.log(u"HTTP error in getURL %s Error: %s" % (url, ex(e)), logger.WARNING)
+ logger.log(u"HTTP error in getURL %s Error: %r" % (url, ex(e)), logger.WARNING)
return None
except requests.exceptions.ConnectionError as e:
- logger.log(u"Connection error to getURL %s Error: %s" % (url, ex(e)), logger.WARNING)
+ logger.log(u"Connection error to getURL %s Error: %r" % (url, ex(e)), logger.WARNING)
return None
except requests.exceptions.Timeout as e:
- logger.log(u"Connection timed out accessing getURL %s Error: %s" % (url, ex(e)), logger.WARNING)
+ logger.log(u"Connection timed out accessing getURL %s Error: %r" % (url, ex(e)), logger.WARNING)
return None
except requests.exceptions.ContentDecodingError:
logger.log(u"Content-Encoding was gzip, but content was not compressed. getURL: %s" % url, logger.DEBUG)
logger.log(traceback.format_exc(), logger.DEBUG)
return None
except Exception as e:
- logger.log(u"Unknown exception in getURL %s Error: %s" % (url, ex(e)), logger.WARNING)
+ logger.log(u"Unknown exception in getURL %s Error: %r" % (url, ex(e)), logger.WARNING)
logger.log(traceback.format_exc(), logger.WARNING)
return None
@@ -1676,8 +1670,7 @@ def download_file(url, filename, session=None, headers={}):
try:
with closing(session.get(url, allow_redirects=True, verify=session.verify)) as resp:
if not resp.ok:
- logger.log(u"Requested download url " + url + " returned status code is " + str(
- resp.status_code) + ': ' + codeDescription(resp.status_code), logger.DEBUG)
+ logger.log(u"Requested download url %s returned status code is %s: %s" % ( url, resp.status_code, codeDescription(resp.status_code) ) , logger.DEBUG)
return False
try:
@@ -1693,23 +1686,23 @@ def download_file(url, filename, session=None, headers={}):
except requests.exceptions.HTTPError as e:
_remove_file_failed(filename)
- logger.log(u"HTTP error " + ex(e) + " while loading download URL " + url, logger.WARNING)
+ logger.log(u"HTTP error %r while loading download URL %s " % (ex(e), url ), logger.WARNING)
return False
except requests.exceptions.ConnectionError as e:
_remove_file_failed(filename)
- logger.log(u"Connection error " + ex(e) + " while loading download URL " + url, logger.WARNING)
+ logger.log(u"Connection error %r while loading download URL %s " % (ex(e), url), logger.WARNING)
return False
except requests.exceptions.Timeout as e:
_remove_file_failed(filename)
- logger.log(u"Connection timed out " + ex(e) + " while loading download URL " + url, logger.WARNING)
+ logger.log(u"Connection timed out %r while loading download URL %s " % (ex(e), url ), logger.WARNING)
return False
except EnvironmentError as e:
_remove_file_failed(filename)
- logger.log(u"Unable to save the file: " + ex(e), logger.WARNING)
+ logger.log(u"Unable to save the file: %r " % ex(e), logger.WARNING)
return False
except Exception:
_remove_file_failed(filename)
- logger.log(u"Unknown exception while loading download URL " + url + ": " + traceback.format_exc(), logger.WARNING)
+ logger.log(u"Unknown exception while loading download URL %s : %r" % ( url, traceback.format_exc() ), logger.WARNING)
return False
return True
@@ -1733,7 +1726,7 @@ def get_size(start_path='.'):
try:
total_size += ek(os.path.getsize, fp)
except OSError as e:
- logger.log('Unable to get size for file %s Error: %s' % (fp, ex(e)), logger.ERROR)
+ logger.log(u"Unable to get size for file %s Error: %r" % (fp, ex(e)), logger.ERROR)
logger.log(traceback.format_exc(), logger.DEBUG)
return total_size
@@ -1855,7 +1848,8 @@ def verify_freespace(src, dest, oldfile=None):
if diskfree > neededspace:
return True
else:
- logger.log("Not enough free space: Needed: " + str(neededspace) + " bytes (" + pretty_filesize(neededspace) + "), found: " + str(diskfree) + " bytes (" + pretty_filesize(diskfree) + ")", logger.WARNING)
+ logger.log("Not enough free space: Needed: %s bytes ( %s ), found: %s bytes ( %s )" % ( neededspace, pretty_filesize(neededspace), diskfree, pretty_filesize(diskfree) ) ,
+ logger.WARNING)
return False
# https://gist.github.com/thatalextaylor/7408395
diff --git a/sickbeard/metadata/kodi_12plus.py b/sickbeard/metadata/kodi_12plus.py
index 3a95ed1d9c388a28d5046e6d62452ec0b727847e..288b5e52dd1253021f25a7cca5ef0c0301603365 100644
--- a/sickbeard/metadata/kodi_12plus.py
+++ b/sickbeard/metadata/kodi_12plus.py
@@ -134,16 +134,16 @@ class KODI_12PlusMetadata(generic.GenericMetadata):
show_obj.indexer).name + ", skipping it", logger.ERROR)
return False
- title = etree.SubElement(tv_node, "title")
if getattr(myShow, 'seriesname', None) is not None:
+ title = etree.SubElement(tv_node, "title")
title.text = myShow["seriesname"]
- rating = etree.SubElement(tv_node, "rating")
if getattr(myShow, 'rating', None) is not None:
+ rating = etree.SubElement(tv_node, "rating")
rating.text = myShow["rating"]
- year = etree.SubElement(tv_node, "year")
if getattr(myShow, 'firstaired', None) is not None:
+ year = etree.SubElement(tv_node, "year")
try:
year_text = str(datetime.datetime.strptime(myShow["firstaired"], dateFormat).year)
if year_text:
@@ -151,48 +151,51 @@ class KODI_12PlusMetadata(generic.GenericMetadata):
except:
pass
- plot = etree.SubElement(tv_node, "plot")
if getattr(myShow, 'overview', None) is not None:
+ plot = etree.SubElement(tv_node, "plot")
plot.text = myShow["overview"]
- episodeguide = etree.SubElement(tv_node, "episodeguide")
- episodeguideurl = etree.SubElement(episodeguide, "url")
if getattr(myShow, 'id', None) is not None:
+ episodeguide = etree.SubElement(tv_node, "episodeguide")
+ episodeguideurl = etree.SubElement(episodeguide, "url")
episodeguideurl.text = sickbeard.indexerApi(show_obj.indexer).config['base_url'] + str(myShow["id"]) + '/all/en.zip'
- mpaa = etree.SubElement(tv_node, "mpaa")
if getattr(myShow, 'contentrating', None) is not None:
+ mpaa = etree.SubElement(tv_node, "mpaa")
mpaa.text = myShow["contentrating"]
- indexerid = etree.SubElement(tv_node, "id")
if getattr(myShow, 'id', None) is not None:
+ indexerid = etree.SubElement(tv_node, "id")
indexerid.text = str(myShow["id"])
- genre = etree.SubElement(tv_node, "genre")
if getattr(myShow, 'genre', None) is not None:
+ genre = etree.SubElement(tv_node, "genre")
if isinstance(myShow["genre"], basestring):
genre.text = " / ".join(x.strip() for x in myShow["genre"].split('|') if x.strip())
- premiered = etree.SubElement(tv_node, "premiered")
if getattr(myShow, 'firstaired', None) is not None:
+ premiered = etree.SubElement(tv_node, "premiered")
premiered.text = myShow["firstaired"]
- studio = etree.SubElement(tv_node, "studio")
if getattr(myShow, 'network', None) is not None:
- studio.text = myShow["network"]
+ studio = etree.SubElement(tv_node, "studio")
+ studio.text = myShow["network"].strip()
if getattr(myShow, '_actors', None) is not None:
for actor in myShow['_actors']:
cur_actor = etree.SubElement(tv_node, "actor")
- cur_actor_name = etree.SubElement(cur_actor, "name")
- cur_actor_name.text = actor['name'].strip()
+ if 'name' in actor and actor['name'].strip():
+ cur_actor_name = etree.SubElement(cur_actor, "name")
+ cur_actor_name.text = actor['name'].strip()
- cur_actor_role = etree.SubElement(cur_actor, "role")
- cur_actor_role.text = actor['role']
+ if 'role' in actor and actor['role'].strip():
+ cur_actor_role = etree.SubElement(cur_actor, "role")
+ cur_actor_role.text = actor['role'].strip()
- cur_actor_thumb = etree.SubElement(cur_actor, "thumb")
- cur_actor_thumb.text = actor['image']
+ if 'image' in actor and actor['image'].strip():
+ cur_actor_thumb = etree.SubElement(cur_actor, "thumb")
+ cur_actor_thumb.text = actor['image'].strip()
# Make it purdy
helpers.indentXML(tv_node)
@@ -262,13 +265,13 @@ class KODI_12PlusMetadata(generic.GenericMetadata):
else:
episode = rootNode
- title = etree.SubElement(episode, "title")
- if curEpToWrite.name != None:
- title.text = curEpToWrite.name
+ if getattr(myEp, 'episodename', None) is not None:
+ title = etree.SubElement(episode, "title")
+ title.text = myEp['episodename']
- showtitle = etree.SubElement(episode, "showtitle")
- if curEpToWrite.show.name != None:
- showtitle.text = curEpToWrite.show.name
+ if getattr(myShow, 'seriesname', None) is not None:
+ showtitle = etree.SubElement(episode, "showtitle")
+ showtitle.text = myShow['seriesname']
season = etree.SubElement(episode, "season")
season.text = str(curEpToWrite.season)
@@ -279,81 +282,74 @@ class KODI_12PlusMetadata(generic.GenericMetadata):
uniqueid = etree.SubElement(episode, "uniqueid")
uniqueid.text = str(curEpToWrite.indexerid)
- aired = etree.SubElement(episode, "aired")
if curEpToWrite.airdate != datetime.date.fromordinal(1):
+ aired = etree.SubElement(episode, "aired")
aired.text = str(curEpToWrite.airdate)
- else:
- aired.text = ''
- plot = etree.SubElement(episode, "plot")
- if curEpToWrite.description != None:
- plot.text = curEpToWrite.description
+ if getattr(myEp, 'overview', None) is not None:
+ plot = etree.SubElement(episode, "plot")
+ plot.text = myEp['overview']
- runtime = etree.SubElement(episode, "runtime")
if curEpToWrite.season != 0:
if getattr(myShow, 'runtime', None) is not None:
+ runtime = etree.SubElement(episode, "runtime")
runtime.text = myShow["runtime"]
- displayseason = etree.SubElement(episode, "displayseason")
if getattr(myEp, 'airsbefore_season', None) is not None:
+ displayseason = etree.SubElement(episode, "displayseason")
displayseason_text = myEp['airsbefore_season']
if displayseason_text != None:
displayseason.text = displayseason_text
- displayepisode = etree.SubElement(episode, "displayepisode")
if getattr(myEp, 'airsbefore_episode', None) is not None:
+ displayepisode = etree.SubElement(episode, "displayepisode")
displayepisode_text = myEp['airsbefore_episode']
if displayepisode_text != None:
displayepisode.text = displayepisode_text
- thumb = etree.SubElement(episode, "thumb")
- thumb_text = getattr(myEp, 'filename', None)
- if thumb_text != None:
- thumb.text = thumb_text
+ if getattr(myEp, 'filename', None) is not None:
+ thumb = etree.SubElement(episode, "thumb")
+ thumb.text = myEp['filename'].strip()
- watched = etree.SubElement(episode, "watched")
- watched.text = 'false'
+ #watched = etree.SubElement(episode, "watched")
+ #watched.text = 'false'
- credits = etree.SubElement(episode, "credits")
- credits_text = getattr(myEp, 'writer', None)
- if credits_text != None:
- credits.text = credits_text
+ if getattr(myEp, 'writer', None) is not None:
+ credits = etree.SubElement(episode, "credits")
+ credits.text = myEp['writer'].strip()
- director = etree.SubElement(episode, "director")
- director_text = getattr(myEp, 'director', None)
- if director_text is not None:
- director.text = director_text
+ if getattr(myEp, 'director', None) is not None:
+ director = etree.SubElement(episode, "director")
+ director.text = myEp['director'].strip()
- rating = etree.SubElement(episode, "rating")
- rating_text = getattr(myEp, 'rating', None)
- if rating_text != None:
- rating.text = rating_text
+ if getattr(myEp, 'rating', None) is not None:
+ rating = etree.SubElement(episode, "rating")
+ rating.text = myEp['rating']
- gueststar_text = getattr(myEp, 'gueststars', None)
- if isinstance(gueststar_text, basestring):
- for actor in (x.strip() for x in gueststar_text.split('|') if x.strip()):
- cur_actor = etree.SubElement(episode, "actor")
- cur_actor_name = etree.SubElement(cur_actor, "name")
- cur_actor_name.text = actor
+ if getattr(myEp, 'gueststars', None) is not None:
+ if isinstance( myEp['gueststars'], basestring):
+ for actor in (x.strip() for x in myEp['gueststars'].split('|') if x.strip()):
+ cur_actor = etree.SubElement(episode, "actor")
+ cur_actor_name = etree.SubElement(cur_actor, "name")
+ cur_actor_name.text = actor
- if getattr(myEp, '_actors', None) is not None:
+ if getattr(myShow, '_actors', None) is not None:
for actor in myShow['_actors']:
cur_actor = etree.SubElement(episode, "actor")
- cur_actor_name = etree.SubElement(cur_actor, "name")
- cur_actor_name_text = actor['name']
- if isinstance(cur_actor_name_text, basestring):
- cur_actor_name.text = cur_actor_name_text.strip()
+ if 'name' in actor and actor['name'].strip():
+ cur_actor_name = etree.SubElement(cur_actor, "name")
+ cur_actor_name.text = actor['name'].strip()
+ else:
+ continue
- cur_actor_role = etree.SubElement(cur_actor, "role")
- cur_actor_role_text = actor['role']
- if cur_actor_role_text != None:
- cur_actor_role.text = cur_actor_role_text
+ if 'role' in actor and actor['role'].strip():
+ cur_actor_role = etree.SubElement(cur_actor, "role")
+ cur_actor_role.text = actor['role'].strip()
- cur_actor_thumb = etree.SubElement(cur_actor, "thumb")
- cur_actor_thumb_text = actor['image']
- if cur_actor_thumb_text != None:
- cur_actor_thumb.text = cur_actor_thumb_text
+ if 'image' in actor and actor['image'].strip():
+ cur_actor_thumb = etree.SubElement(cur_actor, "thumb")
+ cur_actor_thumb.text = actor['image'].strip()
# Make it purdy
helpers.indentXML(rootNode)
diff --git a/sickbeard/postProcessor.py b/sickbeard/postProcessor.py
index 1bef2888c24e13c0052477f699cb4f5943ffc446..df6d9db731cfff5f326b78f0de2258be85e57df4 100644
--- a/sickbeard/postProcessor.py
+++ b/sickbeard/postProcessor.py
@@ -1075,6 +1075,7 @@ class PostProcessor(object):
for cur_ep in [ep_obj] + ep_obj.relatedEps:
with cur_ep.lock:
cur_ep.location = ek(os.path.join, dest_path, new_file_name)
+ cur_ep.refreshSubtitles()
cur_ep.downloadSubtitles(force=True)
# now that processing has finished, we can put the info in the DB. If we do it earlier, then when processing fails, it won't try again.
diff --git a/sickbeard/providers/extratorrent.py b/sickbeard/providers/extratorrent.py
index 193a821a7f7932b2cf0b21bb89c176847caa8105..11872f6e372ae046ce661a2c4d4a5d8097ffc478 100644
--- a/sickbeard/providers/extratorrent.py
+++ b/sickbeard/providers/extratorrent.py
@@ -1,7 +1,7 @@
# Author: duramato <matigonkas@outlook.com>
# Author: miigotu
# URL: https://github.com/SiCKRAGETV/sickrage
-# This file is part of SickRage.
+# This file is part of SickRage.
#
# SickRage is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -80,7 +80,8 @@ class ExtraTorrentProvider(generic.TorrentProvider):
continue
try:
- data = xmltodict.parse(data)
+ # Must replace non-breaking space, as there is no xml DTD
+ data = xmltodict.parse(data.replace(' ',' '))
except ExpatError as e:
logger.log(u"Failed parsing provider. Traceback: %r\n%r" % (traceback.format_exc(), data), logger.ERROR)
continue
diff --git a/sickbeard/providers/fnt.py b/sickbeard/providers/fnt.py
index 6c329cde36cd9247ed79636abb830fe1bd27e0df..1578af190632dc8d2a1c0ad84c2090533cc52eac 100644
--- a/sickbeard/providers/fnt.py
+++ b/sickbeard/providers/fnt.py
@@ -81,7 +81,7 @@ class FNTProvider(generic.TorrentProvider):
logger.log(u"Unable to connect to provider", logger.WARNING)
return False
- if re.search('/account-logout.php', response):
+ if not re.search('Pseudo ou mot de passe non valide', response):
return True
else:
logger.log(u"Invalid username or password. Check your settings", logger.WARNING)
diff --git a/sickbeard/providers/kat.py b/sickbeard/providers/kat.py
index 7a8d86d3f60101506310b27d0247675ce952cf51..e90397007e51d9ab6c969d981580f84c844df5fa 100644
--- a/sickbeard/providers/kat.py
+++ b/sickbeard/providers/kat.py
@@ -88,7 +88,7 @@ class KATProvider(generic.TorrentProvider):
try:
searchURL = self.urls[('search', 'rss')[mode == 'RSS']] + '?' + urlencode(self.search_params)
- logger.log(u"Search URL: %s" % searchURL, logger.DEBUG)
+ logger.log(u"Search URL: %s" % searchURL, logger.DEBUG)
data = self.getURL(searchURL)
#data = self.getURL(self.urls[('search', 'rss')[mode == 'RSS']], params=self.search_params)
if not data:
@@ -96,7 +96,8 @@ class KATProvider(generic.TorrentProvider):
continue
try:
- data = xmltodict.parse(data)
+ # Must replace non-breaking space, as there is no xml DTD
+ data = xmltodict.parse(data.replace(' ',' '))
except ExpatError as e:
logger.log(u"Failed parsing provider. Traceback: %r\n%r" % (traceback.format_exc(), data), logger.ERROR)
continue
diff --git a/sickbeard/providers/nyaatorrents.py b/sickbeard/providers/nyaatorrents.py
index 086c12553eaa3141d8bc6b31319676308eccf3d0..407e212d621cecc266f4f847bb980ceaa5b9b0db 100644
--- a/sickbeard/providers/nyaatorrents.py
+++ b/sickbeard/providers/nyaatorrents.py
@@ -49,6 +49,11 @@ class NyaaProvider(generic.TorrentProvider):
def isEnabled(self):
return self.enabled
+ def getQuality(self, item, anime=False):
+ title = item.get('title')
+ quality = Quality.sceneQuality(title, anime)
+ return quality
+
def findSearchResults(self, show, episodes, search_mode, manualSearch=False, downCurQuality=False):
return generic.TorrentProvider.findSearchResults(self, show, episodes, search_mode, manualSearch, downCurQuality)
diff --git a/sickbeard/scene_exceptions.py b/sickbeard/scene_exceptions.py
index dbf7b665e4f78c736f72c4d791b0e8c53cfe73ea..f3a111eaf062e2dd9104d9cb82af2f6cac67c777 100644
--- a/sickbeard/scene_exceptions.py
+++ b/sickbeard/scene_exceptions.py
@@ -309,7 +309,7 @@ def _xem_exceptions_fetcher():
for indexer in sickbeard.indexerApi().indexers:
logger.log(u"Checking for XEM scene exception updates for " + sickbeard.indexerApi(indexer).name)
- url = "http://thexem.de/map/allNames?origin=%s&seasonNumbers=1&language=us" % sickbeard.indexerApi(indexer).config[
+ url = "http://thexem.de/map/allNames?origin=%s&seasonNumbers=1" % sickbeard.indexerApi(indexer).config[
'xem_origin']
parsedJSON = helpers.getURL(url, session=xem_session, timeout = 90, json=True)
diff --git a/sickbeard/show_name_helpers.py b/sickbeard/show_name_helpers.py
index 9fe166c3b01976b864e263ee74d048075e2b2c41..710652736aa6f94eb39775aab7c19660956b0519 100644
--- a/sickbeard/show_name_helpers.py
+++ b/sickbeard/show_name_helpers.py
@@ -248,6 +248,9 @@ def makeSceneSearchString(show, ep_obj):
if len(ep_obj.show.release_groups.whitelist) > 0:
for keyword in ep_obj.show.release_groups.whitelist:
toReturn.append(keyword + '.' + curShow + '.' + curEpString)
+ elif len(ep_obj.show.release_groups.blacklist) == 0:
+ # If we have neither whitelist or blacklist we just append what we have
+ toReturn.append(curShow + '.' + curEpString)
else:
toReturn.append(curShow + '.' + curEpString)
diff --git a/sickbeard/subtitles.py b/sickbeard/subtitles.py
index 57a6edec1d1f36d5d15f5bc37621ccc7f742ad8c..25747df81af83cb8e13c2e05c2d9dff06ed81c14 100644
--- a/sickbeard/subtitles.py
+++ b/sickbeard/subtitles.py
@@ -18,18 +18,43 @@
import datetime
import sickbeard
+import traceback
+import pkg_resources
+import subliminal
+import subprocess
from sickbeard.common import *
from sickbeard import logger
+from sickbeard import history
from sickbeard import db
from sickrage.helper.common import dateTimeFormat
from sickrage.helper.encoding import ek
from sickrage.helper.exceptions import ex
+from subliminal.api import provider_manager
from enzyme import MKV, MalformedMKVError
from babelfish import Error as BabelfishError, Language, language_converters
-import subliminal
-import subprocess
-subliminal.cache_region.configure('dogpile.cache.memory')
+distribution = pkg_resources.Distribution(location=os.path.dirname(os.path.dirname(__file__)),
+ project_name='fake_entry_points', version='1.0.0')
+
+entry_points = {
+ 'subliminal.providers': [
+ 'addic7ed = subliminal.providers.addic7ed:Addic7edProvider',
+ 'opensubtitles = subliminal.providers.opensubtitles:OpenSubtitlesProvider',
+ 'podnapisi = subliminal.providers.podnapisi:PodnapisiProvider',
+ 'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider',
+ 'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider'
+ ],
+ 'babelfish.language_converters': [
+ 'addic7ed = subliminal.converters.addic7ed:Addic7edConverter',
+ 'tvsubtitles = subliminal.converters.tvsubtitles:TVsubtitlesConverter'
+ ]
+}
+distribution._ep_map = pkg_resources.EntryPoint.parse_map(entry_points, distribution)
+pkg_resources.working_set.add(distribution)
+
+provider_manager.ENTRY_POINT_CACHE.pop('subliminal.providers')
+
+subliminal.region.configure('dogpile.cache.memory')
provider_urls = {
'addic7ed': 'http://www.addic7ed.com',
@@ -41,14 +66,13 @@ provider_urls = {
SINGLE = 'und'
-
def sortedServiceList():
newList = []
lmgtfy = 'http://lmgtfy.com/?q=%s'
curIndex = 0
for curService in sickbeard.SUBTITLES_SERVICES_LIST:
- if curService in subliminal.provider_manager.available_providers:
+ if curService in subliminal.provider_manager.names():
newList.append({'name': curService,
'url': provider_urls[curService] if curService in provider_urls else lmgtfy % curService,
'image': curService + '.png',
@@ -56,7 +80,7 @@ def sortedServiceList():
})
curIndex += 1
- for curService in subliminal.provider_manager.available_providers:
+ for curService in subliminal.provider_manager.names():
if curService not in [x['name'] for x in newList]:
newList.append({'name': curService,
'url': provider_urls[curService] if curService in provider_urls else lmgtfy % curService,
@@ -83,64 +107,122 @@ def isValidLanguage(language):
def getLanguageName(language):
return fromietf(language).name
+def downloadSubtitles(subtitles_info):
+ existing_subtitles = subtitles_info['subtitles']
+ # First of all, check if we need subtitles
+ languages = getNeededLanguages(existing_subtitles)
+ if not languages:
+ logger.log(u'%s: No missing subtitles for S%02dE%02d' % (subtitles_info['show.indexerid'], subtitles_info['season'], subtitles_info['episode']), logger.DEBUG)
+ return existing_subtitles, None
+
+ subtitles_path = getSubtitlesPath(subtitles_info['location']).encode(sickbeard.SYS_ENCODING)
+ video_path = subtitles_info['location'].encode(sickbeard.SYS_ENCODING)
+ providers = getEnabledServiceList()
+
+ try:
+ video = subliminal.scan_video(video_path, subtitles=False, embedded_subtitles=False)
+ except Exception:
+ logger.log(u'%s: Exception caught in subliminal.scan_video for S%02dE%02d' %
+ (subtitles_info['indexerid'], subtitles_info['season'], subtitles_info['episode']), logger.DEBUG)
+ return
+
+ try:
+ # TODO: Add gui option for hearing_impaired parameter ?
+ found_subtitles = subliminal.download_best_subtitles([video], languages=languages, hearing_impaired=False, only_one=not sickbeard.SUBTITLES_MULTI, providers=providers)
+ if not found_subtitles:
+ logger.log(u'%s: No subtitles found for S%02dE%02d on any provider' % (subtitles_info['show.indexerid'], subtitles_info['season'], subtitles_info['episode']), logger.DEBUG)
+ return
+
+ subliminal.save_subtitles(video, found_subtitles[video], directory=subtitles_path, single=not sickbeard.SUBTITLES_MULTI)
+
+ for video, subtitles in found_subtitles.iteritems():
+ for subtitle in subtitles:
+ new_video_path = subtitles_path + "/" + video.name.rsplit("/", 1)[-1]
+ new_subtitles_path = subliminal.subtitle.get_subtitle_path(new_video_path, subtitle.language if sickbeard.SUBTITLES_MULTI else None)
+ sickbeard.helpers.chmodAsParent(new_subtitles_path)
+ sickbeard.helpers.fixSetGroupID(new_subtitles_path)
+
+ if not sickbeard.EMBEDDED_SUBTITLES_ALL and sickbeard.SUBTITLES_EXTRA_SCRIPTS and video_path.endswith(('.mkv','.mp4')):
+ run_subs_extra_scripts(subtitles_info, found_subtitles)
+
+ current_subtitles = subtitlesLanguages(video_path)
+ new_subtitles = frozenset(current_subtitles).difference(existing_subtitles)
+
+ except Exception as e:
+ logger.log("Error occurred when downloading subtitles for: %s" % video_path)
+ logger.log(traceback.format_exc(), logger.ERROR)
+ return
+
+ if sickbeard.SUBTITLES_HISTORY:
+ for video, subtitles in found_subtitles.iteritems():
+ for subtitle in subtitles:
+ logger.log(u'history.logSubtitle %s, %s' % (subtitle.provider_name, subtitle.language.opensubtitles), logger.DEBUG)
+ history.logSubtitle(subtitles_info['show.indexerid'], subtitles_info['season'], subtitles_info['episode'], subtitles_info['status'], subtitle)
+
+ return (current_subtitles, new_subtitles)
+
+def getNeededLanguages(current_subtitles):
+ languages = set()
+ for language in frozenset(wantedLanguages()).difference(current_subtitles):
+ languages.add(fromietf(language))
+
+ return languages
+
# TODO: Filter here for non-languages in sickbeard.SUBTITLES_LANGUAGES
def wantedLanguages(sqlLike = False):
wantedLanguages = [x for x in sorted(sickbeard.SUBTITLES_LANGUAGES) if x in subtitleCodeFilter()]
if sqlLike:
return '%' + ','.join(wantedLanguages) + '%'
+
return wantedLanguages
+def getSubtitlesPath(video_path):
+ if sickbeard.SUBTITLES_DIR and ek(os.path.exists, sickbeard.SUBTITLES_DIR):
+ new_subtitles_path = sickbeard.SUBTITLES_DIR
+ elif sickbeard.SUBTITLES_DIR:
+ new_subtitles_path = ek(os.path.join, ek(os.path.dirname, video_path), sickbeard.SUBTITLES_DIR)
+ dir_exists = sickbeard.helpers.makeDir(new_subtitles_path)
+ if not dir_exists:
+ logger.log(u'Unable to create subtitles folder ' + new_subtitles_path, logger.ERROR)
+ else:
+ sickbeard.helpers.chmodAsParent(new_subtitles_path)
+ else:
+ new_subtitles_path = ek(os.path.join, ek(os.path.dirname, video_path))
+
+ return new_subtitles_path
+
def subtitlesLanguages(video_path):
"""Return a list detected subtitles for the given video file"""
resultList = []
- embedded_subtitle_languages = set()
- # Serch for embedded subtitles
- if not sickbeard.EMBEDDED_SUBTITLES_ALL:
- if video_path.endswith('mkv'):
- try:
- with open(video_path.encode(sickbeard.SYS_ENCODING), 'rb') as f:
- mkv = MKV(f)
- if mkv.subtitle_tracks:
- for st in mkv.subtitle_tracks:
- if st.language:
- try:
- embedded_subtitle_languages.add(Language.fromalpha3b(st.language))
- except BabelfishError:
- logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
- embedded_subtitle_languages.add(Language('und'))
- elif st.name:
- try:
- embedded_subtitle_languages.add(Language.fromname(st.name))
- except BabelfishError:
- logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
- embedded_subtitle_languages.add(Language('und'))
- else:
- embedded_subtitle_languages.add(Language('und'))
- else:
- logger.log('MKV has no subtitle track', logger.DEBUG)
- except MalformedMKVError:
- logger.log('MKV seems to be malformed, ignoring embedded subtitles', logger.WARNING)
+ if not sickbeard.EMBEDDED_SUBTITLES_ALL and video_path.endswith('.mkv'):
+ embedded_subtitle_languages = getEmbeddedLanguages(video_path.encode(sickbeard.SYS_ENCODING))
- # Search subtitles in the absolute path
+ # Search subtitles with the absolute path
if sickbeard.SUBTITLES_DIR and ek(os.path.exists, sickbeard.SUBTITLES_DIR):
video_path = ek(os.path.join, sickbeard.SUBTITLES_DIR, ek(os.path.basename, video_path))
- # Search subtitles in the relative path
+ # Search subtitles with the relative path
elif sickbeard.SUBTITLES_DIR:
+ check_subtitles_path = ek(os.path.join, ek(os.path.dirname, video_path), sickbeard.SUBTITLES_DIR)
+ if not os.path.exists(check_subtitles_path):
+ getSubtitlesPath(video_path)
video_path = ek(os.path.join, ek(os.path.dirname, video_path), sickbeard.SUBTITLES_DIR, ek(os.path.basename, video_path))
+ else:
+ video_path = ek(os.path.join, ek(os.path.dirname, video_path), ek(os.path.basename, video_path))
- external_subtitle_languages = subliminal.video.scan_subtitle_languages(video_path)
- subtitle_languages = external_subtitle_languages.union(embedded_subtitle_languages)
-
- if (len(subtitle_languages) is 1 and len(wantedLanguages()) is 1) and Language('und') in subtitle_languages:
- subtitle_languages.remove(Language('und'))
- subtitle_languages.add(fromietf(wantedLanguages()[0]))
+ if not sickbeard.EMBEDDED_SUBTITLES_ALL and video_path.endswith('.mkv'):
+ external_subtitle_languages = scan_subtitle_languages(video_path)
+ subtitle_languages = external_subtitle_languages.union(embedded_subtitle_languages)
+ else:
+ subtitle_languages = scan_subtitle_languages(video_path)
for language in subtitle_languages:
if hasattr(language, 'opensubtitles') and language.opensubtitles:
resultList.append(language.opensubtitles)
- elif hasattr(language, 'alpha3') and language.alpha3:
- resultList.append(language.alpha3)
+ elif hasattr(language, 'alpha3b') and language.alpha3b:
+ resultList.append(language.alpha3b)
+ elif hasattr(language, 'alpha3t') and language.alpha3t:
+ resultList.append(language.alpha3t)
elif hasattr(language, 'alpha2') and language.alpha2:
resultList.append(language.alpha2)
@@ -151,6 +233,49 @@ def subtitlesLanguages(video_path):
return sorted(resultList)
+def getEmbeddedLanguages(video_path):
+ embedded_subtitle_languages = set()
+ try:
+ with open(video_path, 'rb') as f:
+ mkv = MKV(f)
+ if mkv.subtitle_tracks:
+ for st in mkv.subtitle_tracks:
+ if st.language:
+ try:
+ embedded_subtitle_languages.add(Language.fromalpha3b(st.language))
+ except BabelfishError:
+ logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
+ embedded_subtitle_languages.add(Language('und'))
+ elif st.name:
+ try:
+ embedded_subtitle_languages.add(Language.fromname(st.name))
+ except BabelfishError:
+ logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
+ embedded_subtitle_languages.add(Language('und'))
+ else:
+ embedded_subtitle_languages.add(Language('und'))
+ else:
+ logger.log('MKV has no subtitle track', logger.DEBUG)
+ except MalformedMKVError:
+ logger.log('MKV seems to be malformed, ignoring embedded subtitles', logger.WARNING)
+
+ return embedded_subtitle_languages
+
+def scan_subtitle_languages(path):
+ language_extensions = tuple('.' + c for c in language_converters['opensubtitles'].codes)
+ dirpath, filename = os.path.split(path)
+ subtitles = set()
+ for p in os.listdir(dirpath):
+ if not isinstance(p, bytes) and p.startswith(os.path.splitext(filename)[0]) and p.endswith(subliminal.video.SUBTITLE_EXTENSIONS):
+ if os.path.splitext(p)[0].endswith(language_extensions) and len(os.path.splitext(p)[0].rsplit('.', 1)[1]) is 2:
+ subtitles.add(Language.fromopensubtitles(os.path.splitext(p)[0][-2:]))
+ elif os.path.splitext(p)[0].endswith(language_extensions) and len(os.path.splitext(p)[0].rsplit('.', 1)[1]) is 3:
+ subtitles.add(Language.fromopensubtitles(os.path.splitext(p)[0][-3:]))
+ else:
+ subtitles.add(Language('und'))
+
+ return subtitles
+
# TODO: Return only languages our providers allow
def subtitleLanguageFilter():
return [Language.fromopensubtitles(language) for language in language_converters['opensubtitles'].codes if len(language) == 3]
@@ -226,7 +351,7 @@ class SubtitlesFinder():
logger.log(u'Episode not found', logger.DEBUG)
return
- previous_subtitles = epObj.subtitles
+ existing_subtitles = epObj.subtitles
try:
epObj.downloadSubtitles()
@@ -235,7 +360,7 @@ class SubtitlesFinder():
logger.log(str(e), logger.DEBUG)
return
- newSubtitles = frozenset(epObj.subtitles).difference(previous_subtitles)
+ newSubtitles = frozenset(epObj.subtitles).difference(existing_subtitles)
if newSubtitles:
logger.log(u'Downloaded subtitles for S%02dE%02d in %s' % (epToSub["season"], epToSub["episode"], ', '.join(newSubtitles)))
@@ -266,8 +391,8 @@ def run_subs_extra_scripts(epObj, foundSubs):
elif sickbeard.SUBTITLES_DIR:
subpath = ek(os.path.join, ek(os.path.dirname, subpath), sickbeard.SUBTITLES_DIR, ek(os.path.basename, subpath))
- inner_cmd = script_cmd + [video.name, subpath, sub.language.opensubtitles, epObj.show.name,
- str(epObj.season), str(epObj.episode), epObj.name, str(epObj.show.indexerid)]
+ inner_cmd = script_cmd + [video.name, subpath, sub.language.opensubtitles, epObj['show.name'],
+ str(epObj['season']), str(epObj['episode']), epObj['name'], str(epObj['show.indexerid'])]
# use subprocess to run the command and capture output
logger.log(u"Executing command: %s" % inner_cmd)
diff --git a/sickbeard/tv.py b/sickbeard/tv.py
index 300070eb4b5430953badbfe74e7ab23c099f3c0b..37646016b8aafa73e9238ce2a8063fd6e1dd8d07 100644
--- a/sickbeard/tv.py
+++ b/sickbeard/tv.py
@@ -25,7 +25,6 @@ import re
import glob
import stat
import traceback
-
import sickbeard
try:
@@ -35,8 +34,6 @@ except ImportError:
from name_parser.parser import NameParser, InvalidNameException, InvalidShowException
-import subliminal
-
try:
from send2trash import send2trash
except ImportError:
@@ -504,7 +501,7 @@ class TVShow(object):
curSeason = int(curResult["season"])
curEpisode = int(curResult["episode"])
curShowid = int(curResult['showid'])
-
+
logger.log(u"%s: loading Episodes from DB" % curShowid, logger.DEBUG)
deleteEp = False
@@ -1436,27 +1433,6 @@ class TVEpisode(object):
location = property(lambda self: self._location, _set_location)
- def getSubtitlesPath(self):
- if sickbeard.SUBTITLES_DIR and ek(os.path.exists, sickbeard.SUBTITLES_DIR):
- subs_new_path = sickbeard.SUBTITLES_DIR
- elif sickbeard.SUBTITLES_DIR:
- subs_new_path = ek(os.path.join, ek(os.path.dirname, self.location), sickbeard.SUBTITLES_DIR)
- dir_exists = helpers.makeDir(subs_new_path)
- if not dir_exists:
- logger.log(u'Unable to create subtitles folder ' + subs_new_path, logger.ERROR)
- else:
- helpers.chmodAsParent(subs_new_path)
- else:
- subs_new_path = ek(os.path.join, ek(os.path.dirname, self.location))
- return subs_new_path
-
- def getWantedLanguages(self):
- languages = set()
- for language in frozenset(subtitles.wantedLanguages()).difference(subtitles.subtitlesLanguages(self.location)):
- languages.add(subtitles.fromietf(language))
- self.refreshSubtitles()
- return languages
-
def refreshSubtitles(self):
"""Look for subtitles files and refresh the subtitles property"""
self.subtitles = subtitles.subtitlesLanguages(self.location)
@@ -1469,64 +1445,20 @@ class TVEpisode(object):
logger.log(u"%s: Downloading subtitles for S%02dE%02d" % (self.show.indexerid, self.season, self.episode), logger.DEBUG)
- previous_subtitles = self.subtitles
-
#logging.getLogger('subliminal.api').addHandler(logging.StreamHandler())
#logging.getLogger('subliminal.api').setLevel(logging.DEBUG)
#logging.getLogger('subliminal').addHandler(logging.StreamHandler())
#logging.getLogger('subliminal').setLevel(logging.DEBUG)
- try:
- subs_path = self.getSubtitlesPath();
- languages = self.getWantedLanguages();
- if not languages:
- logger.log(u'%s: No missing subtitles for S%02dE%02d' % (self.show.indexerid, self.season, self.episode), logger.DEBUG)
- return
- providers = sickbeard.subtitles.getEnabledServiceList()
- vname = self.location
- video = None
- try:
- # Never look for subtitles in the same path, as we specify the path later on
- video = subliminal.scan_video(vname.encode(sickbeard.SYS_ENCODING), subtitles=False, embedded_subtitles=False)
- except Exception:
- logger.log(u'%s: Exception caught in subliminal.scan_video for S%02dE%02d' %
- (self.show.indexerid, self.season, self.episode), logger.DEBUG)
- return
+ subtitles_info = {'location': self.location, 'subtitles': self.subtitles, 'show.indexerid': self.show.indexerid, 'season': self.season,
+ 'episode': self.episode, 'name': self.name, 'show.name': self.show.name, 'status': self.status}
- if not video:
- return
-
- # TODO: Add gui option for hearing_impaired parameter ?
- foundSubs = subliminal.download_best_subtitles([video], languages=languages, providers=providers, single=not sickbeard.SUBTITLES_MULTI, hearing_impaired=False)
- if not foundSubs:
- logger.log(u'%s: No subtitles found for S%02dE%02d on any provider' % (self.show.indexerid, self.season, self.episode), logger.DEBUG)
- return
-
- subliminal.save_subtitles(foundSubs, directory=subs_path.encode(sickbeard.SYS_ENCODING), single=not sickbeard.SUBTITLES_MULTI)
-
- for video, subs in foundSubs.iteritems():
- for sub in subs:
- # Get the file name out of video.name and use the path from above
- video_path = subs_path + "/" + video.name.rsplit("/", 1)[-1]
- subpath = subliminal.subtitle.get_subtitle_path(video_path, sub.language if sickbeard.SUBTITLES_MULTI else None)
- helpers.chmodAsParent(subpath)
- helpers.fixSetGroupID(subpath)
-
- if not sickbeard.EMBEDDED_SUBTITLES_ALL and sickbeard.SUBTITLES_EXTRA_SCRIPTS and self.location.endswith(('mkv','mp4')):
- subtitles.run_subs_extra_scripts(self, foundSubs)
-
- except Exception as e:
- logger.log("Error occurred when downloading subtitles for: %s" % self.location)
- logger.log(traceback.format_exc(), logger.ERROR)
- return
-
- self.refreshSubtitles()
+ self.subtitles, newSubtitles = subtitles.downloadSubtitles(subtitles_info)
self.subtitles_searchcount += 1 if self.subtitles_searchcount else 1
self.subtitles_lastsearch = datetime.datetime.now().strftime(dateTimeFormat)
self.saveToDB()
- newSubtitles = frozenset(self.subtitles).difference(previous_subtitles)
if newSubtitles:
subtitleList = ", ".join([subtitles.fromietf(newSub).name for newSub in newSubtitles])
logger.log(u"%s: Downloaded %s subtitles for S%02dE%02d" %
@@ -1537,14 +1469,6 @@ class TVEpisode(object):
logger.log(u"%s: No subtitles downloaded for S%02dE%02d" %
(self.show.indexerid, self.season, self.episode), logger.DEBUG)
- if sickbeard.SUBTITLES_HISTORY:
- for video, subs in foundSubs.iteritems():
- for sub in subs:
- logger.log(u'history.logSubtitle %s, %s' % (sub.provider_name, sub.language.opensubtitles), logger.DEBUG)
- history.logSubtitle(self.show.indexerid, self.season, self.episode, self.status, sub)
-
- return self.subtitles
-
def checkForMetaFiles(self):
oldhasnfo = self.hasnfo