diff --git a/gui/slick/views/layouts/main.mako b/gui/slick/views/layouts/main.mako index 1d5db82d0f0d37e990f55625adf697d2539e6faf..58bedff32f6a5a3b252b282249b8e21a8c645d25 100644 --- a/gui/slick/views/layouts/main.mako +++ b/gui/slick/views/layouts/main.mako @@ -272,7 +272,6 @@ | Daily Search: <span class="footerhighlight">${str(sickbeard.dailySearchScheduler.timeLeft()).split('.')[0]}</span> | Backlog Search: <span class="footerhighlight">${str(sickbeard.backlogSearchScheduler.timeLeft()).split('.')[0]}</span> - % if not sickbeard.BRANCH or sickbeard.BRANCH != 'master' and sickbeard.DEVELOPER and sbLogin: <div> % if has_resource_module: Memory used: <span class="footerhighlight">${sickbeard.helpers.pretty_filesize(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)}</span> | @@ -280,7 +279,6 @@ Load time: <span class="footerhighlight">${"%.4f" % (time() - sbStartTime)}s</span> / Mako: <span class="footerhighlight">${"%.4f" % (time() - makoStartTime)}s</span> | Branch: <span class="footerhighlight">${sickbeard.BRANCH}</span> </div> - % endif </div> </footer> <script type="text/javascript" src="${sbRoot}/js/lib/jquery-1.11.2.min.js?${sbPID}"></script> diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py index b35ebea9f5928b6d374e4623e253b465db0f6bea..b17639c97dd39e065851f12693d144a71bb6ebef 100644 --- a/sickbeard/helpers.py +++ b/sickbeard/helpers.py @@ -1380,7 +1380,7 @@ def getURL(url, post_data=None, params={}, headers={}, timeout=30, session=None, if not resp.ok: logger.log(u"Requested getURL " + url + " returned status code is " + str( resp.status_code) + ': ' + codeDescription(resp.status_code), logger.DEBUG) - return + return None if proxyGlypeProxySSLwarning is not None: if re.search('The site you are attempting to browse is on a secure connection', resp.text): @@ -1389,25 +1389,25 @@ def getURL(url, post_data=None, params={}, headers={}, timeout=30, session=None, if not resp.ok: logger.log(u"GlypeProxySSLwarning: Requested getURL " + url + " returned status code is " + str( resp.status_code) + ': ' + codeDescription(resp.status_code), logger.DEBUG) - return + return None except requests.exceptions.HTTPError as e: logger.log(u"HTTP error in getURL %s Error: %s" % (url, ex(e)), logger.WARNING) - return + return None except requests.exceptions.ConnectionError as e: logger.log(u"Connection error to getURL %s Error: %s" % (url, ex(e)), logger.WARNING) - return + return None except requests.exceptions.Timeout as e: logger.log(u"Connection timed out accessing getURL %s Error: %s" % (url, ex(e)), logger.WARNING) - return + return None except requests.exceptions.ContentDecodingError: logger.log(u"Content-Encoding was gzip, but content was not compressed. getURL: %s" % url, logger.DEBUG) logger.log(traceback.format_exc(), logger.DEBUG) - return + return None except Exception as e: logger.log(u"Unknown exception in getURL %s Error: %s" % (url, ex(e)), logger.WARNING) logger.log(traceback.format_exc(), logger.WARNING) - return + return None attempts = 0 while gzip and len(resp.content) > 1 and resp.content[0] == '\x1f' and resp.content[1] == '\x8b' and attempts < 3: diff --git a/sickbeard/providers/alpharatio.py b/sickbeard/providers/alpharatio.py index 8a3a43d4b0fbe7be2a3d6c4b846526df645c4134..1a8532924f042caeb317ea893ac763bfb8cf282b 100644 --- a/sickbeard/providers/alpharatio.py +++ b/sickbeard/providers/alpharatio.py @@ -82,17 +82,13 @@ class AlphaRatioProvider(generic.TorrentProvider): 'login': 'submit', } - self.session = requests.Session() - - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('Invalid Username/password', response.text) \ - or re.search('<title>Login :: AlphaRatio.cc</title>', response.text) \ - or response.status_code == 401: + if re.search('Invalid Username/password', response) \ + or re.search('<title>Login :: AlphaRatio.cc</title>', response): logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR) return False diff --git a/sickbeard/providers/bitsoup.py b/sickbeard/providers/bitsoup.py index 4aa30ea2de537544c0552e98c04e1b5c2813292b..d56d3417c4ea19d7a25aa9ba6fdc1405cb579a54 100644 --- a/sickbeard/providers/bitsoup.py +++ b/sickbeard/providers/bitsoup.py @@ -22,7 +22,6 @@ import datetime import sickbeard import generic import requests -import urllib from sickbeard.common import Quality from sickbeard import logger @@ -41,12 +40,13 @@ class BitSoupProvider(generic.TorrentProvider): def __init__(self): generic.TorrentProvider.__init__(self, "BitSoup") - self.urls = {'base_url': 'https://www.bitsoup.me', - 'login': 'https://www.bitsoup.me/takelogin.php', - 'detail': 'https://www.bitsoup.me/details.php?id=%s', - 'search': 'https://www.bitsoup.me/browse.php?search=%s%s', - 'download': 'https://bitsoup.me/%s', - } + self.urls = { + 'base_url': 'https://www.bitsoup.me', + 'login': 'https://www.bitsoup.me/takelogin.php', + 'detail': 'https://www.bitsoup.me/details.php?id=%s', + 'search': 'https://www.bitsoup.me/browse.php', + 'download': 'https://bitsoup.me/%s', + } self.url = self.urls['base_url'] @@ -60,7 +60,9 @@ class BitSoupProvider(generic.TorrentProvider): self.cache = BitSoupCache(self) - self.categories = "&c42=1&c45=1&c49=1&c7=1" + self.search_params = { + "c42": 1, "c45": 1, "c49": 1, "c7": 1 + } def isEnabled(self): return self.enabled @@ -81,21 +83,18 @@ class BitSoupProvider(generic.TorrentProvider): def _doLogin(self): - login_params = {'username': self.username, - 'password': self.password, - 'ssl': 'yes' - } - - if not self.session: - self.session = requests.Session() + login_params = { + 'username': self.username, + 'password': self.password, + 'ssl': 'yes' + } - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('Username or password incorrect', response.text): + if re.search('Username or password incorrect', response): logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR) return False @@ -149,7 +148,7 @@ class BitSoupProvider(generic.TorrentProvider): return [search_string] - def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None): + def _doSearch(self, search_strings, search_mode='eponly', epcount=0, age=0, epObj=None): results = [] items = {'Season': [], 'Episode': [], 'RSS': []} @@ -157,17 +156,12 @@ class BitSoupProvider(generic.TorrentProvider): if not self._doLogin(): return results - for mode in search_params.keys(): - for search_string in search_params[mode]: - - if isinstance(search_string, unicode): - search_string = unidecode(search_string) - - searchURL = self.urls['search'] % (urllib.quote(search_string), self.categories) - - logger.log(u"Search string: " + searchURL, logger.DEBUG) + for mode in search_strings.keys(): + for search_string in search_strings[mode]: + logger.log(u"Search string: " + search_string, logger.DEBUG) + self.search_params['search'] = search_string - data = self.getURL(searchURL) + data = self.getURL(self.urls['search'], params=self.search_params) if not data: continue @@ -201,7 +195,7 @@ class BitSoupProvider(generic.TorrentProvider): continue #Filter unseeded torrent - if mode != 'RSS' and (seeders < self.minseed or leechers < self.minleech): + if seeders < self.minseed or leechers < self.minleech: continue if not title or not download_url: @@ -273,8 +267,8 @@ class BitSoupCache(tvcache.TVCache): self.minTime = 20 def _getRSSData(self): - search_params = {'RSS': ['']} - return {'entries': self.provider._doSearch(search_params)} + search_strings = {'RSS': ['']} + return {'entries': self.provider._doSearch(search_strings)} provider = BitSoupProvider() diff --git a/sickbeard/providers/bluetigers.py b/sickbeard/providers/bluetigers.py index d15d0a7e1f9648e0b6fa5479cce2aa5897592a31..e470835f088d62af0416d6d65a662eaeeee4fb9d 100644 --- a/sickbeard/providers/bluetigers.py +++ b/sickbeard/providers/bluetigers.py @@ -23,7 +23,6 @@ import datetime from requests.auth import AuthBase import sickbeard import generic -import urllib import requests from sickbeard.bs4_parser import BS4Parser from sickbeard.common import Quality @@ -52,14 +51,18 @@ class BLUETIGERSProvider(generic.TorrentProvider): self.cache = BLUETIGERSCache(self) - self.urls = {'base_url': 'https://www.bluetigers.ca/', - 'search': 'https://www.bluetigers.ca/torrents-search.php?search=%s%s', - 'login': 'https://www.bluetigers.ca/account-login.php', - 'download': 'https://www.bluetigers.ca/torrents-details.php?id=%s&hit=1', - } + self.urls = { + 'base_url': 'https://www.bluetigers.ca/', + 'search': 'https://www.bluetigers.ca/torrents-search.php', + 'login': 'https://www.bluetigers.ca/account-login.php', + 'download': 'https://www.bluetigers.ca/torrents-details.php?id=%s&hit=1', + } + + self.search_params = { + "c16": 1, "c10": 1, "c130": 1, "c131": 1, "c17": 1, "c18": 1, "c19": 1 + } self.url = self.urls['base_url'] - self.categories = "&c16=1&c10=1&c130=1&c131=1&c17=1&c18=1&c19=1" def isEnabled(self): return self.enabled @@ -72,32 +75,26 @@ class BLUETIGERSProvider(generic.TorrentProvider): return quality def _doLogin(self): - - if any(requests.utils.dict_from_cookiejar(self.session.cookies).values()): return True - - login_params = {'username': self.username, - 'password': self.password, - 'take_login' : '1' - } - - if not self.session: - self.session = requests.Session() + login_params = { + 'username': self.username, + 'password': self.password, + 'take_login' : '1' + } logger.log('Performing authentication to BLUETIGERS', logger.DEBUG) - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('/account-logout.php', response.text): - logger.log(u'Login to ' + self.name + ' was successful.', logger.DEBUG) + if re.search('/account-logout.php', response): + logger.log(u'Login to %s was successful.' % self.name, logger.DEBUG) return True else: - logger.log(u'Login to ' + self.name + ' was unsuccessful.', logger.DEBUG) + logger.log(u'Login to %s was unsuccessful.' % self.name, logger.DEBUG) return False return True @@ -150,26 +147,16 @@ class BLUETIGERSProvider(generic.TorrentProvider): return [search_string] - def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None): - - logger.log(u"_doSearch started with ..." + str(search_params), logger.DEBUG) - + def _doSearch(self, search_strings, search_mode='eponly', epcount=0, age=0, epObj=None): results = [] items = {'Season': [], 'Episode': [], 'RSS': []} - for mode in search_params.keys(): - - for search_string in search_params[mode]: - - if isinstance(search_string, unicode): - search_string = unidecode(search_string) - - - searchURL = self.urls['search'] % (urllib.quote(search_string), self.categories) - - logger.log(u"Search string: " + searchURL, logger.DEBUG) + for mode in search_strings.keys(): + for search_string in search_strings[mode]: + logger.log(u"Search string: " + search_string, logger.DEBUG) + self.search_params['search'] = search_string - data = self.getURL(searchURL) + data = self.getURL(self.urls['search'], params=self.search_params) if not data: continue @@ -267,8 +254,8 @@ class BLUETIGERSCache(tvcache.TVCache): self.minTime = 10 def _getRSSData(self): - search_params = {'RSS': ['']} - return {'entries': self.provider._doSearch(search_params)} + search_strings = {'RSS': ['']} + return {'entries': self.provider._doSearch(search_strings)} provider = BLUETIGERSProvider() diff --git a/sickbeard/providers/fnt.py b/sickbeard/providers/fnt.py index 120f26ef3ceb60c4df0368067acbfa32c744db2c..d7feb2e1344e6d95ca80fb8d1400b7bd2627a143 100644 --- a/sickbeard/providers/fnt.py +++ b/sickbeard/providers/fnt.py @@ -1,5 +1,5 @@ # -*- coding: latin-1 -*- -# Author: raver2046 <raver2046@gmail.com> from djoole <bobby.djoole@gmail.com> +# Author: raver2046 <raver2046@gmail.com> from djoole <bobby.djoole@gmail.com> # URL: http://code.google.com/p/sickbeard/ # # This file is part of Sick Beard. @@ -23,7 +23,6 @@ import datetime from requests.auth import AuthBase import sickbeard import generic -import urllib import requests from sickbeard.bs4_parser import BS4Parser from sickbeard.common import Quality @@ -33,7 +32,6 @@ from sickbeard import show_name_helpers from sickbeard import db from sickbeard import helpers from sickbeard import classes -from unidecode import unidecode from sickbeard.helpers import sanitizeSceneName from sickbeard.exceptions import ex @@ -46,20 +44,24 @@ class FNTProvider(generic.TorrentProvider): self.enabled = False self.username = None self.password = None - self.ratio = None + self.ratio = None self.minseed = None self.minleech = None self.cache = FNTCache(self) self.urls = {'base_url': 'https://fnt.nu', - 'search': 'https://www.fnt.nu/torrents/recherche/?afficher=1&recherche=%s%s', + 'search': 'https://www.fnt.nu/torrents/recherche/', 'login': 'https://fnt.nu/account-login.php', - 'download': 'https://fnt.nu/download.php?id=%s&dl=oui', - } + } self.url = self.urls['base_url'] - self.categories = "&afficher=1&c118=1&c129=1&c119=1&c120=1&c121=1&c126=1&c137=1&c138=1&c146=1&c122=1&c110=1&c109=1&c135=1&c148=1&c153=1&c149=1&c150=1&c154=1&c155=1&c156=1&c114=1&visible=1&freeleech=0&nuke=1&3D=0&sort=size&order=desc" + self.search_params = { + "afficher": 1, "c118": 1, "c129": 1, "c119": 1, "c120": 1, "c121": 1, "c126": 1, + "c137": 1, "c138": 1, "c146": 1, "c122": 1, "c110": 1, "c109": 1, "c135": 1, "c148": 1, + "c153": 1, "c149": 1, "c150": 1, "c154": 1, "c155": 1, "c156": 1, "c114": 1, + "visible": 1, "freeleech": 0, "nuke": 1, "3D": 0, "sort": "size", "order": "desc" + } def isEnabled(self): return self.enabled @@ -71,32 +73,27 @@ class FNTProvider(generic.TorrentProvider): quality = Quality.sceneQuality(item[0], anime) return quality - def _doLogin(self): - + def _doLogin(self): if any(requests.utils.dict_from_cookiejar(self.session.cookies).values()): return True login_params = {'username': self.username, - 'password': self.password, - 'submit' : 'Se loguer' - } - - if not self.session: - self.session = requests.Session() + 'password': self.password, + 'submit' : 'Se loguer' + } logger.log('Performing authentication to FNT', logger.DEBUG) - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('/account-logout.php', response.text): + if re.search('/account-logout.php', response): logger.log(u'Login to ' + self.name + ' was successful.', logger.DEBUG) - return True + return True else: - logger.log(u'Login to ' + self.name + ' was unsuccessful.', logger.DEBUG) + logger.log(u'Login to ' + self.name + ' was unsuccessful.', logger.DEBUG) return False return True @@ -142,85 +139,74 @@ class FNTProvider(generic.TorrentProvider): else: for show_name in set(show_name_helpers.allPossibleShowNames(self.show)): ep_string = sanitizeSceneName(show_name) + '.' + \ - sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season, - 'episodenumber': ep_obj.scene_episode} + ' %s' % add_string + sickbeard.config.naming_ep_type[2] % { + 'seasonnumber': ep_obj.scene_season, + 'episodenumber': ep_obj.scene_episode + } + ' %s' % add_string - search_string['Episode'].append(re.sub('\s+', '.', ep_string)) + search_string['Episode'].append(re.sub(r'\s+', '.', ep_string)) return [search_string] - def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None): - - logger.log(u"_doSearch started with ..." + str(search_params), logger.DEBUG) - + def _doSearch(self, search_strings, search_mode='eponly', epcount=0, age=0, epObj=None): results = [] items = {'Season': [], 'Episode': [], 'RSS': []} - for mode in search_params.keys(): - - - for search_string in search_params[mode]: - - if isinstance(search_string, unicode): - search_string = unidecode(search_string) + for mode in search_strings.keys(): + for search_string in search_strings[mode]: + logger.log(u"Search string: " + search_string, logger.DEBUG) + self.search_params['recherche'] = search_string - - searchURL = self.urls['search'] % (urllib.quote(search_string), self.categories) - - logger.log(u"Search string: " + searchURL, logger.DEBUG) - - - data = self.getURL(searchURL) + data = self.getURL(self.urls['search'], params=self.search_params) if not data: continue try: with BS4Parser(data, features=["html5lib", "permissive"]) as html: result_table = html.find('table', {'id': 'tablealign3bis'}) - + if not result_table: - logger.log(u"The Data returned from " + self.name + " do not contains any torrent", - logger.DEBUG) + logger.log( + u"The Data returned from %s does not contain any torrents" % self.name, logger.DEBUG) continue - + if result_table: - rows = result_table.findAll("tr" , {"class" : "ligntorrent"} ) - + rows = result_table.findAll("tr", {"class" : "ligntorrent"} ) + for row in rows: - link = row.findAll('td')[1].find("a" , href=re.compile("fiche_film") ) - - if link: - - try: - title = link.text - logger.log(u"FNT TITLE : " + title, logger.DEBUG) - download_url = self.urls['base_url'] + "/" + row.find("a",href=re.compile("download\.php"))['href'] - except (AttributeError, TypeError): - continue - - if not title or not download_url: - continue - - try: - id = download_url.replace(self.urls['base_url'] + "/" + 'download.php?id=', '').replace('&dl=oui', '').replace('&dl=oui', '') - logger.log(u"FNT id du torrent " + str(id), logger.DEBUG) - defailseedleech = link['mtcontent'] - seeders = int(defailseedleech.split("<font color='#00b72e'>")[1].split("</font>")[0]) - logger.log(u"FNT seeders : " + str(seeders), logger.DEBUG) - leechers = int(defailseedleech.split("<font color='red'>")[1].split("</font>")[0]) - logger.log(u"FNT leechers : " + str(leechers), logger.DEBUG) - except: - logger.log(u"Unable to parse torrent id & seeders leechers " + self.name + " Traceback: " + traceback.format_exc(), logger.DEBUG) - continue - - #Filter unseeded torrent - if mode != 'RSS' and (seeders < self.minseed or leechers < self.minleech): - continue - - item = title, download_url , id, seeders, leechers - logger.log(u"Found result: " + title.replace(' ','.') + " (" + download_url + ")", logger.DEBUG) - - items[mode].append(item) + link = row.findAll('td')[1].find("a", href=re.compile("fiche_film") ) + + if link: + try: + title = link.text + logger.log(u"FNT TITLE : " + title, logger.DEBUG) + download_url = self.urls['base_url'] + "/" + row.find("a", href=re.compile("download\.php"))['href'] + except (AttributeError, TypeError): + continue + + if not title or not download_url: + continue + + try: + id = download_url.replace(self.urls['base_url'] + "/" + 'download.php?id=', '').replace('&dl=oui', '').replace('&dl=oui', '') + logger.log(u"FNT id du torrent " + str(id), logger.DEBUG) + defailseedleech = link['mtcontent'] + seeders = int(defailseedleech.split("<font color='#00b72e'>")[1].split("</font>")[0]) + logger.log(u"FNT seeders : " + str(seeders), logger.DEBUG) + leechers = int(defailseedleech.split("<font color='red'>")[1].split("</font>")[0]) + logger.log(u"FNT leechers : " + str(leechers), logger.DEBUG) + except: + logger.log(u"Unable to parse torrent id & seeders leechers " + self.name + " Traceback: " + traceback.format_exc(), logger.DEBUG) + continue + + #Filter unseeded torrent + if not seeders or seeders < self.minseed or leechers < self.minleech: + continue + + item = title, download_url , id, seeders, leechers + logger.log(u"Found result: " + title.replace(' ','.') + " (" + download_url + ")", logger.DEBUG) + + items[mode].append(item) except Exception, e: logger.log(u"Failed parsing " + self.name + " Traceback: " + traceback.format_exc(), logger.ERROR) @@ -230,7 +216,7 @@ class FNTProvider(generic.TorrentProvider): return results def _get_title_and_url(self, item): - + title, url, id, seeders, leechers = item if title: @@ -291,8 +277,8 @@ class FNTCache(tvcache.TVCache): self.minTime = 10 def _getRSSData(self): - search_params = {'RSS': ['']} - return {'entries': self.provider._doSearch(search_params)} + search_strings = {'RSS': ['']} + return {'entries': self.provider._doSearch(search_strings)} provider = FNTProvider() diff --git a/sickbeard/providers/frenchtorrentdb.py b/sickbeard/providers/frenchtorrentdb.py index 27de073dc84d710c839b0e76c83dbf25d3bfd240..ee72a6ede24fc8ac2baff5857b2e2746ef9aadab 100644 --- a/sickbeard/providers/frenchtorrentdb.py +++ b/sickbeard/providers/frenchtorrentdb.py @@ -22,11 +22,6 @@ import datetime from requests.auth import AuthBase import sickbeard import generic -import urllib - -import urllib2 -import json -import cookielib from sickbeard.bs4_parser import BS4Parser from sickbeard.common import Quality @@ -36,7 +31,6 @@ from sickbeard import show_name_helpers from sickbeard import db from sickbeard import helpers from sickbeard import classes -from unidecode import unidecode from sickbeard.helpers import sanitizeSceneName class FrenchTorrentDBProvider(generic.TorrentProvider): @@ -47,51 +41,64 @@ class FrenchTorrentDBProvider(generic.TorrentProvider): self.supportsBacklog = True - self.cj = cookielib.CookieJar() - self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj)) + self.urls = { + 'base_url': 'http://www.frenchtorrentdb.com', + } - self.urls = {'base_url': 'http://www.frenchtorrentdb.com', - 'search': 'http://www.frenchtorrentdb.com/?section=TORRENTS&exact=1&name=%s%s&submit=GO', - 'login': 'http://www.frenchtorrentdb.com/?section=LOGIN', - } self.url = self.urls['base_url'] - self.categories = "&adv_cat%5Bs%5D%5B1%5D=95&adv_cat%5Bs%5D%5B2%5D=190&adv_cat%5Bs%5D%5B3%5D=101&adv_cat%5Bs%5D%5B4%5D=191&adv_cat%5Bs%5D%5B5%5D=197&adv_cat%5Bs%5D%5B7%5D=199&adv_cat%5Bs%5D%5B8%5D=201&adv_cat%5Bs%5D%5B9%5D=128" + self.search_params = { + "adv_cat%5Bs%5D%5B1%5D": 95, + "adv_cat%5Bs%5D%5B2%5D": 190, + "adv_cat%5Bs%5D%5B3%5D": 101, + "adv_cat%5Bs%5D%5B4%5D": 191, + "adv_cat%5Bs%5D%5B5%5D": 197, + "adv_cat%5Bs%5D%5B7%5D": 199, + "adv_cat%5Bs%5D%5B8%5D": 201, + "adv_cat%5Bs%5D%5B9%5D": 128, + "section": "TORRENTS", + "exact": 1, + "submit": "GO" + } + self.enabled = False self.username = None self.password = None - self.ratio = None + self.ratio = None self.minseed = None self.minleech = None def isEnabled(self): return self.enabled - + def imageName(self): return 'frenchtorrentdb.png' - + def getQuality(self, item, anime=False): quality = Quality.sceneQuality(item[0], anime) return quality - - def _doLogin(self): - challenge = self.opener.open(self.url + '/?section=LOGIN&challenge=1') + def _doLogin(self): - rawData = challenge.read() + params = { + "section": "LOGIN", + "challenge": 1 + } - data = json.loads(rawData) + data = self.getURL(self.url, params=params, json=True) - data = urllib.urlencode({ + post_data = { 'username' : self.username, 'password' : self.password, 'secure_login': self._getSecureLogin(data['challenge']), 'hash' : data['hash'] - }) + } + + params.pop('challenge') + params['ajax'] = 1 + self.getURL(self.url, params=params, post_data=post_data) - self.opener.open(self.url + '/?section=LOGIN&ajax=1', data).read() - return True - + def _getSecureLogin(self, challenges): def fromCharCode(*args): @@ -114,23 +121,25 @@ class FrenchTorrentDBProvider(generic.TorrentProvider): return p def decodeChallenge(challenge): - challenge = urllib2.unquote(challenge) - regexGetArgs = re.compile('\'([^\']+)\',([0-9]+),([0-9]+),\'([^\']+)\'') + regexGetArgs = re.compile('\'([^\']+)\',([0-9]+),([0-9]+),\'([^\']+)\'') regexIsEncoded = re.compile('decodeURIComponent') - regexUnquote = re.compile('\'') - if (challenge == 'a'): + regexUnquote = re.compile('\'') + if challenge == 'a': return '05f' - if (re.match(regexIsEncoded, challenge) == None): + if re.match(regexIsEncoded, challenge) == None: return re.sub(regexUnquote, '', challenge) args = re.findall(regexGetArgs, challenge) - decoded = decodeString(args[0][0], args[0][1], args[0][2], args[0][3].split('|'), 0, {}) - return urllib2.unquote(decoded.decode('utf8')) + decoded = decodeString( + args[0][0], args[0][1], + args[0][2], args[0][3].split('|'), + 0, {}) + return decoded secureLogin = '' for challenge in challenges: secureLogin += decodeChallenge(challenge) return secureLogin - + def _get_episode_search_strings(self, ep_obj, add_string=''): search_string = {'Episode': []} @@ -157,17 +166,16 @@ class FrenchTorrentDBProvider(generic.TorrentProvider): else: for show_name in set(show_name_helpers.allPossibleShowNames(self.show)): ep_string = sanitizeSceneName(show_name) + '.' + \ - sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season, - 'episodenumber': ep_obj.scene_episode} + ' %s' % add_string + sickbeard.config.naming_ep_type[2] % { + 'seasonnumber': ep_obj.scene_season, + 'episodenumber': ep_obj.scene_episode + } + ' %s' % add_string - search_string['Episode'].append(re.sub('\s+', '.', ep_string)) + search_string['Episode'].append(re.sub(r'\s+', '.', ep_string)) return [search_string] - - def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None): - logger.log(u"_doSearch started with ..." + str(search_params), logger.DEBUG) - + def _doSearch(self, search_strings, search_mode='eponly', epcount=0, age=0, epObj=None): results = [] items = {'Season': [], 'Episode': [], 'RSS': []} @@ -175,20 +183,14 @@ class FrenchTorrentDBProvider(generic.TorrentProvider): if not self._doLogin(): return results - for mode in search_params.keys(): - - for search_string in search_params[mode]: - - if isinstance(search_string, unicode): - search_string = unidecode(search_string) + for mode in search_strings.keys(): + for search_string in search_strings[mode]: + logger.log(u"Search string: " + search_string, logger.DEBUG) + self.search_params['name'] = search_string - searchURL = self.urls['search'] % (urllib.quote(search_string), self.categories) - - logger.log(u"Search string: " + searchURL, logger.DEBUG) - - r = self.opener.open( searchURL ) + r = self.getURL(self.url, params=self.search_params) with BS4Parser(r, features=["html5lib", "permissive"]) as html: - resultsTable = html.find("div", { "class" : "DataGrid" }) + resultsTable = html.find("div", {"class": "DataGrid"}) logger.log(u"Page opened", logger.DEBUG) if resultsTable: @@ -199,10 +201,10 @@ class FrenchTorrentDBProvider(generic.TorrentProvider): link = row.find("a", title=True) title = link['title'] - autogetURL = self.url +'/'+ (row.find("li", { "class" : "torrents_name"}).find('a')['href'][1:]).replace('#FTD_MENU','&menu=4') - r = self.opener.open(autogetURL,'wb').read() + autogetURL = self.url +'/' + (row.find("li", {"class": "torrents_name"}).find('a')['href'][1:]).replace('#FTD_MENU' ,'&menu=4') + r = self.getURL(autogetURL) with BS4Parser(r, features=["html5lib", "permissive"]) as html: - downloadURL = html.find("div", { "class" : "autoget"}).find('a')['href'] + downloadURL = html.find("div", {"class" : "autoget"}).find('a')['href'] item = title, downloadURL logger.log(u"Download URL : " + downloadURL, logger.DEBUG) @@ -211,7 +213,7 @@ class FrenchTorrentDBProvider(generic.TorrentProvider): results += items[mode] return results - + def _get_title_and_url(self, item): title, url = item @@ -224,7 +226,7 @@ class FrenchTorrentDBProvider(generic.TorrentProvider): url = str(url).replace('&', '&') return (title, url) - + def findPropers(self, search_date=datetime.datetime.today()): results = [] @@ -290,7 +292,7 @@ class FrenchTorrentDBCache(tvcache.TVCache): self.minTime = 10 def _getRSSData(self): - search_params = {'RSS': ['']} - return {'entries': self.provider._doSearch(search_params)} + search_strings = {'RSS': ['']} + return {'entries': self.provider._doSearch(search_strings)} provider = FrenchTorrentDBProvider() diff --git a/sickbeard/providers/freshontv.py b/sickbeard/providers/freshontv.py index d2df9f91e39ad257b9d9de74e56d33dfad6fba32..4db36d8372e4008bb3f14f6e21eb91a11387828c 100644 --- a/sickbeard/providers/freshontv.py +++ b/sickbeard/providers/freshontv.py @@ -97,16 +97,12 @@ class FreshOnTVProvider(generic.TorrentProvider): 'login': 'submit' } - if not self.session: - self.session = requests.Session() - - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('/logout.php', response.text): + if re.search('/logout.php', response): logger.log(u'Login to ' + self.name + ' was successful.', logger.DEBUG) try: @@ -124,10 +120,10 @@ class FreshOnTVProvider(generic.TorrentProvider): else: logger.log(u'Login to ' + self.name + ' was unsuccessful.', logger.DEBUG) - if re.search('Username does not exist in the userbase or the account is not confirmed yet.', response.text): + if re.search('Username does not exist in the userbase or the account is not confirmed yet.', response): logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR) - if re.search('DDoS protection by CloudFlare', response.text): + if re.search('DDoS protection by CloudFlare', response): logger.log(u'Unable to login to ' + self.name + ' due to CloudFlare DDoS javascript check.', logger.ERROR) return False diff --git a/sickbeard/providers/generic.py b/sickbeard/providers/generic.py index e66be74c3925670e71fd9dba5d8d28d597057c13..037253b05acbd7a4f0cffd3e47e5a3208c8e05e3 100644 --- a/sickbeard/providers/generic.py +++ b/sickbeard/providers/generic.py @@ -24,11 +24,13 @@ import os import re import itertools import urllib -import random +from random import shuffle +from base64 import b16encode, b32decode -import sickbeard import requests +from hachoir_parser import createParser +import sickbeard from sickbeard import helpers, classes, logger, db from sickbeard.common import MULTI_EP_RESULT, SEASON_RESULT, USER_AGENT from sickbeard import tvcache @@ -36,15 +38,16 @@ from sickbeard import encodingKludge as ek from sickbeard.exceptions import ex from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException from sickbeard.common import Quality +from sickbeard.common import user_agents + -from hachoir_parser import createParser -from base64 import b16encode, b32decode class GenericProvider: NZB = "nzb" TORRENT = "torrent" def __init__(self, name): + # these need to be set in the subclass self.providerType = None self.name = name @@ -71,7 +74,8 @@ class GenericProvider: self.session = requests.Session() - self.headers = {'User-Agent': USER_AGENT} + shuffle(user_agents) + self.headers = {'User-Agent': user_agents[0]} self.btCacheURLS = [ 'http://torcache.net/torrent/{torrent_hash}.torrent', @@ -81,7 +85,7 @@ class GenericProvider: #'http://itorrents.org/torrent/{torrent_hash}.torrent', ] - random.shuffle(self.btCacheURLS) + shuffle(self.btCacheURLS) def getID(self): return GenericProvider.makeID(self.name) diff --git a/sickbeard/providers/hdtorrents.py b/sickbeard/providers/hdtorrents.py index 03582e952476b290caa579fd8134a465684723a3..4c9b17d8d7dbdc9ec2b1ab5712bfb7dfd50c0041 100644 --- a/sickbeard/providers/hdtorrents.py +++ b/sickbeard/providers/hdtorrents.py @@ -42,8 +42,6 @@ class HDTorrentsProvider(generic.TorrentProvider): self.supportsBacklog = True - self.enabled = False - self.session = requests.Session() self.username = None self.password = None self.ratio = None @@ -84,14 +82,12 @@ class HDTorrentsProvider(generic.TorrentProvider): 'pwd': self.password, 'submit': 'Confirm'} - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('You need cookies enabled to log in.', response.text) \ - or response.status_code == 401: + if re.search('You need cookies enabled to log in.', response): logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR) return False @@ -166,10 +162,12 @@ class HDTorrentsProvider(generic.TorrentProvider): empty = html.find('No torrents here') if empty: + logger.log(u"Could not find any torrents", logger.ERROR) continue tables = html.find('table', attrs={'class': 'mainblockcontenttt'}) if not tables: + logger.log(u"Could not find table of torrents mainblockcontenttt", logger.ERROR) continue torrents = tables.findChildren('tr') diff --git a/sickbeard/providers/hounddawgs.py b/sickbeard/providers/hounddawgs.py index 74205967035c0a0e4a10997674b77e64edf3e1c0..a3d355406a578dec5411d16098370a1b051829b3 100644 --- a/sickbeard/providers/hounddawgs.py +++ b/sickbeard/providers/hounddawgs.py @@ -52,15 +52,29 @@ class HoundDawgsProvider(generic.TorrentProvider): self.minleech = None self.cache = HoundDawgsCache(self) - + self.urls = {'base_url': 'https://hounddawgs.org/', - 'search': 'https://hounddawgs.org/torrents.php?type=&userid=&searchstr=%s&searchimdb=&searchtags=&order_by=s3&order_way=desc&%s', + 'search': 'https://hounddawgs.org/torrents.php', 'login': 'https://hounddawgs.org/login.php', } self.url = self.urls['base_url'] - self.categories = "&filter_cat[85]=1&filter_cat[58]=1&filter_cat[57]=1&filter_cat[74]=1&filter_cat[92]=1&filter_cat[93]=1" + self.search_params = { + "filter_cat[85]": 1, + "filter_cat[58]": 1, + "filter_cat[57]": 1, + "filter_cat[74]": 1, + "filter_cat[92]": 1, + "filter_cat[93]": 1, + "order_by": "s3", + "order_way": "desc", + "type": '', + "userid": '', + "searchstr": '', + "searchimdb": '', + "searchtags": '' + } def isEnabled(self): return self.enabled @@ -81,20 +95,16 @@ class HoundDawgsProvider(generic.TorrentProvider): 'login': 'Login', } - self.session = requests.Session() - - try: - self.session.get(self.urls['base_url'], timeout=30) - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + self.getURL(self.urls['base_url'], timeout=30) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to provider.', logger.ERROR) return False - if re.search('Dit brugernavn eller kodeord er forkert.', response.text) \ - or re.search('<title>Login :: HoundDawgs</title>', response.text) \ - or re.search('Dine cookies er ikke aktiveret.', response.text) \ - or response.status_code == 401: - logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR) + if re.search('Dit brugernavn eller kodeord er forkert.', response) \ + or re.search('<title>Login :: HoundDawgs</title>', response) \ + or re.search('Dine cookies er ikke aktiveret.', response): + logger.log(u'Invalid username or password, check your settings', logger.ERROR) return False return True @@ -147,7 +157,7 @@ class HoundDawgsProvider(generic.TorrentProvider): return [search_string] - def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None): + def _doSearch(self, search_strings, search_mode='eponly', epcount=0, age=0, epObj=None): results = [] items = {'Season': [], 'Episode': [], 'RSS': []} @@ -155,21 +165,13 @@ class HoundDawgsProvider(generic.TorrentProvider): if not self._doLogin(): return results - for mode in search_params.keys(): - - for search_string in search_params[mode]: + for mode in search_strings.keys(): + for search_string in search_strings[mode]: + logger.log(u"Search string: " + search_string, logger.DEBUG) + self.search_params['searchstr'] = search_string - if isinstance(search_string, unicode): - search_string = unidecode(search_string) + data = self.getURL(self.urls['search'], params=self.search_params) - #if mode == 'RSS': - #searchURL = self.urls['index'] % self.categories - #else: - searchURL = self.urls['search'] % (urllib.quote(search_string), self.categories) - - logger.log(u"Search string: " + searchURL, logger.DEBUG) - - data = self.getURL(searchURL) strTableStart = "<table class=\"torrent_table" startTableIndex=data.find(strTableStart) trimmedData = data[startTableIndex:] @@ -179,22 +181,22 @@ class HoundDawgsProvider(generic.TorrentProvider): try: with BS4Parser(trimmedData, features=["html5lib", "permissive"]) as html: result_table = html.find('table', {'id': 'torrent_table'}) - + if not result_table: logger.log(u"The Data returned from " + self.name + " do not contains any torrent", logger.DEBUG) continue - + result_tbody = result_table.find('tbody') entries = result_tbody.contents - del entries[1::2] + del entries[1::2] for result in entries[1:]: - + torrent = result.find_all('td') if len(torrent) <= 1: break - + allAs = (torrent[1]).find_all('a') try: @@ -211,10 +213,10 @@ class HoundDawgsProvider(generic.TorrentProvider): title = title.replace("subs.", "") title = title.replace("SUBS.", "") title = title.replace("Subs.", "") - + download_url = self.urls['base_url']+allAs[0].attrs['href'] id = link.replace(self.urls['base_url']+'torrents.php?id=','') - + except (AttributeError, TypeError): continue @@ -288,8 +290,8 @@ class HoundDawgsCache(tvcache.TVCache): self.minTime = 20 def _getRSSData(self): - search_params = {'RSS': ['']} - return {'entries': self.provider._doSearch(search_params)} + search_strings = {'RSS': ['']} + return {'entries': self.provider._doSearch(search_strings)} provider = HoundDawgsProvider() diff --git a/sickbeard/providers/iptorrents.py b/sickbeard/providers/iptorrents.py index f88b71927675dd1826e7588730f2b700f4ed93bb..d2d0d6abfdd8aa69b285bb319e4bde7c5d5abbdd 100644 --- a/sickbeard/providers/iptorrents.py +++ b/sickbeard/providers/iptorrents.py @@ -89,16 +89,14 @@ class IPTorrentsProvider(generic.TorrentProvider): 'login': 'submit', } - try: - self.session.get(self.urls['login'], timeout=30) - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + self.getURL(self.urls['login'], timeout=30) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('tries left', response.text) \ - or re.search('<title>IPT</title>', response.text) \ - or response.status_code == 401: + if re.search('tries left', response) \ + or re.search('<title>IPT</title>', response): logger.log(u'Invalid username or password for ' + self.name + ', Check your settings!', logger.ERROR) return False diff --git a/sickbeard/providers/libertalia.py b/sickbeard/providers/libertalia.py index 5b57592993f17cb4fbd80ec7fdb22ce23ddf8e35..97c720b618c816571280dc467f11464567610ef1 100644 --- a/sickbeard/providers/libertalia.py +++ b/sickbeard/providers/libertalia.py @@ -1,5 +1,5 @@ # -*- coding: latin-1 -*- -# Authors: Raver2046 +# Authors: Raver2046 # adaur # based on tpi.py # URL: http://code.google.com/p/sickbeard/ @@ -42,25 +42,25 @@ from sickbeard.exceptions import ex class LibertaliaProvider(generic.TorrentProvider): def __init__(self): - + generic.TorrentProvider.__init__(self, "Libertalia") self.supportsBacklog = True - + self.cj = cookielib.CookieJar() - + self.url = "https://libertalia.me" self.urlsearch = "https://libertalia.me/torrents.php?name=%s%s" - + self.categories = "&cat%5B%5D=9" - + self.enabled = False self.username = None self.password = None - self.ratio = None + self.ratio = None self.minseed = None self.minleech = None - + def isEnabled(self): return self.enabled @@ -114,89 +114,86 @@ class LibertaliaProvider(generic.TorrentProvider): search_string['Episode'].append(re.sub('\s+', '.', ep_string)) return [search_string] - + def getQuality(self, item, anime=False): quality = Quality.sceneQuality(item[0], anime) return quality - + def _doLogin(self): - + if any(requests.utils.dict_from_cookiejar(self.session.cookies).values()): return True - - self.headers.update({'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.8 (KHTML, like Gecko) Chrome/17.0.940.0 Safari/535.8'}) - + login_params = {'username': self.username, 'password': self.password } logger.log('Performing authentication to Libertalia', logger.DEBUG) - try: - response = self.getURL(self.url + '/login.php', post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.url + '/login.php', post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('upload.php', response.text): + if re.search('upload.php', response): logger.log(u'Login to ' + self.name + ' was successful.', logger.DEBUG) - return True + return True else: - logger.log(u'Login to ' + self.name + ' was unsuccessful.', logger.DEBUG) + logger.log(u'Login to ' + self.name + ' was unsuccessful.', logger.DEBUG) return False return True - + def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None): - + logger.log(u"_doSearch started with ..." + str(search_params), logger.DEBUG) - + results = [] items = {'Season': [], 'Episode': [], 'RSS': []} - + # check for auth if not self._doLogin(): return results - + for mode in search_params.keys(): for search_string in search_params[mode]: if isinstance(search_string, unicode): search_string = unidecode(search_string) - + searchURL = self.urlsearch % (urllib.quote(search_string), self.categories) - + logger.log(u"Search string: " + searchURL, logger.DEBUG) - + data = self.getURL(searchURL) if not data: continue - + with BS4Parser(data, features=["html5lib", "permissive"]) as html: resultsTable = html.find("table", { "class" : "torrent_table" }) if resultsTable: - logger.log(u"Libertalia found result table ! " , logger.DEBUG) + logger.log(u"Libertalia found result table ! " , logger.DEBUG) rows = resultsTable.findAll("tr" , {"class" : "torrent_row new "} ) # torrent_row new - + for row in rows: - - #bypass first row because title only - columns = row.find('td', {"class" : "torrent_name"} ) - logger.log(u"Libertalia found rows ! " , logger.DEBUG) + + #bypass first row because title only + columns = row.find('td', {"class" : "torrent_name"} ) + logger.log(u"Libertalia found rows ! " , logger.DEBUG) isvfclass = row.find('td', {"class" : "sprite-vf"} ) - isvostfrclass = row.find('td', {"class" : "sprite-vostfr"} ) - link = columns.find("a", href=re.compile("torrents")) - if link: + isvostfrclass = row.find('td', {"class" : "sprite-vostfr"} ) + link = columns.find("a", href=re.compile("torrents")) + if link: title = link.text recherched=searchURL.replace(".","(.*)").replace(" ","(.*)").replace("'","(.*)") - logger.log(u"Libertalia title : " + title, logger.DEBUG) - downloadURL = row.find("a",href=re.compile("torrent_pass"))['href'] - logger.log(u"Libertalia download URL : " + downloadURL, logger.DEBUG) + logger.log(u"Libertalia title : " + title, logger.DEBUG) + downloadURL = row.find("a",href=re.compile("torrent_pass"))['href'] + logger.log(u"Libertalia download URL : " + downloadURL, logger.DEBUG) item = title, downloadURL items[mode].append(item) results += items[mode] - + return results def seedRatio(self): @@ -228,7 +225,7 @@ class LibertaliaProvider(generic.TorrentProvider): title, url = self._get_title_and_url(item) results.append(classes.Proper(title, url, datetime.datetime.today(), self.show)) - return results + return results def _get_title_and_url(self, item): @@ -241,6 +238,6 @@ class LibertaliaProvider(generic.TorrentProvider): if url: url = str(url).replace('&', '&') - return (title, url) + return (title, url) provider = LibertaliaProvider() diff --git a/sickbeard/providers/morethantv.py b/sickbeard/providers/morethantv.py index dab0d4c37d565e644965498c36f5becddb5d63aa..b3e29900e89e088f57347618170fa8c8f41c194e 100644 --- a/sickbeard/providers/morethantv.py +++ b/sickbeard/providers/morethantv.py @@ -100,16 +100,12 @@ class MoreThanTVProvider(generic.TorrentProvider): 'login': 'submit' } - if not self.session: - self.session = requests.Session() - - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('Your username or password was incorrect.', response.text): + if re.search('Your username or password was incorrect.', response): logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR) return False @@ -230,7 +226,7 @@ class MoreThanTVProvider(generic.TorrentProvider): except (AttributeError, TypeError): continue - + #Filter unseeded torrent if mode != 'RSS' and (seeders < self.minseed or leechers < self.minleech): continue diff --git a/sickbeard/providers/nextgen.py b/sickbeard/providers/nextgen.py index c28d6a8c94851cf9135c97c384138434021a92d9..ed4420d58a035b47f7a433fd02737cd447d4b982 100644 --- a/sickbeard/providers/nextgen.py +++ b/sickbeard/providers/nextgen.py @@ -92,10 +92,9 @@ class NextGenProvider(generic.TorrentProvider): def _doLogin(self): now = time.time() - if self.login_opener and self.last_login_check < (now - 3600): try: - output = self.login_opener.open(self.urls['test']) + output = self.getURL(self.urls['test']) if self.loginSuccess(output): self.last_login_check = now return True @@ -109,13 +108,10 @@ class NextGenProvider(generic.TorrentProvider): try: login_params = self.getLoginParams() - self.session = requests.Session() - self.session.headers.update( - {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20130519 Firefox/24.0)'}) - data = self.session.get(self.urls['login_page']) - with BS4Parser(data.content.decode('iso-8859-1')) as bs: + data = self.getURL(self.urls['login_page']) + with BS4Parser(data) as bs: csrfraw = bs.find('form', attrs={'id': 'login'})['action'] - output = self.getURL(self.urls['base_url'] + csrfraw, post_data=login_params) + output = self.getURL(self.urls['base_url'] + csrfraw, post_data=login_params) if self.loginSuccess(output): self.last_login_check = now @@ -175,7 +171,7 @@ class NextGenProvider(generic.TorrentProvider): sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season, 'episodenumber': ep_obj.scene_episode} + ' %s' % add_string - search_string['Episode'].append(re.sub('\s+', ' ', ep_string)) + search_string['Episode'].append(re.sub(r'\s+', ' ', ep_string)) return [search_string] diff --git a/sickbeard/providers/rarbg.py b/sickbeard/providers/rarbg.py index aecde2ee7e1d52ed5fe9bbb9f3ffbc84eba48dfd..3e974e3beba00504b2960965db6be221f4f09576 100644 --- a/sickbeard/providers/rarbg.py +++ b/sickbeard/providers/rarbg.py @@ -35,7 +35,6 @@ from sickbeard import db from sickbeard import helpers from sickbeard import classes from sickbeard.exceptions import ex -from requests.exceptions import RequestException from sickbeard.indexers.indexer_config import INDEXER_TVDB @@ -86,8 +85,6 @@ class RarbgProvider(generic.TorrentProvider): self.cache = RarbgCache(self) - self.headers = {'User-Agent': USER_AGENT} - def isEnabled(self): return self.enabled @@ -100,26 +97,19 @@ class RarbgProvider(generic.TorrentProvider): resp_json = None - try: - response = self.session.get(self.urls['token'], timeout=30, headers=self.headers) - response.raise_for_status() - resp_json = response.json() - except (RequestException) as e: - logger.log(u'Unable to connect to {name} provider: {error}'.format(name=self.name, error=ex(e)), logger.ERROR) + response = self.getURL(self.urls['token'], timeout=30, json=True) + if not response: + logger.log(u'Unable to connect to %s provider.' % self.name, logger.WARNING) return False - if not resp_json: - logger.log(u'{name} provider: empty json response'.format(name=self.name), logger.ERROR) - return False - else: - try: - if resp_json['token']: - self.token = resp_json['token'] - self.tokenExpireDate = datetime.datetime.now() + datetime.timedelta(minutes=14) - return True - except Exception as e: - logger.log(u'{name} provider: No token found'.format(name=self.name), logger.ERROR) - logger.log(u'{name} provider: No token found: {error}'.format(name=self.name, error=ex(e)), logger.DEBUG) + try: + if response['token']: + self.token = response['token'] + self.tokenExpireDate = datetime.datetime.now() + datetime.timedelta(minutes=14) + return True + except Exception as e: + logger.log(u'%s provider: No token found' % self.name, logger.WARNING) + logger.log(u'%s provider: No token found: %s' % (self.name, ex(e)), logger.DEBUG) return False diff --git a/sickbeard/providers/scc.py b/sickbeard/providers/scc.py index 5f92049006ec20c958863bc5c86649940a7ea502..4f5caea412366732f8dacc57c25383326454cb28 100644 --- a/sickbeard/providers/scc.py +++ b/sickbeard/providers/scc.py @@ -88,15 +88,13 @@ class SCCProvider(generic.TorrentProvider): } - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('Username or password incorrect', response.text) \ - or re.search('<title>SceneAccess \| Login</title>', response.text) \ - or response.status_code == 401: + if re.search('Username or password incorrect', response) \ + or re.search('<title>SceneAccess \| Login</title>', response): logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR) return False diff --git a/sickbeard/providers/scenetime.py b/sickbeard/providers/scenetime.py index aa3de5ce80c00482da38f1155cf7201987e1a9df..515d91448bc4396a19eed81a47d6a786afb85d1e 100644 --- a/sickbeard/providers/scenetime.py +++ b/sickbeard/providers/scenetime.py @@ -81,15 +81,12 @@ class SceneTimeProvider(generic.TorrentProvider): 'password': self.password } - self.session = requests.Session() - - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('Username or password incorrect', response.text): + if re.search('Username or password incorrect', response): logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR) return False @@ -175,7 +172,7 @@ class SceneTimeProvider(generic.TorrentProvider): logger.log(u"The Data returned from %s does not contain any torrent links" % self.name, logger.DEBUG) continue - + # Scenetime apparently uses different number of cells in #torrenttable based # on who you are. This works around that by extracting labels from the first # <tr> and using their index to find the correct download/seeders/leechers td. @@ -192,10 +189,10 @@ class SceneTimeProvider(generic.TorrentProvider): try: title = link.contents[0].get_text() - filename = "%s.torrent" % title.replace(" ", ".") - + filename = "%s.torrent" % title.replace(" ", ".") + download_url = self.urls['download'] % (torrent_id, filename) - + id = int(torrent_id) seeders = int(cells[labels.index('Seeders')].get_text()) leechers = int(cells[labels.index('Leechers')].get_text()) diff --git a/sickbeard/providers/speedcd.py b/sickbeard/providers/speedcd.py index a805591e74d962c956f564c956e06c3df42e7d09..3c8d281bb59e6e9afb1373b29e770c869dc94ac1 100644 --- a/sickbeard/providers/speedcd.py +++ b/sickbeard/providers/speedcd.py @@ -79,14 +79,12 @@ class SpeedCDProvider(generic.TorrentProvider): 'password': self.password } - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.',logger.ERROR) return False - if re.search('Incorrect username or Password. Please try again.', response.text) \ - or response.status_code == 401: + if re.search('Incorrect username or Password. Please try again.', response): logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR) return False diff --git a/sickbeard/providers/t411.py b/sickbeard/providers/t411.py index 11e9adabfcf844a0ea255a39af933417d55e24ae..508189fb579ff495be5b2880e8ec57d3d17a93a0 100644 --- a/sickbeard/providers/t411.py +++ b/sickbeard/providers/t411.py @@ -62,8 +62,6 @@ class T411Provider(generic.TorrentProvider): self.subcategories = [433, 637, 455, 639] - self.session = requests.Session() - def isEnabled(self): return self.enabled @@ -86,10 +84,9 @@ class T411Provider(generic.TorrentProvider): logger.log('Performing authentication to T411', logger.DEBUG) - try: - response = helpers.getURL(self.urls['login_page'], post_data=login_params, timeout=30, session=self.session, json=True) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.WARNING) + response = helpers.getURL(self.urls['login_page'], post_data=login_params, timeout=30, json=True) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.WARNING) return False if response and 'token' in response: diff --git a/sickbeard/providers/titansoftv.py b/sickbeard/providers/titansoftv.py index d1d9c255aa8f2ed433e541bab6629ef93bfad7c1..f71fc29f2ea1eef727bfa5c5b91f9e3cdc731daa 100644 --- a/sickbeard/providers/titansoftv.py +++ b/sickbeard/providers/titansoftv.py @@ -37,14 +37,13 @@ class TitansOfTVProvider(generic.TorrentProvider): self.cache = TitansOfTVCache(self) self.url = 'http://titansof.tv/api/torrents' self.download_url = 'http://titansof.tv/api/torrents/%s/download?apikey=%s' - self.session = requests.Session() - + def isEnabled(self): return self.enabled - + def imageName(self): return 'titansoftv.png' - + def seedRatio(self): return self.ratio diff --git a/sickbeard/providers/tntvillage.py b/sickbeard/providers/tntvillage.py index 501ea167695b83c2b9b9fc570761b54583c31a8b..6690b693a5578258ddaef0a22e6f20e29494ef3c 100644 --- a/sickbeard/providers/tntvillage.py +++ b/sickbeard/providers/tntvillage.py @@ -103,7 +103,7 @@ class TNTVillageProvider(generic.TorrentProvider): 'Anime' : 7, 'Programmi e Film TV' : 1, 'Documentari' : 14, - 'All' : 0, + 'All' : 0, } self.urls = {'base_url' : 'http://forum.tntvillage.scambioetico.org', @@ -150,15 +150,13 @@ class TNTVillageProvider(generic.TorrentProvider): 'submit': 'Connettiti al Forum', } - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except RequestException as e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('Sono stati riscontrati i seguenti errori', response.text) \ - or re.search('<title>Connettiti</title>', response.text) \ - or response.status_code == 401: + if re.search('Sono stati riscontrati i seguenti errori', response) \ + or re.search('<title>Connettiti</title>', response): logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR) return False @@ -242,7 +240,7 @@ class TNTVillageProvider(generic.TorrentProvider): file_quality='' img_all = (torrent_rows.find_all('td'))[1].find_all('img') - + if len(img_all) > 0: for img_type in img_all: try: @@ -310,7 +308,7 @@ class TNTVillageProvider(generic.TorrentProvider): if not subFound and re.search("ita", name, re.I): logger.log(u"Found Italian release", logger.DEBUG) italian = True - + return italian def _is_season_pack(self, name): @@ -361,7 +359,7 @@ class TNTVillageProvider(generic.TorrentProvider): for x in range(0,y): z=x*20 if last_page: - break + break if mode != 'RSS': searchURL = (self.urls['search_page'] + '&filter={2}').format(z,self.categories,search_string) @@ -419,7 +417,7 @@ class TNTVillageProvider(generic.TorrentProvider): break if Quality.nameQuality(title) == Quality.UNKNOWN: - title += filename_qt + title += filename_qt if not self._is_italian(result) and not self.subtitle: logger.log(u"Subtitled, skipping " + title + "(" + searchURL + ")", logger.DEBUG) diff --git a/sickbeard/providers/torrentbytes.py b/sickbeard/providers/torrentbytes.py index 165fc2d476289c2697e03fee0d2bc4f986725068..0fca46100bb3aff331e301203a145f2bfc0b36d6 100644 --- a/sickbeard/providers/torrentbytes.py +++ b/sickbeard/providers/torrentbytes.py @@ -82,15 +82,12 @@ class TorrentBytesProvider(generic.TorrentProvider): 'login': 'Log in!' } - self.session = requests.Session() - - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('Username or password incorrect', response.text): + if re.search('Username or password incorrect', response): logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR) return False diff --git a/sickbeard/providers/torrentday.py b/sickbeard/providers/torrentday.py index fb3763b90438ff2cc68e9dccb05c66098c5dc6f5..fe711df62e391ac9a48dd891a9995c84fae1a38a 100644 --- a/sickbeard/providers/torrentday.py +++ b/sickbeard/providers/torrentday.py @@ -90,23 +90,15 @@ class TorrentDayProvider(generic.TorrentProvider): 'submit.y': 0 } - if not self.session: - self.session = requests.Session() - - try: - response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('You tried too often', response.text): + if re.search('You tried too often', response): logger.log(u'Too many login access for ' + self.name + ', can''t retrive any data', logger.ERROR) return False - if response.status_code == 401: - logger.log(u'Invalid username or password for ' + self.name + ', Check your settings!', logger.ERROR) - return False - try: if requests.utils.dict_from_cookiejar(self.session.cookies)['uid'] and requests.utils.dict_from_cookiejar(self.session.cookies)['pass']: self._uid = requests.utils.dict_from_cookiejar(self.session.cookies)['uid'] diff --git a/sickbeard/providers/torrentleech.py b/sickbeard/providers/torrentleech.py index 90c2b96df523ced9b2ebd488e29be6d7e5425048..ff888b4edb93f76256330d2719012155c6708249 100644 --- a/sickbeard/providers/torrentleech.py +++ b/sickbeard/providers/torrentleech.py @@ -86,7 +86,7 @@ class TorrentLeechProvider(generic.TorrentProvider): response = self.getURL(self.urls['login'], post_data=login_params, timeout=30) if not response: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False if re.search('Invalid Username/password', response) or re.search('<title>Login :: TorrentLeech.org</title>', response): diff --git a/sickbeard/providers/xthor.py b/sickbeard/providers/xthor.py index 07c1afeb3cfff563bf01b9b97f8033063836a806..f39771e71ffffa0dcd94bcb1eca3db0543327ff5 100644 --- a/sickbeard/providers/xthor.py +++ b/sickbeard/providers/xthor.py @@ -38,13 +38,13 @@ from sickbeard.exceptions import ex class XthorProvider(generic.TorrentProvider): def __init__(self): - + generic.TorrentProvider.__init__(self, "Xthor") self.supportsBacklog = True - + self.cj = cookielib.CookieJar() - + self.url = "https://xthor.bz" self.urlsearch = "https://xthor.bz/browse.php?search=%s%s" self.categories = "&searchin=title&incldead=0" @@ -53,13 +53,13 @@ class XthorProvider(generic.TorrentProvider): self.username = None self.password = None self.ratio = None - + def isEnabled(self): return self.enabled def imageName(self): return 'xthor.png' - + def _get_season_search_strings(self, ep_obj): search_string = {'Season': []} @@ -107,7 +107,7 @@ class XthorProvider(generic.TorrentProvider): search_string['Episode'].append(re.sub('\s+', '.', ep_string)) return [search_string] - + def _get_title_and_url(self, item): title, url = item @@ -119,66 +119,60 @@ class XthorProvider(generic.TorrentProvider): if url: url = str(url).replace('&', '&') - return (title, url) - + return (title, url) + def getQuality(self, item, anime=False): quality = Quality.sceneQuality(item[0], anime) return quality - + def _doLogin(self): - + if any(requests.utils.dict_from_cookiejar(self.session.cookies).values()): return True - self.headers.update({'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.8 (KHTML, like Gecko) Chrome/17.0.940.0 Safari/535.8'}) - login_params = {'username': self.username, 'password': self.password, 'submitme': 'X' } - - if not self.session: - self.session = requests.Session() - + logger.log('Performing authentication to Xthor', logger.DEBUG) - - try: - response = self.getURL(self.url + '/takelogin.php', post_data=login_params, timeout=30) - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: - logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR) + + response = self.getURL(self.url + '/takelogin.php', post_data=login_params, timeout=30) + if not response: + logger.log(u'Unable to connect to ' + self.name + ' provider.', logger.ERROR) return False - if re.search('donate.php', response.text): + if re.search('donate.php', response): logger.log(u'Login to ' + self.name + ' was successful.', logger.DEBUG) - return True + return True else: - logger.log(u'Login to ' + self.name + ' was unsuccessful.', logger.DEBUG) + logger.log(u'Login to ' + self.name + ' was unsuccessful.', logger.DEBUG) return False - return True + return True def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None): logger.log(u"_doSearch started with ..." + str(search_params), logger.DEBUG) - + results = [] items = {'Season': [], 'Episode': [], 'RSS': []} - + # check for auth if not self._doLogin(): return results - + for mode in search_params.keys(): for search_string in search_params[mode]: if isinstance(search_string, unicode): search_string = unidecode(search_string) - + searchURL = self.urlsearch % (urllib.quote(search_string), self.categories) - + logger.log(u"Search string: " + searchURL, logger.DEBUG) - + data = self.getURL(searchURL) if not data: @@ -189,17 +183,17 @@ class XthorProvider(generic.TorrentProvider): if resultsTable: rows = resultsTable.findAll("tr") for row in rows: - link = row.find("a",href=re.compile("details.php")) - if link: + link = row.find("a",href=re.compile("details.php")) + if link: title = link.text - logger.log(u"Xthor title : " + title, logger.DEBUG) - downloadURL = self.url + '/' + row.find("a",href=re.compile("download.php"))['href'] - logger.log(u"Xthor download URL : " + downloadURL, logger.DEBUG) + logger.log(u"Xthor title : " + title, logger.DEBUG) + downloadURL = self.url + '/' + row.find("a",href=re.compile("download.php"))['href'] + logger.log(u"Xthor download URL : " + downloadURL, logger.DEBUG) item = title, downloadURL items[mode].append(item) results += items[mode] - return results - + return results + def seedRatio(self): return self.ratio @@ -229,6 +223,6 @@ class XthorProvider(generic.TorrentProvider): title, url = self._get_title_and_url(item) results.append(classes.Proper(title, url, datetime.datetime.today(), self.show)) - return results + return results provider = XthorProvider() diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index 22d03b213e62dc5538395a08624c0c9ec4b6bcd5..215b69fe8d713a247c9b04e6d3478c55e646ad8b 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -2529,7 +2529,7 @@ class HomeAddShows(Home): except Exception as e: popular_shows = None - return t.render(submenu = self.HomeMenu(), popular_shows=popular_shows, imdb_exception=e) + return t.render(title="Popular Shows", header="Popular Shows", submenu = self.HomeMenu(), popular_shows=popular_shows, imdb_exception=e) def addShowToBlacklist(self, indexer_id):