Private GIT

Skip to content
Snippets Groups Projects
Commit 665ead8f authored by Dustyn Gibson's avatar Dustyn Gibson
Browse files

Fixup HDTorrents provider

parent 8d4ceab6
Branches
Tags
No related merge requests found
Show whitespace changes
Inline Side-by-side
sickbeard/providers/hdtorrents.py
+ 119
− 144
View file @ 665ead8f
......@@ -19,7 +19,6 @@
import re
import traceback
import datetime
import urlparse
import sickbeard
import generic
......@@ -33,12 +32,14 @@ from sickbeard import helpers
from sickbeard import show_name_helpers
from sickbeard.exceptions import ex, AuthException
from sickbeard import clients
from lib import requests
from lib.requests import exceptions
from sickbeard.bs4_parser import BS4Parser
import requests
from requests import exceptions
from bs4 import BeautifulSoup as soup
#from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
from requests.auth import AuthBase
from datetime import datetime
class HDTorrentsProvider(generic.TorrentProvider):
def __init__(self):
......@@ -48,20 +49,19 @@ class HDTorrentsProvider(generic.TorrentProvider):
self.supportsBacklog = True
self.enabled = False
self._uid = None
self._hash = None
#self._uid = None
#self._hash = None
self.session = requests.Session()
self.username = None
self.password = None
self.ratio = None
self.minseed = None
self.minleech = None
self.urls = {'base_url': 'https://hdts.ru/index.php',
'login': 'https://hdts.ru/login.php',
'detail': 'https://hdts.ru/details.php?id=%s',
'search': 'https://hdts.ru/torrents.php?search=%s&active=1&options=0%s',
'download': 'https://hdts.ru/%s',
'home': 'https://hdts.ru/%s'
self.urls = {'base_url': 'https://hd-torrents.org',
'login': 'https://hd-torrents.org/login.php',
'search': 'https://hd-torrents.org/torrents.php?search=%s&active=1&options=0%s',
'home': 'https://hd-torrents.org/%s'
}
self.url = self.urls['base_url']
......@@ -70,7 +70,7 @@ class HDTorrentsProvider(generic.TorrentProvider):
self.categories = "&category[]=59&category[]=60&category[]=30&category[]=38"
self.cookies = None
#self.cookies = None
def isEnabled(self):
return self.enabled
......@@ -78,11 +78,6 @@ class HDTorrentsProvider(generic.TorrentProvider):
def imageName(self):
return 'hdtorrents.png'
def getQuality(self, item, anime=False):
quality = Quality.sceneQuality(item[0])
return quality
def _checkAuth(self):
if not self.username or not self.password:
......@@ -95,9 +90,10 @@ class HDTorrentsProvider(generic.TorrentProvider):
if any(requests.utils.dict_from_cookiejar(self.session.cookies).values()):
return True
if self._uid and self._hash:
# requests automatically handles cookies.
#if self._uid and self._hash:
requests.utils.add_dict_to_cookiejar(self.session.cookies, self.cookies)
# requests.utils.add_dict_to_cookiejar(self.session.cookies, self.cookies)
else:
......@@ -117,179 +113,140 @@ class HDTorrentsProvider(generic.TorrentProvider):
logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR)
return False
self._uid = requests.utils.dict_from_cookiejar(self.session.cookies)['uid']
self._hash = requests.utils.dict_from_cookiejar(self.session.cookies)['pass']
self.cookies = {'uid': self._uid,
'pass': self._hash
}
#self._uid = requests.utils.dict_from_cookiejar(self.session.cookies)['uid']
#self._hash = requests.utils.dict_from_cookiejar(self.session.cookies)['pass']
#self.cookies = {'uid': self._uid,
# 'pass': self._hash
#}
return True
def _get_season_search_strings(self, ep_obj):
if not ep_obj:
return search_strings
search_string = {'Season': []}
search_strings = []
for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
if ep_obj.show.air_by_date or ep_obj.show.sports:
ep_string = show_name + ' ' + str(ep_obj.airdate).split('-')[0]
elif ep_obj.show.anime:
ep_string = show_name + ' ' + "%d" % ep_obj.scene_absolute_number
else:
ep_string = show_name + ' S%02d' % int(ep_obj.scene_season) #1) showName SXX
ep_string = show_name + ' S%02d' % ep_obj.scene_season
search_string['Season'].append(ep_string)
search_strings.append(ep_string)
return [search_string]
return [search_strings]
def _get_episode_search_strings(self, ep_obj, add_string=''):
search_string = {'Episode': []}
if not ep_obj:
return []
return search_strings
if self.show.air_by_date:
search_strings = []
for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
if self.show.air_by_date:
ep_string = sanitizeSceneName(show_name) + ' ' + \
str(ep_obj.airdate).replace('-', '|')
search_string['Episode'].append(ep_string)
elif self.show.sports:
for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
ep_string = sanitizeSceneName(show_name) + ' ' + \
str(ep_obj.airdate).replace('-', '|') + '|' + \
ep_obj.airdate.strftime('%b')
search_string['Episode'].append(ep_string)
elif self.show.anime:
for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
ep_string = sanitizeSceneName(show_name) + ' ' + \
"%i" % int(ep_obj.scene_absolute_number)
search_string['Episode'].append(ep_string)
else:
for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
ep_string = show_name_helpers.sanitizeSceneName(show_name) + ' ' + \
sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season,
'episodenumber': ep_obj.scene_episode} + ' %s' % add_string
'episodenumber': ep_obj.scene_episode}
if add_string:
ep_string += ' %s' % add_string
search_string['Episode'].append(re.sub('\s+', ' ', ep_string))
search_strings.append(ep_string)
return [search_string]
return [search_strings]
def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None):
results = []
items = {'Season': [], 'Episode': [], 'RSS': []}
if not self._doLogin():
return results
for mode in search_params.keys():
for search_string in search_params[mode]:
for search_string in search_params if search_params else '':
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
if search_string == '':
continue
search_string = str(search_string).replace('.', ' ')
searchURL = self.urls['search'] % (urllib.quote(search_string), self.categories)
searchURL = self.urls['search'] % (urllib.quote_plus(search_string.replace('.', ' ')), self.categories)
logger.log(u"Search string: " + searchURL, logger.DEBUG)
data = self.getURL(searchURL)
if not data:
logger.log(u'No grabs for you', logger.DEBUG)
continue
# Remove HDTorrents NEW list
split_data = data.partition('<!-- Show New Torrents After Last Visit -->\n\n\n\n')
data = split_data[2]
try:
with BS4Parser(data, features=["html5lib", "permissive"]) as html:
#Get first entry in table
entries = html.find_all('td', attrs={'align': 'center'})
if html.find(text='No torrents here...'):
logger.log(u"No results found for: " + search_string + " (" + searchURL + ")", logger.DEBUG)
html = soup(data)
if not html:
continue
if not entries:
logger.log(u"The Data returned from " + self.name + " do not contains any torrent",
logger.DEBUG)
empty = html.find('No torrents here')
if empty:
continue
try:
title = entries[22].find('a')['title'].replace('History - ','').replace('Blu-ray', 'bd50')
url = self.urls['home'] % entries[15].find('a')['href']
download_url = self.urls['home'] % entries[15].find('a')['href']
id = entries[23].find('div')['id']
seeders = int(entries[20].get_text())
leechers = int(entries[21].get_text())
except (AttributeError, TypeError):
continue
if mode != 'RSS' and (seeders < self.minseed or leechers < self.minleech):
tables = html.find('table', attrs={'class': 'mainblockcontenttt'})
if not tables:
continue
if not title or not download_url:
torrents = tables.findChildren('tr')
if not torrents:
continue
item = title, download_url, id, seeders, leechers
logger.log(u"Found result: " + title.replace(' ','.') + " (" + searchURL + ")", logger.DEBUG)
items[mode].append(item)
#Now attempt to get any others
result_table = html.find('table', attrs={'class': 'mainblockcontenttt'})
if not result_table:
continue
entries = result_table.find_all('td', attrs={'align': 'center', 'class': 'listas'})
if not entries:
# Skip column headers
for result in torrents[1:]:
try:
cells = result.findChildren('td', attrs={'class': re.compile(r'(green|yellow|red|mainblockcontent)')})
if not cells:
continue
for result in entries:
block2 = result.find_parent('tr').find_next_sibling('tr')
if not block2:
title = url = seeders = leechers = None
size = 0
for cell in cells:
if None is title and cell.get('title') and cell.get('title') in 'Download':
title = re.search('f=(.*).torrent', cell.a['href']).group(1).replace('+', '.')
url = self.urls['home'] % cell.a['href']
if None is seeders and cell.get('class')[0] and cell.get('class')[0] in 'green' 'yellow' 'red':
seeders = int(cell.text)
elif None is leechers and cell.get('class')[0] and cell.get('class')[0] in 'green' 'yellow' 'red':
leechers = int(cell.text)
# Skip torrents released before the episode aired (fakes)
if re.match('..:..:.. ..:..:....', cells[6].text):
if (datetime.strptime(cells[6].text, '%H:%M:%S %m/%d/%Y') -
datetime.combine(ep_obj.airdate, datetime.min.time())).days < 0:
continue
cells = block2.find_all('td')
try:
title = cells[1].find('b').get_text().strip('\t ').replace('Blu-ray', 'bd50')
url = self.urls['home'] % cells[4].find('a')['href']
download_url = self.urls['home'] % cells[4].find('a')['href']
detail = cells[1].find('a')['href']
id = detail.replace('details.php?id=', '')
seeders = int(cells[9].get_text())
leechers = int(cells[10].get_text())
except (AttributeError, TypeError):
continue
# Need size for failed downloads handling
if re.match('[0-9]+,?\.?[0-9]* [KkMmGg]+[Bb]+', cells[7].text):
size = self._convertSize(cells[7].text)
if mode != 'RSS' and (seeders < self.minseed or leechers < self.minleech):
except (AttributeError, TypeError, KeyError, ValueError):
continue
if not title or not download_url:
if not title or not url or not seeders or not leechers or not size or \
seeders < self.minseed or leechers < self.minleech:
continue
item = title, download_url, id, seeders, leechers
logger.log(u"Found result: " + title.replace(' ','.') + " (" + searchURL + ")", logger.DEBUG)
items[mode].append(item)
except Exception, e:
logger.log(u"Failed parsing " + self.name + " Traceback: " + traceback.format_exc(), logger.ERROR)
item = title, url, seeders, leechers, size
logger.log(u"Found result: " + title + " (" + searchURL + ")", logger.DEBUG)
#For each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[3], reverse=True)
results += items[mode]
results.append(item)
results.sort(key=lambda tup: tup[3], reverse=True)
return results
def _get_title_and_url(self, item):
title, url, id, seeders, leechers = item
title, url, seeders, leechers, size = item
if title:
title = self._clean_title_from_provider(title)
......@@ -299,7 +256,13 @@ class HDTorrentsProvider(generic.TorrentProvider):
return (title, url)
def findPropers(self, search_date=datetime.datetime.today()):
def _get_size(self, item):
title, url, seeders, leechers, size = item
return size
def findPropers(self, search_date=datetime.today()):
results = []
......@@ -324,19 +287,31 @@ class HDTorrentsProvider(generic.TorrentProvider):
for item in self._doSearch(proper_searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today(), self.show))
results.append(classes.Proper(title, url, datetime.today(), self.show))
repack_searchString = self._get_episode_search_strings(curEp, add_string='REPACK')
for item in self._doSearch(repack_searchString[0]):
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today(), self.show))
results.append(classes.Proper(title, url, datetime.today(), self.show))
return results
def seedRatio(self):
return self.ratio
def _convertSize(self, size):
size, modifier = size.split(' ')
size = float(size)
if modifier in 'KB':
size = size * 1024
elif modifier in 'MB':
size = size * 1024**2
elif modifier in 'GB':
size = size * 1024**3
elif modifier in 'TB':
size = size * 1024**4
return size
class HDTorrentsCache(tvcache.TVCache):
def __init__(self, provider):
......@@ -347,7 +322,7 @@ class HDTorrentsCache(tvcache.TVCache):
self.minTime = 20
def _getRSSData(self):
search_params = {'RSS': []}
search_params = []
return {'entries': self.provider._doSearch(search_params)}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment

Another place to be (or not)