Merge pull request #1961 from miigotu/indexer-fix

Limit newznab searches to 400 results, clean up logging

sickbeard/common.py

@@ -23,7 +23,10 @@ import re
 import uuid
 
 INSTANCE_ID = str(uuid.uuid1())
-USER_AGENT = ('SickRage/(' + platform.system() + '; ' + platform.release() + '; ' + INSTANCE_ID + ')')
+#Use Sick Beard USER_AGENT until they stop throttling us,
+#newznab searching has long been fixed, but we now limit it to 400 results just as they do.
+#USER_AGENT = ('SickRage/(' + platform.system() + '; ' + platform.release() + '; ' + INSTANCE_ID + ')')
+USER_AGENT = 'Sick Beard/alpha2-master' + ' (' + platform.system() + ' ' + platform.release() + ')'
 
 mediaExtensions = ['avi', 'mkv', 'mpg', 'mpeg', 'wmv',
                    'ogm', 'mp4', 'iso', 'img', 'divx',

sickbeard/providers/newznab.py

@@ -270,7 +270,8 @@ class NewznabProvider(generic.NZBProvider):
         results = []
         offset = total = 0
 
-        while (total >= offset) and (offset < 1000):
+        # Limit to 400 results, like Sick Beard does, to prevent throttling
+        while (total > offset) and (offset <= 400):
             search_url = self.url + 'api?' + urllib.urlencode(params)
             logger.log(u"Search url: " + search_url, logger.DEBUG)
 
@@ -306,16 +307,13 @@ class NewznabProvider(generic.NZBProvider):
                 break
 
             params['offset'] += params['limit']
-            if (total > int(params['offset'])):
+            if (total > int(params['offset'])) and (int(params['offset']) <= 400):
                 offset = int(params['offset'])
                 # if there are more items available then the amount given in one call, grab some more
-                logger.log(str(
-                    total - int(params['offset'])) + " more items to be fetched from provider. Fetching another " + str(
-                    params['limit']) + " items.", logger.DEBUG)
+                logger.log(u'%d' % (total - offset) + ' more items to be fetched from provider.' +
+                'Fetching another %d' % int(params['limit']) + ' items.', logger.DEBUG)
             else:
-                logger.log(str(
-                    total - int(params['offset'])) + " No more searches needed, could find anything I was looking for! " + str(
-                    params['limit']) + " items.", logger.DEBUG)
+                logger.log(u'No more searches needed.', logger.DEBUG)
                 break
 
             time.sleep(0.2)
@@ -389,8 +387,8 @@ class NewznabCache(tvcache.TVCache):
 
         tvcache.TVCache.__init__(self, provider)
 
-        # only poll newznab providers every 15 minutes max
-        self.minTime = 15
+        # only poll newznab providers every 30 minutes max, doubled so we don't get throttled again.
+        self.minTime = 30
 
     def _getRSSData(self):
 

tests/all_tests.py

@@ -29,7 +29,6 @@ sys.path.insert(1, os.path.join(tests_dir, '..'))
 
 class AllTests(unittest.TestCase):
     #Block issue_submitter_tests to avoid issue tracker spam on every build
-    #Block feedparser_tests because http://lolo.sickbeard.com/ has changed api, which makes the test fail
     blacklist = [tests_dir + 'all_tests.py', tests_dir + 'issue_submitter_tests.py']
     def setUp(self):
         self.test_file_strings = [ x for x in glob.glob(tests_dir + '*_tests.py') if not x in self.blacklist ]

tests/test_lib.py

@@ -85,7 +85,7 @@ sickbeard.NAMING_MULTI_EP = 1
 
 
 sickbeard.PROVIDER_ORDER = ["sick_beard_index"]
-sickbeard.newznabProviderList = providers.getNewznabProviderList("'Sick Beard Index|http://lolo.sickbeard.com/|0|5030,5040,5060|0|eponly|0!!!NZBs.org|https://nzbs.org/||5030,5040,5060,5070,5090|0|eponly|0!!!Usenet-Crawler|https://www.usenet-crawler.com/||5030,5040,5060|0|eponly|0'")
+sickbeard.newznabProviderList = providers.getNewznabProviderList("'Sick Beard Index|http://lolo.sickbeard.com/|0|5030,5040|0|eponly|0|0|0!!!NZBs.org|https://nzbs.org/||5030,5040,5060,5070,5090|0|eponly|0|0|0!!!Usenet-Crawler|https://www.usenet-crawler.com/||5030,5040,5060|0|eponly|0|0|0'")
 sickbeard.providerList = providers.makeProviderList()
 
 sickbeard.PROG_DIR = os.path.abspath('..')