Change USER_AGENT back to SR now that problems are fixed. Hopefully we stop being blocked.

Set user agent in feedcache/feedparser
Better logging in generic provider
Fix typo in scene_numbering ( Thanks @thezoggy )
Fix some paths in tests, and fix some status'/db structures in tests. Also commit to db between tests.

sickbeard/common.py

@@ -23,10 +23,7 @@ import re
 import uuid
 
 INSTANCE_ID = str(uuid.uuid1())
-#Use Sick Beard USER_AGENT until they stop throttling us,
-#newznab searching has long been fixed, but we now limit it to 400 results just as they do.
-#USER_AGENT = ('SickRage/(' + platform.system() + '; ' + platform.release() + '; ' + INSTANCE_ID + ')')
-USER_AGENT = 'Sick Beard/alpha2-master' + ' (' + platform.system() + ' ' + platform.release() + ')'
+USER_AGENT = ('SickRage/(' + platform.system() + '; ' + platform.release() + '; ' + INSTANCE_ID + ')')
 
 mediaExtensions = ['avi', 'mkv', 'mpg', 'mpeg', 'wmv',
                    'ogm', 'mp4', 'iso', 'img', 'divx',

sickbeard/providers/generic.py

@@ -174,20 +174,21 @@ class GenericProvider:
             return
 
         for url in urls:
-            if helpers.download_file(url, filename, session=self.session):
             logger.log(u"Downloading a result from " + self.name + " at " + url)
-
+            if helpers.download_file(url, filename, session=self.session):
+                if self._verify_download(filename):
                     if self.providerType == GenericProvider.TORRENT:
                         logger.log(u"Saved magnet link to " + filename, logger.INFO)
                     else:
                         logger.log(u"Saved result to " + filename, logger.INFO)
-
-                if self._verify_download(filename):
                     return True
                 else:
+                    logger.log(u"Could not download %s" % url, logger.WARNING)
                     helpers._remove_file_failed(filename)
 
-        logger.log(u"Failed to download result", logger.WARNING)
+        if len(urls):
+            logger.log(u"Failed to download any results", logger.WARNING)
+
         return False
 
     def _verify_download(self, file_name=None):

sickbeard/rssfeeds.py

@@ -38,7 +38,7 @@ class RSSFeeds:
             url += urllib.urlencode(post_data)
 
         try:
-            resp = Cache(self.rssDB).fetch(url, force_update=True, request_headers=request_headers, handlers=handlers)
+            resp = Cache(self.rssDB, userAgent=sickbeard.common.USER_AGENT).fetch(url, force_update=True, request_headers=request_headers, handlers=handlers)
         finally:
             self.rssDB.close()
 

sickbeard/scene_numbering.py

@@ -498,7 +498,7 @@ def xem_refresh(indexer_id, indexer, force=False):
         try:
             parsedJSON = sickbeard.helpers.getURL(url, json=True)
             if not parsedJSON or parsedJSON == '':
-                logger.log(u'No XEN data for show "%s on %s"' % (indexer_id, sickbeard.indexerApi(indexer).name,), logger.INFO)
+                logger.log(u'No XEM data for show "%s on %s"' % (indexer_id, sickbeard.indexerApi(indexer).name,), logger.INFO)
                 return
 
             if 'success' in parsedJSON['result']:

tests/test_lib.py

@@ -43,7 +43,7 @@ shutil.copyfile = lib.shutil_custom.copyfile_custom
 #=================
 # test globals
 #=================
-TESTDIR = os.path.abspath('.')
+TESTDIR = os.path.abspath(os.path.dirname(__file__))
 TESTDBNAME = "sickbeard.db"
 TESTCACHEDBNAME = "cache.db"
 TESTFAILEDDBNAME = "failed.db"
@@ -88,8 +88,8 @@ sickbeard.PROVIDER_ORDER = ["sick_beard_index"]
 sickbeard.newznabProviderList = providers.getNewznabProviderList("'Sick Beard Index|http://lolo.sickbeard.com/|0|5030,5040|0|eponly|0|0|0!!!NZBs.org|https://nzbs.org/||5030,5040,5060,5070,5090|0|eponly|0|0|0!!!Usenet-Crawler|https://www.usenet-crawler.com/||5030,5040,5060|0|eponly|0|0|0'")
 sickbeard.providerList = providers.makeProviderList()
 
-sickbeard.PROG_DIR = os.path.abspath('..')
-sickbeard.DATA_DIR = sickbeard.PROG_DIR
+sickbeard.PROG_DIR = os.path.abspath(os.path.join(TESTDIR, '..'))
+sickbeard.DATA_DIR = TESTDIR
 sickbeard.CONFIG_FILE = os.path.join(sickbeard.DATA_DIR, "config.ini")
 sickbeard.CFG = ConfigObj(sickbeard.CONFIG_FILE)
 
@@ -140,34 +140,36 @@ class SickbeardTestDBCase(unittest.TestCase):
         tearDown_test_episode_file()
         tearDown_test_show_dir()
 
-
 class TestDBConnection(db.DBConnection, object):
 
     def __init__(self, dbFileName=TESTDBNAME):
         dbFileName = os.path.join(TESTDIR, dbFileName)
         super(TestDBConnection, self).__init__(dbFileName)
 
-
 class TestCacheDBConnection(TestDBConnection, object):
-
      def __init__(self, providerName):
         db.DBConnection.__init__(self, os.path.join(TESTDIR, TESTCACHEDBNAME))
 
         # Create the table if it's not already there
         try:
-            sql = "CREATE TABLE " + providerName + " (name TEXT, season NUMERIC, episodes TEXT, indexerid NUMERIC, url TEXT, time NUMERIC, quality TEXT);"
+            if not self.hasTable(providerName):
+                sql = "CREATE TABLE [" + providerName + "] (name TEXT, season NUMERIC, episodes TEXT, indexerid NUMERIC, url TEXT, time NUMERIC, quality TEXT, release_group TEXT)"
                 self.connection.execute(sql)
                 self.connection.commit()
-        except sqlite3.OperationalError, e:
-            if str(e) != "table " + providerName + " already exists":
+        except Exception, e:
+            if str(e) != "table [" + providerName + "] already exists":
                 raise
 
+            # add version column to table if missing
+            if not self.hasColumn(providerName, 'version'):
+                self.addColumn(providerName, 'version', "NUMERIC", "-1")
+
         # Create the table if it's not already there
         try:
             sql = "CREATE TABLE lastUpdate (provider TEXT, time NUMERIC);"
             self.connection.execute(sql)
             self.connection.commit()
-        except sqlite3.OperationalError, e:
+        except Exception, e:
             if str(e) != "table lastUpdate already exists":
                 raise
 
@@ -195,31 +197,29 @@ def setUp_test_db():
 
 
 def tearDown_test_db():
-    """Deletes the test db
-        although this seams not to work on my system it leaves me with an zero kb file
-    """
-
-    #uncomment next line so leave the db intact between test and at the end
-    #return False
-
-    for current_db in [ TESTDBNAME, TESTCACHEDBNAME, TESTFAILEDDBNAME ]:
-        for file_name in [ os.path.join(TESTDIR, current_db), os.path.join(TESTDIR, current_db + '-journal') ]:
-            if os.path.exists(file_name):
-                try:
-                    os.remove(file_name)
-                except (IOError, OSError) as e:
-                    print 'ERROR: Failed to remove ' + file_name
-                    print ex(e)
-
+    from sickbeard.db import db_cons
+    for connection in db_cons:
+        db_cons[connection].commit()
+#        db_cons[connection].close()
+
+#    for current_db in [ TESTDBNAME, TESTCACHEDBNAME, TESTFAILEDDBNAME ]:
+#        file_name = os.path.join(TESTDIR, current_db)
+#        if os.path.exists(file_name):
+#            try:
+#                os.remove(file_name)
+#            except Exception as e:
+#                print 'ERROR: Failed to remove ' + file_name
+#                print ex(e)
 
 def setUp_test_episode_file():
     if not os.path.exists(FILEDIR):
         os.makedirs(FILEDIR)
 
     try:
-        with open(FILEPATH, 'w') as f:
+        with open(FILEPATH, 'wb') as f:
             f.write("foo bar")
-    except EnvironmentError:
+            f.flush()
+    except Exception:
         print "Unable to set up test episode"
         raise
 

tests/tv_tests.py

@@ -43,7 +43,7 @@ class TVShowTests(test.SickbeardTestDBCase):
         show.network = "cbs"
         show.genre = "crime"
         show.runtime = 40
-        show.status = "5"
+        show.status = "Ended"
         show.default_ep_status = "5"
         show.airs = "monday"
         show.startyear = 1987
@@ -92,7 +92,7 @@ class TVTests(test.SickbeardTestDBCase):
         show.network = "cbs"
         show.genre = "crime"
         show.runtime = 40
-        show.status = "5"
+        show.status = "Ended"
         show.default_ep_status = "5"
         show.airs = "monday"
         show.startyear = 1987