diff --git a/.travis.yml b/.travis.yml
index dc702dde4bb4d50b6565b1c5e9ba7912f3bf021c..6d85ba9e19dafdf3b94eac202e71ab15d9eecf5d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,8 +4,8 @@ python:
- 2.7
branches:
- only:
- - develop
+ except:
+ - master
cache: pip
diff --git a/CHANGES.md b/CHANGES.md
index 08717d2e6cc6e435417546424a2c417cd972508c..4446ffc221b0d80ab1aa6dbf2559eb93ad270339 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,179 @@
+### 4.0.29 (2015-06-26)
+
+[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.28...v4.0.29)
+
+* Fixed: "Failed to load URL" problems
+* Fixed: Spam and search string in SCC
+* Fixed: Missing import
+* Fixed: Name_cache lock
+* Fixed: Missing import in SCC
+* Fixed: Bad encoding
+* Fixed: ResultType not in class Proper
+* Added: NZB.Cat Provider
+* Added: Sleep between result url hits
+* Added: Scene exceptions as submodules
+* Removed: Verify off in order to fix requests
+* Removed: Https from rarbg token
+* Change: Travis can now cache pip installs between builds
+* Change: Made scene exceptions and network timezones paths be absolute
+
+### 4.0.28 (2015-06-23)
+
+[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.27...v4.0.28)
+
+* Fixed: SiCKRAGETV/sickrage-issues#1747
+* Fixed: torrentday
+* Fixed: Release group sometimes not showing in compact history
+* Fixed: Results from cache not honoring ignored/required words
+* Fixed: Newznab limmiting/fixes
+* Fixed: Missed import
+* Fixed: Missing import for show_name_helpers
+* Added: Trakt empty token error message
+* Added: Build caching for travis.
+* Disabled: feedparser test
+* Cleaned: Name cache
+* Change: Strip ". -" from names after removeWords
+* Change: Propers code for newznab
+* Change: Use womble for feedparser test
+* Change: Make sure we are cleaning up destination filenames before renaming
+* Change: Set maxage to today-airdate to limit results to relevant results
+* Change: Test for write/permissions problems and warn
+* Change: Filter .torrent and .nzb from pp list of files
+* Change: Sleep between proper snatches
+* Change: Change USER_AGENT back to SR
+* Changed cpu_presets
+* Updated: webserve.py
+* Updated: tornado to 4.1.0 from 4.1.0dev1
+
+### 4.0.27 (2015-06-18)
+
+[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.26...v4.0.27)
+
+* Fixed: webapi CMD_ShowPause not updating database
+* Fixed: HDT removing first letter of the file
+* Fixed: Searches not running at all
+* Fixed: Properly import sickbeard.exceptions.ex
+* Fixed: Remove default ep status
+* Fixed: Fixed /SiCKRAGETV/sickrage-issues#1814 & /SiCKRAGETV/sickrage-issues#1743
+* Added: alt.binaries.teevee
+* Change: Try to get the best quality from the best/archived list on first snatch, but accept the highest from the any/allowed list otherwise
+* Change: Limit newznab searches to 400 results & clean up its logging
+* Change: Warn on bad gzip header response, instead of error
+* Removed eztv and ezrss due to general scene skepticism about the shady nature of their takeover
+* Removed OldPirateBay provider
+* Removed "Default Episode Status"
+* Updated RARBG to the new API (v2)
+
+### 4.0.26 (2015-06-14)
+
+[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.25...v4.0.26)
+
+* Fixed: RemoveWords, ignoreWords, requiredWords fixes
+* Fixed: WindowsError undefined on linux, but WindowsError and IOError are just alias' of OSError
+* Added: '[vtv]' and '-20-40' to non-rlsgroups, Strip '[ www.TorrentDay.com ] - ' and '[ www.Cpasbien.pw ] ' release_name prefixes
+* Change: Allow SD as archive quality in custom qualities, SD is better than Unknown
+* Change: Allow select SDTV for Best/Archive quality in MassEdit
+
+### 4.0.25 (2015-06-12)
+
+[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.24...v4.0.25)
+
+* Fixed: Verify_freespace check and return False on isFileLocked check
+* Fixed: SQLite3 on Debian8
+* Fixed: Torrage and zoink.ch, because torcache has a timer and returns html instead of a torrent
+* Added: Check for locked files when PPing
+* Added ranked and sorting options to RARBG
+* Change: Return all results from rarbg, not only internal.
+* Change: Log regex mode only on error, disable unnecessary pp messages.
+* Change: Sort rarbg results by seeders, not by newest
+* Updated thepiratebay url
+* Updated RARBG category to 'tv'
+
+### 4.0.24 (2015-06-08)
+
+[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.23...v4.0.24)
+
+* Fixed: Potential bad episode airdates preventing SR from starting
+* Added: Small change for additional layout.Change: Cleaned up logging
+* Removed the dropdown menu under "Shows"
+* Updated the initial schema for failed.db and cache.db
+
+### 4.0.23 (2015-06-02)
+
+[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.22...v4.0.23)
+
+* Changed image download issue from error to warning
+* Changed urls to use http github.io pages from our repos
+* Updated regexes.py
+
+### 4.0.22 (2015-05-28)
+
+[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.21...v4.0.22)
+
+* Fixed: 'utf8' codec can't decode byte while reading the DB
+* Fixed SiCKRAGETV/sickrage-issues#1691
+* Added failed option to PP API
+* Added Add size attribute in providers.generic for nzb providers
+* Feature: Trakt PIN Auth
+* Revert: "Update template"
+* Change: Also catch BadStatusLine
+* Change: Lowercase date fuzzies were ugly, and the 'last' keyword suggests > 7 days ago
+* Change: Uppercase 'in' and 'a' in fuzzy dates
+* Updated both requests to 2.6.0
+
+### 4.0.21 (2015-05-17)
+
+[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.20...v4.0.21)
+
+* Fixed quality colors on history and displayShow pages
+* Fixed manual adding of group to black or white list
+* Added several network logos
+* Added the ability to browse possible links found in the homepage for the EZTV provider
+* Changed the InitScripts to use variable names and file names referencing SickRage, instead of SickBeard, to prevent conflicts, and improve readability
+* Feature: Add Fanart to local image cache
+
+### 4.0.20 (2015-05-10)
+
+[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.19...v4.0.20)
+
+* Fixed incorrect behaviour in the web API when a show search returns 0 results
+* Fixed funky Quality Names
+* Fixed the sb.searchindexers function of the web API so it doesn't immediately return after finding no results from the first indexer
+* Fixed html changes in http://eztv.ch
+* Fixed 'NoneType' object has no attribute 'whitelist
+* Added flexibility when determine title from link in eztvapi.re
+* Added missing "The Anime Network" logo
+* Enable trending shows only if Trakt is enabled
+* Change: SCC doesn't support searching dates with pipe characters. Use '.' instead
+* Change: Move init/upstart scripts to their own folder to organize and clean up the source
+* Change: Replace SSL Error with a url with information. Disable issue submission of such errors
+* Change: Re-enable feedparser test, as lolo.sickbeard.com is operational again
+* Changed sr_tvrage_scene_exceptions to use SiCKRAGETV repo instead of echel0n's
+* Removed pyOpenSSL from libs. Cryptography is only needed if you use pyOpenSSL 0.14 or newer. Instead `pip install pyopenssl==0.13.1`
+* Removed torrage.com as the domain expired, zoink.ch has nginx misconfigured or service stopped
+* Updated Regex to be more specific to for replacement
+
+### 4.0.19 (2015-05-04)
+
+[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.18...v4.0.19)
+
+* Fixed SSL errors
+* Fixed travis reporting failed builds as successful
+* Fix: If re.search returns None due to no match, .group() will cause an exception
+* Fix: Center Search icon on displayShowTable
+* Added pyOpenSSL SNI test
+* Added ndg-httpsclient, pyOpenSSL, and pyasn1 to enable SNI
+* Added some missing Network logos.
+* Enable trending shows only if Trakt is enabled
+* Change: Use included libs
+* Change: Simplify script, replace sickbeard with sickrage to avoid conflicts
+* Reverted "Update requests library from v2.5.1 to v2.6.2 ff71b25e"
+* Removed reference to readme-FailedDownloads.md, which doesnt exist
+* Removed unused libs
+* Updated certifi certificates
+* Updated requests library from v2.5.1 to v2.6.2
+* Updated mainDB.InitialSchema to v42 to prevent db upgrade on a freshly built database
+
### 4.0.18 (2015-04-26)
[full changelog](https://github.com/SiCKRAGETV/SickRage/compare/v4.0.17...v4.0.18)
diff --git a/SickBeard.py b/SickBeard.py
index ad8b41a7ec6582ea5b0bb067310e1ff0db47ee70..ceec62f029385f704df1df1ddcb1a81e52b56c7b 100755
--- a/SickBeard.py
+++ b/SickBeard.py
@@ -376,12 +376,6 @@ class SickRage(object):
# Fire up all our threads
sickbeard.start()
- # Build internal name cache
- name_cache.buildNameCache()
-
- # refresh network timezones
- network_timezones.update_network_dict()
-
# sure, why not?
if sickbeard.USE_FAILED_DOWNLOADS:
failed_history.trimHistory()
diff --git a/autoProcessTV/autoProcessTV.py b/autoProcessTV/autoProcessTV.py
index d8abf60e53b4c4aeb96754dc8601d50a6577b6bc..d8d56e3f0f936912cdf94b95806b1eda5bc411c0 100755
--- a/autoProcessTV/autoProcessTV.py
+++ b/autoProcessTV/autoProcessTV.py
@@ -26,7 +26,7 @@ import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), 'lib')))
try:
- from lib import requests
+ import requests
except ImportError:
print ("You need to install python requests library")
sys.exit(1)
diff --git a/autoProcessTV/mediaToSickbeard.py b/autoProcessTV/mediaToSickbeard.py
index 60847bfea23d761b41297402ef649bb054a2b016..ba0d3cafecb06db8af00001aeba78969add90a7b 100755
--- a/autoProcessTV/mediaToSickbeard.py
+++ b/autoProcessTV/mediaToSickbeard.py
@@ -10,7 +10,7 @@ sys.path.append(os.path.join( sickbeardPath, 'lib'))
sys.path.append(sickbeardPath)
configFilename = os.path.join(sickbeardPath, "config.ini")
-from lib import requests
+import requests
config = ConfigParser.ConfigParser()
diff --git a/gui/slick/images/flags/af.png b/gui/slick/images/flags/afr.png
similarity index 100%
rename from gui/slick/images/flags/af.png
rename to gui/slick/images/flags/afr.png
diff --git a/gui/slick/images/flags/am.png b/gui/slick/images/flags/amh.png
similarity index 100%
rename from gui/slick/images/flags/am.png
rename to gui/slick/images/flags/amh.png
diff --git a/gui/slick/images/flags/ar.png b/gui/slick/images/flags/ara.png
similarity index 100%
rename from gui/slick/images/flags/ar.png
rename to gui/slick/images/flags/ara.png
diff --git a/gui/slick/images/flags/an.png b/gui/slick/images/flags/arg.png
similarity index 100%
rename from gui/slick/images/flags/an.png
rename to gui/slick/images/flags/arg.png
diff --git a/gui/slick/images/flags/as.png b/gui/slick/images/flags/asm.png
similarity index 100%
rename from gui/slick/images/flags/as.png
rename to gui/slick/images/flags/asm.png
diff --git a/gui/slick/images/flags/ae.png b/gui/slick/images/flags/ave.png
similarity index 100%
rename from gui/slick/images/flags/ae.png
rename to gui/slick/images/flags/ave.png
diff --git a/gui/slick/images/flags/ay.png b/gui/slick/images/flags/aym.png
similarity index 100%
rename from gui/slick/images/flags/ay.png
rename to gui/slick/images/flags/aym.png
diff --git a/gui/slick/images/flags/az.png b/gui/slick/images/flags/aze.png
similarity index 100%
rename from gui/slick/images/flags/az.png
rename to gui/slick/images/flags/aze.png
diff --git a/gui/slick/images/flags/ba.png b/gui/slick/images/flags/bak.png
similarity index 100%
rename from gui/slick/images/flags/ba.png
rename to gui/slick/images/flags/bak.png
diff --git a/gui/slick/images/flags/bm.png b/gui/slick/images/flags/bam.png
similarity index 100%
rename from gui/slick/images/flags/bm.png
rename to gui/slick/images/flags/bam.png
diff --git a/gui/slick/images/flags/be.png b/gui/slick/images/flags/bel.png
similarity index 100%
rename from gui/slick/images/flags/be.png
rename to gui/slick/images/flags/bel.png
diff --git a/gui/slick/images/flags/bn.png b/gui/slick/images/flags/ben.png
similarity index 100%
rename from gui/slick/images/flags/bn.png
rename to gui/slick/images/flags/ben.png
diff --git a/gui/slick/images/flags/bi.png b/gui/slick/images/flags/bis.png
similarity index 100%
rename from gui/slick/images/flags/bi.png
rename to gui/slick/images/flags/bis.png
diff --git a/gui/slick/images/flags/bo.png b/gui/slick/images/flags/bod.png
similarity index 100%
rename from gui/slick/images/flags/bo.png
rename to gui/slick/images/flags/bod.png
diff --git a/gui/slick/images/flags/bs.png b/gui/slick/images/flags/bos.png
similarity index 100%
rename from gui/slick/images/flags/bs.png
rename to gui/slick/images/flags/bos.png
diff --git a/gui/slick/images/flags/br.png b/gui/slick/images/flags/bre.png
similarity index 100%
rename from gui/slick/images/flags/br.png
rename to gui/slick/images/flags/bre.png
diff --git a/gui/slick/images/flags/bg.png b/gui/slick/images/flags/bul.png
similarity index 100%
rename from gui/slick/images/flags/bg.png
rename to gui/slick/images/flags/bul.png
diff --git a/gui/slick/images/flags/ca.png b/gui/slick/images/flags/cat.png
similarity index 100%
rename from gui/slick/images/flags/ca.png
rename to gui/slick/images/flags/cat.png
diff --git a/gui/slick/images/flags/cs.png b/gui/slick/images/flags/ces.png
similarity index 100%
rename from gui/slick/images/flags/cs.png
rename to gui/slick/images/flags/ces.png
diff --git a/gui/slick/images/flags/ch.png b/gui/slick/images/flags/cha.png
similarity index 100%
rename from gui/slick/images/flags/ch.png
rename to gui/slick/images/flags/cha.png
diff --git a/gui/slick/images/flags/cu.png b/gui/slick/images/flags/chu.png
similarity index 100%
rename from gui/slick/images/flags/cu.png
rename to gui/slick/images/flags/chu.png
diff --git a/gui/slick/images/flags/cv.png b/gui/slick/images/flags/chv.png
similarity index 100%
rename from gui/slick/images/flags/cv.png
rename to gui/slick/images/flags/chv.png
diff --git a/gui/slick/images/flags/kw.png b/gui/slick/images/flags/cor.png
similarity index 100%
rename from gui/slick/images/flags/kw.png
rename to gui/slick/images/flags/cor.png
diff --git a/gui/slick/images/flags/co.png b/gui/slick/images/flags/cos.png
similarity index 100%
rename from gui/slick/images/flags/co.png
rename to gui/slick/images/flags/cos.png
diff --git a/gui/slick/images/flags/cr.png b/gui/slick/images/flags/cre.png
similarity index 100%
rename from gui/slick/images/flags/cr.png
rename to gui/slick/images/flags/cre.png
diff --git a/gui/slick/images/flags/cy.png b/gui/slick/images/flags/cym.png
similarity index 100%
rename from gui/slick/images/flags/cy.png
rename to gui/slick/images/flags/cym.png
diff --git a/gui/slick/images/flags/da.png b/gui/slick/images/flags/dan.png
similarity index 100%
rename from gui/slick/images/flags/da.png
rename to gui/slick/images/flags/dan.png
diff --git a/gui/slick/images/flags/de.png b/gui/slick/images/flags/deu.png
similarity index 100%
rename from gui/slick/images/flags/de.png
rename to gui/slick/images/flags/deu.png
diff --git a/gui/slick/images/flags/dz.png b/gui/slick/images/flags/dzo.png
similarity index 100%
rename from gui/slick/images/flags/dz.png
rename to gui/slick/images/flags/dzo.png
diff --git a/gui/slick/images/flags/el.png b/gui/slick/images/flags/ell.png
similarity index 100%
rename from gui/slick/images/flags/el.png
rename to gui/slick/images/flags/ell.png
diff --git a/gui/slick/images/flags/en.png b/gui/slick/images/flags/eng.png
similarity index 100%
rename from gui/slick/images/flags/en.png
rename to gui/slick/images/flags/eng.png
diff --git a/gui/slick/images/flags/eo.png b/gui/slick/images/flags/epo.png
similarity index 100%
rename from gui/slick/images/flags/eo.png
rename to gui/slick/images/flags/epo.png
diff --git a/gui/slick/images/flags/et.png b/gui/slick/images/flags/est.png
similarity index 100%
rename from gui/slick/images/flags/et.png
rename to gui/slick/images/flags/est.png
diff --git a/gui/slick/images/flags/ee.png b/gui/slick/images/flags/ewe.png
similarity index 100%
rename from gui/slick/images/flags/ee.png
rename to gui/slick/images/flags/ewe.png
diff --git a/gui/slick/images/flags/fo.png b/gui/slick/images/flags/fao.png
similarity index 100%
rename from gui/slick/images/flags/fo.png
rename to gui/slick/images/flags/fao.png
diff --git a/gui/slick/images/flags/fa.png b/gui/slick/images/flags/fas.png
similarity index 100%
rename from gui/slick/images/flags/fa.png
rename to gui/slick/images/flags/fas.png
diff --git a/gui/slick/images/flags/fj.png b/gui/slick/images/flags/fij.png
similarity index 100%
rename from gui/slick/images/flags/fj.png
rename to gui/slick/images/flags/fij.png
diff --git a/gui/slick/images/flags/fi.png b/gui/slick/images/flags/fin.png
similarity index 100%
rename from gui/slick/images/flags/fi.png
rename to gui/slick/images/flags/fin.png
diff --git a/gui/slick/images/flags/fr.png b/gui/slick/images/flags/fra.png
similarity index 100%
rename from gui/slick/images/flags/fr.png
rename to gui/slick/images/flags/fra.png
diff --git a/gui/slick/images/flags/gd.png b/gui/slick/images/flags/gla.png
similarity index 100%
rename from gui/slick/images/flags/gd.png
rename to gui/slick/images/flags/gla.png
diff --git a/gui/slick/images/flags/ga.png b/gui/slick/images/flags/gle.png
similarity index 100%
rename from gui/slick/images/flags/ga.png
rename to gui/slick/images/flags/gle.png
diff --git a/gui/slick/images/flags/gl.png b/gui/slick/images/flags/glg.png
similarity index 100%
rename from gui/slick/images/flags/gl.png
rename to gui/slick/images/flags/glg.png
diff --git a/gui/slick/images/flags/gn.png b/gui/slick/images/flags/grn.png
similarity index 100%
rename from gui/slick/images/flags/gn.png
rename to gui/slick/images/flags/grn.png
diff --git a/gui/slick/images/flags/gu.png b/gui/slick/images/flags/guj.png
similarity index 100%
rename from gui/slick/images/flags/gu.png
rename to gui/slick/images/flags/guj.png
diff --git a/gui/slick/images/flags/ht.png b/gui/slick/images/flags/hat.png
similarity index 100%
rename from gui/slick/images/flags/ht.png
rename to gui/slick/images/flags/hat.png
diff --git a/gui/slick/images/flags/he.png b/gui/slick/images/flags/heb.png
similarity index 100%
rename from gui/slick/images/flags/he.png
rename to gui/slick/images/flags/heb.png
diff --git a/gui/slick/images/flags/hi.png b/gui/slick/images/flags/hin.png
similarity index 100%
rename from gui/slick/images/flags/hi.png
rename to gui/slick/images/flags/hin.png
diff --git a/gui/slick/images/flags/hr.png b/gui/slick/images/flags/hrv.png
similarity index 100%
rename from gui/slick/images/flags/hr.png
rename to gui/slick/images/flags/hrv.png
diff --git a/gui/slick/images/flags/hu.png b/gui/slick/images/flags/hun.png
similarity index 100%
rename from gui/slick/images/flags/hu.png
rename to gui/slick/images/flags/hun.png
diff --git a/gui/slick/images/flags/hy.png b/gui/slick/images/flags/hye.png
similarity index 100%
rename from gui/slick/images/flags/hy.png
rename to gui/slick/images/flags/hye.png
diff --git a/gui/slick/images/flags/io.png b/gui/slick/images/flags/ido.png
similarity index 100%
rename from gui/slick/images/flags/io.png
rename to gui/slick/images/flags/ido.png
diff --git a/gui/slick/images/flags/ie.png b/gui/slick/images/flags/ile.png
similarity index 100%
rename from gui/slick/images/flags/ie.png
rename to gui/slick/images/flags/ile.png
diff --git a/gui/slick/images/flags/id.png b/gui/slick/images/flags/ind.png
similarity index 100%
rename from gui/slick/images/flags/id.png
rename to gui/slick/images/flags/ind.png
diff --git a/gui/slick/images/flags/is.png b/gui/slick/images/flags/isl.png
similarity index 100%
rename from gui/slick/images/flags/is.png
rename to gui/slick/images/flags/isl.png
diff --git a/gui/slick/images/flags/it.png b/gui/slick/images/flags/ita.png
similarity index 100%
rename from gui/slick/images/flags/it.png
rename to gui/slick/images/flags/ita.png
diff --git a/gui/slick/images/flags/ja.png b/gui/slick/images/flags/jpn.png
similarity index 100%
rename from gui/slick/images/flags/ja.png
rename to gui/slick/images/flags/jpn.png
diff --git a/gui/slick/images/flags/kn.png b/gui/slick/images/flags/kan.png
similarity index 100%
rename from gui/slick/images/flags/kn.png
rename to gui/slick/images/flags/kan.png
diff --git a/gui/slick/images/flags/ka.png b/gui/slick/images/flags/kat.png
similarity index 100%
rename from gui/slick/images/flags/ka.png
rename to gui/slick/images/flags/kat.png
diff --git a/gui/slick/images/flags/kr.png b/gui/slick/images/flags/kau.png
similarity index 100%
rename from gui/slick/images/flags/kr.png
rename to gui/slick/images/flags/kau.png
diff --git a/gui/slick/images/flags/kk.png b/gui/slick/images/flags/kaz.png
similarity index 100%
rename from gui/slick/images/flags/kk.png
rename to gui/slick/images/flags/kaz.png
diff --git a/gui/slick/images/flags/km.png b/gui/slick/images/flags/khm.png
similarity index 100%
rename from gui/slick/images/flags/km.png
rename to gui/slick/images/flags/khm.png
diff --git a/gui/slick/images/flags/ki.png b/gui/slick/images/flags/kik.png
similarity index 100%
rename from gui/slick/images/flags/ki.png
rename to gui/slick/images/flags/kik.png
diff --git a/gui/slick/images/flags/rw.png b/gui/slick/images/flags/kin.png
similarity index 100%
rename from gui/slick/images/flags/rw.png
rename to gui/slick/images/flags/kin.png
diff --git a/gui/slick/images/flags/ky.png b/gui/slick/images/flags/kir.png
similarity index 100%
rename from gui/slick/images/flags/ky.png
rename to gui/slick/images/flags/kir.png
diff --git a/gui/slick/images/flags/kg.png b/gui/slick/images/flags/kon.png
similarity index 100%
rename from gui/slick/images/flags/kg.png
rename to gui/slick/images/flags/kon.png
diff --git a/gui/slick/images/flags/ko.png b/gui/slick/images/flags/kor.png
similarity index 100%
rename from gui/slick/images/flags/ko.png
rename to gui/slick/images/flags/kor.png
diff --git a/gui/slick/images/flags/la.png b/gui/slick/images/flags/lat.png
similarity index 100%
rename from gui/slick/images/flags/la.png
rename to gui/slick/images/flags/lat.png
diff --git a/gui/slick/images/flags/lv.png b/gui/slick/images/flags/lav.png
similarity index 100%
rename from gui/slick/images/flags/lv.png
rename to gui/slick/images/flags/lav.png
diff --git a/gui/slick/images/flags/li.png b/gui/slick/images/flags/lim.png
similarity index 100%
rename from gui/slick/images/flags/li.png
rename to gui/slick/images/flags/lim.png
diff --git a/gui/slick/images/flags/lt.png b/gui/slick/images/flags/lit.png
similarity index 100%
rename from gui/slick/images/flags/lt.png
rename to gui/slick/images/flags/lit.png
diff --git a/gui/slick/images/flags/lb.png b/gui/slick/images/flags/ltz.png
similarity index 100%
rename from gui/slick/images/flags/lb.png
rename to gui/slick/images/flags/ltz.png
diff --git a/gui/slick/images/flags/lu.png b/gui/slick/images/flags/lub.png
similarity index 100%
rename from gui/slick/images/flags/lu.png
rename to gui/slick/images/flags/lub.png
diff --git a/gui/slick/images/flags/mh.png b/gui/slick/images/flags/mah.png
similarity index 100%
rename from gui/slick/images/flags/mh.png
rename to gui/slick/images/flags/mah.png
diff --git a/gui/slick/images/flags/ml.png b/gui/slick/images/flags/mal.png
similarity index 100%
rename from gui/slick/images/flags/ml.png
rename to gui/slick/images/flags/mal.png
diff --git a/gui/slick/images/flags/mr.png b/gui/slick/images/flags/mar.png
similarity index 100%
rename from gui/slick/images/flags/mr.png
rename to gui/slick/images/flags/mar.png
diff --git a/gui/slick/images/flags/mk.png b/gui/slick/images/flags/mkd.png
similarity index 100%
rename from gui/slick/images/flags/mk.png
rename to gui/slick/images/flags/mkd.png
diff --git a/gui/slick/images/flags/mg.png b/gui/slick/images/flags/mlg.png
similarity index 100%
rename from gui/slick/images/flags/mg.png
rename to gui/slick/images/flags/mlg.png
diff --git a/gui/slick/images/flags/mt.png b/gui/slick/images/flags/mlt.png
similarity index 100%
rename from gui/slick/images/flags/mt.png
rename to gui/slick/images/flags/mlt.png
diff --git a/gui/slick/images/flags/mn.png b/gui/slick/images/flags/mon.png
similarity index 100%
rename from gui/slick/images/flags/mn.png
rename to gui/slick/images/flags/mon.png
diff --git a/gui/slick/images/flags/ms.png b/gui/slick/images/flags/msa.png
similarity index 100%
rename from gui/slick/images/flags/ms.png
rename to gui/slick/images/flags/msa.png
diff --git a/gui/slick/images/flags/my.png b/gui/slick/images/flags/mya.png
similarity index 100%
rename from gui/slick/images/flags/my.png
rename to gui/slick/images/flags/mya.png
diff --git a/gui/slick/images/flags/na.png b/gui/slick/images/flags/nau.png
similarity index 100%
rename from gui/slick/images/flags/na.png
rename to gui/slick/images/flags/nau.png
diff --git a/gui/slick/images/flags/nr.png b/gui/slick/images/flags/nbl.png
similarity index 100%
rename from gui/slick/images/flags/nr.png
rename to gui/slick/images/flags/nbl.png
diff --git a/gui/slick/images/flags/ng.png b/gui/slick/images/flags/ndo.png
similarity index 100%
rename from gui/slick/images/flags/ng.png
rename to gui/slick/images/flags/ndo.png
diff --git a/gui/slick/images/flags/ne.png b/gui/slick/images/flags/nep.png
similarity index 100%
rename from gui/slick/images/flags/ne.png
rename to gui/slick/images/flags/nep.png
diff --git a/gui/slick/images/flags/nl.png b/gui/slick/images/flags/nld.png
similarity index 100%
rename from gui/slick/images/flags/nl.png
rename to gui/slick/images/flags/nld.png
diff --git a/gui/slick/images/flags/no.png b/gui/slick/images/flags/nor.png
similarity index 100%
rename from gui/slick/images/flags/no.png
rename to gui/slick/images/flags/nor.png
diff --git a/gui/slick/images/flags/oc.png b/gui/slick/images/flags/oci.png
similarity index 100%
rename from gui/slick/images/flags/oc.png
rename to gui/slick/images/flags/oci.png
diff --git a/gui/slick/images/flags/om.png b/gui/slick/images/flags/orm.png
similarity index 100%
rename from gui/slick/images/flags/om.png
rename to gui/slick/images/flags/orm.png
diff --git a/gui/slick/images/flags/pa.png b/gui/slick/images/flags/pan.png
similarity index 100%
rename from gui/slick/images/flags/pa.png
rename to gui/slick/images/flags/pan.png
diff --git a/gui/slick/images/flags/pl.png b/gui/slick/images/flags/pol.png
similarity index 100%
rename from gui/slick/images/flags/pl.png
rename to gui/slick/images/flags/pol.png
diff --git a/gui/slick/images/flags/pt.png b/gui/slick/images/flags/por.png
similarity index 100%
rename from gui/slick/images/flags/pt.png
rename to gui/slick/images/flags/por.png
diff --git a/gui/slick/images/flags/ps.png b/gui/slick/images/flags/pus.png
similarity index 100%
rename from gui/slick/images/flags/ps.png
rename to gui/slick/images/flags/pus.png
diff --git a/gui/slick/images/flags/ro.png b/gui/slick/images/flags/ron.png
similarity index 100%
rename from gui/slick/images/flags/ro.png
rename to gui/slick/images/flags/ron.png
diff --git a/gui/slick/images/flags/ru.png b/gui/slick/images/flags/rus.png
similarity index 100%
rename from gui/slick/images/flags/ru.png
rename to gui/slick/images/flags/rus.png
diff --git a/gui/slick/images/flags/sg.png b/gui/slick/images/flags/sag.png
similarity index 100%
rename from gui/slick/images/flags/sg.png
rename to gui/slick/images/flags/sag.png
diff --git a/gui/slick/images/flags/sa.png b/gui/slick/images/flags/san.png
similarity index 100%
rename from gui/slick/images/flags/sa.png
rename to gui/slick/images/flags/san.png
diff --git a/gui/slick/images/flags/si.png b/gui/slick/images/flags/sin.png
similarity index 100%
rename from gui/slick/images/flags/si.png
rename to gui/slick/images/flags/sin.png
diff --git a/gui/slick/images/flags/sk.png b/gui/slick/images/flags/slk.png
similarity index 100%
rename from gui/slick/images/flags/sk.png
rename to gui/slick/images/flags/slk.png
diff --git a/gui/slick/images/flags/sl.png b/gui/slick/images/flags/slv.png
similarity index 100%
rename from gui/slick/images/flags/sl.png
rename to gui/slick/images/flags/slv.png
diff --git a/gui/slick/images/flags/se.png b/gui/slick/images/flags/sme.png
similarity index 100%
rename from gui/slick/images/flags/se.png
rename to gui/slick/images/flags/sme.png
diff --git a/gui/slick/images/flags/sm.png b/gui/slick/images/flags/smo.png
similarity index 100%
rename from gui/slick/images/flags/sm.png
rename to gui/slick/images/flags/smo.png
diff --git a/gui/slick/images/flags/sn.png b/gui/slick/images/flags/sna.png
similarity index 100%
rename from gui/slick/images/flags/sn.png
rename to gui/slick/images/flags/sna.png
diff --git a/gui/slick/images/flags/sd.png b/gui/slick/images/flags/snd.png
similarity index 100%
rename from gui/slick/images/flags/sd.png
rename to gui/slick/images/flags/snd.png
diff --git a/gui/slick/images/flags/so.png b/gui/slick/images/flags/som.png
similarity index 100%
rename from gui/slick/images/flags/so.png
rename to gui/slick/images/flags/som.png
diff --git a/gui/slick/images/flags/st.png b/gui/slick/images/flags/sot.png
similarity index 100%
rename from gui/slick/images/flags/st.png
rename to gui/slick/images/flags/sot.png
diff --git a/gui/slick/images/flags/es.png b/gui/slick/images/flags/spa.png
similarity index 100%
rename from gui/slick/images/flags/es.png
rename to gui/slick/images/flags/spa.png
diff --git a/gui/slick/images/flags/sq.png b/gui/slick/images/flags/sqi.png
similarity index 100%
rename from gui/slick/images/flags/sq.png
rename to gui/slick/images/flags/sqi.png
diff --git a/gui/slick/images/flags/sc.png b/gui/slick/images/flags/srd.png
similarity index 100%
rename from gui/slick/images/flags/sc.png
rename to gui/slick/images/flags/srd.png
diff --git a/gui/slick/images/flags/sr.png b/gui/slick/images/flags/srp.png
similarity index 100%
rename from gui/slick/images/flags/sr.png
rename to gui/slick/images/flags/srp.png
diff --git a/gui/slick/images/flags/sv.png b/gui/slick/images/flags/swe.png
similarity index 100%
rename from gui/slick/images/flags/sv.png
rename to gui/slick/images/flags/swe.png
diff --git a/gui/slick/images/flags/tt.png b/gui/slick/images/flags/tat.png
similarity index 100%
rename from gui/slick/images/flags/tt.png
rename to gui/slick/images/flags/tat.png
diff --git a/gui/slick/images/flags/tg.png b/gui/slick/images/flags/tgk.png
similarity index 100%
rename from gui/slick/images/flags/tg.png
rename to gui/slick/images/flags/tgk.png
diff --git a/gui/slick/images/flags/tl.png b/gui/slick/images/flags/tgl.png
similarity index 100%
rename from gui/slick/images/flags/tl.png
rename to gui/slick/images/flags/tgl.png
diff --git a/gui/slick/images/flags/th.png b/gui/slick/images/flags/tha.png
similarity index 100%
rename from gui/slick/images/flags/th.png
rename to gui/slick/images/flags/tha.png
diff --git a/gui/slick/images/flags/to.png b/gui/slick/images/flags/ton.png
similarity index 100%
rename from gui/slick/images/flags/to.png
rename to gui/slick/images/flags/ton.png
diff --git a/gui/slick/images/flags/tn.png b/gui/slick/images/flags/tsn.png
similarity index 100%
rename from gui/slick/images/flags/tn.png
rename to gui/slick/images/flags/tsn.png
diff --git a/gui/slick/images/flags/tk.png b/gui/slick/images/flags/tuk.png
similarity index 100%
rename from gui/slick/images/flags/tk.png
rename to gui/slick/images/flags/tuk.png
diff --git a/gui/slick/images/flags/tr.png b/gui/slick/images/flags/tur.png
similarity index 100%
rename from gui/slick/images/flags/tr.png
rename to gui/slick/images/flags/tur.png
diff --git a/gui/slick/images/flags/tw.png b/gui/slick/images/flags/twi.png
similarity index 100%
rename from gui/slick/images/flags/tw.png
rename to gui/slick/images/flags/twi.png
diff --git a/gui/slick/images/flags/ug.png b/gui/slick/images/flags/uig.png
similarity index 100%
rename from gui/slick/images/flags/ug.png
rename to gui/slick/images/flags/uig.png
diff --git a/gui/slick/images/flags/uk.png b/gui/slick/images/flags/ukr.png
similarity index 100%
rename from gui/slick/images/flags/uk.png
rename to gui/slick/images/flags/ukr.png
diff --git a/gui/slick/images/flags/und.png b/gui/slick/images/flags/und.png
new file mode 100644
index 0000000000000000000000000000000000000000..af9249bc317b724a8923e1447c60977dc9a91827
Binary files /dev/null and b/gui/slick/images/flags/und.png differ
diff --git a/gui/slick/images/flags/uz.png b/gui/slick/images/flags/uzb.png
similarity index 100%
rename from gui/slick/images/flags/uz.png
rename to gui/slick/images/flags/uzb.png
diff --git a/gui/slick/images/flags/ve.png b/gui/slick/images/flags/ven.png
similarity index 100%
rename from gui/slick/images/flags/ve.png
rename to gui/slick/images/flags/ven.png
diff --git a/gui/slick/images/flags/vi.png b/gui/slick/images/flags/vie.png
similarity index 100%
rename from gui/slick/images/flags/vi.png
rename to gui/slick/images/flags/vie.png
diff --git a/gui/slick/images/flags/za.png b/gui/slick/images/flags/zha.png
similarity index 100%
rename from gui/slick/images/flags/za.png
rename to gui/slick/images/flags/zha.png
diff --git a/gui/slick/images/flags/zh.png b/gui/slick/images/flags/zho.png
similarity index 100%
rename from gui/slick/images/flags/zh.png
rename to gui/slick/images/flags/zho.png
diff --git a/gui/slick/images/providers/6box.png b/gui/slick/images/providers/6box.png
index 60eb5858f35ba2532f9de19260735b1b77af3204..42819b21a34ae46042651d074bb1cab9c70b0b81 100644
Binary files a/gui/slick/images/providers/6box.png and b/gui/slick/images/providers/6box.png differ
diff --git a/gui/slick/images/providers/6box_me.png b/gui/slick/images/providers/6box_me.png
new file mode 100644
index 0000000000000000000000000000000000000000..42819b21a34ae46042651d074bb1cab9c70b0b81
Binary files /dev/null and b/gui/slick/images/providers/6box_me.png differ
diff --git a/gui/slick/images/providers/BLUETIGERS.png b/gui/slick/images/providers/BLUETIGERS.png
new file mode 100644
index 0000000000000000000000000000000000000000..e6f83e734de85b96f2577e7a35e0437a48d242f4
Binary files /dev/null and b/gui/slick/images/providers/BLUETIGERS.png differ
diff --git a/gui/slick/images/providers/FNT.png b/gui/slick/images/providers/FNT.png
new file mode 100644
index 0000000000000000000000000000000000000000..af30de65eb256df3ea3b25a7f2c52cc6aece0e94
Binary files /dev/null and b/gui/slick/images/providers/FNT.png differ
diff --git a/gui/slick/images/providers/althub.png b/gui/slick/images/providers/althub.png
new file mode 100644
index 0000000000000000000000000000000000000000..015ed41dcb5f85773e27f0469ec225cfd29936be
Binary files /dev/null and b/gui/slick/images/providers/althub.png differ
diff --git a/gui/slick/images/providers/althub_co_za.png b/gui/slick/images/providers/althub_co_za.png
new file mode 100644
index 0000000000000000000000000000000000000000..015ed41dcb5f85773e27f0469ec225cfd29936be
Binary files /dev/null and b/gui/slick/images/providers/althub_co_za.png differ
diff --git a/gui/slick/images/providers/bitsoup.png b/gui/slick/images/providers/bitsoup.png
index c20762e91976edd648835b99f3fd27ee15222136..a787b8e53ef7f1fec555c38302cb46aa429cdbfe 100644
Binary files a/gui/slick/images/providers/bitsoup.png and b/gui/slick/images/providers/bitsoup.png differ
diff --git a/gui/slick/images/providers/btdigg.png b/gui/slick/images/providers/btdigg.png
new file mode 100644
index 0000000000000000000000000000000000000000..aa33b2a5a880f81b439d52de9bc4131f6ab30a39
Binary files /dev/null and b/gui/slick/images/providers/btdigg.png differ
diff --git a/gui/slick/images/providers/newz_complex.png b/gui/slick/images/providers/newz_complex.png
new file mode 100644
index 0000000000000000000000000000000000000000..7d01d6dd46b25860902857a527cad7dc1c637ed0
Binary files /dev/null and b/gui/slick/images/providers/newz_complex.png differ
diff --git a/gui/slick/images/providers/newz_complex_org.png b/gui/slick/images/providers/newz_complex_org.png
new file mode 100644
index 0000000000000000000000000000000000000000..7d01d6dd46b25860902857a527cad7dc1c637ed0
Binary files /dev/null and b/gui/slick/images/providers/newz_complex_org.png differ
diff --git a/gui/slick/images/providers/nmatrix.png b/gui/slick/images/providers/nmatrix.png
new file mode 100644
index 0000000000000000000000000000000000000000..015ed41dcb5f85773e27f0469ec225cfd29936be
Binary files /dev/null and b/gui/slick/images/providers/nmatrix.png differ
diff --git a/gui/slick/images/providers/nmatrix_co_za.png b/gui/slick/images/providers/nmatrix_co_za.png
index 316fd5bf4f5698de6165b1814b216794aa5bb868..015ed41dcb5f85773e27f0469ec225cfd29936be 100644
Binary files a/gui/slick/images/providers/nmatrix_co_za.png and b/gui/slick/images/providers/nmatrix_co_za.png differ
diff --git a/gui/slick/images/providers/nzb_cat.png b/gui/slick/images/providers/nzb_cat.png
new file mode 100644
index 0000000000000000000000000000000000000000..2aacec919c6326747e6cd7f82fe66c11a277cbe1
Binary files /dev/null and b/gui/slick/images/providers/nzb_cat.png differ
diff --git a/gui/slick/images/providers/nzbfriends.png b/gui/slick/images/providers/nzbfriends.png
new file mode 100644
index 0000000000000000000000000000000000000000..f50befc3df5f9aa67d736fd3f4f3f456be9c1659
Binary files /dev/null and b/gui/slick/images/providers/nzbfriends.png differ
diff --git a/gui/slick/images/providers/nzbgeek.png b/gui/slick/images/providers/nzbgeek.png
index 0a2ddb86ae3071e9442968947c4b9e42fd4221a1..0eb8739b921af7e01a271de85281a8add01cb993 100644
Binary files a/gui/slick/images/providers/nzbgeek.png and b/gui/slick/images/providers/nzbgeek.png differ
diff --git a/gui/slick/images/providers/nzbgeek_info.png b/gui/slick/images/providers/nzbgeek_info.png
index 0a2ddb86ae3071e9442968947c4b9e42fd4221a1..0eb8739b921af7e01a271de85281a8add01cb993 100644
Binary files a/gui/slick/images/providers/nzbgeek_info.png and b/gui/slick/images/providers/nzbgeek_info.png differ
diff --git a/gui/slick/images/providers/nzbid.png b/gui/slick/images/providers/nzbid.png
new file mode 100644
index 0000000000000000000000000000000000000000..ceeffce156d7cbcbb84388d1a31afae3bc9077fd
Binary files /dev/null and b/gui/slick/images/providers/nzbid.png differ
diff --git a/gui/slick/images/providers/nzbid_org.png b/gui/slick/images/providers/nzbid_org.png
index 486ca108da8a0c75880d5b070cfe8a91b89de42a..ceeffce156d7cbcbb84388d1a31afae3bc9077fd 100644
Binary files a/gui/slick/images/providers/nzbid_org.png and b/gui/slick/images/providers/nzbid_org.png differ
diff --git a/gui/slick/images/providers/nzbindex.png b/gui/slick/images/providers/nzbindex.png
new file mode 100644
index 0000000000000000000000000000000000000000..3fc4fa99df13448d3b22360fda0e86894909d144
Binary files /dev/null and b/gui/slick/images/providers/nzbindex.png differ
diff --git a/gui/slick/images/providers/nzbindex_in.png b/gui/slick/images/providers/nzbindex_in.png
index f958657d45fa3928ca049ec34b429ad8be91f595..c99d23fcbb8886edc26bb2aa4249c94e70ac6603 100644
Binary files a/gui/slick/images/providers/nzbindex_in.png and b/gui/slick/images/providers/nzbindex_in.png differ
diff --git a/gui/slick/images/providers/nzbplanet.png b/gui/slick/images/providers/nzbplanet.png
index 92c9e0d73cf3c749b7715a512cc2fc59cddf551a..1b9a63450291f48ea47e30c67c062f88f87d71af 100644
Binary files a/gui/slick/images/providers/nzbplanet.png and b/gui/slick/images/providers/nzbplanet.png differ
diff --git a/gui/slick/images/providers/nzbplanet_net.png b/gui/slick/images/providers/nzbplanet_net.png
index 3b46d6bfcb5d60e1a55a915d40ad7cf7707f0f18..1b9a63450291f48ea47e30c67c062f88f87d71af 100644
Binary files a/gui/slick/images/providers/nzbplanet_net.png and b/gui/slick/images/providers/nzbplanet_net.png differ
diff --git a/gui/slick/images/providers/opennzb.png b/gui/slick/images/providers/opennzb.png
new file mode 100644
index 0000000000000000000000000000000000000000..1b9a63450291f48ea47e30c67c062f88f87d71af
Binary files /dev/null and b/gui/slick/images/providers/opennzb.png differ
diff --git a/gui/slick/images/providers/opennzb_net.png b/gui/slick/images/providers/opennzb_net.png
index 3b46d6bfcb5d60e1a55a915d40ad7cf7707f0f18..1b9a63450291f48ea47e30c67c062f88f87d71af 100644
Binary files a/gui/slick/images/providers/opennzb_net.png and b/gui/slick/images/providers/opennzb_net.png differ
diff --git a/gui/slick/images/subtitles/bierdopje.png b/gui/slick/images/subtitles/bierdopje.png
deleted file mode 100644
index 349eb3bc88c197c8b916f94781291284f637fb34..0000000000000000000000000000000000000000
Binary files a/gui/slick/images/subtitles/bierdopje.png and /dev/null differ
diff --git a/gui/slick/images/subtitles/podnapisi.png b/gui/slick/images/subtitles/podnapisi.png
new file mode 100644
index 0000000000000000000000000000000000000000..0f6bddc4b7df25903f3f3f0ae0dd7f450a570d43
Binary files /dev/null and b/gui/slick/images/subtitles/podnapisi.png differ
diff --git a/gui/slick/images/subtitles/podnapisiweb.png b/gui/slick/images/subtitles/podnapisiweb.png
deleted file mode 100644
index c640d0db289a978589e5e97355f1319e0a07e237..0000000000000000000000000000000000000000
Binary files a/gui/slick/images/subtitles/podnapisiweb.png and /dev/null differ
diff --git a/gui/slick/images/subtitles/subscenter.png b/gui/slick/images/subtitles/subscenter.png
index 84389c5763885d840c67b07cae71f25aa3967faa..3b9157c8183632ed797c6d914c79c27a82e0efdb 100644
Binary files a/gui/slick/images/subtitles/subscenter.png and b/gui/slick/images/subtitles/subscenter.png differ
diff --git a/gui/slick/images/subtitles/thesubdb.png b/gui/slick/images/subtitles/thesubdb.png
index 69409549684a7f58a1bccf7eeab7cf0333058add..48ea0913d31f468b7112a2c7bbee56937ae866c2 100644
Binary files a/gui/slick/images/subtitles/thesubdb.png and b/gui/slick/images/subtitles/thesubdb.png differ
diff --git a/gui/slick/interfaces/default/apiBuilder.tmpl b/gui/slick/interfaces/default/apiBuilder.tmpl
index 6ee5ca8db6148de31134893b7f5ff877fcf0a0cc..79eefee603375306575bdc1df1c2835d66893a29 100644
--- a/gui/slick/interfaces/default/apiBuilder.tmpl
+++ b/gui/slick/interfaces/default/apiBuilder.tmpl
@@ -32,6 +32,7 @@ addListGroup("api", "Command");
addOption("Command", "SickRage", "?cmd=sb", 1); //make default
addList("Command", "SickRage.AddRootDir", "?cmd=sb.addrootdir", "sb.addrootdir", "", "", "action");
+addOption("Command", "SickRage.CheckVersion", "?cmd=sb.checkversion", "", "", "action");
addOption("Command", "SickRage.CheckScheduler", "?cmd=sb.checkscheduler", "", "", "action");
addList("Command", "SickRage.DeleteRootDir", "?cmd=sb.deleterootdir", "sb.deleterootdir", "", "", "action");
addOption("Command", "SickRage.ForceSearch", "?cmd=sb.forcesearch", "", "", "action");
@@ -44,6 +45,7 @@ addOption("Command", "SickRage.Restart", "?cmd=sb.restart", "", "", "action");
addList("Command", "SickRage.searchindexers", "?cmd=sb.searchindexers", "sb.searchindexers", "", "", "action");
addList("Command", "SickRage.SetDefaults", "?cmd=sb.setdefaults", "sb.setdefaults", "", "", "action");
addOption("Command", "SickRage.Shutdown", "?cmd=sb.shutdown", "", "", "action");
+addOption("Command", "SickRage.Update", "?cmd=sb.update", "", "", "action");
addList("Command", "Coming Episodes", "?cmd=future", "future");
addList("Command", "Episode", "?cmd=episode", "episode");
addList("Command", "Episode.Search", "?cmd=episode.search", "episode.search", "", "", "action");
diff --git a/gui/slick/interfaces/default/config_general.tmpl b/gui/slick/interfaces/default/config_general.tmpl
index d7c04f944fbe44df8c83df0ef25662d75b1b986a..ada2e4513eaa3e3d9a012c51d14f9960dd593b70 100644
--- a/gui/slick/interfaces/default/config_general.tmpl
+++ b/gui/slick/interfaces/default/config_general.tmpl
@@ -773,4 +773,4 @@
//-->
</script>
-#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_bottom.tmpl')
\ No newline at end of file
+#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_bottom.tmpl')
diff --git a/gui/slick/interfaces/default/config_notifications.tmpl b/gui/slick/interfaces/default/config_notifications.tmpl
index eb392ee48353c8e26150ff84f7b883ec56fcb07c..0a1816b6f6fcebd6de716863fd0feb05c25e5f12 100644
--- a/gui/slick/interfaces/default/config_notifications.tmpl
+++ b/gui/slick/interfaces/default/config_notifications.tmpl
@@ -39,7 +39,7 @@
</div>
<fieldset class="component-group-list">
<div class="field-pair">
- <label class="cleafix" for="use_kodi">
+ <label class="clearfix" for="use_kodi">
<span class="component-title">Enable</span>
<span class="component-desc">
<input type="checkbox" class="enabler" name="use_kodi" id="use_kodi" #if $sickbeard.USE_KODI then "checked=\"checked\"" else ""# />
@@ -911,6 +911,16 @@
<span class="component-desc"><a href="<%= anon_url('https://pushover.net/apps/clone/sickrage') %>" rel="noreferrer" onclick="window.open(this.href, '_blank'); return false;"><b>Click here</b></a> to create a Pushover API key</span>
</label>
</div>
+ <div class="field-pair">
+ <label for="pushover_device">
+ <span class="component-title">Pushover devices</span>
+ <input type="text" name="pushover_device" id="pushover_device" value="$sickbeard.PUSHOVER_DEVICE" class="form-control input-sm input250" />
+ </label>
+ <label>
+ <span class="component-title"> </span>
+ <span class="component-desc">comma separated list of pushover devices you want to send notifications to</span>
+ </label>
+ </div>
<div class="testNotification" id="testPushover-result">Click below to test.</div>
<input class="btn" type="button" value="Test Pushover" id="testPushover" />
<input type="submit" class="config_submitter btn" value="Save Changes" />
@@ -1462,7 +1472,7 @@
<input type="button" class="btn hide" value="Authorize SickRage" id="authTrakt" />
<div class="field-pair">
<label for="trakt_disable_ssl_verify">
- <span class="component-title">Disable SSL Verification:</span>
+ <span class="component-title">Disable SSL Verification</span>
<span class="component-desc">
<input type="checkbox" class="enabler" name="trakt_disable_ssl_verify" id="trakt_disable_ssl_verify" #if $sickbeard.TRAKT_DISABLE_SSL_VERIFY then "checked=\"checked\"" else ""# />
<p>Disable SSL certificate verification for broken SSL installs (like QNAP NAS)</p>
@@ -1471,7 +1481,7 @@
</div>
<div class="field-pair">
<label for="trakt_timeout">
- <span class="component-title">API Timeout:</span>
+ <span class="component-title">API Timeout</span>
<input type="text" name="trakt_timeout" id="trakt_timeout" value="$sickbeard.TRAKT_TIMEOUT" class="form-control input-sm input75" />
</label>
<p>
@@ -1482,7 +1492,7 @@
</div>
<div class="field-pair">
<label for="trakt_default_indexer">
- <span class="component-title">Default indexer:</span>
+ <span class="component-title">Default indexer</span>
<span class="component-desc">
<select id="trakt_default_indexer" name="trakt_default_indexer" class="form-control input-sm">
#for $indexer in $sickbeard.indexerApi().indexers
@@ -1494,7 +1504,7 @@
</div>
<div class="field-pair">
<label for="trakt_sync">
- <span class="component-title">Sync libraries:</span>
+ <span class="component-title">Sync libraries</span>
<span class="component-desc">
<input type="checkbox" class="enabler" name="trakt_sync" id="trakt_sync" #if $sickbeard.TRAKT_SYNC then "checked=\"checked\"" else ""# />
<p>sync your SickRage show library with your trakt show library.</p>
@@ -1504,7 +1514,7 @@
<div id="content_trakt_sync">
<div class="field-pair">
<label for="trakt_sync_remove">
- <span class="component-title">Remove Episodes From Collection:</span>
+ <span class="component-title">Remove Episodes From Collection</span>
<span class="component-desc">
<input type="checkbox" name="trakt_sync_remove" id="trakt_sync_remove" #if $sickbeard.TRAKT_SYNC_REMOVE then "checked=\"checked\"" else ""# />
<p>Remove an Episode from your Trakt Collection if it is not in your SickRage Library.</p>
@@ -1514,7 +1524,7 @@
</div>
<div class="field-pair">
<label for="trakt_sync_watchlist">
- <span class="component-title">Sync watchlist:</span>
+ <span class="component-title">Sync watchlist</span>
<span class="component-desc">
<input type="checkbox" class="enabler" name="trakt_sync_watchlist" id="trakt_sync_watchlist" #if $sickbeard.TRAKT_SYNC_WATCHLIST then "checked=\"checked\"" else ""# />
<p>sync your SickRage show watchlist with your trakt show watchlist (either Show and Episode).</p>
@@ -1525,7 +1535,7 @@
<div id="content_trakt_sync_watchlist">
<div class="field-pair">
<label for="trakt_method_add">
- <span class="component-title">Watchlist add method:</span>
+ <span class="component-title">Watchlist add method</span>
<select id="trakt_method_add" name="trakt_method_add" class="form-control input-sm">
<option value="0" #if $sickbeard.TRAKT_METHOD_ADD == 0 then "selected=\"selected\"" else ""#>Skip All</option>
<option value="1" #if $sickbeard.TRAKT_METHOD_ADD == 1 then "selected=\"selected\"" else ""#>Download Pilot Only</option>
@@ -1539,35 +1549,51 @@
</div>
<div class="field-pair">
<label for="trakt_remove_watchlist">
- <span class="component-title">Remove episode:</span>
+ <span class="component-title">Remove episode</span>
<span class="component-desc">
<input type="checkbox" name="trakt_remove_watchlist" id="trakt_remove_watchlist" #if $sickbeard.TRAKT_REMOVE_WATCHLIST then "checked=\"checked\"" else ""# />
<p>remove an episode from your watchlist after it is downloaded.</p>
</span>
</label>
- </div>
+ </div>
<div class="field-pair">
<label for="trakt_remove_serieslist">
- <span class="component-title">Remove series:</span>
+ <span class="component-title">Remove series</span>
<span class="component-desc">
<input type="checkbox" name="trakt_remove_serieslist" id="trakt_remove_serieslist" #if $sickbeard.TRAKT_REMOVE_SERIESLIST then "checked=\"checked\"" else ""# />
<p>remove the whole series from your watchlist after any download.</p>
</span>
</label>
</div>
+ <div class="field-pair">
+ <label for="trakt_remove_show_from_sickrage">
+ <span class="component-title">Remove watched show:</span>
+ <span class="component-desc">
+ <input type="checkbox" name="trakt_remove_show_from_sickrage" id="trakt_remove_show_from_sickrage" #if $sickbeard.TRAKT_REMOVE_SHOW_FROM_SICKRAGE then "checked=\"checked\"" else ""# />
+ <p>remove the show from sickrage if it's ended and completely watched</p>
+ </span>
+ </label>
+ </div>
<div class="field-pair">
<label for="trakt_start_paused">
- <span class="component-title">Start paused:</span>
+ <span class="component-title">Start paused</span>
+#if not $sickbeard.TRAKT_USE_ROLLING_DOWNLOAD
<span class="component-desc">
<input type="checkbox" name="trakt_start_paused" id="trakt_start_paused" #if $sickbeard.TRAKT_START_PAUSED then "checked=\"checked\"" else ""# />
<p>show's grabbed from your trakt watchlist start paused.</p>
</span>
+#else
+ <span class="component-desc">
+ <input type="checkbox" name="trakt_start_paused" id="trakt_start_paused" #if $sickbeard.TRAKT_START_PAUSED then "checked=\"checked\"" else ""# disabled="disable"/>
+ <p>show's grabbed from your trakt watchlist start paused.</p>
+ </span>
+#end if
</label>
</div>
</div>
<div class="field-pair">
<label for="trakt_blacklist_name">
- <span class="component-title">Trakt blackList name:</span>
+ <span class="component-title">Trakt blackList name</span>
<input type="text" name="trakt_blacklist_name" id="trakt_blacklist_name" value="$sickbeard.TRAKT_BLACKLIST_NAME" class="form-control input-sm input150" />
</label>
<label>
@@ -1577,7 +1603,7 @@
</div>
<div class="field-pair">
<label for="trakt_use_rolling_download">
- <span class="component-title">Use rolling download:</span>
+ <span class="component-title">Use rolling download</span>
<span class="component-desc">
<input type="checkbox" class="enabler" name="trakt_use_rolling_download" id="trakt_use_rolling_download" #if $sickbeard.TRAKT_USE_ROLLING_DOWNLOAD then "checked=\"checked\"" else ""# />
<p>Collect defined number of episodes after last watched one</p>
@@ -1587,18 +1613,18 @@
<div id="content_trakt_use_rolling_download">
<div class="field-pair">
<label for="trakt_rolling_num_ep">
- <span class="component-title">Number of Episode:</span>
+ <span class="component-title">Number of episodes</span>
<span class="component-desc">
<input type="number" name="trakt_rolling_num_ep" id="trakt_rolling_num_ep" value="$sickbeard.TRAKT_ROLLING_NUM_EP" class="form-control input-sm input75"/>
</label>
<label>
<span class="component-title"> </span>
- <span class="component-desc">numebr of episode that SickBeard try to download fron last watched episode</span>
+ <span class="component-desc">Episodes that Sickrage will download based on the last watched episode</span>
</label>
</div>
<div class="field-pair">
<label for="trakt_rolling_frequency">
- <span class="component-title">Rolling frequency check:</span>
+ <span class="component-title">Rolling frequency check</span>
<input type="text" name="trakt_rolling_frequency" id="trakt_rolling_frequency" value="$sickbeard.TRAKT_ROLLING_FREQUENCY" class="form-control input-sm input250" />
</label>
<p>
@@ -1607,13 +1633,13 @@
</div>
<div class="field-pair">
<label for="trakt_rolling_add_paused">
- <span class="component-title">Should new show to be added paused?:</span>
+ <span class="component-title">Add new show as paused</span>
<span class="component-desc">
<input type="checkbox" name="trakt_rolling_add_paused" id="trakt_rolling_add_paused" #if $sickbeard.TRAKT_ROLLING_ADD_PAUSED then "checked=\"checked\"" else ""# />
</label>
<label>
<span class="component-title"> </span>
- <span class="component-desc">This feauture will try to snatch <i>number of episode</i> if the show is active. Whould you like to add new show in paused mode(this override previous choice)?</span>
+ <span class="component-desc">Stop rolling download to start to download episode.</span>
</label>
</div>
</div>
diff --git a/gui/slick/interfaces/default/config_subtitles.tmpl b/gui/slick/interfaces/default/config_subtitles.tmpl
index dd91d46968e62e5560bb5d97b31a97c7317038e7..91d2139f8d96da70e8ea5e5bfbd8b33570731d76 100644
--- a/gui/slick/interfaces/default/config_subtitles.tmpl
+++ b/gui/slick/interfaces/default/config_subtitles.tmpl
@@ -1,7 +1,7 @@
#from sickbeard import subtitles
#import sickbeard
#from sickbeard.helpers import anon_url
-
+#from babelfish import Language
#set global $title="Config - Subtitles"
#set global $header="Subtitles"
@@ -19,7 +19,7 @@
\$(document).ready(function() {
\$("#subtitles_languages").tokenInput(
[
- <%=",\r\n".join("{id: \"" + lang[2] + "\", name: \"" + lang[3] + "\"}" for lang in subtitles.subtitleLanguageFilter())%>
+ <%=",\r\n".join("{id: \"" + lang.alpha3 + "\", name: \"" + lang.name + "\"}" for lang in subtitles.subtitleLanguageFilter())%>
],
{
method: "POST",
@@ -29,7 +29,7 @@
[
<%=
- ",\r\n".join("{id: \"" + lang + "\", name: \"" + subtitles.getLanguageName(lang) + "\"}" for lang in sickbeard.SUBTITLES_LANGUAGES) if sickbeard.SUBTITLES_LANGUAGES != '' else ''
+ ",\r\n".join("{id: \"" + Language.fromietf(lang).alpha3 + "\", name: \"" + Language.fromietf(lang).name + "\"}" for lang in subtitles.wantedLanguages()) if subtitles.wantedLanguages() else ''
%>
]
}
@@ -143,20 +143,17 @@
<fieldset class="component-group-list" style="margin-left: 50px; margin-top:36px">
<ul id="service_order_list">
#for $curService in $sickbeard.subtitles.sortedServiceList():
- #set $curName = $curService.id
- <li class="ui-state-default" id="$curName">
- <input type="checkbox" id="enable_$curName" class="service_enabler" #if $curService.enabled then "checked=\"checked\"" else ""#/>
- #set $provider_url = $curService.url
- <a href="<%= anon_url(provider_url) %>" class="imgLink" target="_new">
- <img src="$sbRoot/images/subtitles/$curService.image" alt="$curService.name" title="$curService.name" width="16" height="16" style="vertical-align:middle;"/>
- </a>
- <span style="vertical-align:middle;">$curService.name.capitalize()</span>
- #if not $curService.api_based then "*" else ""#
+ <li class="ui-state-default" id="$curService['name']">
+ <input type="checkbox" id="enable_$curService['name']" class="service_enabler" #if $curService['enabled'] then "checked=\"checked\"" else ""#/>
+ <a href="<%= anon_url(curService['url']) %>" class="imgLink" target="_new">
+ <img src="$sbRoot/images/subtitles/$curService.image" alt="$curService['url']" title="$curService['url']" width="16" height="16" style="vertical-align:middle;"/>
+ </a>
+ <span style="vertical-align:middle;">$curService['name'].capitalize()</span>
<span class="ui-icon ui-icon-arrowthick-2-n-s pull-right" style="vertical-align:middle;"></span>
</li>
#end for
</ul>
- <input type="hidden" name="service_order" id="service_order" value="<%=" ".join([x.get('id')+':'+str(int(x.get('enabled'))) for x in sickbeard.subtitles.sortedServiceList()])%>"/>
+ <input type="hidden" name="service_order" id="service_order" value="<%=" ".join(['%s:%d' % (x['name'], x['enabled']) for x in sickbeard.subtitles.sortedServiceList()])%>"/>
<br/><input type="submit" class="btn config_submitter" value="Save Changes" /><br/>
</fieldset>
diff --git a/gui/slick/interfaces/default/displayShow.tmpl b/gui/slick/interfaces/default/displayShow.tmpl
index 92005c5fea5ecba4993cb49c2f2da5fa63c2e7c1..0312c0d05f0968027fc4ca22932831d1cd11b19a 100644
--- a/gui/slick/interfaces/default/displayShow.tmpl
+++ b/gui/slick/interfaces/default/displayShow.tmpl
@@ -3,11 +3,12 @@
#import sickbeard.helpers
#from sickbeard.common import *
#from sickbeard.helpers import anon_url
-#from lib import subliminal
+#import subliminal
#import os.path, os
#import datetime
#import urllib
#import ntpath
+#import babelfish
#set global $title=$show.name
##set global $header = '<a></a>' %
@@ -131,7 +132,7 @@
#end if
#if not $sickbeard.DISPLAY_SHOW_SPECIALS and $season_special:
- $seasonResults.pop(-1)
+ #$seasonResults.pop(-1)
#end if
<span class="h2footer displayspecials pull-right">
@@ -218,13 +219,13 @@
#if not $show.imdbid
#if $show.genre:
#for $genre in $show.genre[1:-1].split('|')
- <a href="<%= anon_url('http://trakt.tv/shows/popular/', genre.lower()) %>" target="_blank" title="View other popular $genre shows on trakt.tv."><li>$genre</li></a>
+ <a href="<%= anon_url('http://trakt.tv/shows/popular/?genres=', genre.lower()) %>" target="_blank" title="View other popular $genre shows on trakt.tv."><li>$genre</li></a>
#end for
#end if
#end if
#if 'year' in $show.imdb_info:
#for $imdbgenre in $show.imdb_info['genres'].replace('Sci-Fi','Science-Fiction').split('|')
- <a href="<%= anon_url('http://trakt.tv/shows/popular/', imdbgenre.lower()) %>" target="_blank" title="View other popular $imdbgenre shows on trakt.tv."><li>$imdbgenre</li></a>
+ <a href="<%= anon_url('http://trakt.tv/shows/popular/?genres=', imdbgenre.lower()) %>" target="_blank" title="View other popular $imdbgenre shows on trakt.tv."><li>$imdbgenre</li></a>
#end for
#end if
</ul>
@@ -285,9 +286,10 @@
</table>
<table style="width:180px; float: right; vertical-align: middle; height: 100%;">
- <tr><td class="showLegend">Info Language:</td><td><img src="$sbRoot/images/flags/${show.lang}.png" width="16" height="11" alt="$show.lang" title="$show.lang" /></td></tr>
+ #set $info_flag = $babelfish.Language.fromietf($show.lang).alpha3 if $subtitles.isValidLanguage($show.lang) else 'unknown'
+ <tr><td class="showLegend">Info Language:</td><td><img src="$sbRoot/images/flags/${info_flag}.png" width="16" height="11" alt="$show.lang" title="$show.lang" onError="this.onerror=null;this.src='$sbRoot/images/flags/unknown.png';"/></td></tr>
#if $sickbeard.USE_SUBTITLES
- <tr><td class="showLegend">Subtitles: </td><td><img src="$sbRoot/images/#if int($show.subtitles) == 1 then "yes16.png\" alt=\"Y" else "no16.png\" alt=\"N"#" width="16" height="16" /></td></tr>
+ <tr><td class="showLegend">Subtitles: </td><td><img src="$sbRoot/images/#if $show.subtitles then "yes16.png\" alt=\"Y" else "no16.png\" alt=\"N"#" width="16" height="16" /></td></tr>
#end if
<tr><td class="showLegend">Flat Folders: </td><td><img src="$sbRoot/images/#if $show.flatten_folders == 1 or $sickbeard.NAMING_FORCE_FOLDERS then "yes16.png\" alt=\"Y" else "no16.png\" alt=\"N"#" width="16" height="16" /></td></tr>
<tr><td class="showLegend">Paused: </td><td><img src="$sbRoot/images/#if int($show.paused) == 1 then "yes16.png\" alt=\"Y" else "no16.png\" alt=\"N"#" width="16" height="16" /></td></tr>
@@ -578,15 +580,10 @@
#end if
</td>
<td class="col-subtitles" align="center">
- #if $epResult["subtitles"]:
- #for $sub_lang in subliminal.language.language_list([x.strip() for x in $epResult["subtitles"].split(',') if x != ""]):
- #if sub_lang.alpha2 != ""
- <img src="$sbRoot/images/flags/${sub_lang.alpha2}.png" width="16" height="11" alt="${sub_lang}" />
- #else
- <img src="$sbRoot/images/flags/unknown.png" width="16" height="11" alt="Unknown" />
- #end if
- #end for
- #end if
+ #for $sub_lang in [babelfish.Language.fromietf(x) for x in $epResult["subtitles"].split(',') if $epResult["subtitles"]]:
+ #set $flag = $sub_lang.alpha3 if $hasattr($sub_lang, 'alpha3') and $sub_lang.alpha3 else $sub_lang.alpha2 if $hasattr($sub_lang, 'alpha2') and $sub_lang.alpha2 else 'unknown'
+ <img src="$sbRoot/images/flags/${flag}.png" width="16" height="11" alt="${sub_lang.name}" onError="this.onerror=null;this.src='$sbRoot/images/flags/unknown.png';" />
+ #end for
</td>
#set $curStatus, $curQuality = $Quality.splitCompositeStatus(int($epResult["status"]))
#if $curQuality != Quality.NONE:
@@ -602,7 +599,7 @@
<a class="epSearch" id="#echo $str($show.indexerid)+'x'+$str(epResult["season"])+'x'+$str(epResult["episode"])#" name="#echo $str($show.indexerid)+'x'+$str(epResult["season"])+'x'+$str(epResult["episode"])#" href="searchEpisode?show=$show.indexerid&season=$epResult["season"]&episode=$epResult["episode"]"><img src="$sbRoot/images/search16.png" width="16" height="16" alt="search" title="Manual Search" /></a>
#end if
#end if
- #if $sickbeard.USE_SUBTITLES and $show.subtitles and len(set(str($epResult["subtitles"]).split(',')).intersection(set($subtitles.wantedLanguages()))) < len($subtitles.wantedLanguages()) and $epResult["location"]
+ #if $sickbeard.USE_SUBTITLES and $show.subtitles and $epResult["location"] and frozenset($subtitles.wantedLanguages()).difference($epResult["subtitles"].split(',')):
<a class="epSubtitlesSearch" href="searchEpisodeSubtitles?show=$show.indexerid&season=$epResult["season"]&episode=$epResult["episode"]"><img src="$sbRoot/images/closed_captioning.png" height="16" alt="search subtitles" title="Search Subtitles" /></a>
#end if
</td>
diff --git a/gui/slick/interfaces/default/editShow.tmpl b/gui/slick/interfaces/default/editShow.tmpl
index 3f959330ee025f79605b30f7a6d9640a31a7a1dd..c1d7c0721400bea3ba04152585044df99c652275 100644
--- a/gui/slick/interfaces/default/editShow.tmpl
+++ b/gui/slick/interfaces/default/editShow.tmpl
@@ -63,16 +63,14 @@ This will <b>affect the episode show search</b> on nzb and torrent provider.<br
#include $os.path.join($sickbeard.PROG_DIR, "gui/slick/interfaces/default/inc_qualityChooser.tmpl")
<br />
-<!--
<b>Default Episode Status:</b><br />
-(this will set a default status of already aired episodes)<br />
+(this will set the status for future episodes)<br />
<select name="defaultEpStatus" id="defaultEpStatusSelect" class="form-control form-control-inline input-sm">
#for $curStatus in [$WANTED, $SKIPPED, $ARCHIVED, $IGNORED]:
<option value="$curStatus" #if $curStatus == $show.default_ep_status then 'selected="selected"' else ''#>$statusStrings[$curStatus]</option>
#end for
</select><br />
<br />
--->
<b>Info Language:</b><br />
(this will only affect the language of the retrieved metadata file contents and episode filenames)<br />
diff --git a/gui/slick/interfaces/default/history.tmpl b/gui/slick/interfaces/default/history.tmpl
index 2123adea52b91ad1c39ce4feeef528e71bd9ab78..904ac9b5ae2fde77bb0f863b10ea2b81f12f5275 100644
--- a/gui/slick/interfaces/default/history.tmpl
+++ b/gui/slick/interfaces/default/history.tmpl
@@ -131,13 +131,9 @@
<td class="tvShow" width="35%"><a href="$sbRoot/home/displayShow?show=$hItem["showid"]#season-$hItem["season"]">$hItem["show_name"] - <%="S%02i" % int(hItem["season"])+"E%02i" % int(hItem["episode"]) %>#if "proper" in $hItem["resource"].lower() or "repack" in $hItem["resource"].lower() then ' <span class="quality Proper">Proper</span>' else ""#</a></td>
<td align="center" #if $curStatus == SUBTITLED then 'class="subtitles_column"' else ''#>
#if $curStatus == SUBTITLED:
- #if $sickbeard.SUBTITLES_MULTI:
- <img width="16" height="11" style="vertical-align:middle;" src="$sbRoot/images/flags/<%= hItem["resource"][len(hItem["resource"])-6:len(hItem["resource"])-4]+'.png'%>" onError="this.onerror=null;this.src='$sbRoot/images/flags/unknown.png';">
- #else
- <img width="16" height="11" style="vertical-align:middle;" src="$sbRoot/images/flags/unknown.png">
- #end if
- #end if
- <span style="cursor: help; vertical-align:middle;" title="$os.path.basename($hItem["resource"])">$statusStrings[$curStatus]</span>
+ <img width="16" height="11" style="vertical-align:middle;" src="$sbRoot/images/flags/${hItem['resource']}.png" onError="this.onerror=null;this.src='$sbRoot/images/flags/unknown.png';">
+ #end if
+ <span style="cursor: help; vertical-align:middle;" title="$os.path.basename($hItem['resource'])">$statusStrings[$curStatus]</span>
</td>
<td align="center">
#if $curStatus == DOWNLOADED:
@@ -153,11 +149,11 @@
#else:
<img src="$sbRoot/images/providers/missing.png" width="16" height="16" style="vertical-align:middle;" title="missing provider"/> <span style="vertical-align:middle;">Missing Provider</span>
#end if
- #else:
- <img src="$sbRoot/images/subtitles/<%=hItem["provider"]+'.png' %>" width="16" height="16" style="vertical-align:middle;" /> <span style="vertical-align:middle;"><%=hItem["provider"].capitalize()%></span>
- #end if
+ #else:
+ <img src="$sbRoot/images/subtitles/${hItem['provider']}.png" width="16" height="16" style="vertical-align:middle;" /> <span style="vertical-align:middle;"><%=hItem["provider"].capitalize()%></span>
#end if
#end if
+ #end if
</td>
<span style="display: none;">$curQuality</span>
<td align="center"><span class="quality $Quality.qualityStrings[$curQuality].replace("720p","HD720p").replace("1080p","HD1080p").replace("HDTV", "HD720p")">$Quality.qualityStrings[$curQuality]</span></td>
@@ -226,9 +222,9 @@
#for $action in sorted($hItem["actions"]):
#set $curStatus, $curQuality = $Quality.splitCompositeStatus(int($action["action"]))
#if $curStatus == SUBTITLED:
- <img src="$sbRoot/images/subtitles/<%=action["provider"]+'.png' %>" width="16" height="16" style="vertical-align:middle;" alt="$action["provider"]" title="<%=action["provider"].capitalize()%>: $os.path.basename($action["resource"])"/>
+ <img src="$sbRoot/images/subtitles/${action['provider']}.png" width="16" height="16" style="vertical-align:middle;" alt="$action["provider"]" title="<%=action["provider"].capitalize()%>: $os.path.basename($action["resource"])"/>
<span style="vertical-align:middle;"> / </span>
- <img width="16" height="11" style="vertical-align:middle;" src="$sbRoot/images/flags/<%= action["resource"][len(action["resource"])-6:len(action["resource"])-4]+'.png'%>" style="vertical-align: middle !important;">
+ <img width="16" height="11" style="vertical-align:middle;" src="$sbRoot/images/flags/${action['resource']}.png" onError="this.onerror=null;this.src='$sbRoot/images/flags/unknown.png';" style="vertical-align: middle !important;">
#end if
#end for
diff --git a/gui/slick/interfaces/default/inc_addShowOptions.tmpl b/gui/slick/interfaces/default/inc_addShowOptions.tmpl
index fd3a4917ea06d75ec91a26bd790f25ebe7811922..668c8411f5c7c3b5a1c8734b43a306d80bf1147a 100644
--- a/gui/slick/interfaces/default/inc_addShowOptions.tmpl
+++ b/gui/slick/interfaces/default/inc_addShowOptions.tmpl
@@ -16,7 +16,7 @@
<div class="field-pair">
<label for="statusSelect">
- <span class="component-title">Set the initial status<br /> of already aired episodes</span>
+ <span class="component-title">Status for previously aired episodes</span>
<span class="component-desc">
<select name="defaultStatus" id="statusSelect" class="form-control form-control-inline input-sm">
#for $curStatus in [$SKIPPED, $WANTED, $ARCHIVED, $IGNORED]:
@@ -26,7 +26,6 @@
</span>
</label>
</div>
-
<div class="field-pair alt">
<label for="flatten_folders" class="clearfix">
<span class="component-title">Flatten Folders</span>
@@ -80,4 +79,4 @@
#include $os.path.join($sickbeard.PROG_DIR, "gui/slick/interfaces/default/inc_blackwhitelist.tmpl")
#else
<input type="hidden" name="anime" id="anime" value="0" />
-#end if
\ No newline at end of file
+#end if
diff --git a/gui/slick/interfaces/default/manage_subtitleMissed.tmpl b/gui/slick/interfaces/default/manage_subtitleMissed.tmpl
index 2b0713adc4b27dc00234c1a5f30bbc4c5a24f653..ecc04db93c9cb61acbf7a4197cfe2f7aa9a6cced 100644
--- a/gui/slick/interfaces/default/manage_subtitleMissed.tmpl
+++ b/gui/slick/interfaces/default/manage_subtitleMissed.tmpl
@@ -1,6 +1,8 @@
-#import sickbeard
-#from lib import subliminal
+#from sickbeard import subtitles
+#import subliminal
+#import babelfish
#import datetime
+#import sickbeard
#from sickbeard import common
#set global $title="Episode Overview"
#set global $header="Episode Overview"
@@ -17,7 +19,7 @@
<h1 class="title">$title</h1>
#end if
#if $whichSubs:
-#set subsLanguage = $subliminal.language.Language($whichSubs) if not $whichSubs == 'all' else 'All'
+#set subsLanguage = $babelfish.Language.fromietf($whichSubs).name if not $whichSubs == 'all' else 'All'
#end if
#if not $whichSubs or ($whichSubs and not $ep_counts):
@@ -29,8 +31,8 @@
<form action="$sbRoot/manage/subtitleMissed" method="get">
Manage episodes without <select name="whichSubs" class="form-control form-control-inline input-sm">
<option value="all">All</option>
-#for $sub_lang in $subliminal.language.language_list($sickbeard.SUBTITLES_LANGUAGES):
-<option value="$sub_lang.alpha2">$sub_lang</option>
+#for $sub_lang in [$babelfish.Language.fromietf(x) for x in $subtitles.wantedLanguages]:
+<option value="$sub_lang.alpha3">$sub_lang.name</option>
#end for
</select>
subtitles
diff --git a/lib/adba/aniDBfileInfo.py b/lib/adba/aniDBfileInfo.py
index ba715b86a20f4b1eb35b2c6142c72d9bf0800263..fb7bd606bcb16a705f646fcc6eb9fda71fb2a3e7 100644
--- a/lib/adba/aniDBfileInfo.py
+++ b/lib/adba/aniDBfileInfo.py
@@ -23,7 +23,7 @@ import xml.etree.cElementTree as etree
import time
# http://www.radicand.org/blog/orz/2010/2/21/edonkey2000-hash-in-python/
-from lib import requests
+import requests
def get_file_hash(filePath):
diff --git a/lib/babelfish/__init__.py b/lib/babelfish/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..559705a25eb5be475f312d785d1f5a5cacb205b2
--- /dev/null
+++ b/lib/babelfish/__init__.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+__title__ = 'babelfish'
+__version__ = '0.5.5-dev'
+__author__ = 'Antoine Bertin'
+__license__ = 'BSD'
+__copyright__ = 'Copyright 2015 the BabelFish authors'
+
+import sys
+
+if sys.version_info[0] >= 3:
+ basestr = str
+else:
+ basestr = basestring
+
+from .converters import (LanguageConverter, LanguageReverseConverter, LanguageEquivalenceConverter, CountryConverter,
+ CountryReverseConverter)
+from .country import country_converters, COUNTRIES, COUNTRY_MATRIX, Country
+from .exceptions import Error, LanguageConvertError, LanguageReverseError, CountryConvertError, CountryReverseError
+from .language import language_converters, LANGUAGES, LANGUAGE_MATRIX, Language
+from .script import SCRIPTS, SCRIPT_MATRIX, Script
diff --git a/lib/babelfish/converters/__init__.py b/lib/babelfish/converters/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..feb687b0e3b5d51ce59aa3e405e6c95663e8d039
--- /dev/null
+++ b/lib/babelfish/converters/__init__.py
@@ -0,0 +1,287 @@
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+import collections
+from pkg_resources import iter_entry_points, EntryPoint
+from ..exceptions import LanguageConvertError, LanguageReverseError
+
+
+# from https://github.com/kennethreitz/requests/blob/master/requests/structures.py
+class CaseInsensitiveDict(collections.MutableMapping):
+ """A case-insensitive ``dict``-like object.
+
+ Implements all methods and operations of
+ ``collections.MutableMapping`` as well as dict's ``copy``. Also
+ provides ``lower_items``.
+
+ All keys are expected to be strings. The structure remembers the
+ case of the last key to be set, and ``iter(instance)``,
+ ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()``
+ will contain case-sensitive keys. However, querying and contains
+ testing is case insensitive:
+
+ cid = CaseInsensitiveDict()
+ cid['English'] = 'eng'
+ cid['ENGLISH'] == 'eng' # True
+ list(cid) == ['English'] # True
+
+ If the constructor, ``.update``, or equality comparison
+ operations are given keys that have equal ``.lower()``s, the
+ behavior is undefined.
+
+ """
+ def __init__(self, data=None, **kwargs):
+ self._store = dict()
+ if data is None:
+ data = {}
+ self.update(data, **kwargs)
+
+ def __setitem__(self, key, value):
+ # Use the lowercased key for lookups, but store the actual
+ # key alongside the value.
+ self._store[key.lower()] = (key, value)
+
+ def __getitem__(self, key):
+ return self._store[key.lower()][1]
+
+ def __delitem__(self, key):
+ del self._store[key.lower()]
+
+ def __iter__(self):
+ return (casedkey for casedkey, mappedvalue in self._store.values())
+
+ def __len__(self):
+ return len(self._store)
+
+ def lower_items(self):
+ """Like iteritems(), but with all lowercase keys."""
+ return (
+ (lowerkey, keyval[1])
+ for (lowerkey, keyval)
+ in self._store.items()
+ )
+
+ def __eq__(self, other):
+ if isinstance(other, collections.Mapping):
+ other = CaseInsensitiveDict(other)
+ else:
+ return NotImplemented
+ # Compare insensitively
+ return dict(self.lower_items()) == dict(other.lower_items())
+
+ # Copy is required
+ def copy(self):
+ return CaseInsensitiveDict(self._store.values())
+
+ def __repr__(self):
+ return '%s(%r)' % (self.__class__.__name__, dict(self.items()))
+
+
+class LanguageConverter(object):
+ """A :class:`LanguageConverter` supports converting an alpha3 language code with an
+ alpha2 country code and a script code into a custom code
+
+ .. attribute:: codes
+
+ Set of possible custom codes
+
+ """
+ def convert(self, alpha3, country=None, script=None):
+ """Convert an alpha3 language code with an alpha2 country code and a script code
+ into a custom code
+
+ :param string alpha3: ISO-639-3 language code
+ :param country: ISO-3166 country code, if any
+ :type country: string or None
+ :param script: ISO-15924 script code, if any
+ :type script: string or None
+ :return: the corresponding custom code
+ :rtype: string
+ :raise: :class:`~babelfish.exceptions.LanguageConvertError`
+
+ """
+ raise NotImplementedError
+
+
+class LanguageReverseConverter(LanguageConverter):
+ """A :class:`LanguageConverter` able to reverse a custom code into a alpha3
+ ISO-639-3 language code, alpha2 ISO-3166-1 country code and ISO-15924 script code
+
+ """
+ def reverse(self, code):
+ """Reverse a custom code into alpha3, country and script code
+
+ :param string code: custom code to reverse
+ :return: the corresponding alpha3 ISO-639-3 language code, alpha2 ISO-3166-1 country code and ISO-15924 script code
+ :rtype: tuple
+ :raise: :class:`~babelfish.exceptions.LanguageReverseError`
+
+ """
+ raise NotImplementedError
+
+
+class LanguageEquivalenceConverter(LanguageReverseConverter):
+ """A :class:`LanguageEquivalenceConverter` is a utility class that allows you to easily define a
+ :class:`LanguageReverseConverter` by only specifying the dict from alpha3 to their corresponding symbols.
+
+ You must specify the dict of equivalence as a class variable named SYMBOLS.
+
+ If you also set the class variable CASE_SENSITIVE to ``True`` then the reverse conversion function will be
+ case-sensitive (it is case-insensitive by default).
+
+ Example::
+
+ class MyCodeConverter(babelfish.LanguageEquivalenceConverter):
+ CASE_SENSITIVE = True
+ SYMBOLS = {'fra': 'mycode1', 'eng': 'mycode2'}
+
+ """
+ CASE_SENSITIVE = False
+
+ def __init__(self):
+ self.codes = set()
+ self.to_symbol = {}
+ if self.CASE_SENSITIVE:
+ self.from_symbol = {}
+ else:
+ self.from_symbol = CaseInsensitiveDict()
+
+ for alpha3, symbol in self.SYMBOLS.items():
+ self.to_symbol[alpha3] = symbol
+ self.from_symbol[symbol] = (alpha3, None, None)
+ self.codes.add(symbol)
+
+ def convert(self, alpha3, country=None, script=None):
+ try:
+ return self.to_symbol[alpha3]
+ except KeyError:
+ raise LanguageConvertError(alpha3, country, script)
+
+ def reverse(self, code):
+ try:
+ return self.from_symbol[code]
+ except KeyError:
+ raise LanguageReverseError(code)
+
+
+class CountryConverter(object):
+ """A :class:`CountryConverter` supports converting an alpha2 country code
+ into a custom code
+
+ .. attribute:: codes
+
+ Set of possible custom codes
+
+ """
+ def convert(self, alpha2):
+ """Convert an alpha2 country code into a custom code
+
+ :param string alpha2: ISO-3166-1 language code
+ :return: the corresponding custom code
+ :rtype: string
+ :raise: :class:`~babelfish.exceptions.CountryConvertError`
+
+ """
+ raise NotImplementedError
+
+
+class CountryReverseConverter(CountryConverter):
+ """A :class:`CountryConverter` able to reverse a custom code into a alpha2
+ ISO-3166-1 country code
+
+ """
+ def reverse(self, code):
+ """Reverse a custom code into alpha2 code
+
+ :param string code: custom code to reverse
+ :return: the corresponding alpha2 ISO-3166-1 country code
+ :rtype: string
+ :raise: :class:`~babelfish.exceptions.CountryReverseError`
+
+ """
+ raise NotImplementedError
+
+
+class ConverterManager(object):
+ """Manager for babelfish converters behaving like a dict with lazy loading
+
+ Loading is done in this order:
+
+ * Entry point converters
+ * Registered converters
+ * Internal converters
+
+ .. attribute:: entry_point
+
+ The entry point where to look for converters
+
+ .. attribute:: internal_converters
+
+ Internal converters with entry point syntax
+
+ """
+ entry_point = ''
+ internal_converters = []
+
+ def __init__(self):
+ #: Registered converters with entry point syntax
+ self.registered_converters = []
+
+ #: Loaded converters
+ self.converters = {}
+
+ def __getitem__(self, name):
+ """Get a converter, lazy loading it if necessary"""
+ if name in self.converters:
+ return self.converters[name]
+ for ep in iter_entry_points(self.entry_point):
+ if ep.name == name:
+ self.converters[ep.name] = ep.load()()
+ return self.converters[ep.name]
+ for ep in (EntryPoint.parse(c) for c in self.registered_converters + self.internal_converters):
+ if ep.name == name:
+ # `require` argument of ep.load() is deprecated in newer versions of setuptools
+ if hasattr(ep, 'resolve'):
+ plugin = ep.resolve()
+ elif hasattr(ep, '_load'):
+ plugin = ep._load()
+ else:
+ plugin = ep.load(require=False)
+ self.converters[ep.name] = plugin()
+ return self.converters[ep.name]
+ raise KeyError(name)
+
+ def __setitem__(self, name, converter):
+ """Load a converter"""
+ self.converters[name] = converter
+
+ def __delitem__(self, name):
+ """Unload a converter"""
+ del self.converters[name]
+
+ def __iter__(self):
+ """Iterator over loaded converters"""
+ return iter(self.converters)
+
+ def register(self, entry_point):
+ """Register a converter
+
+ :param string entry_point: converter to register (entry point syntax)
+ :raise: ValueError if already registered
+
+ """
+ if entry_point in self.registered_converters:
+ raise ValueError('Already registered')
+ self.registered_converters.insert(0, entry_point)
+
+ def unregister(self, entry_point):
+ """Unregister a converter
+
+ :param string entry_point: converter to unregister (entry point syntax)
+
+ """
+ self.registered_converters.remove(entry_point)
+
+ def __contains__(self, name):
+ return name in self.converters
diff --git a/lib/babelfish/converters/alpha2.py b/lib/babelfish/converters/alpha2.py
new file mode 100644
index 0000000000000000000000000000000000000000..aca973ddffe196381db31a48136809528e312130
--- /dev/null
+++ b/lib/babelfish/converters/alpha2.py
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+from . import LanguageEquivalenceConverter
+from ..language import LANGUAGE_MATRIX
+
+
+class Alpha2Converter(LanguageEquivalenceConverter):
+ CASE_SENSITIVE = True
+ SYMBOLS = {}
+ for iso_language in LANGUAGE_MATRIX:
+ if iso_language.alpha2:
+ SYMBOLS[iso_language.alpha3] = iso_language.alpha2
diff --git a/lib/babelfish/converters/alpha3b.py b/lib/babelfish/converters/alpha3b.py
new file mode 100644
index 0000000000000000000000000000000000000000..e90c5f5ea232297927f5bd64eaf045ac920f128d
--- /dev/null
+++ b/lib/babelfish/converters/alpha3b.py
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+from . import LanguageEquivalenceConverter
+from ..language import LANGUAGE_MATRIX
+
+
+class Alpha3BConverter(LanguageEquivalenceConverter):
+ CASE_SENSITIVE = True
+ SYMBOLS = {}
+ for iso_language in LANGUAGE_MATRIX:
+ if iso_language.alpha3b:
+ SYMBOLS[iso_language.alpha3] = iso_language.alpha3b
diff --git a/lib/babelfish/converters/alpha3t.py b/lib/babelfish/converters/alpha3t.py
new file mode 100644
index 0000000000000000000000000000000000000000..6de6e4c643a274e3ffa70791a63a6451a2872a84
--- /dev/null
+++ b/lib/babelfish/converters/alpha3t.py
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+from . import LanguageEquivalenceConverter
+from ..language import LANGUAGE_MATRIX
+
+
+class Alpha3TConverter(LanguageEquivalenceConverter):
+ CASE_SENSITIVE = True
+ SYMBOLS = {}
+ for iso_language in LANGUAGE_MATRIX:
+ if iso_language.alpha3t:
+ SYMBOLS[iso_language.alpha3] = iso_language.alpha3t
diff --git a/lib/babelfish/converters/countryname.py b/lib/babelfish/converters/countryname.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff36c878d06deeefe0c6aa048d0225b617a6af6a
--- /dev/null
+++ b/lib/babelfish/converters/countryname.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+from . import CountryReverseConverter, CaseInsensitiveDict
+from ..country import COUNTRY_MATRIX
+from ..exceptions import CountryConvertError, CountryReverseError
+
+
+class CountryNameConverter(CountryReverseConverter):
+ def __init__(self):
+ self.codes = set()
+ self.to_name = {}
+ self.from_name = CaseInsensitiveDict()
+ for country in COUNTRY_MATRIX:
+ self.codes.add(country.name)
+ self.to_name[country.alpha2] = country.name
+ self.from_name[country.name] = country.alpha2
+
+ def convert(self, alpha2):
+ if alpha2 not in self.to_name:
+ raise CountryConvertError(alpha2)
+ return self.to_name[alpha2]
+
+ def reverse(self, name):
+ if name not in self.from_name:
+ raise CountryReverseError(name)
+ return self.from_name[name]
diff --git a/lib/babelfish/converters/name.py b/lib/babelfish/converters/name.py
new file mode 100644
index 0000000000000000000000000000000000000000..8dd865b7b132b9b36d7a2b02a01d72507f11704b
--- /dev/null
+++ b/lib/babelfish/converters/name.py
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+from . import LanguageEquivalenceConverter
+from ..language import LANGUAGE_MATRIX
+
+
+class NameConverter(LanguageEquivalenceConverter):
+ CASE_SENSITIVE = False
+ SYMBOLS = {}
+ for iso_language in LANGUAGE_MATRIX:
+ if iso_language.name:
+ SYMBOLS[iso_language.alpha3] = iso_language.name
diff --git a/lib/babelfish/converters/opensubtitles.py b/lib/babelfish/converters/opensubtitles.py
new file mode 100644
index 0000000000000000000000000000000000000000..101c40fda1b94b8d1b985383650984fbcd3a8b27
--- /dev/null
+++ b/lib/babelfish/converters/opensubtitles.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+from . import LanguageReverseConverter, CaseInsensitiveDict
+from ..exceptions import LanguageReverseError
+from ..language import language_converters
+
+
+class OpenSubtitlesConverter(LanguageReverseConverter):
+ def __init__(self):
+ self.alpha3b_converter = language_converters['alpha3b']
+ self.alpha2_converter = language_converters['alpha2']
+ self.to_opensubtitles = {('por', 'BR'): 'pob', ('gre', None): 'ell', ('srp', None): 'scc', ('srp', 'ME'): 'mne'}
+ self.from_opensubtitles = CaseInsensitiveDict({'pob': ('por', 'BR'), 'pb': ('por', 'BR'), 'ell': ('ell', None),
+ 'scc': ('srp', None), 'mne': ('srp', 'ME')})
+ self.codes = (self.alpha2_converter.codes | self.alpha3b_converter.codes | set(['pob', 'pb', 'scc', 'mne']))
+
+ def convert(self, alpha3, country=None, script=None):
+ alpha3b = self.alpha3b_converter.convert(alpha3, country, script)
+ if (alpha3b, country) in self.to_opensubtitles:
+ return self.to_opensubtitles[(alpha3b, country)]
+ return alpha3b
+
+ def reverse(self, opensubtitles):
+ if opensubtitles in self.from_opensubtitles:
+ return self.from_opensubtitles[opensubtitles]
+ for conv in [self.alpha3b_converter, self.alpha2_converter]:
+ try:
+ return conv.reverse(opensubtitles)
+ except LanguageReverseError:
+ pass
+ raise LanguageReverseError(opensubtitles)
diff --git a/lib/babelfish/converters/scope.py b/lib/babelfish/converters/scope.py
new file mode 100644
index 0000000000000000000000000000000000000000..73540063cf7ffff10a4044ebd2cb64735f401aa6
--- /dev/null
+++ b/lib/babelfish/converters/scope.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+from . import LanguageConverter
+from ..exceptions import LanguageConvertError
+from ..language import LANGUAGE_MATRIX
+
+
+class ScopeConverter(LanguageConverter):
+ FULLNAME = {'I': 'individual', 'M': 'macrolanguage', 'S': 'special'}
+ SYMBOLS = {}
+ for iso_language in LANGUAGE_MATRIX:
+ SYMBOLS[iso_language.alpha3] = iso_language.scope
+ codes = set(SYMBOLS.values())
+
+ def convert(self, alpha3, country=None, script=None):
+ if self.SYMBOLS[alpha3] in self.FULLNAME:
+ return self.FULLNAME[self.SYMBOLS[alpha3]]
+ raise LanguageConvertError(alpha3, country, script)
diff --git a/lib/babelfish/converters/type.py b/lib/babelfish/converters/type.py
new file mode 100644
index 0000000000000000000000000000000000000000..3b7378c22c76a18771adca2261fb35ba068d1e5c
--- /dev/null
+++ b/lib/babelfish/converters/type.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+from . import LanguageConverter
+from ..exceptions import LanguageConvertError
+from ..language import LANGUAGE_MATRIX
+
+
+class LanguageTypeConverter(LanguageConverter):
+ FULLNAME = {'A': 'ancient', 'C': 'constructed', 'E': 'extinct', 'H': 'historical', 'L': 'living', 'S': 'special'}
+ SYMBOLS = {}
+ for iso_language in LANGUAGE_MATRIX:
+ SYMBOLS[iso_language.alpha3] = iso_language.type
+ codes = set(SYMBOLS.values())
+
+ def convert(self, alpha3, country=None, script=None):
+ if self.SYMBOLS[alpha3] in self.FULLNAME:
+ return self.FULLNAME[self.SYMBOLS[alpha3]]
+ raise LanguageConvertError(alpha3, country, script)
diff --git a/lib/babelfish/country.py b/lib/babelfish/country.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce32d9b50519b057454e452acbc5b59d75dc5295
--- /dev/null
+++ b/lib/babelfish/country.py
@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+from collections import namedtuple
+from functools import partial
+from pkg_resources import resource_stream # @UnresolvedImport
+from .converters import ConverterManager
+from . import basestr
+
+
+COUNTRIES = {}
+COUNTRY_MATRIX = []
+
+#: The namedtuple used in the :data:`COUNTRY_MATRIX`
+IsoCountry = namedtuple('IsoCountry', ['name', 'alpha2'])
+
+f = resource_stream('babelfish', 'data/iso-3166-1.txt')
+f.readline()
+for l in f:
+ iso_country = IsoCountry(*l.decode('utf-8').strip().split(';'))
+ COUNTRIES[iso_country.alpha2] = iso_country.name
+ COUNTRY_MATRIX.append(iso_country)
+f.close()
+
+
+class CountryConverterManager(ConverterManager):
+ """:class:`~babelfish.converters.ConverterManager` for country converters"""
+ entry_point = 'babelfish.country_converters'
+ internal_converters = ['name = babelfish.converters.countryname:CountryNameConverter']
+
+country_converters = CountryConverterManager()
+
+
+class CountryMeta(type):
+ """The :class:`Country` metaclass
+
+ Dynamically redirect :meth:`Country.frommycode` to :meth:`Country.fromcode` with the ``mycode`` `converter`
+
+ """
+ def __getattr__(cls, name):
+ if name.startswith('from'):
+ return partial(cls.fromcode, converter=name[4:])
+ return type.__getattribute__(cls, name)
+
+
+class Country(CountryMeta(str('CountryBase'), (object,), {})):
+ """A country on Earth
+
+ A country is represented by a 2-letter code from the ISO-3166 standard
+
+ :param string country: 2-letter ISO-3166 country code
+
+ """
+ def __init__(self, country):
+ if country not in COUNTRIES:
+ raise ValueError('%r is not a valid country' % country)
+
+ #: ISO-3166 2-letter country code
+ self.alpha2 = country
+
+ @classmethod
+ def fromcode(cls, code, converter):
+ """Create a :class:`Country` by its `code` using `converter` to
+ :meth:`~babelfish.converters.CountryReverseConverter.reverse` it
+
+ :param string code: the code to reverse
+ :param string converter: name of the :class:`~babelfish.converters.CountryReverseConverter` to use
+ :return: the corresponding :class:`Country` instance
+ :rtype: :class:`Country`
+
+ """
+ return cls(country_converters[converter].reverse(code))
+
+ def __getstate__(self):
+ return self.alpha2
+
+ def __setstate__(self, state):
+ self.alpha2 = state
+
+ def __getattr__(self, name):
+ return country_converters[name].convert(self.alpha2)
+
+ def __hash__(self):
+ return hash(self.alpha2)
+
+ def __eq__(self, other):
+ if isinstance(other, basestr):
+ return str(self) == other
+ if not isinstance(other, Country):
+ return False
+ return self.alpha2 == other.alpha2
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __repr__(self):
+ return '<Country [%s]>' % self
+
+ def __str__(self):
+ return self.alpha2
diff --git a/lib/babelfish/data/get_files.py b/lib/babelfish/data/get_files.py
new file mode 100644
index 0000000000000000000000000000000000000000..aaa090cccc0ee6aa898fae793811e65b0fa4e501
--- /dev/null
+++ b/lib/babelfish/data/get_files.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+import os.path
+import tempfile
+import zipfile
+import requests
+
+
+DATA_DIR = os.path.dirname(__file__)
+
+# iso-3166-1.txt
+print('Downloading ISO-3166-1 standard (ISO country codes)...')
+with open(os.path.join(DATA_DIR, 'iso-3166-1.txt'), 'w') as f:
+ r = requests.get('http://www.iso.org/iso/home/standards/country_codes/country_names_and_code_elements_txt.htm')
+ f.write(r.content.strip())
+
+# iso-639-3.tab
+print('Downloading ISO-639-3 standard (ISO language codes)...')
+with tempfile.TemporaryFile() as f:
+ r = requests.get('http://www-01.sil.org/iso639-3/iso-639-3_Code_Tables_20130531.zip')
+ f.write(r.content)
+ with zipfile.ZipFile(f) as z:
+ z.extract('iso-639-3.tab', DATA_DIR)
+
+# iso-15924
+print('Downloading ISO-15924 standard (ISO script codes)...')
+with tempfile.TemporaryFile() as f:
+ r = requests.get('http://www.unicode.org/iso15924/iso15924.txt.zip')
+ f.write(r.content)
+ with zipfile.ZipFile(f) as z:
+ z.extract('iso15924-utf8-20131012.txt', DATA_DIR)
+
+# opensubtitles supported languages
+print('Downloading OpenSubtitles supported languages...')
+with open(os.path.join(DATA_DIR, 'opensubtitles_languages.txt'), 'w') as f:
+ r = requests.get('http://www.opensubtitles.org/addons/export_languages.php')
+ f.write(r.content)
+
+print('Done!')
diff --git a/lib/babelfish/data/iso-3166-1.txt b/lib/babelfish/data/iso-3166-1.txt
new file mode 100644
index 0000000000000000000000000000000000000000..da10507229d5a6f03eea21ce7952324afb82b49e
--- /dev/null
+++ b/lib/babelfish/data/iso-3166-1.txt
@@ -0,0 +1,250 @@
+Country Name;ISO 3166-1-alpha-2 code
+AFGHANISTAN;AF
+ÅLAND ISLANDS;AX
+ALBANIA;AL
+ALGERIA;DZ
+AMERICAN SAMOA;AS
+ANDORRA;AD
+ANGOLA;AO
+ANGUILLA;AI
+ANTARCTICA;AQ
+ANTIGUA AND BARBUDA;AG
+ARGENTINA;AR
+ARMENIA;AM
+ARUBA;AW
+AUSTRALIA;AU
+AUSTRIA;AT
+AZERBAIJAN;AZ
+BAHAMAS;BS
+BAHRAIN;BH
+BANGLADESH;BD
+BARBADOS;BB
+BELARUS;BY
+BELGIUM;BE
+BELIZE;BZ
+BENIN;BJ
+BERMUDA;BM
+BHUTAN;BT
+BOLIVIA, PLURINATIONAL STATE OF;BO
+BONAIRE, SINT EUSTATIUS AND SABA;BQ
+BOSNIA AND HERZEGOVINA;BA
+BOTSWANA;BW
+BOUVET ISLAND;BV
+BRAZIL;BR
+BRITISH INDIAN OCEAN TERRITORY;IO
+BRUNEI DARUSSALAM;BN
+BULGARIA;BG
+BURKINA FASO;BF
+BURUNDI;BI
+CAMBODIA;KH
+CAMEROON;CM
+CANADA;CA
+CAPE VERDE;CV
+CAYMAN ISLANDS;KY
+CENTRAL AFRICAN REPUBLIC;CF
+CHAD;TD
+CHILE;CL
+CHINA;CN
+CHRISTMAS ISLAND;CX
+COCOS (KEELING) ISLANDS;CC
+COLOMBIA;CO
+COMOROS;KM
+CONGO;CG
+CONGO, THE DEMOCRATIC REPUBLIC OF THE;CD
+COOK ISLANDS;CK
+COSTA RICA;CR
+CÔTE D'IVOIRE;CI
+CROATIA;HR
+CUBA;CU
+CURAÇAO;CW
+CYPRUS;CY
+CZECH REPUBLIC;CZ
+DENMARK;DK
+DJIBOUTI;DJ
+DOMINICA;DM
+DOMINICAN REPUBLIC;DO
+ECUADOR;EC
+EGYPT;EG
+EL SALVADOR;SV
+EQUATORIAL GUINEA;GQ
+ERITREA;ER
+ESTONIA;EE
+ETHIOPIA;ET
+FALKLAND ISLANDS (MALVINAS);FK
+FAROE ISLANDS;FO
+FIJI;FJ
+FINLAND;FI
+FRANCE;FR
+FRENCH GUIANA;GF
+FRENCH POLYNESIA;PF
+FRENCH SOUTHERN TERRITORIES;TF
+GABON;GA
+GAMBIA;GM
+GEORGIA;GE
+GERMANY;DE
+GHANA;GH
+GIBRALTAR;GI
+GREECE;GR
+GREENLAND;GL
+GRENADA;GD
+GUADELOUPE;GP
+GUAM;GU
+GUATEMALA;GT
+GUERNSEY;GG
+GUINEA;GN
+GUINEA-BISSAU;GW
+GUYANA;GY
+HAITI;HT
+HEARD ISLAND AND MCDONALD ISLANDS;HM
+HOLY SEE (VATICAN CITY STATE);VA
+HONDURAS;HN
+HONG KONG;HK
+HUNGARY;HU
+ICELAND;IS
+INDIA;IN
+INDONESIA;ID
+IRAN, ISLAMIC REPUBLIC OF;IR
+IRAQ;IQ
+IRELAND;IE
+ISLE OF MAN;IM
+ISRAEL;IL
+ITALY;IT
+JAMAICA;JM
+JAPAN;JP
+JERSEY;JE
+JORDAN;JO
+KAZAKHSTAN;KZ
+KENYA;KE
+KIRIBATI;KI
+KOREA, DEMOCRATIC PEOPLE'S REPUBLIC OF;KP
+KOREA, REPUBLIC OF;KR
+KUWAIT;KW
+KYRGYZSTAN;KG
+LAO PEOPLE'S DEMOCRATIC REPUBLIC;LA
+LATVIA;LV
+LEBANON;LB
+LESOTHO;LS
+LIBERIA;LR
+LIBYA;LY
+LIECHTENSTEIN;LI
+LITHUANIA;LT
+LUXEMBOURG;LU
+MACAO;MO
+MACEDONIA, THE FORMER YUGOSLAV REPUBLIC OF;MK
+MADAGASCAR;MG
+MALAWI;MW
+MALAYSIA;MY
+MALDIVES;MV
+MALI;ML
+MALTA;MT
+MARSHALL ISLANDS;MH
+MARTINIQUE;MQ
+MAURITANIA;MR
+MAURITIUS;MU
+MAYOTTE;YT
+MEXICO;MX
+MICRONESIA, FEDERATED STATES OF;FM
+MOLDOVA, REPUBLIC OF;MD
+MONACO;MC
+MONGOLIA;MN
+MONTENEGRO;ME
+MONTSERRAT;MS
+MOROCCO;MA
+MOZAMBIQUE;MZ
+MYANMAR;MM
+NAMIBIA;NA
+NAURU;NR
+NEPAL;NP
+NETHERLANDS;NL
+NEW CALEDONIA;NC
+NEW ZEALAND;NZ
+NICARAGUA;NI
+NIGER;NE
+NIGERIA;NG
+NIUE;NU
+NORFOLK ISLAND;NF
+NORTHERN MARIANA ISLANDS;MP
+NORWAY;NO
+OMAN;OM
+PAKISTAN;PK
+PALAU;PW
+PALESTINE, STATE OF;PS
+PANAMA;PA
+PAPUA NEW GUINEA;PG
+PARAGUAY;PY
+PERU;PE
+PHILIPPINES;PH
+PITCAIRN;PN
+POLAND;PL
+PORTUGAL;PT
+PUERTO RICO;PR
+QATAR;QA
+RÉUNION;RE
+ROMANIA;RO
+RUSSIAN FEDERATION;RU
+RWANDA;RW
+SAINT BARTHÉLEMY;BL
+SAINT HELENA, ASCENSION AND TRISTAN DA CUNHA;SH
+SAINT KITTS AND NEVIS;KN
+SAINT LUCIA;LC
+SAINT MARTIN (FRENCH PART);MF
+SAINT PIERRE AND MIQUELON;PM
+SAINT VINCENT AND THE GRENADINES;VC
+SAMOA;WS
+SAN MARINO;SM
+SAO TOME AND PRINCIPE;ST
+SAUDI ARABIA;SA
+SENEGAL;SN
+SERBIA;RS
+SEYCHELLES;SC
+SIERRA LEONE;SL
+SINGAPORE;SG
+SINT MAARTEN (DUTCH PART);SX
+SLOVAKIA;SK
+SLOVENIA;SI
+SOLOMON ISLANDS;SB
+SOMALIA;SO
+SOUTH AFRICA;ZA
+SOUTH GEORGIA AND THE SOUTH SANDWICH ISLANDS;GS
+SOUTH SUDAN;SS
+SPAIN;ES
+SRI LANKA;LK
+SUDAN;SD
+SURINAME;SR
+SVALBARD AND JAN MAYEN;SJ
+SWAZILAND;SZ
+SWEDEN;SE
+SWITZERLAND;CH
+SYRIAN ARAB REPUBLIC;SY
+TAIWAN, PROVINCE OF CHINA;TW
+TAJIKISTAN;TJ
+TANZANIA, UNITED REPUBLIC OF;TZ
+THAILAND;TH
+TIMOR-LESTE;TL
+TOGO;TG
+TOKELAU;TK
+TONGA;TO
+TRINIDAD AND TOBAGO;TT
+TUNISIA;TN
+TURKEY;TR
+TURKMENISTAN;TM
+TURKS AND CAICOS ISLANDS;TC
+TUVALU;TV
+UGANDA;UG
+UKRAINE;UA
+UNITED ARAB EMIRATES;AE
+UNITED KINGDOM;GB
+UNITED STATES;US
+UNITED STATES MINOR OUTLYING ISLANDS;UM
+URUGUAY;UY
+UZBEKISTAN;UZ
+VANUATU;VU
+VENEZUELA, BOLIVARIAN REPUBLIC OF;VE
+VIET NAM;VN
+VIRGIN ISLANDS, BRITISH;VG
+VIRGIN ISLANDS, U.S.;VI
+WALLIS AND FUTUNA;WF
+WESTERN SAHARA;EH
+YEMEN;YE
+ZAMBIA;ZM
+ZIMBABWE;ZW
\ No newline at end of file
diff --git a/lib/babelfish/data/iso-639-3.tab b/lib/babelfish/data/iso-639-3.tab
new file mode 100644
index 0000000000000000000000000000000000000000..f66d683be4d8e7adab9f2322663a7c96d584d51e
--- /dev/null
+++ b/lib/babelfish/data/iso-639-3.tab
@@ -0,0 +1,7875 @@
+Id Part2B Part2T Part1 Scope Language_Type Ref_Name Comment
+aaa I L Ghotuo
+aab I L Alumu-Tesu
+aac I L Ari
+aad I L Amal
+aae I L Arbëreshë Albanian
+aaf I L Aranadan
+aag I L Ambrak
+aah I L Abu' Arapesh
+aai I L Arifama-Miniafia
+aak I L Ankave
+aal I L Afade
+aam I L Aramanik
+aan I L Anambé
+aao I L Algerian Saharan Arabic
+aap I L Pará Arára
+aaq I E Eastern Abnaki
+aar aar aar aa I L Afar
+aas I L Aasáx
+aat I L Arvanitika Albanian
+aau I L Abau
+aaw I L Solong
+aax I L Mandobo Atas
+aaz I L Amarasi
+aba I L Abé
+abb I L Bankon
+abc I L Ambala Ayta
+abd I L Manide
+abe I E Western Abnaki
+abf I L Abai Sungai
+abg I L Abaga
+abh I L Tajiki Arabic
+abi I L Abidji
+abj I E Aka-Bea
+abk abk abk ab I L Abkhazian
+abl I L Lampung Nyo
+abm I L Abanyom
+abn I L Abua
+abo I L Abon
+abp I L Abellen Ayta
+abq I L Abaza
+abr I L Abron
+abs I L Ambonese Malay
+abt I L Ambulas
+abu I L Abure
+abv I L Baharna Arabic
+abw I L Pal
+abx I L Inabaknon
+aby I L Aneme Wake
+abz I L Abui
+aca I L Achagua
+acb I L Áncá
+acd I L Gikyode
+ace ace ace I L Achinese
+acf I L Saint Lucian Creole French
+ach ach ach I L Acoli
+aci I E Aka-Cari
+ack I E Aka-Kora
+acl I E Akar-Bale
+acm I L Mesopotamian Arabic
+acn I L Achang
+acp I L Eastern Acipa
+acq I L Ta'izzi-Adeni Arabic
+acr I L Achi
+acs I E Acroá
+act I L Achterhoeks
+acu I L Achuar-Shiwiar
+acv I L Achumawi
+acw I L Hijazi Arabic
+acx I L Omani Arabic
+acy I L Cypriot Arabic
+acz I L Acheron
+ada ada ada I L Adangme
+adb I L Adabe
+add I L Dzodinka
+ade I L Adele
+adf I L Dhofari Arabic
+adg I L Andegerebinha
+adh I L Adhola
+adi I L Adi
+adj I L Adioukrou
+adl I L Galo
+adn I L Adang
+ado I L Abu
+adp I L Adap
+adq I L Adangbe
+adr I L Adonara
+ads I L Adamorobe Sign Language
+adt I L Adnyamathanha
+adu I L Aduge
+adw I L Amundava
+adx I L Amdo Tibetan
+ady ady ady I L Adyghe
+adz I L Adzera
+aea I E Areba
+aeb I L Tunisian Arabic
+aec I L Saidi Arabic
+aed I L Argentine Sign Language
+aee I L Northeast Pashayi
+aek I L Haeke
+ael I L Ambele
+aem I L Arem
+aen I L Armenian Sign Language
+aeq I L Aer
+aer I L Eastern Arrernte
+aes I E Alsea
+aeu I L Akeu
+aew I L Ambakich
+aey I L Amele
+aez I L Aeka
+afb I L Gulf Arabic
+afd I L Andai
+afe I L Putukwam
+afg I L Afghan Sign Language
+afh afh afh I C Afrihili
+afi I L Akrukay
+afk I L Nanubae
+afn I L Defaka
+afo I L Eloyi
+afp I L Tapei
+afr afr afr af I L Afrikaans
+afs I L Afro-Seminole Creole
+aft I L Afitti
+afu I L Awutu
+afz I L Obokuitai
+aga I E Aguano
+agb I L Legbo
+agc I L Agatu
+agd I L Agarabi
+age I L Angal
+agf I L Arguni
+agg I L Angor
+agh I L Ngelima
+agi I L Agariya
+agj I L Argobba
+agk I L Isarog Agta
+agl I L Fembe
+agm I L Angaataha
+agn I L Agutaynen
+ago I L Tainae
+agq I L Aghem
+agr I L Aguaruna
+ags I L Esimbi
+agt I L Central Cagayan Agta
+agu I L Aguacateco
+agv I L Remontado Dumagat
+agw I L Kahua
+agx I L Aghul
+agy I L Southern Alta
+agz I L Mt. Iriga Agta
+aha I L Ahanta
+ahb I L Axamb
+ahg I L Qimant
+ahh I L Aghu
+ahi I L Tiagbamrin Aizi
+ahk I L Akha
+ahl I L Igo
+ahm I L Mobumrin Aizi
+ahn I L Àhàn
+aho I E Ahom
+ahp I L Aproumu Aizi
+ahr I L Ahirani
+ahs I L Ashe
+aht I L Ahtena
+aia I L Arosi
+aib I L Ainu (China)
+aic I L Ainbai
+aid I E Alngith
+aie I L Amara
+aif I L Agi
+aig I L Antigua and Barbuda Creole English
+aih I L Ai-Cham
+aii I L Assyrian Neo-Aramaic
+aij I L Lishanid Noshan
+aik I L Ake
+ail I L Aimele
+aim I L Aimol
+ain ain ain I L Ainu (Japan)
+aio I L Aiton
+aip I L Burumakok
+aiq I L Aimaq
+air I L Airoran
+ais I L Nataoran Amis
+ait I E Arikem
+aiw I L Aari
+aix I L Aighon
+aiy I L Ali
+aja I L Aja (Sudan)
+ajg I L Aja (Benin)
+aji I L Ajië
+ajn I L Andajin
+ajp I L South Levantine Arabic
+ajt I L Judeo-Tunisian Arabic
+aju I L Judeo-Moroccan Arabic
+ajw I E Ajawa
+ajz I L Amri Karbi
+aka aka aka ak M L Akan
+akb I L Batak Angkola
+akc I L Mpur
+akd I L Ukpet-Ehom
+ake I L Akawaio
+akf I L Akpa
+akg I L Anakalangu
+akh I L Angal Heneng
+aki I L Aiome
+akj I E Aka-Jeru
+akk akk akk I A Akkadian
+akl I L Aklanon
+akm I E Aka-Bo
+ako I L Akurio
+akp I L Siwu
+akq I L Ak
+akr I L Araki
+aks I L Akaselem
+akt I L Akolet
+aku I L Akum
+akv I L Akhvakh
+akw I L Akwa
+akx I E Aka-Kede
+aky I E Aka-Kol
+akz I L Alabama
+ala I L Alago
+alc I L Qawasqar
+ald I L Alladian
+ale ale ale I L Aleut
+alf I L Alege
+alh I L Alawa
+ali I L Amaimon
+alj I L Alangan
+alk I L Alak
+all I L Allar
+alm I L Amblong
+aln I L Gheg Albanian
+alo I L Larike-Wakasihu
+alp I L Alune
+alq I L Algonquin
+alr I L Alutor
+als I L Tosk Albanian
+alt alt alt I L Southern Altai
+alu I L 'Are'are
+alw I L Alaba-K’abeena
+alx I L Amol
+aly I L Alyawarr
+alz I L Alur
+ama I E Amanayé
+amb I L Ambo
+amc I L Amahuaca
+ame I L Yanesha'
+amf I L Hamer-Banna
+amg I L Amurdak
+amh amh amh am I L Amharic
+ami I L Amis
+amj I L Amdang
+amk I L Ambai
+aml I L War-Jaintia
+amm I L Ama (Papua New Guinea)
+amn I L Amanab
+amo I L Amo
+amp I L Alamblak
+amq I L Amahai
+amr I L Amarakaeri
+ams I L Southern Amami-Oshima
+amt I L Amto
+amu I L Guerrero Amuzgo
+amv I L Ambelau
+amw I L Western Neo-Aramaic
+amx I L Anmatyerre
+amy I L Ami
+amz I E Atampaya
+ana I E Andaqui
+anb I E Andoa
+anc I L Ngas
+and I L Ansus
+ane I L Xârâcùù
+anf I L Animere
+ang ang ang I H Old English (ca. 450-1100)
+anh I L Nend
+ani I L Andi
+anj I L Anor
+ank I L Goemai
+anl I L Anu-Hkongso Chin
+anm I L Anal
+ann I L Obolo
+ano I L Andoque
+anp anp anp I L Angika
+anq I L Jarawa (India)
+anr I L Andh
+ans I E Anserma
+ant I L Antakarinya
+anu I L Anuak
+anv I L Denya
+anw I L Anaang
+anx I L Andra-Hus
+any I L Anyin
+anz I L Anem
+aoa I L Angolar
+aob I L Abom
+aoc I L Pemon
+aod I L Andarum
+aoe I L Angal Enen
+aof I L Bragat
+aog I L Angoram
+aoh I E Arma
+aoi I L Anindilyakwa
+aoj I L Mufian
+aok I L Arhö
+aol I L Alor
+aom I L Ömie
+aon I L Bumbita Arapesh
+aor I E Aore
+aos I L Taikat
+aot I L A'tong
+aou I L A'ou
+aox I L Atorada
+aoz I L Uab Meto
+apb I L Sa'a
+apc I L North Levantine Arabic
+apd I L Sudanese Arabic
+ape I L Bukiyip
+apf I L Pahanan Agta
+apg I L Ampanang
+aph I L Athpariya
+api I L Apiaká
+apj I L Jicarilla Apache
+apk I L Kiowa Apache
+apl I L Lipan Apache
+apm I L Mescalero-Chiricahua Apache
+apn I L Apinayé
+apo I L Ambul
+app I L Apma
+apq I L A-Pucikwar
+apr I L Arop-Lokep
+aps I L Arop-Sissano
+apt I L Apatani
+apu I L Apurinã
+apv I E Alapmunte
+apw I L Western Apache
+apx I L Aputai
+apy I L Apalaí
+apz I L Safeyoka
+aqc I L Archi
+aqd I L Ampari Dogon
+aqg I L Arigidi
+aqm I L Atohwaim
+aqn I L Northern Alta
+aqp I E Atakapa
+aqr I L Arhâ
+aqz I L Akuntsu
+ara ara ara ar M L Arabic
+arb I L Standard Arabic
+arc arc arc I A Official Aramaic (700-300 BCE)
+ard I E Arabana
+are I L Western Arrarnta
+arg arg arg an I L Aragonese
+arh I L Arhuaco
+ari I L Arikara
+arj I E Arapaso
+ark I L Arikapú
+arl I L Arabela
+arn arn arn I L Mapudungun
+aro I L Araona
+arp arp arp I L Arapaho
+arq I L Algerian Arabic
+arr I L Karo (Brazil)
+ars I L Najdi Arabic
+aru I E Aruá (Amazonas State)
+arv I L Arbore
+arw arw arw I L Arawak
+arx I L Aruá (Rodonia State)
+ary I L Moroccan Arabic
+arz I L Egyptian Arabic
+asa I L Asu (Tanzania)
+asb I L Assiniboine
+asc I L Casuarina Coast Asmat
+asd I L Asas
+ase I L American Sign Language
+asf I L Australian Sign Language
+asg I L Cishingini
+ash I E Abishira
+asi I L Buruwai
+asj I L Sari
+ask I L Ashkun
+asl I L Asilulu
+asm asm asm as I L Assamese
+asn I L Xingú Asuriní
+aso I L Dano
+asp I L Algerian Sign Language
+asq I L Austrian Sign Language
+asr I L Asuri
+ass I L Ipulo
+ast ast ast I L Asturian
+asu I L Tocantins Asurini
+asv I L Asoa
+asw I L Australian Aborigines Sign Language
+asx I L Muratayak
+asy I L Yaosakor Asmat
+asz I L As
+ata I L Pele-Ata
+atb I L Zaiwa
+atc I E Atsahuaca
+atd I L Ata Manobo
+ate I L Atemble
+atg I L Ivbie North-Okpela-Arhe
+ati I L Attié
+atj I L Atikamekw
+atk I L Ati
+atl I L Mt. Iraya Agta
+atm I L Ata
+atn I L Ashtiani
+ato I L Atong
+atp I L Pudtol Atta
+atq I L Aralle-Tabulahan
+atr I L Waimiri-Atroari
+ats I L Gros Ventre
+att I L Pamplona Atta
+atu I L Reel
+atv I L Northern Altai
+atw I L Atsugewi
+atx I L Arutani
+aty I L Aneityum
+atz I L Arta
+aua I L Asumboa
+aub I L Alugu
+auc I L Waorani
+aud I L Anuta
+aue I L =/Kx'au//'ein
+aug I L Aguna
+auh I L Aushi
+aui I L Anuki
+auj I L Awjilah
+auk I L Heyo
+aul I L Aulua
+aum I L Asu (Nigeria)
+aun I L Molmo One
+auo I E Auyokawa
+aup I L Makayam
+auq I L Anus
+aur I L Aruek
+aut I L Austral
+auu I L Auye
+auw I L Awyi
+aux I E Aurá
+auy I L Awiyaana
+auz I L Uzbeki Arabic
+ava ava ava av I L Avaric
+avb I L Avau
+avd I L Alviri-Vidari
+ave ave ave ae I A Avestan
+avi I L Avikam
+avk I C Kotava
+avl I L Eastern Egyptian Bedawi Arabic
+avm I E Angkamuthi
+avn I L Avatime
+avo I E Agavotaguerra
+avs I E Aushiri
+avt I L Au
+avu I L Avokaya
+avv I L Avá-Canoeiro
+awa awa awa I L Awadhi
+awb I L Awa (Papua New Guinea)
+awc I L Cicipu
+awe I L Awetí
+awg I E Anguthimri
+awh I L Awbono
+awi I L Aekyom
+awk I E Awabakal
+awm I L Arawum
+awn I L Awngi
+awo I L Awak
+awr I L Awera
+aws I L South Awyu
+awt I L Araweté
+awu I L Central Awyu
+awv I L Jair Awyu
+aww I L Awun
+awx I L Awara
+awy I L Edera Awyu
+axb I E Abipon
+axe I E Ayerrerenge
+axg I E Mato Grosso Arára
+axk I L Yaka (Central African Republic)
+axl I E Lower Southern Aranda
+axm I H Middle Armenian
+axx I L Xârâgurè
+aya I L Awar
+ayb I L Ayizo Gbe
+ayc I L Southern Aymara
+ayd I E Ayabadhu
+aye I L Ayere
+ayg I L Ginyanga
+ayh I L Hadrami Arabic
+ayi I L Leyigha
+ayk I L Akuku
+ayl I L Libyan Arabic
+aym aym aym ay M L Aymara
+ayn I L Sanaani Arabic
+ayo I L Ayoreo
+ayp I L North Mesopotamian Arabic
+ayq I L Ayi (Papua New Guinea)
+ayr I L Central Aymara
+ays I L Sorsogon Ayta
+ayt I L Magbukun Ayta
+ayu I L Ayu
+ayy I E Tayabas Ayta
+ayz I L Mai Brat
+aza I L Azha
+azb I L South Azerbaijani
+azd I L Eastern Durango Nahuatl
+aze aze aze az M L Azerbaijani
+azg I L San Pedro Amuzgos Amuzgo
+azj I L North Azerbaijani
+azm I L Ipalapa Amuzgo
+azn I L Western Durango Nahuatl
+azo I L Awing
+azt I L Faire Atta
+azz I L Highland Puebla Nahuatl
+baa I L Babatana
+bab I L Bainouk-Gunyuño
+bac I L Badui
+bae I E Baré
+baf I L Nubaca
+bag I L Tuki
+bah I L Bahamas Creole English
+baj I L Barakai
+bak bak bak ba I L Bashkir
+bal bal bal M L Baluchi
+bam bam bam bm I L Bambara
+ban ban ban I L Balinese
+bao I L Waimaha
+bap I L Bantawa
+bar I L Bavarian
+bas bas bas I L Basa (Cameroon)
+bau I L Bada (Nigeria)
+bav I L Vengo
+baw I L Bambili-Bambui
+bax I L Bamun
+bay I L Batuley
+bba I L Baatonum
+bbb I L Barai
+bbc I L Batak Toba
+bbd I L Bau
+bbe I L Bangba
+bbf I L Baibai
+bbg I L Barama
+bbh I L Bugan
+bbi I L Barombi
+bbj I L Ghomálá'
+bbk I L Babanki
+bbl I L Bats
+bbm I L Babango
+bbn I L Uneapa
+bbo I L Northern Bobo Madaré
+bbp I L West Central Banda
+bbq I L Bamali
+bbr I L Girawa
+bbs I L Bakpinka
+bbt I L Mburku
+bbu I L Kulung (Nigeria)
+bbv I L Karnai
+bbw I L Baba
+bbx I L Bubia
+bby I L Befang
+bbz I L Babalia Creole Arabic
+bca I L Central Bai
+bcb I L Bainouk-Samik
+bcc I L Southern Balochi
+bcd I L North Babar
+bce I L Bamenyam
+bcf I L Bamu
+bcg I L Baga Binari
+bch I L Bariai
+bci I L Baoulé
+bcj I L Bardi
+bck I L Bunaba
+bcl I L Central Bikol
+bcm I L Bannoni
+bcn I L Bali (Nigeria)
+bco I L Kaluli
+bcp I L Bali (Democratic Republic of Congo)
+bcq I L Bench
+bcr I L Babine
+bcs I L Kohumono
+bct I L Bendi
+bcu I L Awad Bing
+bcv I L Shoo-Minda-Nye
+bcw I L Bana
+bcy I L Bacama
+bcz I L Bainouk-Gunyaamolo
+bda I L Bayot
+bdb I L Basap
+bdc I L Emberá-Baudó
+bdd I L Bunama
+bde I L Bade
+bdf I L Biage
+bdg I L Bonggi
+bdh I L Baka (Sudan)
+bdi I L Burun
+bdj I L Bai
+bdk I L Budukh
+bdl I L Indonesian Bajau
+bdm I L Buduma
+bdn I L Baldemu
+bdo I L Morom
+bdp I L Bende
+bdq I L Bahnar
+bdr I L West Coast Bajau
+bds I L Burunge
+bdt I L Bokoto
+bdu I L Oroko
+bdv I L Bodo Parja
+bdw I L Baham
+bdx I L Budong-Budong
+bdy I L Bandjalang
+bdz I L Badeshi
+bea I L Beaver
+beb I L Bebele
+bec I L Iceve-Maci
+bed I L Bedoanas
+bee I L Byangsi
+bef I L Benabena
+beg I L Belait
+beh I L Biali
+bei I L Bekati'
+bej bej bej I L Beja
+bek I L Bebeli
+bel bel bel be I L Belarusian
+bem bem bem I L Bemba (Zambia)
+ben ben ben bn I L Bengali
+beo I L Beami
+bep I L Besoa
+beq I L Beembe
+bes I L Besme
+bet I L Guiberoua Béte
+beu I L Blagar
+bev I L Daloa Bété
+bew I L Betawi
+bex I L Jur Modo
+bey I L Beli (Papua New Guinea)
+bez I L Bena (Tanzania)
+bfa I L Bari
+bfb I L Pauri Bareli
+bfc I L Northern Bai
+bfd I L Bafut
+bfe I L Betaf
+bff I L Bofi
+bfg I L Busang Kayan
+bfh I L Blafe
+bfi I L British Sign Language
+bfj I L Bafanji
+bfk I L Ban Khor Sign Language
+bfl I L Banda-Ndélé
+bfm I L Mmen
+bfn I L Bunak
+bfo I L Malba Birifor
+bfp I L Beba
+bfq I L Badaga
+bfr I L Bazigar
+bfs I L Southern Bai
+bft I L Balti
+bfu I L Gahri
+bfw I L Bondo
+bfx I L Bantayanon
+bfy I L Bagheli
+bfz I L Mahasu Pahari
+bga I L Gwamhi-Wuri
+bgb I L Bobongko
+bgc I L Haryanvi
+bgd I L Rathwi Bareli
+bge I L Bauria
+bgf I L Bangandu
+bgg I L Bugun
+bgi I L Giangan
+bgj I L Bangolan
+bgk I L Bit
+bgl I L Bo (Laos)
+bgm I L Baga Mboteni
+bgn I L Western Balochi
+bgo I L Baga Koga
+bgp I L Eastern Balochi
+bgq I L Bagri
+bgr I L Bawm Chin
+bgs I L Tagabawa
+bgt I L Bughotu
+bgu I L Mbongno
+bgv I L Warkay-Bipim
+bgw I L Bhatri
+bgx I L Balkan Gagauz Turkish
+bgy I L Benggoi
+bgz I L Banggai
+bha I L Bharia
+bhb I L Bhili
+bhc I L Biga
+bhd I L Bhadrawahi
+bhe I L Bhaya
+bhf I L Odiai
+bhg I L Binandere
+bhh I L Bukharic
+bhi I L Bhilali
+bhj I L Bahing
+bhl I L Bimin
+bhm I L Bathari
+bhn I L Bohtan Neo-Aramaic
+bho bho bho I L Bhojpuri
+bhp I L Bima
+bhq I L Tukang Besi South
+bhr I L Bara Malagasy
+bhs I L Buwal
+bht I L Bhattiyali
+bhu I L Bhunjia
+bhv I L Bahau
+bhw I L Biak
+bhx I L Bhalay
+bhy I L Bhele
+bhz I L Bada (Indonesia)
+bia I L Badimaya
+bib I L Bissa
+bic I L Bikaru
+bid I L Bidiyo
+bie I L Bepour
+bif I L Biafada
+big I L Biangai
+bij I L Vaghat-Ya-Bijim-Legeri
+bik bik bik M L Bikol
+bil I L Bile
+bim I L Bimoba
+bin bin bin I L Bini
+bio I L Nai
+bip I L Bila
+biq I L Bipi
+bir I L Bisorio
+bis bis bis bi I L Bislama
+bit I L Berinomo
+biu I L Biete
+biv I L Southern Birifor
+biw I L Kol (Cameroon)
+bix I L Bijori
+biy I L Birhor
+biz I L Baloi
+bja I L Budza
+bjb I E Banggarla
+bjc I L Bariji
+bje I L Biao-Jiao Mien
+bjf I L Barzani Jewish Neo-Aramaic
+bjg I L Bidyogo
+bjh I L Bahinemo
+bji I L Burji
+bjj I L Kanauji
+bjk I L Barok
+bjl I L Bulu (Papua New Guinea)
+bjm I L Bajelani
+bjn I L Banjar
+bjo I L Mid-Southern Banda
+bjp I L Fanamaket
+bjr I L Binumarien
+bjs I L Bajan
+bjt I L Balanta-Ganja
+bju I L Busuu
+bjv I L Bedjond
+bjw I L Bakwé
+bjx I L Banao Itneg
+bjy I E Bayali
+bjz I L Baruga
+bka I L Kyak
+bkc I L Baka (Cameroon)
+bkd I L Binukid
+bkf I L Beeke
+bkg I L Buraka
+bkh I L Bakoko
+bki I L Baki
+bkj I L Pande
+bkk I L Brokskat
+bkl I L Berik
+bkm I L Kom (Cameroon)
+bkn I L Bukitan
+bko I L Kwa'
+bkp I L Boko (Democratic Republic of Congo)
+bkq I L Bakairí
+bkr I L Bakumpai
+bks I L Northern Sorsoganon
+bkt I L Boloki
+bku I L Buhid
+bkv I L Bekwarra
+bkw I L Bekwel
+bkx I L Baikeno
+bky I L Bokyi
+bkz I L Bungku
+bla bla bla I L Siksika
+blb I L Bilua
+blc I L Bella Coola
+bld I L Bolango
+ble I L Balanta-Kentohe
+blf I L Buol
+blg I L Balau
+blh I L Kuwaa
+bli I L Bolia
+blj I L Bolongan
+blk I L Pa'o Karen
+bll I E Biloxi
+blm I L Beli (Sudan)
+bln I L Southern Catanduanes Bikol
+blo I L Anii
+blp I L Blablanga
+blq I L Baluan-Pam
+blr I L Blang
+bls I L Balaesang
+blt I L Tai Dam
+blv I L Bolo
+blw I L Balangao
+blx I L Mag-Indi Ayta
+bly I L Notre
+blz I L Balantak
+bma I L Lame
+bmb I L Bembe
+bmc I L Biem
+bmd I L Baga Manduri
+bme I L Limassa
+bmf I L Bom
+bmg I L Bamwe
+bmh I L Kein
+bmi I L Bagirmi
+bmj I L Bote-Majhi
+bmk I L Ghayavi
+bml I L Bomboli
+bmm I L Northern Betsimisaraka Malagasy
+bmn I E Bina (Papua New Guinea)
+bmo I L Bambalang
+bmp I L Bulgebi
+bmq I L Bomu
+bmr I L Muinane
+bms I L Bilma Kanuri
+bmt I L Biao Mon
+bmu I L Somba-Siawari
+bmv I L Bum
+bmw I L Bomwali
+bmx I L Baimak
+bmy I L Bemba (Democratic Republic of Congo)
+bmz I L Baramu
+bna I L Bonerate
+bnb I L Bookan
+bnc M L Bontok
+bnd I L Banda (Indonesia)
+bne I L Bintauna
+bnf I L Masiwang
+bng I L Benga
+bni I L Bangi
+bnj I L Eastern Tawbuid
+bnk I L Bierebo
+bnl I L Boon
+bnm I L Batanga
+bnn I L Bunun
+bno I L Bantoanon
+bnp I L Bola
+bnq I L Bantik
+bnr I L Butmas-Tur
+bns I L Bundeli
+bnu I L Bentong
+bnv I L Bonerif
+bnw I L Bisis
+bnx I L Bangubangu
+bny I L Bintulu
+bnz I L Beezen
+boa I L Bora
+bob I L Aweer
+bod tib bod bo I L Tibetan
+boe I L Mundabli
+bof I L Bolon
+bog I L Bamako Sign Language
+boh I L Boma
+boi I E Barbareño
+boj I L Anjam
+bok I L Bonjo
+bol I L Bole
+bom I L Berom
+bon I L Bine
+boo I L Tiemacèwè Bozo
+bop I L Bonkiman
+boq I L Bogaya
+bor I L Borôro
+bos bos bos bs I L Bosnian
+bot I L Bongo
+bou I L Bondei
+bov I L Tuwuli
+bow I E Rema
+box I L Buamu
+boy I L Bodo (Central African Republic)
+boz I L Tiéyaxo Bozo
+bpa I L Daakaka
+bpb I E Barbacoas
+bpd I L Banda-Banda
+bpg I L Bonggo
+bph I L Botlikh
+bpi I L Bagupi
+bpj I L Binji
+bpk I L Orowe
+bpl I L Broome Pearling Lugger Pidgin
+bpm I L Biyom
+bpn I L Dzao Min
+bpo I L Anasi
+bpp I L Kaure
+bpq I L Banda Malay
+bpr I L Koronadal Blaan
+bps I L Sarangani Blaan
+bpt I E Barrow Point
+bpu I L Bongu
+bpv I L Bian Marind
+bpw I L Bo (Papua New Guinea)
+bpx I L Palya Bareli
+bpy I L Bishnupriya
+bpz I L Bilba
+bqa I L Tchumbuli
+bqb I L Bagusa
+bqc I L Boko (Benin)
+bqd I L Bung
+bqf I E Baga Kaloum
+bqg I L Bago-Kusuntu
+bqh I L Baima
+bqi I L Bakhtiari
+bqj I L Bandial
+bqk I L Banda-Mbrès
+bql I L Bilakura
+bqm I L Wumboko
+bqn I L Bulgarian Sign Language
+bqo I L Balo
+bqp I L Busa
+bqq I L Biritai
+bqr I L Burusu
+bqs I L Bosngun
+bqt I L Bamukumbit
+bqu I L Boguru
+bqv I L Koro Wachi
+bqw I L Buru (Nigeria)
+bqx I L Baangi
+bqy I L Bengkala Sign Language
+bqz I L Bakaka
+bra bra bra I L Braj
+brb I L Lave
+brc I E Berbice Creole Dutch
+brd I L Baraamu
+bre bre bre br I L Breton
+brf I L Bera
+brg I L Baure
+brh I L Brahui
+bri I L Mokpwe
+brj I L Bieria
+brk I E Birked
+brl I L Birwa
+brm I L Barambu
+brn I L Boruca
+bro I L Brokkat
+brp I L Barapasi
+brq I L Breri
+brr I L Birao
+brs I L Baras
+brt I L Bitare
+bru I L Eastern Bru
+brv I L Western Bru
+brw I L Bellari
+brx I L Bodo (India)
+bry I L Burui
+brz I L Bilbil
+bsa I L Abinomn
+bsb I L Brunei Bisaya
+bsc I L Bassari
+bse I L Wushi
+bsf I L Bauchi
+bsg I L Bashkardi
+bsh I L Kati
+bsi I L Bassossi
+bsj I L Bangwinji
+bsk I L Burushaski
+bsl I E Basa-Gumna
+bsm I L Busami
+bsn I L Barasana-Eduria
+bso I L Buso
+bsp I L Baga Sitemu
+bsq I L Bassa
+bsr I L Bassa-Kontagora
+bss I L Akoose
+bst I L Basketo
+bsu I L Bahonsuai
+bsv I E Baga Sobané
+bsw I L Baiso
+bsx I L Yangkam
+bsy I L Sabah Bisaya
+bta I L Bata
+btc I L Bati (Cameroon)
+btd I L Batak Dairi
+bte I E Gamo-Ningi
+btf I L Birgit
+btg I L Gagnoa Bété
+bth I L Biatah Bidayuh
+bti I L Burate
+btj I L Bacanese Malay
+btl I L Bhatola
+btm I L Batak Mandailing
+btn I L Ratagnon
+bto I L Rinconada Bikol
+btp I L Budibud
+btq I L Batek
+btr I L Baetora
+bts I L Batak Simalungun
+btt I L Bete-Bendi
+btu I L Batu
+btv I L Bateri
+btw I L Butuanon
+btx I L Batak Karo
+bty I L Bobot
+btz I L Batak Alas-Kluet
+bua bua bua M L Buriat
+bub I L Bua
+buc I L Bushi
+bud I L Ntcham
+bue I E Beothuk
+buf I L Bushoong
+bug bug bug I L Buginese
+buh I L Younuo Bunu
+bui I L Bongili
+buj I L Basa-Gurmana
+buk I L Bugawac
+bul bul bul bg I L Bulgarian
+bum I L Bulu (Cameroon)
+bun I L Sherbro
+buo I L Terei
+bup I L Busoa
+buq I L Brem
+bus I L Bokobaru
+but I L Bungain
+buu I L Budu
+buv I L Bun
+buw I L Bubi
+bux I L Boghom
+buy I L Bullom So
+buz I L Bukwen
+bva I L Barein
+bvb I L Bube
+bvc I L Baelelea
+bvd I L Baeggu
+bve I L Berau Malay
+bvf I L Boor
+bvg I L Bonkeng
+bvh I L Bure
+bvi I L Belanda Viri
+bvj I L Baan
+bvk I L Bukat
+bvl I L Bolivian Sign Language
+bvm I L Bamunka
+bvn I L Buna
+bvo I L Bolgo
+bvp I L Bumang
+bvq I L Birri
+bvr I L Burarra
+bvt I L Bati (Indonesia)
+bvu I L Bukit Malay
+bvv I E Baniva
+bvw I L Boga
+bvx I L Dibole
+bvy I L Baybayanon
+bvz I L Bauzi
+bwa I L Bwatoo
+bwb I L Namosi-Naitasiri-Serua
+bwc I L Bwile
+bwd I L Bwaidoka
+bwe I L Bwe Karen
+bwf I L Boselewa
+bwg I L Barwe
+bwh I L Bishuo
+bwi I L Baniwa
+bwj I L Láá Láá Bwamu
+bwk I L Bauwaki
+bwl I L Bwela
+bwm I L Biwat
+bwn I L Wunai Bunu
+bwo I L Boro (Ethiopia)
+bwp I L Mandobo Bawah
+bwq I L Southern Bobo Madaré
+bwr I L Bura-Pabir
+bws I L Bomboma
+bwt I L Bafaw-Balong
+bwu I L Buli (Ghana)
+bww I L Bwa
+bwx I L Bu-Nao Bunu
+bwy I L Cwi Bwamu
+bwz I L Bwisi
+bxa I L Tairaha
+bxb I L Belanda Bor
+bxc I L Molengue
+bxd I L Pela
+bxe I L Birale
+bxf I L Bilur
+bxg I L Bangala
+bxh I L Buhutu
+bxi I E Pirlatapa
+bxj I L Bayungu
+bxk I L Bukusu
+bxl I L Jalkunan
+bxm I L Mongolia Buriat
+bxn I L Burduna
+bxo I L Barikanchi
+bxp I L Bebil
+bxq I L Beele
+bxr I L Russia Buriat
+bxs I L Busam
+bxu I L China Buriat
+bxv I L Berakou
+bxw I L Bankagooma
+bxx I L Borna (Democratic Republic of Congo)
+bxz I L Binahari
+bya I L Batak
+byb I L Bikya
+byc I L Ubaghara
+byd I L Benyadu'
+bye I L Pouye
+byf I L Bete
+byg I E Baygo
+byh I L Bhujel
+byi I L Buyu
+byj I L Bina (Nigeria)
+byk I L Biao
+byl I L Bayono
+bym I L Bidyara
+byn byn byn I L Bilin
+byo I L Biyo
+byp I L Bumaji
+byq I E Basay
+byr I L Baruya
+bys I L Burak
+byt I E Berti
+byv I L Medumba
+byw I L Belhariya
+byx I L Qaqet
+byy I L Buya
+byz I L Banaro
+bza I L Bandi
+bzb I L Andio
+bzc I L Southern Betsimisaraka Malagasy
+bzd I L Bribri
+bze I L Jenaama Bozo
+bzf I L Boikin
+bzg I L Babuza
+bzh I L Mapos Buang
+bzi I L Bisu
+bzj I L Belize Kriol English
+bzk I L Nicaragua Creole English
+bzl I L Boano (Sulawesi)
+bzm I L Bolondo
+bzn I L Boano (Maluku)
+bzo I L Bozaba
+bzp I L Kemberano
+bzq I L Buli (Indonesia)
+bzr I E Biri
+bzs I L Brazilian Sign Language
+bzt I C Brithenig
+bzu I L Burmeso
+bzv I L Naami
+bzw I L Basa (Nigeria)
+bzx I L Kɛlɛngaxo Bozo
+bzy I L Obanliku
+bzz I L Evant
+caa I L Chortí
+cab I L Garifuna
+cac I L Chuj
+cad cad cad I L Caddo
+cae I L Lehar
+caf I L Southern Carrier
+cag I L Nivaclé
+cah I L Cahuarano
+caj I E Chané
+cak I L Kaqchikel
+cal I L Carolinian
+cam I L Cemuhî
+can I L Chambri
+cao I L Chácobo
+cap I L Chipaya
+caq I L Car Nicobarese
+car car car I L Galibi Carib
+cas I L Tsimané
+cat cat cat ca I L Catalan
+cav I L Cavineña
+caw I L Callawalla
+cax I L Chiquitano
+cay I L Cayuga
+caz I E Canichana
+cbb I L Cabiyarí
+cbc I L Carapana
+cbd I L Carijona
+cbe I E Chipiajes
+cbg I L Chimila
+cbh I E Cagua
+cbi I L Chachi
+cbj I L Ede Cabe
+cbk I L Chavacano
+cbl I L Bualkhaw Chin
+cbn I L Nyahkur
+cbo I L Izora
+cbr I L Cashibo-Cacataibo
+cbs I L Cashinahua
+cbt I L Chayahuita
+cbu I L Candoshi-Shapra
+cbv I L Cacua
+cbw I L Kinabalian
+cby I L Carabayo
+cca I E Cauca
+ccc I L Chamicuro
+ccd I L Cafundo Creole
+cce I L Chopi
+ccg I L Samba Daka
+cch I L Atsam
+ccj I L Kasanga
+ccl I L Cutchi-Swahili
+ccm I L Malaccan Creole Malay
+cco I L Comaltepec Chinantec
+ccp I L Chakma
+ccr I E Cacaopera
+cda I L Choni
+cde I L Chenchu
+cdf I L Chiru
+cdg I L Chamari
+cdh I L Chambeali
+cdi I L Chodri
+cdj I L Churahi
+cdm I L Chepang
+cdn I L Chaudangsi
+cdo I L Min Dong Chinese
+cdr I L Cinda-Regi-Tiyal
+cds I L Chadian Sign Language
+cdy I L Chadong
+cdz I L Koda
+cea I E Lower Chehalis
+ceb ceb ceb I L Cebuano
+ceg I L Chamacoco
+cek I L Eastern Khumi Chin
+cen I L Cen
+ces cze ces cs I L Czech
+cet I L Centúúm
+cfa I L Dijim-Bwilim
+cfd I L Cara
+cfg I L Como Karim
+cfm I L Falam Chin
+cga I L Changriwa
+cgc I L Kagayanen
+cgg I L Chiga
+cgk I L Chocangacakha
+cha cha cha ch I L Chamorro
+chb chb chb I E Chibcha
+chc I E Catawba
+chd I L Highland Oaxaca Chontal
+che che che ce I L Chechen
+chf I L Tabasco Chontal
+chg chg chg I E Chagatai
+chh I L Chinook
+chj I L Ojitlán Chinantec
+chk chk chk I L Chuukese
+chl I L Cahuilla
+chm chm chm M L Mari (Russia)
+chn chn chn I L Chinook jargon
+cho cho cho I L Choctaw
+chp chp chp I L Chipewyan
+chq I L Quiotepec Chinantec
+chr chr chr I L Cherokee
+cht I E Cholón
+chu chu chu cu I A Church Slavic
+chv chv chv cv I L Chuvash
+chw I L Chuwabu
+chx I L Chantyal
+chy chy chy I L Cheyenne
+chz I L Ozumacín Chinantec
+cia I L Cia-Cia
+cib I L Ci Gbe
+cic I L Chickasaw
+cid I E Chimariko
+cie I L Cineni
+cih I L Chinali
+cik I L Chitkuli Kinnauri
+cim I L Cimbrian
+cin I L Cinta Larga
+cip I L Chiapanec
+cir I L Tiri
+ciw I L Chippewa
+ciy I L Chaima
+cja I L Western Cham
+cje I L Chru
+cjh I E Upper Chehalis
+cji I L Chamalal
+cjk I L Chokwe
+cjm I L Eastern Cham
+cjn I L Chenapian
+cjo I L Ashéninka Pajonal
+cjp I L Cabécar
+cjs I L Shor
+cjv I L Chuave
+cjy I L Jinyu Chinese
+ckb I L Central Kurdish
+ckh I L Chak
+ckl I L Cibak
+ckn I L Kaang Chin
+cko I L Anufo
+ckq I L Kajakse
+ckr I L Kairak
+cks I L Tayo
+ckt I L Chukot
+cku I L Koasati
+ckv I L Kavalan
+ckx I L Caka
+cky I L Cakfem-Mushere
+ckz I L Cakchiquel-Quiché Mixed Language
+cla I L Ron
+clc I L Chilcotin
+cld I L Chaldean Neo-Aramaic
+cle I L Lealao Chinantec
+clh I L Chilisso
+cli I L Chakali
+clj I L Laitu Chin
+clk I L Idu-Mishmi
+cll I L Chala
+clm I L Clallam
+clo I L Lowland Oaxaca Chontal
+clt I L Lautu Chin
+clu I L Caluyanun
+clw I L Chulym
+cly I L Eastern Highland Chatino
+cma I L Maa
+cme I L Cerma
+cmg I H Classical Mongolian
+cmi I L Emberá-Chamí
+cml I L Campalagian
+cmm I E Michigamea
+cmn I L Mandarin Chinese
+cmo I L Central Mnong
+cmr I L Mro-Khimi Chin
+cms I A Messapic
+cmt I L Camtho
+cna I L Changthang
+cnb I L Chinbon Chin
+cnc I L Côông
+cng I L Northern Qiang
+cnh I L Haka Chin
+cni I L Asháninka
+cnk I L Khumi Chin
+cnl I L Lalana Chinantec
+cno I L Con
+cns I L Central Asmat
+cnt I L Tepetotutla Chinantec
+cnu I L Chenoua
+cnw I L Ngawn Chin
+cnx I H Middle Cornish
+coa I L Cocos Islands Malay
+cob I E Chicomuceltec
+coc I L Cocopa
+cod I L Cocama-Cocamilla
+coe I L Koreguaje
+cof I L Colorado
+cog I L Chong
+coh I L Chonyi-Dzihana-Kauma
+coj I E Cochimi
+cok I L Santa Teresa Cora
+col I L Columbia-Wenatchi
+com I L Comanche
+con I L Cofán
+coo I L Comox
+cop cop cop I E Coptic
+coq I E Coquille
+cor cor cor kw I L Cornish
+cos cos cos co I L Corsican
+cot I L Caquinte
+cou I L Wamey
+cov I L Cao Miao
+cow I E Cowlitz
+cox I L Nanti
+coy I E Coyaima
+coz I L Chochotec
+cpa I L Palantla Chinantec
+cpb I L Ucayali-Yurúa Ashéninka
+cpc I L Ajyíninka Apurucayali
+cpg I E Cappadocian Greek
+cpi I L Chinese Pidgin English
+cpn I L Cherepon
+cpo I L Kpeego
+cps I L Capiznon
+cpu I L Pichis Ashéninka
+cpx I L Pu-Xian Chinese
+cpy I L South Ucayali Ashéninka
+cqd I L Chuanqiandian Cluster Miao
+cqu I L Chilean Quechua
+cra I L Chara
+crb I E Island Carib
+crc I L Lonwolwol
+crd I L Coeur d'Alene
+cre cre cre cr M L Cree
+crf I E Caramanta
+crg I L Michif
+crh crh crh I L Crimean Tatar
+cri I L Sãotomense
+crj I L Southern East Cree
+crk I L Plains Cree
+crl I L Northern East Cree
+crm I L Moose Cree
+crn I L El Nayar Cora
+cro I L Crow
+crq I L Iyo'wujwa Chorote
+crr I E Carolina Algonquian
+crs I L Seselwa Creole French
+crt I L Iyojwa'ja Chorote
+crv I L Chaura
+crw I L Chrau
+crx I L Carrier
+cry I L Cori
+crz I E Cruzeño
+csa I L Chiltepec Chinantec
+csb csb csb I L Kashubian
+csc I L Catalan Sign Language
+csd I L Chiangmai Sign Language
+cse I L Czech Sign Language
+csf I L Cuba Sign Language
+csg I L Chilean Sign Language
+csh I L Asho Chin
+csi I E Coast Miwok
+csj I L Songlai Chin
+csk I L Jola-Kasa
+csl I L Chinese Sign Language
+csm I L Central Sierra Miwok
+csn I L Colombian Sign Language
+cso I L Sochiapam Chinantec
+csq I L Croatia Sign Language
+csr I L Costa Rican Sign Language
+css I E Southern Ohlone
+cst I L Northern Ohlone
+csv I L Sumtu Chin
+csw I L Swampy Cree
+csy I L Siyin Chin
+csz I L Coos
+cta I L Tataltepec Chatino
+ctc I L Chetco
+ctd I L Tedim Chin
+cte I L Tepinapa Chinantec
+ctg I L Chittagonian
+cth I L Thaiphum Chin
+ctl I L Tlacoatzintepec Chinantec
+ctm I E Chitimacha
+ctn I L Chhintange
+cto I L Emberá-Catío
+ctp I L Western Highland Chatino
+cts I L Northern Catanduanes Bikol
+ctt I L Wayanad Chetti
+ctu I L Chol
+ctz I L Zacatepec Chatino
+cua I L Cua
+cub I L Cubeo
+cuc I L Usila Chinantec
+cug I L Cung
+cuh I L Chuka
+cui I L Cuiba
+cuj I L Mashco Piro
+cuk I L San Blas Kuna
+cul I L Culina
+cum I E Cumeral
+cuo I E Cumanagoto
+cup I E Cupeño
+cuq I L Cun
+cur I L Chhulung
+cut I L Teutila Cuicatec
+cuu I L Tai Ya
+cuv I L Cuvok
+cuw I L Chukwa
+cux I L Tepeuxila Cuicatec
+cvg I L Chug
+cvn I L Valle Nacional Chinantec
+cwa I L Kabwa
+cwb I L Maindo
+cwd I L Woods Cree
+cwe I L Kwere
+cwg I L Chewong
+cwt I L Kuwaataay
+cya I L Nopala Chatino
+cyb I E Cayubaba
+cym wel cym cy I L Welsh
+cyo I L Cuyonon
+czh I L Huizhou Chinese
+czk I E Knaanic
+czn I L Zenzontepec Chatino
+czo I L Min Zhong Chinese
+czt I L Zotung Chin
+daa I L Dangaléat
+dac I L Dambi
+dad I L Marik
+dae I L Duupa
+dag I L Dagbani
+dah I L Gwahatike
+dai I L Day
+daj I L Dar Fur Daju
+dak dak dak I L Dakota
+dal I L Dahalo
+dam I L Damakawa
+dan dan dan da I L Danish
+dao I L Daai Chin
+daq I L Dandami Maria
+dar dar dar I L Dargwa
+das I L Daho-Doo
+dau I L Dar Sila Daju
+dav I L Taita
+daw I L Davawenyo
+dax I L Dayi
+daz I L Dao
+dba I L Bangime
+dbb I L Deno
+dbd I L Dadiya
+dbe I L Dabe
+dbf I L Edopi
+dbg I L Dogul Dom Dogon
+dbi I L Doka
+dbj I L Ida'an
+dbl I L Dyirbal
+dbm I L Duguri
+dbn I L Duriankere
+dbo I L Dulbu
+dbp I L Duwai
+dbq I L Daba
+dbr I L Dabarre
+dbt I L Ben Tey Dogon
+dbu I L Bondum Dom Dogon
+dbv I L Dungu
+dbw I L Bankan Tey Dogon
+dby I L Dibiyaso
+dcc I L Deccan
+dcr I E Negerhollands
+dda I E Dadi Dadi
+ddd I L Dongotono
+dde I L Doondo
+ddg I L Fataluku
+ddi I L West Goodenough
+ddj I L Jaru
+ddn I L Dendi (Benin)
+ddo I L Dido
+ddr I E Dhudhuroa
+dds I L Donno So Dogon
+ddw I L Dawera-Daweloor
+dec I L Dagik
+ded I L Dedua
+dee I L Dewoin
+def I L Dezfuli
+deg I L Degema
+deh I L Dehwari
+dei I L Demisa
+dek I L Dek
+del del del M L Delaware
+dem I L Dem
+den den den M L Slave (Athapascan)
+dep I E Pidgin Delaware
+deq I L Dendi (Central African Republic)
+der I L Deori
+des I L Desano
+deu ger deu de I L German
+dev I L Domung
+dez I L Dengese
+dga I L Southern Dagaare
+dgb I L Bunoge Dogon
+dgc I L Casiguran Dumagat Agta
+dgd I L Dagaari Dioula
+dge I L Degenan
+dgg I L Doga
+dgh I L Dghwede
+dgi I L Northern Dagara
+dgk I L Dagba
+dgl I L Andaandi
+dgn I E Dagoman
+dgo I L Dogri (individual language)
+dgr dgr dgr I L Dogrib
+dgs I L Dogoso
+dgt I E Ndra'ngith
+dgu I L Degaru
+dgw I E Daungwurrung
+dgx I L Doghoro
+dgz I L Daga
+dhd I L Dhundari
+dhg I L Djangu
+dhi I L Dhimal
+dhl I L Dhalandji
+dhm I L Zemba
+dhn I L Dhanki
+dho I L Dhodia
+dhr I L Dhargari
+dhs I L Dhaiso
+dhu I E Dhurga
+dhv I L Dehu
+dhw I L Dhanwar (Nepal)
+dhx I L Dhungaloo
+dia I L Dia
+dib I L South Central Dinka
+dic I L Lakota Dida
+did I L Didinga
+dif I E Dieri
+dig I L Digo
+dih I L Kumiai
+dii I L Dimbong
+dij I L Dai
+dik I L Southwestern Dinka
+dil I L Dilling
+dim I L Dime
+din din din M L Dinka
+dio I L Dibo
+dip I L Northeastern Dinka
+diq I L Dimli (individual language)
+dir I L Dirim
+dis I L Dimasa
+dit I E Dirari
+diu I L Diriku
+div div div dv I L Dhivehi
+diw I L Northwestern Dinka
+dix I L Dixon Reef
+diy I L Diuwe
+diz I L Ding
+dja I E Djadjawurrung
+djb I L Djinba
+djc I L Dar Daju Daju
+djd I L Djamindjung
+dje I L Zarma
+djf I E Djangun
+dji I L Djinang
+djj I L Djeebbana
+djk I L Eastern Maroon Creole
+djm I L Jamsay Dogon
+djn I L Djauan
+djo I L Jangkang
+djr I L Djambarrpuyngu
+dju I L Kapriman
+djw I E Djawi
+dka I L Dakpakha
+dkk I L Dakka
+dkr I L Kuijau
+dks I L Southeastern Dinka
+dkx I L Mazagway
+dlg I L Dolgan
+dlk I L Dahalik
+dlm I E Dalmatian
+dln I L Darlong
+dma I L Duma
+dmb I L Mombo Dogon
+dmc I L Gavak
+dmd I E Madhi Madhi
+dme I L Dugwor
+dmg I L Upper Kinabatangan
+dmk I L Domaaki
+dml I L Dameli
+dmm I L Dama
+dmo I L Kemedzung
+dmr I L East Damar
+dms I L Dampelas
+dmu I L Dubu
+dmv I L Dumpas
+dmw I L Mudburra
+dmx I L Dema
+dmy I L Demta
+dna I L Upper Grand Valley Dani
+dnd I L Daonda
+dne I L Ndendeule
+dng I L Dungan
+dni I L Lower Grand Valley Dani
+dnj I L Dan
+dnk I L Dengka
+dnn I L Dzùùngoo
+dnr I L Danaru
+dnt I L Mid Grand Valley Dani
+dnu I L Danau
+dnv I L Danu
+dnw I L Western Dani
+dny I L Dení
+doa I L Dom
+dob I L Dobu
+doc I L Northern Dong
+doe I L Doe
+dof I L Domu
+doh I L Dong
+doi doi doi M L Dogri (macrolanguage)
+dok I L Dondo
+dol I L Doso
+don I L Toura (Papua New Guinea)
+doo I L Dongo
+dop I L Lukpa
+doq I L Dominican Sign Language
+dor I L Dori'o
+dos I L Dogosé
+dot I L Dass
+dov I L Dombe
+dow I L Doyayo
+dox I L Bussa
+doy I L Dompo
+doz I L Dorze
+dpp I L Papar
+drb I L Dair
+drc I L Minderico
+drd I L Darmiya
+dre I L Dolpo
+drg I L Rungus
+dri I L C'lela
+drl I L Paakantyi
+drn I L West Damar
+dro I L Daro-Matu Melanau
+drq I E Dura
+drr I E Dororo
+drs I L Gedeo
+drt I L Drents
+dru I L Rukai
+dry I L Darai
+dsb dsb dsb I L Lower Sorbian
+dse I L Dutch Sign Language
+dsh I L Daasanach
+dsi I L Disa
+dsl I L Danish Sign Language
+dsn I L Dusner
+dso I L Desiya
+dsq I L Tadaksahak
+dta I L Daur
+dtb I L Labuk-Kinabatangan Kadazan
+dtd I L Ditidaht
+dth I E Adithinngithigh
+dti I L Ana Tinga Dogon
+dtk I L Tene Kan Dogon
+dtm I L Tomo Kan Dogon
+dto I L Tommo So Dogon
+dtp I L Central Dusun
+dtr I L Lotud
+dts I L Toro So Dogon
+dtt I L Toro Tegu Dogon
+dtu I L Tebul Ure Dogon
+dty I L Dotyali
+dua dua dua I L Duala
+dub I L Dubli
+duc I L Duna
+dud I L Hun-Saare
+due I L Umiray Dumaget Agta
+duf I L Dumbea
+dug I L Duruma
+duh I L Dungra Bhil
+dui I L Dumun
+duj I L Dhuwal
+duk I L Uyajitaya
+dul I L Alabat Island Agta
+dum dum dum I H Middle Dutch (ca. 1050-1350)
+dun I L Dusun Deyah
+duo I L Dupaninan Agta
+dup I L Duano
+duq I L Dusun Malang
+dur I L Dii
+dus I L Dumi
+duu I L Drung
+duv I L Duvle
+duw I L Dusun Witu
+dux I L Duungooma
+duy I E Dicamay Agta
+duz I E Duli
+dva I L Duau
+dwa I L Diri
+dwr I L Dawro
+dws I C Dutton World Speedwords
+dww I L Dawawa
+dya I L Dyan
+dyb I E Dyaberdyaber
+dyd I E Dyugun
+dyg I E Villa Viciosa Agta
+dyi I L Djimini Senoufo
+dym I L Yanda Dom Dogon
+dyn I L Dyangadi
+dyo I L Jola-Fonyi
+dyu dyu dyu I L Dyula
+dyy I L Dyaabugay
+dza I L Tunzu
+dzd I L Daza
+dze I E Djiwarli
+dzg I L Dazaga
+dzl I L Dzalakha
+dzn I L Dzando
+dzo dzo dzo dz I L Dzongkha
+eaa I E Karenggapa
+ebg I L Ebughu
+ebk I L Eastern Bontok
+ebo I L Teke-Ebo
+ebr I L Ebrié
+ebu I L Embu
+ecr I A Eteocretan
+ecs I L Ecuadorian Sign Language
+ecy I A Eteocypriot
+eee I L E
+efa I L Efai
+efe I L Efe
+efi efi efi I L Efik
+ega I L Ega
+egl I L Emilian
+ego I L Eggon
+egy egy egy I A Egyptian (Ancient)
+ehu I L Ehueun
+eip I L Eipomek
+eit I L Eitiep
+eiv I L Askopan
+eja I L Ejamat
+eka eka eka I L Ekajuk
+ekc I E Eastern Karnic
+eke I L Ekit
+ekg I L Ekari
+eki I L Eki
+ekk I L Standard Estonian
+ekl I L Kol (Bangladesh)
+ekm I L Elip
+eko I L Koti
+ekp I L Ekpeye
+ekr I L Yace
+eky I L Eastern Kayah
+ele I L Elepi
+elh I L El Hugeirat
+eli I E Nding
+elk I L Elkei
+ell gre ell el I L Modern Greek (1453-)
+elm I L Eleme
+elo I L El Molo
+elu I L Elu
+elx elx elx I A Elamite
+ema I L Emai-Iuleha-Ora
+emb I L Embaloh
+eme I L Emerillon
+emg I L Eastern Meohang
+emi I L Mussau-Emira
+emk I L Eastern Maninkakan
+emm I E Mamulique
+emn I L Eman
+emo I E Emok
+emp I L Northern Emberá
+ems I L Pacific Gulf Yupik
+emu I L Eastern Muria
+emw I L Emplawas
+emx I L Erromintxela
+emy I E Epigraphic Mayan
+ena I L Apali
+enb I L Markweeta
+enc I L En
+end I L Ende
+enf I L Forest Enets
+eng eng eng en I L English
+enh I L Tundra Enets
+enm enm enm I H Middle English (1100-1500)
+enn I L Engenni
+eno I L Enggano
+enq I L Enga
+enr I L Emumu
+enu I L Enu
+env I L Enwan (Edu State)
+enw I L Enwan (Akwa Ibom State)
+eot I L Beti (Côte d'Ivoire)
+epi I L Epie
+epo epo epo eo I C Esperanto
+era I L Eravallan
+erg I L Sie
+erh I L Eruwa
+eri I L Ogea
+erk I L South Efate
+ero I L Horpa
+err I E Erre
+ers I L Ersu
+ert I L Eritai
+erw I L Erokwanas
+ese I L Ese Ejja
+esh I L Eshtehardi
+esi I L North Alaskan Inupiatun
+esk I L Northwest Alaska Inupiatun
+esl I L Egypt Sign Language
+esm I E Esuma
+esn I L Salvadoran Sign Language
+eso I L Estonian Sign Language
+esq I E Esselen
+ess I L Central Siberian Yupik
+est est est et M L Estonian
+esu I L Central Yupik
+etb I L Etebi
+etc I E Etchemin
+eth I L Ethiopian Sign Language
+etn I L Eton (Vanuatu)
+eto I L Eton (Cameroon)
+etr I L Edolo
+ets I L Yekhee
+ett I A Etruscan
+etu I L Ejagham
+etx I L Eten
+etz I L Semimi
+eus baq eus eu I L Basque
+eve I L Even
+evh I L Uvbie
+evn I L Evenki
+ewe ewe ewe ee I L Ewe
+ewo ewo ewo I L Ewondo
+ext I L Extremaduran
+eya I E Eyak
+eyo I L Keiyo
+eza I L Ezaa
+eze I L Uzekwe
+faa I L Fasu
+fab I L Fa d'Ambu
+fad I L Wagi
+faf I L Fagani
+fag I L Finongan
+fah I L Baissa Fali
+fai I L Faiwol
+faj I L Faita
+fak I L Fang (Cameroon)
+fal I L South Fali
+fam I L Fam
+fan fan fan I L Fang (Equatorial Guinea)
+fao fao fao fo I L Faroese
+fap I L Palor
+far I L Fataleka
+fas per fas fa M L Persian
+fat fat fat I L Fanti
+fau I L Fayu
+fax I L Fala
+fay I L Southwestern Fars
+faz I L Northwestern Fars
+fbl I L West Albay Bikol
+fcs I L Quebec Sign Language
+fer I L Feroge
+ffi I L Foia Foia
+ffm I L Maasina Fulfulde
+fgr I L Fongoro
+fia I L Nobiin
+fie I L Fyer
+fij fij fij fj I L Fijian
+fil fil fil I L Filipino
+fin fin fin fi I L Finnish
+fip I L Fipa
+fir I L Firan
+fit I L Tornedalen Finnish
+fiw I L Fiwaga
+fkk I L Kirya-Konzəl
+fkv I L Kven Finnish
+fla I L Kalispel-Pend d'Oreille
+flh I L Foau
+fli I L Fali
+fll I L North Fali
+fln I E Flinders Island
+flr I L Fuliiru
+fly I L Tsotsitaal
+fmp I L Fe'fe'
+fmu I L Far Western Muria
+fng I L Fanagalo
+fni I L Fania
+fod I L Foodo
+foi I L Foi
+fom I L Foma
+fon fon fon I L Fon
+for I L Fore
+fos I E Siraya
+fpe I L Fernando Po Creole English
+fqs I L Fas
+fra fre fra fr I L French
+frc I L Cajun French
+frd I L Fordata
+frk I E Frankish
+frm frm frm I H Middle French (ca. 1400-1600)
+fro fro fro I H Old French (842-ca. 1400)
+frp I L Arpitan
+frq I L Forak
+frr frr frr I L Northern Frisian
+frs frs frs I L Eastern Frisian
+frt I L Fortsenal
+fry fry fry fy I L Western Frisian
+fse I L Finnish Sign Language
+fsl I L French Sign Language
+fss I L Finland-Swedish Sign Language
+fub I L Adamawa Fulfulde
+fuc I L Pulaar
+fud I L East Futuna
+fue I L Borgu Fulfulde
+fuf I L Pular
+fuh I L Western Niger Fulfulde
+fui I L Bagirmi Fulfulde
+fuj I L Ko
+ful ful ful ff M L Fulah
+fum I L Fum
+fun I L Fulniô
+fuq I L Central-Eastern Niger Fulfulde
+fur fur fur I L Friulian
+fut I L Futuna-Aniwa
+fuu I L Furu
+fuv I L Nigerian Fulfulde
+fuy I L Fuyug
+fvr I L Fur
+fwa I L Fwâi
+fwe I L Fwe
+gaa gaa gaa I L Ga
+gab I L Gabri
+gac I L Mixed Great Andamanese
+gad I L Gaddang
+gae I L Guarequena
+gaf I L Gende
+gag I L Gagauz
+gah I L Alekano
+gai I L Borei
+gaj I L Gadsup
+gak I L Gamkonora
+gal I L Galolen
+gam I L Kandawo
+gan I L Gan Chinese
+gao I L Gants
+gap I L Gal
+gaq I L Gata'
+gar I L Galeya
+gas I L Adiwasi Garasia
+gat I L Kenati
+gau I L Mudhili Gadaba
+gaw I L Nobonob
+gax I L Borana-Arsi-Guji Oromo
+gay gay gay I L Gayo
+gaz I L West Central Oromo
+gba gba gba M L Gbaya (Central African Republic)
+gbb I L Kaytetye
+gbd I L Karadjeri
+gbe I L Niksek
+gbf I L Gaikundi
+gbg I L Gbanziri
+gbh I L Defi Gbe
+gbi I L Galela
+gbj I L Bodo Gadaba
+gbk I L Gaddi
+gbl I L Gamit
+gbm I L Garhwali
+gbn I L Mo'da
+gbo I L Northern Grebo
+gbp I L Gbaya-Bossangoa
+gbq I L Gbaya-Bozoum
+gbr I L Gbagyi
+gbs I L Gbesi Gbe
+gbu I L Gagadu
+gbv I L Gbanu
+gbw I L Gabi-Gabi
+gbx I L Eastern Xwla Gbe
+gby I L Gbari
+gbz I L Zoroastrian Dari
+gcc I L Mali
+gcd I E Ganggalida
+gce I E Galice
+gcf I L Guadeloupean Creole French
+gcl I L Grenadian Creole English
+gcn I L Gaina
+gcr I L Guianese Creole French
+gct I L Colonia Tovar German
+gda I L Gade Lohar
+gdb I L Pottangi Ollar Gadaba
+gdc I E Gugu Badhun
+gdd I L Gedaged
+gde I L Gude
+gdf I L Guduf-Gava
+gdg I L Ga'dang
+gdh I L Gadjerawang
+gdi I L Gundi
+gdj I L Gurdjar
+gdk I L Gadang
+gdl I L Dirasha
+gdm I L Laal
+gdn I L Umanakaina
+gdo I L Ghodoberi
+gdq I L Mehri
+gdr I L Wipi
+gds I L Ghandruk Sign Language
+gdt I E Kungardutyi
+gdu I L Gudu
+gdx I L Godwari
+gea I L Geruma
+geb I L Kire
+gec I L Gboloo Grebo
+ged I L Gade
+geg I L Gengle
+geh I L Hutterite German
+gei I L Gebe
+gej I L Gen
+gek I L Yiwom
+gel I L ut-Ma'in
+geq I L Geme
+ges I L Geser-Gorom
+gew I L Gera
+gex I L Garre
+gey I L Enya
+gez gez gez I A Geez
+gfk I L Patpatar
+gft I E Gafat
+gfx I L Mangetti Dune !Xung
+gga I L Gao
+ggb I L Gbii
+ggd I E Gugadj
+gge I L Guragone
+ggg I L Gurgula
+ggk I E Kungarakany
+ggl I L Ganglau
+ggm I E Gugu Mini
+ggn I L Eastern Gurung
+ggo I L Southern Gondi
+ggt I L Gitua
+ggu I L Gagu
+ggw I L Gogodala
+gha I L Ghadamès
+ghc I E Hiberno-Scottish Gaelic
+ghe I L Southern Ghale
+ghh I L Northern Ghale
+ghk I L Geko Karen
+ghl I L Ghulfan
+ghn I L Ghanongga
+gho I E Ghomara
+ghr I L Ghera
+ghs I L Guhu-Samane
+ght I L Kuke
+gia I L Kitja
+gib I L Gibanawa
+gic I L Gail
+gid I L Gidar
+gig I L Goaria
+gih I L Githabul
+gil gil gil I L Gilbertese
+gim I L Gimi (Eastern Highlands)
+gin I L Hinukh
+gip I L Gimi (West New Britain)
+giq I L Green Gelao
+gir I L Red Gelao
+gis I L North Giziga
+git I L Gitxsan
+giu I L Mulao
+giw I L White Gelao
+gix I L Gilima
+giy I L Giyug
+giz I L South Giziga
+gji I L Geji
+gjk I L Kachi Koli
+gjm I E Gunditjmara
+gjn I L Gonja
+gju I L Gujari
+gka I L Guya
+gke I L Ndai
+gkn I L Gokana
+gko I E Kok-Nar
+gkp I L Guinea Kpelle
+gla gla gla gd I L Scottish Gaelic
+glc I L Bon Gula
+gld I L Nanai
+gle gle gle ga I L Irish
+glg glg glg gl I L Galician
+glh I L Northwest Pashayi
+gli I E Guliguli
+glj I L Gula Iro
+glk I L Gilaki
+gll I E Garlali
+glo I L Galambu
+glr I L Glaro-Twabo
+glu I L Gula (Chad)
+glv glv glv gv I L Manx
+glw I L Glavda
+gly I E Gule
+gma I E Gambera
+gmb I L Gula'alaa
+gmd I L Mághdì
+gmh gmh gmh I H Middle High German (ca. 1050-1500)
+gml I H Middle Low German
+gmm I L Gbaya-Mbodomo
+gmn I L Gimnime
+gmu I L Gumalu
+gmv I L Gamo
+gmx I L Magoma
+gmy I A Mycenaean Greek
+gmz I L Mgbolizhia
+gna I L Kaansa
+gnb I L Gangte
+gnc I E Guanche
+gnd I L Zulgo-Gemzek
+gne I L Ganang
+gng I L Ngangam
+gnh I L Lere
+gni I L Gooniyandi
+gnk I L //Gana
+gnl I E Gangulu
+gnm I L Ginuman
+gnn I L Gumatj
+gno I L Northern Gondi
+gnq I L Gana
+gnr I E Gureng Gureng
+gnt I L Guntai
+gnu I L Gnau
+gnw I L Western Bolivian Guaraní
+gnz I L Ganzi
+goa I L Guro
+gob I L Playero
+goc I L Gorakor
+god I L Godié
+goe I L Gongduk
+gof I L Gofa
+gog I L Gogo
+goh goh goh I H Old High German (ca. 750-1050)
+goi I L Gobasi
+goj I L Gowlan
+gok I L Gowli
+gol I L Gola
+gom I L Goan Konkani
+gon gon gon M L Gondi
+goo I L Gone Dau
+gop I L Yeretuar
+goq I L Gorap
+gor gor gor I L Gorontalo
+gos I L Gronings
+got got got I A Gothic
+gou I L Gavar
+gow I L Gorowa
+gox I L Gobu
+goy I L Goundo
+goz I L Gozarkhani
+gpa I L Gupa-Abawa
+gpe I L Ghanaian Pidgin English
+gpn I L Taiap
+gqa I L Ga'anda
+gqi I L Guiqiong
+gqn I E Guana (Brazil)
+gqr I L Gor
+gqu I L Qau
+gra I L Rajput Garasia
+grb grb grb M L Grebo
+grc grc grc I H Ancient Greek (to 1453)
+grd I L Guruntum-Mbaaru
+grg I L Madi
+grh I L Gbiri-Niragu
+gri I L Ghari
+grj I L Southern Grebo
+grm I L Kota Marudu Talantang
+grn grn grn gn M L Guarani
+gro I L Groma
+grq I L Gorovu
+grr I L Taznatit
+grs I L Gresi
+grt I L Garo
+gru I L Kistane
+grv I L Central Grebo
+grw I L Gweda
+grx I L Guriaso
+gry I L Barclayville Grebo
+grz I L Guramalum
+gse I L Ghanaian Sign Language
+gsg I L German Sign Language
+gsl I L Gusilay
+gsm I L Guatemalan Sign Language
+gsn I L Gusan
+gso I L Southwest Gbaya
+gsp I L Wasembo
+gss I L Greek Sign Language
+gsw gsw gsw I L Swiss German
+gta I L Guató
+gti I L Gbati-ri
+gtu I E Aghu-Tharnggala
+gua I L Shiki
+gub I L Guajajára
+guc I L Wayuu
+gud I L Yocoboué Dida
+gue I L Gurinji
+guf I L Gupapuyngu
+gug I L Paraguayan Guaraní
+guh I L Guahibo
+gui I L Eastern Bolivian Guaraní
+guj guj guj gu I L Gujarati
+guk I L Gumuz
+gul I L Sea Island Creole English
+gum I L Guambiano
+gun I L Mbyá Guaraní
+guo I L Guayabero
+gup I L Gunwinggu
+guq I L Aché
+gur I L Farefare
+gus I L Guinean Sign Language
+gut I L Maléku Jaíka
+guu I L Yanomamö
+guv I E Gey
+guw I L Gun
+gux I L Gourmanchéma
+guz I L Gusii
+gva I L Guana (Paraguay)
+gvc I L Guanano
+gve I L Duwet
+gvf I L Golin
+gvj I L Guajá
+gvl I L Gulay
+gvm I L Gurmana
+gvn I L Kuku-Yalanji
+gvo I L Gavião Do Jiparaná
+gvp I L Pará Gavião
+gvr I L Western Gurung
+gvs I L Gumawana
+gvy I E Guyani
+gwa I L Mbato
+gwb I L Gwa
+gwc I L Kalami
+gwd I L Gawwada
+gwe I L Gweno
+gwf I L Gowro
+gwg I L Moo
+gwi gwi gwi I L Gwichʼin
+gwj I L /Gwi
+gwm I E Awngthim
+gwn I L Gwandara
+gwr I L Gwere
+gwt I L Gawar-Bati
+gwu I E Guwamu
+gww I L Kwini
+gwx I L Gua
+gxx I L Wè Southern
+gya I L Northwest Gbaya
+gyb I L Garus
+gyd I L Kayardild
+gye I L Gyem
+gyf I E Gungabula
+gyg I L Gbayi
+gyi I L Gyele
+gyl I L Gayil
+gym I L Ngäbere
+gyn I L Guyanese Creole English
+gyr I L Guarayu
+gyy I E Gunya
+gza I L Ganza
+gzi I L Gazi
+gzn I L Gane
+haa I L Han
+hab I L Hanoi Sign Language
+hac I L Gurani
+had I L Hatam
+hae I L Eastern Oromo
+haf I L Haiphong Sign Language
+hag I L Hanga
+hah I L Hahon
+hai hai hai M L Haida
+haj I L Hajong
+hak I L Hakka Chinese
+hal I L Halang
+ham I L Hewa
+han I L Hangaza
+hao I L Hakö
+hap I L Hupla
+haq I L Ha
+har I L Harari
+has I L Haisla
+hat hat hat ht I L Haitian
+hau hau hau ha I L Hausa
+hav I L Havu
+haw haw haw I L Hawaiian
+hax I L Southern Haida
+hay I L Haya
+haz I L Hazaragi
+hba I L Hamba
+hbb I L Huba
+hbn I L Heiban
+hbo I H Ancient Hebrew
+hbs sh M L Serbo-Croatian Code element for 639-1 has been deprecated
+hbu I L Habu
+hca I L Andaman Creole Hindi
+hch I L Huichol
+hdn I L Northern Haida
+hds I L Honduras Sign Language
+hdy I L Hadiyya
+hea I L Northern Qiandong Miao
+heb heb heb he I L Hebrew
+hed I L Herdé
+heg I L Helong
+heh I L Hehe
+hei I L Heiltsuk
+hem I L Hemba
+her her her hz I L Herero
+hgm I L Hai//om
+hgw I L Haigwai
+hhi I L Hoia Hoia
+hhr I L Kerak
+hhy I L Hoyahoya
+hia I L Lamang
+hib I E Hibito
+hid I L Hidatsa
+hif I L Fiji Hindi
+hig I L Kamwe
+hih I L Pamosu
+hii I L Hinduri
+hij I L Hijuk
+hik I L Seit-Kaitetu
+hil hil hil I L Hiligaynon
+hin hin hin hi I L Hindi
+hio I L Tsoa
+hir I L Himarimã
+hit hit hit I A Hittite
+hiw I L Hiw
+hix I L Hixkaryána
+hji I L Haji
+hka I L Kahe
+hke I L Hunde
+hkk I L Hunjara-Kaina Ke
+hks I L Hong Kong Sign Language
+hla I L Halia
+hlb I L Halbi
+hld I L Halang Doan
+hle I L Hlersu
+hlt I L Matu Chin
+hlu I A Hieroglyphic Luwian
+hma I L Southern Mashan Hmong
+hmb I L Humburi Senni Songhay
+hmc I L Central Huishui Hmong
+hmd I L Large Flowery Miao
+hme I L Eastern Huishui Hmong
+hmf I L Hmong Don
+hmg I L Southwestern Guiyang Hmong
+hmh I L Southwestern Huishui Hmong
+hmi I L Northern Huishui Hmong
+hmj I L Ge
+hmk I E Maek
+hml I L Luopohe Hmong
+hmm I L Central Mashan Hmong
+hmn hmn hmn M L Hmong
+hmo hmo hmo ho I L Hiri Motu
+hmp I L Northern Mashan Hmong
+hmq I L Eastern Qiandong Miao
+hmr I L Hmar
+hms I L Southern Qiandong Miao
+hmt I L Hamtai
+hmu I L Hamap
+hmv I L Hmong Dô
+hmw I L Western Mashan Hmong
+hmy I L Southern Guiyang Hmong
+hmz I L Hmong Shua
+hna I L Mina (Cameroon)
+hnd I L Southern Hindko
+hne I L Chhattisgarhi
+hnh I L //Ani
+hni I L Hani
+hnj I L Hmong Njua
+hnn I L Hanunoo
+hno I L Northern Hindko
+hns I L Caribbean Hindustani
+hnu I L Hung
+hoa I L Hoava
+hob I L Mari (Madang Province)
+hoc I L Ho
+hod I E Holma
+hoe I L Horom
+hoh I L Hobyót
+hoi I L Holikachuk
+hoj I L Hadothi
+hol I L Holu
+hom I E Homa
+hoo I L Holoholo
+hop I L Hopi
+hor I E Horo
+hos I L Ho Chi Minh City Sign Language
+hot I L Hote
+hov I L Hovongan
+how I L Honi
+hoy I L Holiya
+hoz I L Hozo
+hpo I L Hpon
+hps I L Hawai'i Pidgin Sign Language
+hra I L Hrangkhol
+hrc I L Niwer Mil
+hre I L Hre
+hrk I L Haruku
+hrm I L Horned Miao
+hro I L Haroi
+hrp I E Nhirrpi
+hrt I L Hértevin
+hru I L Hruso
+hrv hrv hrv hr I L Croatian
+hrw I L Warwar Feni
+hrx I L Hunsrik
+hrz I L Harzani
+hsb hsb hsb I L Upper Sorbian
+hsh I L Hungarian Sign Language
+hsl I L Hausa Sign Language
+hsn I L Xiang Chinese
+hss I L Harsusi
+hti I L Hoti
+hto I L Minica Huitoto
+hts I L Hadza
+htu I L Hitu
+htx I A Middle Hittite
+hub I L Huambisa
+huc I L =/Hua
+hud I L Huaulu
+hue I L San Francisco Del Mar Huave
+huf I L Humene
+hug I L Huachipaeri
+huh I L Huilliche
+hui I L Huli
+huj I L Northern Guiyang Hmong
+huk I L Hulung
+hul I L Hula
+hum I L Hungana
+hun hun hun hu I L Hungarian
+huo I L Hu
+hup hup hup I L Hupa
+huq I L Tsat
+hur I L Halkomelem
+hus I L Huastec
+hut I L Humla
+huu I L Murui Huitoto
+huv I L San Mateo Del Mar Huave
+huw I E Hukumina
+hux I L Nüpode Huitoto
+huy I L Hulaulá
+huz I L Hunzib
+hvc I L Haitian Vodoun Culture Language
+hve I L San Dionisio Del Mar Huave
+hvk I L Haveke
+hvn I L Sabu
+hvv I L Santa María Del Mar Huave
+hwa I L Wané
+hwc I L Hawai'i Creole English
+hwo I L Hwana
+hya I L Hya
+hye arm hye hy I L Armenian
+iai I L Iaai
+ian I L Iatmul
+iap I L Iapama
+iar I L Purari
+iba iba iba I L Iban
+ibb I L Ibibio
+ibd I L Iwaidja
+ibe I L Akpes
+ibg I L Ibanag
+ibl I L Ibaloi
+ibm I L Agoi
+ibn I L Ibino
+ibo ibo ibo ig I L Igbo
+ibr I L Ibuoro
+ibu I L Ibu
+iby I L Ibani
+ica I L Ede Ica
+ich I L Etkywan
+icl I L Icelandic Sign Language
+icr I L Islander Creole English
+ida I L Idakho-Isukha-Tiriki
+idb I L Indo-Portuguese
+idc I L Idon
+idd I L Ede Idaca
+ide I L Idere
+idi I L Idi
+ido ido ido io I C Ido
+idr I L Indri
+ids I L Idesa
+idt I L Idaté
+idu I L Idoma
+ifa I L Amganad Ifugao
+ifb I L Batad Ifugao
+ife I L Ifè
+iff I E Ifo
+ifk I L Tuwali Ifugao
+ifm I L Teke-Fuumu
+ifu I L Mayoyao Ifugao
+ify I L Keley-I Kallahan
+igb I L Ebira
+ige I L Igede
+igg I L Igana
+igl I L Igala
+igm I L Kanggape
+ign I L Ignaciano
+igo I L Isebe
+igs I C Interglossa
+igw I L Igwe
+ihb I L Iha Based Pidgin
+ihi I L Ihievbe
+ihp I L Iha
+ihw I E Bidhawal
+iii iii iii ii I L Sichuan Yi
+iin I E Thiin
+ijc I L Izon
+ije I L Biseni
+ijj I L Ede Ije
+ijn I L Kalabari
+ijs I L Southeast Ijo
+ike I L Eastern Canadian Inuktitut
+iki I L Iko
+ikk I L Ika
+ikl I L Ikulu
+iko I L Olulumo-Ikom
+ikp I L Ikpeshi
+ikr I E Ikaranggal
+ikt I L Inuinnaqtun
+iku iku iku iu M L Inuktitut
+ikv I L Iku-Gora-Ankwa
+ikw I L Ikwere
+ikx I L Ik
+ikz I L Ikizu
+ila I L Ile Ape
+ilb I L Ila
+ile ile ile ie I C Interlingue
+ilg I E Garig-Ilgar
+ili I L Ili Turki
+ilk I L Ilongot
+ill I L Iranun
+ilo ilo ilo I L Iloko
+ils I L International Sign
+ilu I L Ili'uun
+ilv I L Ilue
+ima I L Mala Malasar
+ime I L Imeraguen
+imi I L Anamgura
+iml I E Miluk
+imn I L Imonda
+imo I L Imbongu
+imr I L Imroing
+ims I A Marsian
+imy I A Milyan
+ina ina ina ia I C Interlingua (International Auxiliary Language Association)
+inb I L Inga
+ind ind ind id I L Indonesian
+ing I L Degexit'an
+inh inh inh I L Ingush
+inj I L Jungle Inga
+inl I L Indonesian Sign Language
+inm I A Minaean
+inn I L Isinai
+ino I L Inoke-Yate
+inp I L Iñapari
+ins I L Indian Sign Language
+int I L Intha
+inz I E Ineseño
+ior I L Inor
+iou I L Tuma-Irumu
+iow I E Iowa-Oto
+ipi I L Ipili
+ipk ipk ipk ik M L Inupiaq
+ipo I L Ipiko
+iqu I L Iquito
+iqw I L Ikwo
+ire I L Iresim
+irh I L Irarutu
+iri I L Irigwe
+irk I L Iraqw
+irn I L Irántxe
+irr I L Ir
+iru I L Irula
+irx I L Kamberau
+iry I L Iraya
+isa I L Isabi
+isc I L Isconahua
+isd I L Isnag
+ise I L Italian Sign Language
+isg I L Irish Sign Language
+ish I L Esan
+isi I L Nkem-Nkum
+isk I L Ishkashimi
+isl ice isl is I L Icelandic
+ism I L Masimasi
+isn I L Isanzu
+iso I L Isoko
+isr I L Israeli Sign Language
+ist I L Istriot
+isu I L Isu (Menchum Division)
+ita ita ita it I L Italian
+itb I L Binongan Itneg
+ite I E Itene
+iti I L Inlaod Itneg
+itk I L Judeo-Italian
+itl I L Itelmen
+itm I L Itu Mbon Uzo
+ito I L Itonama
+itr I L Iteri
+its I L Isekiri
+itt I L Maeng Itneg
+itv I L Itawit
+itw I L Ito
+itx I L Itik
+ity I L Moyadan Itneg
+itz I L Itzá
+ium I L Iu Mien
+ivb I L Ibatan
+ivv I L Ivatan
+iwk I L I-Wak
+iwm I L Iwam
+iwo I L Iwur
+iws I L Sepik Iwam
+ixc I L Ixcatec
+ixl I L Ixil
+iya I L Iyayu
+iyo I L Mesaka
+iyx I L Yaka (Congo)
+izh I L Ingrian
+izr I L Izere
+izz I L Izii
+jaa I L Jamamadí
+jab I L Hyam
+jac I L Popti'
+jad I L Jahanka
+jae I L Yabem
+jaf I L Jara
+jah I L Jah Hut
+jaj I L Zazao
+jak I L Jakun
+jal I L Yalahatan
+jam I L Jamaican Creole English
+jan I E Jandai
+jao I L Yanyuwa
+jaq I L Yaqay
+jas I L New Caledonian Javanese
+jat I L Jakati
+jau I L Yaur
+jav jav jav jv I L Javanese
+jax I L Jambi Malay
+jay I L Yan-nhangu
+jaz I L Jawe
+jbe I L Judeo-Berber
+jbi I E Badjiri
+jbj I L Arandai
+jbk I L Barikewa
+jbn I L Nafusi
+jbo jbo jbo I C Lojban
+jbr I L Jofotek-Bromnya
+jbt I L Jabutí
+jbu I L Jukun Takum
+jbw I E Yawijibaya
+jcs I L Jamaican Country Sign Language
+jct I L Krymchak
+jda I L Jad
+jdg I L Jadgali
+jdt I L Judeo-Tat
+jeb I L Jebero
+jee I L Jerung
+jeg I L Jeng
+jeh I L Jeh
+jei I L Yei
+jek I L Jeri Kuo
+jel I L Yelmek
+jen I L Dza
+jer I L Jere
+jet I L Manem
+jeu I L Jonkor Bourmataguil
+jgb I E Ngbee
+jge I L Judeo-Georgian
+jgk I L Gwak
+jgo I L Ngomba
+jhi I L Jehai
+jhs I L Jhankot Sign Language
+jia I L Jina
+jib I L Jibu
+jic I L Tol
+jid I L Bu
+jie I L Jilbe
+jig I L Djingili
+jih I L sTodsde
+jii I L Jiiddu
+jil I L Jilim
+jim I L Jimi (Cameroon)
+jio I L Jiamao
+jiq I L Guanyinqiao
+jit I L Jita
+jiu I L Youle Jinuo
+jiv I L Shuar
+jiy I L Buyuan Jinuo
+jjr I L Bankal
+jkm I L Mobwa Karen
+jko I L Kubo
+jkp I L Paku Karen
+jkr I L Koro (India)
+jku I L Labir
+jle I L Ngile
+jls I L Jamaican Sign Language
+jma I L Dima
+jmb I L Zumbun
+jmc I L Machame
+jmd I L Yamdena
+jmi I L Jimi (Nigeria)
+jml I L Jumli
+jmn I L Makuri Naga
+jmr I L Kamara
+jms I L Mashi (Nigeria)
+jmw I L Mouwase
+jmx I L Western Juxtlahuaca Mixtec
+jna I L Jangshung
+jnd I L Jandavra
+jng I E Yangman
+jni I L Janji
+jnj I L Yemsa
+jnl I L Rawat
+jns I L Jaunsari
+job I L Joba
+jod I L Wojenaka
+jor I E Jorá
+jos I L Jordanian Sign Language
+jow I L Jowulu
+jpa I H Jewish Palestinian Aramaic
+jpn jpn jpn ja I L Japanese
+jpr jpr jpr I L Judeo-Persian
+jqr I L Jaqaru
+jra I L Jarai
+jrb jrb jrb M L Judeo-Arabic
+jrr I L Jiru
+jrt I L Jorto
+jru I L Japrería
+jsl I L Japanese Sign Language
+jua I L Júma
+jub I L Wannu
+juc I E Jurchen
+jud I L Worodougou
+juh I L Hõne
+jui I E Ngadjuri
+juk I L Wapan
+jul I L Jirel
+jum I L Jumjum
+jun I L Juang
+juo I L Jiba
+jup I L Hupdë
+jur I L Jurúna
+jus I L Jumla Sign Language
+jut I L Jutish
+juu I L Ju
+juw I L Wãpha
+juy I L Juray
+jvd I E Javindo
+jvn I L Caribbean Javanese
+jwi I L Jwira-Pepesa
+jya I L Jiarong
+jye I L Judeo-Yemeni Arabic
+jyy I L Jaya
+kaa kaa kaa I L Kara-Kalpak
+kab kab kab I L Kabyle
+kac kac kac I L Kachin
+kad I L Adara
+kae I E Ketangalan
+kaf I L Katso
+kag I L Kajaman
+kah I L Kara (Central African Republic)
+kai I L Karekare
+kaj I L Jju
+kak I L Kayapa Kallahan
+kal kal kal kl I L Kalaallisut
+kam kam kam I L Kamba (Kenya)
+kan kan kan kn I L Kannada
+kao I L Xaasongaxango
+kap I L Bezhta
+kaq I L Capanahua
+kas kas kas ks I L Kashmiri
+kat geo kat ka I L Georgian
+kau kau kau kr M L Kanuri
+kav I L Katukína
+kaw kaw kaw I A Kawi
+kax I L Kao
+kay I L Kamayurá
+kaz kaz kaz kk I L Kazakh
+kba I E Kalarko
+kbb I E Kaxuiâna
+kbc I L Kadiwéu
+kbd kbd kbd I L Kabardian
+kbe I L Kanju
+kbf I E Kakauhua
+kbg I L Khamba
+kbh I L Camsá
+kbi I L Kaptiau
+kbj I L Kari
+kbk I L Grass Koiari
+kbl I L Kanembu
+kbm I L Iwal
+kbn I L Kare (Central African Republic)
+kbo I L Keliko
+kbp I L Kabiyè
+kbq I L Kamano
+kbr I L Kafa
+kbs I L Kande
+kbt I L Abadi
+kbu I L Kabutra
+kbv I L Dera (Indonesia)
+kbw I L Kaiep
+kbx I L Ap Ma
+kby I L Manga Kanuri
+kbz I L Duhwa
+kca I L Khanty
+kcb I L Kawacha
+kcc I L Lubila
+kcd I L Ngkâlmpw Kanum
+kce I L Kaivi
+kcf I L Ukaan
+kcg I L Tyap
+kch I L Vono
+kci I L Kamantan
+kcj I L Kobiana
+kck I L Kalanga
+kcl I L Kela (Papua New Guinea)
+kcm I L Gula (Central African Republic)
+kcn I L Nubi
+kco I L Kinalakna
+kcp I L Kanga
+kcq I L Kamo
+kcr I L Katla
+kcs I L Koenoem
+kct I L Kaian
+kcu I L Kami (Tanzania)
+kcv I L Kete
+kcw I L Kabwari
+kcx I L Kachama-Ganjule
+kcy I L Korandje
+kcz I L Konongo
+kda I E Worimi
+kdc I L Kutu
+kdd I L Yankunytjatjara
+kde I L Makonde
+kdf I L Mamusi
+kdg I L Seba
+kdh I L Tem
+kdi I L Kumam
+kdj I L Karamojong
+kdk I L Numèè
+kdl I L Tsikimba
+kdm I L Kagoma
+kdn I L Kunda
+kdp I L Kaningdon-Nindem
+kdq I L Koch
+kdr I L Karaim
+kdt I L Kuy
+kdu I L Kadaru
+kdw I L Koneraw
+kdx I L Kam
+kdy I L Keder
+kdz I L Kwaja
+kea I L Kabuverdianu
+keb I L Kélé
+kec I L Keiga
+ked I L Kerewe
+kee I L Eastern Keres
+kef I L Kpessi
+keg I L Tese
+keh I L Keak
+kei I L Kei
+kej I L Kadar
+kek I L Kekchí
+kel I L Kela (Democratic Republic of Congo)
+kem I L Kemak
+ken I L Kenyang
+keo I L Kakwa
+kep I L Kaikadi
+keq I L Kamar
+ker I L Kera
+kes I L Kugbo
+ket I L Ket
+keu I L Akebu
+kev I L Kanikkaran
+kew I L West Kewa
+kex I L Kukna
+key I L Kupia
+kez I L Kukele
+kfa I L Kodava
+kfb I L Northwestern Kolami
+kfc I L Konda-Dora
+kfd I L Korra Koraga
+kfe I L Kota (India)
+kff I L Koya
+kfg I L Kudiya
+kfh I L Kurichiya
+kfi I L Kannada Kurumba
+kfj I L Kemiehua
+kfk I L Kinnauri
+kfl I L Kung
+kfm I L Khunsari
+kfn I L Kuk
+kfo I L Koro (Côte d'Ivoire)
+kfp I L Korwa
+kfq I L Korku
+kfr I L Kachchi
+kfs I L Bilaspuri
+kft I L Kanjari
+kfu I L Katkari
+kfv I L Kurmukar
+kfw I L Kharam Naga
+kfx I L Kullu Pahari
+kfy I L Kumaoni
+kfz I L Koromfé
+kga I L Koyaga
+kgb I L Kawe
+kgc I L Kasseng
+kgd I L Kataang
+kge I L Komering
+kgf I L Kube
+kgg I L Kusunda
+kgi I L Selangor Sign Language
+kgj I L Gamale Kham
+kgk I L Kaiwá
+kgl I E Kunggari
+kgm I E Karipúna
+kgn I L Karingani
+kgo I L Krongo
+kgp I L Kaingang
+kgq I L Kamoro
+kgr I L Abun
+kgs I L Kumbainggar
+kgt I L Somyev
+kgu I L Kobol
+kgv I L Karas
+kgw I L Karon Dori
+kgx I L Kamaru
+kgy I L Kyerung
+kha kha kha I L Khasi
+khb I L Lü
+khc I L Tukang Besi North
+khd I L Bädi Kanum
+khe I L Korowai
+khf I L Khuen
+khg I L Khams Tibetan
+khh I L Kehu
+khj I L Kuturmi
+khk I L Halh Mongolian
+khl I L Lusi
+khm khm khm km I L Central Khmer
+khn I L Khandesi
+kho kho kho I A Khotanese
+khp I L Kapori
+khq I L Koyra Chiini Songhay
+khr I L Kharia
+khs I L Kasua
+kht I L Khamti
+khu I L Nkhumbi
+khv I L Khvarshi
+khw I L Khowar
+khx I L Kanu
+khy I L Kele (Democratic Republic of Congo)
+khz I L Keapara
+kia I L Kim
+kib I L Koalib
+kic I L Kickapoo
+kid I L Koshin
+kie I L Kibet
+kif I L Eastern Parbate Kham
+kig I L Kimaama
+kih I L Kilmeri
+kii I E Kitsai
+kij I L Kilivila
+kik kik kik ki I L Kikuyu
+kil I L Kariya
+kim I L Karagas
+kin kin kin rw I L Kinyarwanda
+kio I L Kiowa
+kip I L Sheshi Kham
+kiq I L Kosadle
+kir kir kir ky I L Kirghiz
+kis I L Kis
+kit I L Agob
+kiu I L Kirmanjki (individual language)
+kiv I L Kimbu
+kiw I L Northeast Kiwai
+kix I L Khiamniungan Naga
+kiy I L Kirikiri
+kiz I L Kisi
+kja I L Mlap
+kjb I L Q'anjob'al
+kjc I L Coastal Konjo
+kjd I L Southern Kiwai
+kje I L Kisar
+kjf I L Khalaj
+kjg I L Khmu
+kjh I L Khakas
+kji I L Zabana
+kjj I L Khinalugh
+kjk I L Highland Konjo
+kjl I L Western Parbate Kham
+kjm I L Kháng
+kjn I L Kunjen
+kjo I L Harijan Kinnauri
+kjp I L Pwo Eastern Karen
+kjq I L Western Keres
+kjr I L Kurudu
+kjs I L East Kewa
+kjt I L Phrae Pwo Karen
+kju I L Kashaya
+kjx I L Ramopa
+kjy I L Erave
+kjz I L Bumthangkha
+kka I L Kakanda
+kkb I L Kwerisa
+kkc I L Odoodee
+kkd I L Kinuku
+kke I L Kakabe
+kkf I L Kalaktang Monpa
+kkg I L Mabaka Valley Kalinga
+kkh I L Khün
+kki I L Kagulu
+kkj I L Kako
+kkk I L Kokota
+kkl I L Kosarek Yale
+kkm I L Kiong
+kkn I L Kon Keu
+kko I L Karko
+kkp I L Gugubera
+kkq I L Kaiku
+kkr I L Kir-Balar
+kks I L Giiwo
+kkt I L Koi
+kku I L Tumi
+kkv I L Kangean
+kkw I L Teke-Kukuya
+kkx I L Kohin
+kky I L Guguyimidjir
+kkz I L Kaska
+kla I E Klamath-Modoc
+klb I L Kiliwa
+klc I L Kolbila
+kld I L Gamilaraay
+kle I L Kulung (Nepal)
+klf I L Kendeje
+klg I L Tagakaulo
+klh I L Weliki
+kli I L Kalumpang
+klj I L Turkic Khalaj
+klk I L Kono (Nigeria)
+kll I L Kagan Kalagan
+klm I L Migum
+kln M L Kalenjin
+klo I L Kapya
+klp I L Kamasa
+klq I L Rumu
+klr I L Khaling
+kls I L Kalasha
+klt I L Nukna
+klu I L Klao
+klv I L Maskelynes
+klw I L Lindu
+klx I L Koluwawa
+kly I L Kalao
+klz I L Kabola
+kma I L Konni
+kmb kmb kmb I L Kimbundu
+kmc I L Southern Dong
+kmd I L Majukayang Kalinga
+kme I L Bakole
+kmf I L Kare (Papua New Guinea)
+kmg I L Kâte
+kmh I L Kalam
+kmi I L Kami (Nigeria)
+kmj I L Kumarbhag Paharia
+kmk I L Limos Kalinga
+kml I L Tanudan Kalinga
+kmm I L Kom (India)
+kmn I L Awtuw
+kmo I L Kwoma
+kmp I L Gimme
+kmq I L Kwama
+kmr I L Northern Kurdish
+kms I L Kamasau
+kmt I L Kemtuik
+kmu I L Kanite
+kmv I L Karipúna Creole French
+kmw I L Komo (Democratic Republic of Congo)
+kmx I L Waboda
+kmy I L Koma
+kmz I L Khorasani Turkish
+kna I L Dera (Nigeria)
+knb I L Lubuagan Kalinga
+knc I L Central Kanuri
+knd I L Konda
+kne I L Kankanaey
+knf I L Mankanya
+kng I L Koongo
+kni I L Kanufi
+knj I L Western Kanjobal
+knk I L Kuranko
+knl I L Keninjal
+knm I L Kanamarí
+knn I L Konkani (individual language)
+kno I L Kono (Sierra Leone)
+knp I L Kwanja
+knq I L Kintaq
+knr I L Kaningra
+kns I L Kensiu
+knt I L Panoan Katukína
+knu I L Kono (Guinea)
+knv I L Tabo
+knw I L Kung-Ekoka
+knx I L Kendayan
+kny I L Kanyok
+knz I L Kalamsé
+koa I L Konomala
+koc I E Kpati
+kod I L Kodi
+koe I L Kacipo-Balesi
+kof I E Kubi
+kog I L Cogui
+koh I L Koyo
+koi I L Komi-Permyak
+koj I L Sara Dunjo
+kok kok kok M L Konkani (macrolanguage)
+kol I L Kol (Papua New Guinea)
+kom kom kom kv M L Komi
+kon kon kon kg M L Kongo
+koo I L Konzo
+kop I L Waube
+koq I L Kota (Gabon)
+kor kor kor ko I L Korean
+kos kos kos I L Kosraean
+kot I L Lagwan
+kou I L Koke
+kov I L Kudu-Camo
+kow I L Kugama
+kox I E Coxima
+koy I L Koyukon
+koz I L Korak
+kpa I L Kutto
+kpb I L Mullu Kurumba
+kpc I L Curripaco
+kpd I L Koba
+kpe kpe kpe M L Kpelle
+kpf I L Komba
+kpg I L Kapingamarangi
+kph I L Kplang
+kpi I L Kofei
+kpj I L Karajá
+kpk I L Kpan
+kpl I L Kpala
+kpm I L Koho
+kpn I E Kepkiriwát
+kpo I L Ikposo
+kpq I L Korupun-Sela
+kpr I L Korafe-Yegha
+kps I L Tehit
+kpt I L Karata
+kpu I L Kafoa
+kpv I L Komi-Zyrian
+kpw I L Kobon
+kpx I L Mountain Koiali
+kpy I L Koryak
+kpz I L Kupsabiny
+kqa I L Mum
+kqb I L Kovai
+kqc I L Doromu-Koki
+kqd I L Koy Sanjaq Surat
+kqe I L Kalagan
+kqf I L Kakabai
+kqg I L Khe
+kqh I L Kisankasa
+kqi I L Koitabu
+kqj I L Koromira
+kqk I L Kotafon Gbe
+kql I L Kyenele
+kqm I L Khisa
+kqn I L Kaonde
+kqo I L Eastern Krahn
+kqp I L Kimré
+kqq I L Krenak
+kqr I L Kimaragang
+kqs I L Northern Kissi
+kqt I L Klias River Kadazan
+kqu I E Seroa
+kqv I L Okolod
+kqw I L Kandas
+kqx I L Mser
+kqy I L Koorete
+kqz I E Korana
+kra I L Kumhali
+krb I E Karkin
+krc krc krc I L Karachay-Balkar
+krd I L Kairui-Midiki
+kre I L Panará
+krf I L Koro (Vanuatu)
+krh I L Kurama
+kri I L Krio
+krj I L Kinaray-A
+krk I E Kerek
+krl krl krl I L Karelian
+krm I L Krim
+krn I L Sapo
+krp I L Korop
+krr I L Kru'ng 2
+krs I L Gbaya (Sudan)
+krt I L Tumari Kanuri
+kru kru kru I L Kurukh
+krv I L Kavet
+krw I L Western Krahn
+krx I L Karon
+kry I L Kryts
+krz I L Sota Kanum
+ksa I L Shuwa-Zamani
+ksb I L Shambala
+ksc I L Southern Kalinga
+ksd I L Kuanua
+kse I L Kuni
+ksf I L Bafia
+ksg I L Kusaghe
+ksh I L Kölsch
+ksi I L Krisa
+ksj I L Uare
+ksk I L Kansa
+ksl I L Kumalu
+ksm I L Kumba
+ksn I L Kasiguranin
+kso I L Kofa
+ksp I L Kaba
+ksq I L Kwaami
+ksr I L Borong
+kss I L Southern Kisi
+kst I L Winyé
+ksu I L Khamyang
+ksv I L Kusu
+ksw I L S'gaw Karen
+ksx I L Kedang
+ksy I L Kharia Thar
+ksz I L Kodaku
+kta I L Katua
+ktb I L Kambaata
+ktc I L Kholok
+ktd I L Kokata
+kte I L Nubri
+ktf I L Kwami
+ktg I E Kalkutung
+kth I L Karanga
+kti I L North Muyu
+ktj I L Plapo Krumen
+ktk I E Kaniet
+ktl I L Koroshi
+ktm I L Kurti
+ktn I L Karitiâna
+kto I L Kuot
+ktp I L Kaduo
+ktq I E Katabaga
+ktr I L Kota Marudu Tinagas
+kts I L South Muyu
+ktt I L Ketum
+ktu I L Kituba (Democratic Republic of Congo)
+ktv I L Eastern Katu
+ktw I E Kato
+ktx I L Kaxararí
+kty I L Kango (Bas-Uélé District)
+ktz I L Ju/'hoan
+kua kua kua kj I L Kuanyama
+kub I L Kutep
+kuc I L Kwinsu
+kud I L 'Auhelawa
+kue I L Kuman
+kuf I L Western Katu
+kug I L Kupa
+kuh I L Kushi
+kui I L Kuikúro-Kalapálo
+kuj I L Kuria
+kuk I L Kepo'
+kul I L Kulere
+kum kum kum I L Kumyk
+kun I L Kunama
+kuo I L Kumukio
+kup I L Kunimaipa
+kuq I L Karipuna
+kur kur kur ku M L Kurdish
+kus I L Kusaal
+kut kut kut I L Kutenai
+kuu I L Upper Kuskokwim
+kuv I L Kur
+kuw I L Kpagua
+kux I L Kukatja
+kuy I L Kuuku-Ya'u
+kuz I E Kunza
+kva I L Bagvalal
+kvb I L Kubu
+kvc I L Kove
+kvd I L Kui (Indonesia)
+kve I L Kalabakan
+kvf I L Kabalai
+kvg I L Kuni-Boazi
+kvh I L Komodo
+kvi I L Kwang
+kvj I L Psikye
+kvk I L Korean Sign Language
+kvl I L Kayaw
+kvm I L Kendem
+kvn I L Border Kuna
+kvo I L Dobel
+kvp I L Kompane
+kvq I L Geba Karen
+kvr I L Kerinci
+kvs I L Kunggara
+kvt I L Lahta Karen
+kvu I L Yinbaw Karen
+kvv I L Kola
+kvw I L Wersing
+kvx I L Parkari Koli
+kvy I L Yintale Karen
+kvz I L Tsakwambo
+kwa I L Dâw
+kwb I L Kwa
+kwc I L Likwala
+kwd I L Kwaio
+kwe I L Kwerba
+kwf I L Kwara'ae
+kwg I L Sara Kaba Deme
+kwh I L Kowiai
+kwi I L Awa-Cuaiquer
+kwj I L Kwanga
+kwk I L Kwakiutl
+kwl I L Kofyar
+kwm I L Kwambi
+kwn I L Kwangali
+kwo I L Kwomtari
+kwp I L Kodia
+kwq I L Kwak
+kwr I L Kwer
+kws I L Kwese
+kwt I L Kwesten
+kwu I L Kwakum
+kwv I L Sara Kaba Náà
+kww I L Kwinti
+kwx I L Khirwar
+kwy I L San Salvador Kongo
+kwz I E Kwadi
+kxa I L Kairiru
+kxb I L Krobu
+kxc I L Konso
+kxd I L Brunei
+kxe I L Kakihum
+kxf I L Manumanaw Karen
+kxh I L Karo (Ethiopia)
+kxi I L Keningau Murut
+kxj I L Kulfa
+kxk I L Zayein Karen
+kxl I L Nepali Kurux
+kxm I L Northern Khmer
+kxn I L Kanowit-Tanjong Melanau
+kxo I E Kanoé
+kxp I L Wadiyara Koli
+kxq I L Smärky Kanum
+kxr I L Koro (Papua New Guinea)
+kxs I L Kangjia
+kxt I L Koiwat
+kxu I L Kui (India)
+kxv I L Kuvi
+kxw I L Konai
+kxx I L Likuba
+kxy I L Kayong
+kxz I L Kerewo
+kya I L Kwaya
+kyb I L Butbut Kalinga
+kyc I L Kyaka
+kyd I L Karey
+kye I L Krache
+kyf I L Kouya
+kyg I L Keyagana
+kyh I L Karok
+kyi I L Kiput
+kyj I L Karao
+kyk I L Kamayo
+kyl I L Kalapuya
+kym I L Kpatili
+kyn I L Northern Binukidnon
+kyo I L Kelon
+kyp I L Kang
+kyq I L Kenga
+kyr I L Kuruáya
+kys I L Baram Kayan
+kyt I L Kayagar
+kyu I L Western Kayah
+kyv I L Kayort
+kyw I L Kudmali
+kyx I L Rapoisi
+kyy I L Kambaira
+kyz I L Kayabí
+kza I L Western Karaboro
+kzb I L Kaibobo
+kzc I L Bondoukou Kulango
+kzd I L Kadai
+kze I L Kosena
+kzf I L Da'a Kaili
+kzg I L Kikai
+kzi I L Kelabit
+kzj I L Coastal Kadazan
+kzk I E Kazukuru
+kzl I L Kayeli
+kzm I L Kais
+kzn I L Kokola
+kzo I L Kaningi
+kzp I L Kaidipang
+kzq I L Kaike
+kzr I L Karang
+kzs I L Sugut Dusun
+kzt I L Tambunan Dusun
+kzu I L Kayupulau
+kzv I L Komyandaret
+kzw I E Karirí-Xocó
+kzx I L Kamarian
+kzy I L Kango (Tshopo District)
+kzz I L Kalabra
+laa I L Southern Subanen
+lab I A Linear A
+lac I L Lacandon
+lad lad lad I L Ladino
+lae I L Pattani
+laf I L Lafofa
+lag I L Langi
+lah lah lah M L Lahnda
+lai I L Lambya
+laj I L Lango (Uganda)
+lak I L Laka (Nigeria)
+lal I L Lalia
+lam lam lam I L Lamba
+lan I L Laru
+lao lao lao lo I L Lao
+lap I L Laka (Chad)
+laq I L Qabiao
+lar I L Larteh
+las I L Lama (Togo)
+lat lat lat la I A Latin
+lau I L Laba
+lav lav lav lv M L Latvian
+law I L Lauje
+lax I L Tiwa
+lay I L Lama (Myanmar)
+laz I E Aribwatsa
+lba I E Lui
+lbb I L Label
+lbc I L Lakkia
+lbe I L Lak
+lbf I L Tinani
+lbg I L Laopang
+lbi I L La'bi
+lbj I L Ladakhi
+lbk I L Central Bontok
+lbl I L Libon Bikol
+lbm I L Lodhi
+lbn I L Lamet
+lbo I L Laven
+lbq I L Wampar
+lbr I L Lohorung
+lbs I L Libyan Sign Language
+lbt I L Lachi
+lbu I L Labu
+lbv I L Lavatbura-Lamusong
+lbw I L Tolaki
+lbx I L Lawangan
+lby I E Lamu-Lamu
+lbz I L Lardil
+lcc I L Legenyem
+lcd I L Lola
+lce I L Loncong
+lcf I L Lubu
+lch I L Luchazi
+lcl I L Lisela
+lcm I L Tungag
+lcp I L Western Lawa
+lcq I L Luhu
+lcs I L Lisabata-Nuniali
+lda I L Kla-Dan
+ldb I L Dũya
+ldd I L Luri
+ldg I L Lenyima
+ldh I L Lamja-Dengsa-Tola
+ldi I L Laari
+ldj I L Lemoro
+ldk I L Leelau
+ldl I L Kaan
+ldm I L Landoma
+ldn I C Láadan
+ldo I L Loo
+ldp I L Tso
+ldq I L Lufu
+lea I L Lega-Shabunda
+leb I L Lala-Bisa
+lec I L Leco
+led I L Lendu
+lee I L Lyélé
+lef I L Lelemi
+leg I L Lengua
+leh I L Lenje
+lei I L Lemio
+lej I L Lengola
+lek I L Leipon
+lel I L Lele (Democratic Republic of Congo)
+lem I L Nomaande
+len I E Lenca
+leo I L Leti (Cameroon)
+lep I L Lepcha
+leq I L Lembena
+ler I L Lenkau
+les I L Lese
+let I L Lesing-Gelimi
+leu I L Kara (Papua New Guinea)
+lev I L Lamma
+lew I L Ledo Kaili
+lex I L Luang
+ley I L Lemolang
+lez lez lez I L Lezghian
+lfa I L Lefa
+lfn I C Lingua Franca Nova
+lga I L Lungga
+lgb I L Laghu
+lgg I L Lugbara
+lgh I L Laghuu
+lgi I L Lengilu
+lgk I L Lingarak
+lgl I L Wala
+lgm I L Lega-Mwenga
+lgn I L Opuuo
+lgq I L Logba
+lgr I L Lengo
+lgt I L Pahi
+lgu I L Longgu
+lgz I L Ligenza
+lha I L Laha (Viet Nam)
+lhh I L Laha (Indonesia)
+lhi I L Lahu Shi
+lhl I L Lahul Lohar
+lhm I L Lhomi
+lhn I L Lahanan
+lhp I L Lhokpu
+lhs I E Mlahsö
+lht I L Lo-Toga
+lhu I L Lahu
+lia I L West-Central Limba
+lib I L Likum
+lic I L Hlai
+lid I L Nyindrou
+lie I L Likila
+lif I L Limbu
+lig I L Ligbi
+lih I L Lihir
+lii I L Lingkhim
+lij I L Ligurian
+lik I L Lika
+lil I L Lillooet
+lim lim lim li I L Limburgan
+lin lin lin ln I L Lingala
+lio I L Liki
+lip I L Sekpele
+liq I L Libido
+lir I L Liberian English
+lis I L Lisu
+lit lit lit lt I L Lithuanian
+liu I L Logorik
+liv I L Liv
+liw I L Col
+lix I L Liabuku
+liy I L Banda-Bambari
+liz I L Libinza
+lja I E Golpa
+lje I L Rampi
+lji I L Laiyolo
+ljl I L Li'o
+ljp I L Lampung Api
+ljw I L Yirandali
+ljx I E Yuru
+lka I L Lakalei
+lkb I L Kabras
+lkc I L Kucong
+lkd I L Lakondê
+lke I L Kenyi
+lkh I L Lakha
+lki I L Laki
+lkj I L Remun
+lkl I L Laeko-Libuat
+lkm I E Kalaamaya
+lkn I L Lakon
+lko I L Khayo
+lkr I L Päri
+lks I L Kisa
+lkt I L Lakota
+lku I E Kungkari
+lky I L Lokoya
+lla I L Lala-Roba
+llb I L Lolo
+llc I L Lele (Guinea)
+lld I L Ladin
+lle I L Lele (Papua New Guinea)
+llf I E Hermit
+llg I L Lole
+llh I L Lamu
+lli I L Teke-Laali
+llj I E Ladji Ladji
+llk I E Lelak
+lll I L Lilau
+llm I L Lasalimu
+lln I L Lele (Chad)
+llo I L Khlor
+llp I L North Efate
+llq I L Lolak
+lls I L Lithuanian Sign Language
+llu I L Lau
+llx I L Lauan
+lma I L East Limba
+lmb I L Merei
+lmc I E Limilngan
+lmd I L Lumun
+lme I L Pévé
+lmf I L South Lembata
+lmg I L Lamogai
+lmh I L Lambichhong
+lmi I L Lombi
+lmj I L West Lembata
+lmk I L Lamkang
+lml I L Hano
+lmm I L Lamam
+lmn I L Lambadi
+lmo I L Lombard
+lmp I L Limbum
+lmq I L Lamatuka
+lmr I L Lamalera
+lmu I L Lamenu
+lmv I L Lomaiviti
+lmw I L Lake Miwok
+lmx I L Laimbue
+lmy I L Lamboya
+lmz I E Lumbee
+lna I L Langbashe
+lnb I L Mbalanhu
+lnd I L Lundayeh
+lng I A Langobardic
+lnh I L Lanoh
+lni I L Daantanai'
+lnj I E Leningitij
+lnl I L South Central Banda
+lnm I L Langam
+lnn I L Lorediakarkar
+lno I L Lango (Sudan)
+lns I L Lamnso'
+lnu I L Longuda
+lnw I E Lanima
+lnz I L Lonzo
+loa I L Loloda
+lob I L Lobi
+loc I L Inonhan
+loe I L Saluan
+lof I L Logol
+log I L Logo
+loh I L Narim
+loi I L Loma (Côte d'Ivoire)
+loj I L Lou
+lok I L Loko
+lol lol lol I L Mongo
+lom I L Loma (Liberia)
+lon I L Malawi Lomwe
+loo I L Lombo
+lop I L Lopa
+loq I L Lobala
+lor I L Téén
+los I L Loniu
+lot I L Otuho
+lou I L Louisiana Creole French
+lov I L Lopi
+low I L Tampias Lobu
+lox I L Loun
+loy I L Loke
+loz loz loz I L Lozi
+lpa I L Lelepa
+lpe I L Lepki
+lpn I L Long Phuri Naga
+lpo I L Lipo
+lpx I L Lopit
+lra I L Rara Bakati'
+lrc I L Northern Luri
+lre I E Laurentian
+lrg I E Laragia
+lri I L Marachi
+lrk I L Loarki
+lrl I L Lari
+lrm I L Marama
+lrn I L Lorang
+lro I L Laro
+lrr I L Southern Yamphu
+lrt I L Larantuka Malay
+lrv I L Larevat
+lrz I L Lemerig
+lsa I L Lasgerdi
+lsd I L Lishana Deni
+lse I L Lusengo
+lsg I L Lyons Sign Language
+lsh I L Lish
+lsi I L Lashi
+lsl I L Latvian Sign Language
+lsm I L Saamia
+lso I L Laos Sign Language
+lsp I L Panamanian Sign Language
+lsr I L Aruop
+lss I L Lasi
+lst I L Trinidad and Tobago Sign Language
+lsy I L Mauritian Sign Language
+ltc I H Late Middle Chinese
+ltg I L Latgalian
+lti I L Leti (Indonesia)
+ltn I L Latundê
+lto I L Tsotso
+lts I L Tachoni
+ltu I L Latu
+ltz ltz ltz lb I L Luxembourgish
+lua lua lua I L Luba-Lulua
+lub lub lub lu I L Luba-Katanga
+luc I L Aringa
+lud I L Ludian
+lue I L Luvale
+luf I L Laua
+lug lug lug lg I L Ganda
+lui lui lui I L Luiseno
+luj I L Luna
+luk I L Lunanakha
+lul I L Olu'bo
+lum I L Luimbi
+lun lun lun I L Lunda
+luo luo luo I L Luo (Kenya and Tanzania)
+lup I L Lumbu
+luq I L Lucumi
+lur I L Laura
+lus lus lus I L Lushai
+lut I L Lushootseed
+luu I L Lumba-Yakkha
+luv I L Luwati
+luw I L Luo (Cameroon)
+luy M L Luyia
+luz I L Southern Luri
+lva I L Maku'a
+lvk I L Lavukaleve
+lvs I L Standard Latvian
+lvu I L Levuka
+lwa I L Lwalu
+lwe I L Lewo Eleng
+lwg I L Wanga
+lwh I L White Lachi
+lwl I L Eastern Lawa
+lwm I L Laomian
+lwo I L Luwo
+lwt I L Lewotobi
+lwu I L Lawu
+lww I L Lewo
+lya I L Layakha
+lyg I L Lyngngam
+lyn I L Luyana
+lzh I H Literary Chinese
+lzl I L Litzlitz
+lzn I L Leinong Naga
+lzz I L Laz
+maa I L San Jerónimo Tecóatl Mazatec
+mab I L Yutanduchi Mixtec
+mad mad mad I L Madurese
+mae I L Bo-Rukul
+maf I L Mafa
+mag mag mag I L Magahi
+mah mah mah mh I L Marshallese
+mai mai mai I L Maithili
+maj I L Jalapa De Díaz Mazatec
+mak mak mak I L Makasar
+mal mal mal ml I L Malayalam
+mam I L Mam
+man man man M L Mandingo
+maq I L Chiquihuitlán Mazatec
+mar mar mar mr I L Marathi
+mas mas mas I L Masai
+mat I L San Francisco Matlatzinca
+mau I L Huautla Mazatec
+mav I L Sateré-Mawé
+maw I L Mampruli
+max I L North Moluccan Malay
+maz I L Central Mazahua
+mba I L Higaonon
+mbb I L Western Bukidnon Manobo
+mbc I L Macushi
+mbd I L Dibabawon Manobo
+mbe I E Molale
+mbf I L Baba Malay
+mbh I L Mangseng
+mbi I L Ilianen Manobo
+mbj I L Nadëb
+mbk I L Malol
+mbl I L Maxakalí
+mbm I L Ombamba
+mbn I L Macaguán
+mbo I L Mbo (Cameroon)
+mbp I L Malayo
+mbq I L Maisin
+mbr I L Nukak Makú
+mbs I L Sarangani Manobo
+mbt I L Matigsalug Manobo
+mbu I L Mbula-Bwazza
+mbv I L Mbulungish
+mbw I L Maring
+mbx I L Mari (East Sepik Province)
+mby I L Memoni
+mbz I L Amoltepec Mixtec
+mca I L Maca
+mcb I L Machiguenga
+mcc I L Bitur
+mcd I L Sharanahua
+mce I L Itundujia Mixtec
+mcf I L Matsés
+mcg I L Mapoyo
+mch I L Maquiritari
+mci I L Mese
+mcj I L Mvanip
+mck I L Mbunda
+mcl I E Macaguaje
+mcm I L Malaccan Creole Portuguese
+mcn I L Masana
+mco I L Coatlán Mixe
+mcp I L Makaa
+mcq I L Ese
+mcr I L Menya
+mcs I L Mambai
+mct I L Mengisa
+mcu I L Cameroon Mambila
+mcv I L Minanibai
+mcw I L Mawa (Chad)
+mcx I L Mpiemo
+mcy I L South Watut
+mcz I L Mawan
+mda I L Mada (Nigeria)
+mdb I L Morigi
+mdc I L Male (Papua New Guinea)
+mdd I L Mbum
+mde I L Maba (Chad)
+mdf mdf mdf I L Moksha
+mdg I L Massalat
+mdh I L Maguindanaon
+mdi I L Mamvu
+mdj I L Mangbetu
+mdk I L Mangbutu
+mdl I L Maltese Sign Language
+mdm I L Mayogo
+mdn I L Mbati
+mdp I L Mbala
+mdq I L Mbole
+mdr mdr mdr I L Mandar
+mds I L Maria (Papua New Guinea)
+mdt I L Mbere
+mdu I L Mboko
+mdv I L Santa Lucía Monteverde Mixtec
+mdw I L Mbosi
+mdx I L Dizin
+mdy I L Male (Ethiopia)
+mdz I L Suruí Do Pará
+mea I L Menka
+meb I L Ikobi
+mec I L Mara
+med I L Melpa
+mee I L Mengen
+mef I L Megam
+meh I L Southwestern Tlaxiaco Mixtec
+mei I L Midob
+mej I L Meyah
+mek I L Mekeo
+mel I L Central Melanau
+mem I E Mangala
+men men men I L Mende (Sierra Leone)
+meo I L Kedah Malay
+mep I L Miriwung
+meq I L Merey
+mer I L Meru
+mes I L Masmaje
+met I L Mato
+meu I L Motu
+mev I L Mano
+mew I L Maaka
+mey I L Hassaniyya
+mez I L Menominee
+mfa I L Pattani Malay
+mfb I L Bangka
+mfc I L Mba
+mfd I L Mendankwe-Nkwen
+mfe I L Morisyen
+mff I L Naki
+mfg I L Mogofin
+mfh I L Matal
+mfi I L Wandala
+mfj I L Mefele
+mfk I L North Mofu
+mfl I L Putai
+mfm I L Marghi South
+mfn I L Cross River Mbembe
+mfo I L Mbe
+mfp I L Makassar Malay
+mfq I L Moba
+mfr I L Marithiel
+mfs I L Mexican Sign Language
+mft I L Mokerang
+mfu I L Mbwela
+mfv I L Mandjak
+mfw I E Mulaha
+mfx I L Melo
+mfy I L Mayo
+mfz I L Mabaan
+mga mga mga I H Middle Irish (900-1200)
+mgb I L Mararit
+mgc I L Morokodo
+mgd I L Moru
+mge I L Mango
+mgf I L Maklew
+mgg I L Mpumpong
+mgh I L Makhuwa-Meetto
+mgi I L Lijili
+mgj I L Abureni
+mgk I L Mawes
+mgl I L Maleu-Kilenge
+mgm I L Mambae
+mgn I L Mbangi
+mgo I L Meta'
+mgp I L Eastern Magar
+mgq I L Malila
+mgr I L Mambwe-Lungu
+mgs I L Manda (Tanzania)
+mgt I L Mongol
+mgu I L Mailu
+mgv I L Matengo
+mgw I L Matumbi
+mgy I L Mbunga
+mgz I L Mbugwe
+mha I L Manda (India)
+mhb I L Mahongwe
+mhc I L Mocho
+mhd I L Mbugu
+mhe I L Besisi
+mhf I L Mamaa
+mhg I L Margu
+mhh I L Maskoy Pidgin
+mhi I L Ma'di
+mhj I L Mogholi
+mhk I L Mungaka
+mhl I L Mauwake
+mhm I L Makhuwa-Moniga
+mhn I L Mócheno
+mho I L Mashi (Zambia)
+mhp I L Balinese Malay
+mhq I L Mandan
+mhr I L Eastern Mari
+mhs I L Buru (Indonesia)
+mht I L Mandahuaca
+mhu I L Digaro-Mishmi
+mhw I L Mbukushu
+mhx I L Maru
+mhy I L Ma'anyan
+mhz I L Mor (Mor Islands)
+mia I L Miami
+mib I L Atatláhuca Mixtec
+mic mic mic I L Mi'kmaq
+mid I L Mandaic
+mie I L Ocotepec Mixtec
+mif I L Mofu-Gudur
+mig I L San Miguel El Grande Mixtec
+mih I L Chayuco Mixtec
+mii I L Chigmecatitlán Mixtec
+mij I L Abar
+mik I L Mikasuki
+mil I L Peñoles Mixtec
+mim I L Alacatlatzala Mixtec
+min min min I L Minangkabau
+mio I L Pinotepa Nacional Mixtec
+mip I L Apasco-Apoala Mixtec
+miq I L Mískito
+mir I L Isthmus Mixe
+mis mis mis S S Uncoded languages
+mit I L Southern Puebla Mixtec
+miu I L Cacaloxtepec Mixtec
+miw I L Akoye
+mix I L Mixtepec Mixtec
+miy I L Ayutla Mixtec
+miz I L Coatzospan Mixtec
+mjc I L San Juan Colorado Mixtec
+mjd I L Northwest Maidu
+mje I E Muskum
+mjg I L Tu
+mjh I L Mwera (Nyasa)
+mji I L Kim Mun
+mjj I L Mawak
+mjk I L Matukar
+mjl I L Mandeali
+mjm I L Medebur
+mjn I L Ma (Papua New Guinea)
+mjo I L Malankuravan
+mjp I L Malapandaram
+mjq I E Malaryan
+mjr I L Malavedan
+mjs I L Miship
+mjt I L Sauria Paharia
+mju I L Manna-Dora
+mjv I L Mannan
+mjw I L Karbi
+mjx I L Mahali
+mjy I E Mahican
+mjz I L Majhi
+mka I L Mbre
+mkb I L Mal Paharia
+mkc I L Siliput
+mkd mac mkd mk I L Macedonian
+mke I L Mawchi
+mkf I L Miya
+mkg I L Mak (China)
+mki I L Dhatki
+mkj I L Mokilese
+mkk I L Byep
+mkl I L Mokole
+mkm I L Moklen
+mkn I L Kupang Malay
+mko I L Mingang Doso
+mkp I L Moikodi
+mkq I E Bay Miwok
+mkr I L Malas
+mks I L Silacayoapan Mixtec
+mkt I L Vamale
+mku I L Konyanka Maninka
+mkv I L Mafea
+mkw I L Kituba (Congo)
+mkx I L Kinamiging Manobo
+mky I L East Makian
+mkz I L Makasae
+mla I L Malo
+mlb I L Mbule
+mlc I L Cao Lan
+mle I L Manambu
+mlf I L Mal
+mlg mlg mlg mg M L Malagasy
+mlh I L Mape
+mli I L Malimpung
+mlj I L Miltu
+mlk I L Ilwana
+mll I L Malua Bay
+mlm I L Mulam
+mln I L Malango
+mlo I L Mlomp
+mlp I L Bargam
+mlq I L Western Maninkakan
+mlr I L Vame
+mls I L Masalit
+mlt mlt mlt mt I L Maltese
+mlu I L To'abaita
+mlv I L Motlav
+mlw I L Moloko
+mlx I L Malfaxal
+mlz I L Malaynon
+mma I L Mama
+mmb I L Momina
+mmc I L Michoacán Mazahua
+mmd I L Maonan
+mme I L Mae
+mmf I L Mundat
+mmg I L North Ambrym
+mmh I L Mehináku
+mmi I L Musar
+mmj I L Majhwar
+mmk I L Mukha-Dora
+mml I L Man Met
+mmm I L Maii
+mmn I L Mamanwa
+mmo I L Mangga Buang
+mmp I L Siawi
+mmq I L Musak
+mmr I L Western Xiangxi Miao
+mmt I L Malalamai
+mmu I L Mmaala
+mmv I E Miriti
+mmw I L Emae
+mmx I L Madak
+mmy I L Migaama
+mmz I L Mabaale
+mna I L Mbula
+mnb I L Muna
+mnc mnc mnc I L Manchu
+mnd I L Mondé
+mne I L Naba
+mnf I L Mundani
+mng I L Eastern Mnong
+mnh I L Mono (Democratic Republic of Congo)
+mni mni mni I L Manipuri
+mnj I L Munji
+mnk I L Mandinka
+mnl I L Tiale
+mnm I L Mapena
+mnn I L Southern Mnong
+mnp I L Min Bei Chinese
+mnq I L Minriq
+mnr I L Mono (USA)
+mns I L Mansi
+mnu I L Mer
+mnv I L Rennell-Bellona
+mnw I L Mon
+mnx I L Manikion
+mny I L Manyawa
+mnz I L Moni
+moa I L Mwan
+moc I L Mocoví
+mod I E Mobilian
+moe I L Montagnais
+mog I L Mongondow
+moh moh moh I L Mohawk
+moi I L Mboi
+moj I L Monzombo
+mok I L Morori
+mom I E Mangue
+mon mon mon mn M L Mongolian
+moo I L Monom
+mop I L Mopán Maya
+moq I L Mor (Bomberai Peninsula)
+mor I L Moro
+mos mos mos I L Mossi
+mot I L Barí
+mou I L Mogum
+mov I L Mohave
+mow I L Moi (Congo)
+mox I L Molima
+moy I L Shekkacho
+moz I L Mukulu
+mpa I L Mpoto
+mpb I L Mullukmulluk
+mpc I L Mangarayi
+mpd I L Machinere
+mpe I L Majang
+mpg I L Marba
+mph I L Maung
+mpi I L Mpade
+mpj I L Martu Wangka
+mpk I L Mbara (Chad)
+mpl I L Middle Watut
+mpm I L Yosondúa Mixtec
+mpn I L Mindiri
+mpo I L Miu
+mpp I L Migabac
+mpq I L Matís
+mpr I L Vangunu
+mps I L Dadibi
+mpt I L Mian
+mpu I L Makuráp
+mpv I L Mungkip
+mpw I L Mapidian
+mpx I L Misima-Panaeati
+mpy I L Mapia
+mpz I L Mpi
+mqa I L Maba (Indonesia)
+mqb I L Mbuko
+mqc I L Mangole
+mqe I L Matepi
+mqf I L Momuna
+mqg I L Kota Bangun Kutai Malay
+mqh I L Tlazoyaltepec Mixtec
+mqi I L Mariri
+mqj I L Mamasa
+mqk I L Rajah Kabunsuwan Manobo
+mql I L Mbelime
+mqm I L South Marquesan
+mqn I L Moronene
+mqo I L Modole
+mqp I L Manipa
+mqq I L Minokok
+mqr I L Mander
+mqs I L West Makian
+mqt I L Mok
+mqu I L Mandari
+mqv I L Mosimo
+mqw I L Murupi
+mqx I L Mamuju
+mqy I L Manggarai
+mqz I L Pano
+mra I L Mlabri
+mrb I L Marino
+mrc I L Maricopa
+mrd I L Western Magar
+mre I E Martha's Vineyard Sign Language
+mrf I L Elseng
+mrg I L Mising
+mrh I L Mara Chin
+mri mao mri mi I L Maori
+mrj I L Western Mari
+mrk I L Hmwaveke
+mrl I L Mortlockese
+mrm I L Merlav
+mrn I L Cheke Holo
+mro I L Mru
+mrp I L Morouas
+mrq I L North Marquesan
+mrr I L Maria (India)
+mrs I L Maragus
+mrt I L Marghi Central
+mru I L Mono (Cameroon)
+mrv I L Mangareva
+mrw I L Maranao
+mrx I L Maremgi
+mry I L Mandaya
+mrz I L Marind
+msa may msa ms M L Malay (macrolanguage)
+msb I L Masbatenyo
+msc I L Sankaran Maninka
+msd I L Yucatec Maya Sign Language
+mse I L Musey
+msf I L Mekwei
+msg I L Moraid
+msh I L Masikoro Malagasy
+msi I L Sabah Malay
+msj I L Ma (Democratic Republic of Congo)
+msk I L Mansaka
+msl I L Molof
+msm I L Agusan Manobo
+msn I L Vurës
+mso I L Mombum
+msp I E Maritsauá
+msq I L Caac
+msr I L Mongolian Sign Language
+mss I L West Masela
+msu I L Musom
+msv I L Maslam
+msw I L Mansoanka
+msx I L Moresada
+msy I L Aruamu
+msz I L Momare
+mta I L Cotabato Manobo
+mtb I L Anyin Morofo
+mtc I L Munit
+mtd I L Mualang
+mte I L Mono (Solomon Islands)
+mtf I L Murik (Papua New Guinea)
+mtg I L Una
+mth I L Munggui
+mti I L Maiwa (Papua New Guinea)
+mtj I L Moskona
+mtk I L Mbe'
+mtl I L Montol
+mtm I E Mator
+mtn I E Matagalpa
+mto I L Totontepec Mixe
+mtp I L Wichí Lhamtés Nocten
+mtq I L Muong
+mtr I L Mewari
+mts I L Yora
+mtt I L Mota
+mtu I L Tututepec Mixtec
+mtv I L Asaro'o
+mtw I L Southern Binukidnon
+mtx I L Tidaá Mixtec
+mty I L Nabi
+mua I L Mundang
+mub I L Mubi
+muc I L Ajumbu
+mud I L Mednyj Aleut
+mue I L Media Lengua
+mug I L Musgu
+muh I L Mündü
+mui I L Musi
+muj I L Mabire
+muk I L Mugom
+mul mul mul S S Multiple languages
+mum I L Maiwala
+muo I L Nyong
+mup I L Malvi
+muq I L Eastern Xiangxi Miao
+mur I L Murle
+mus mus mus I L Creek
+mut I L Western Muria
+muu I L Yaaku
+muv I L Muthuvan
+mux I L Bo-Ung
+muy I L Muyang
+muz I L Mursi
+mva I L Manam
+mvb I E Mattole
+mvd I L Mamboru
+mve I L Marwari (Pakistan)
+mvf I L Peripheral Mongolian
+mvg I L Yucuañe Mixtec
+mvh I L Mulgi
+mvi I L Miyako
+mvk I L Mekmek
+mvl I E Mbara (Australia)
+mvm I L Muya
+mvn I L Minaveha
+mvo I L Marovo
+mvp I L Duri
+mvq I L Moere
+mvr I L Marau
+mvs I L Massep
+mvt I L Mpotovoro
+mvu I L Marfa
+mvv I L Tagal Murut
+mvw I L Machinga
+mvx I L Meoswar
+mvy I L Indus Kohistani
+mvz I L Mesqan
+mwa I L Mwatebu
+mwb I L Juwal
+mwc I L Are
+mwe I L Mwera (Chimwera)
+mwf I L Murrinh-Patha
+mwg I L Aiklep
+mwh I L Mouk-Aria
+mwi I L Labo
+mwj I L Maligo
+mwk I L Kita Maninkakan
+mwl mwl mwl I L Mirandese
+mwm I L Sar
+mwn I L Nyamwanga
+mwo I L Central Maewo
+mwp I L Kala Lagaw Ya
+mwq I L Mün Chin
+mwr mwr mwr M L Marwari
+mws I L Mwimbi-Muthambi
+mwt I L Moken
+mwu I E Mittu
+mwv I L Mentawai
+mww I L Hmong Daw
+mwx I L Mediak
+mwy I L Mosiro
+mwz I L Moingi
+mxa I L Northwest Oaxaca Mixtec
+mxb I L Tezoatlán Mixtec
+mxc I L Manyika
+mxd I L Modang
+mxe I L Mele-Fila
+mxf I L Malgbe
+mxg I L Mbangala
+mxh I L Mvuba
+mxi I E Mozarabic
+mxj I L Miju-Mishmi
+mxk I L Monumbo
+mxl I L Maxi Gbe
+mxm I L Meramera
+mxn I L Moi (Indonesia)
+mxo I L Mbowe
+mxp I L Tlahuitoltepec Mixe
+mxq I L Juquila Mixe
+mxr I L Murik (Malaysia)
+mxs I L Huitepec Mixtec
+mxt I L Jamiltepec Mixtec
+mxu I L Mada (Cameroon)
+mxv I L Metlatónoc Mixtec
+mxw I L Namo
+mxx I L Mahou
+mxy I L Southeastern Nochixtlán Mixtec
+mxz I L Central Masela
+mya bur mya my I L Burmese
+myb I L Mbay
+myc I L Mayeka
+myd I L Maramba
+mye I L Myene
+myf I L Bambassi
+myg I L Manta
+myh I L Makah
+myi I L Mina (India)
+myj I L Mangayat
+myk I L Mamara Senoufo
+myl I L Moma
+mym I L Me'en
+myo I L Anfillo
+myp I L Pirahã
+myr I L Muniche
+mys I E Mesmes
+myu I L Mundurukú
+myv myv myv I L Erzya
+myw I L Muyuw
+myx I L Masaaba
+myy I L Macuna
+myz I H Classical Mandaic
+mza I L Santa María Zacatepec Mixtec
+mzb I L Tumzabt
+mzc I L Madagascar Sign Language
+mzd I L Malimba
+mze I L Morawa
+mzg I L Monastic Sign Language
+mzh I L Wichí Lhamtés Güisnay
+mzi I L Ixcatlán Mazatec
+mzj I L Manya
+mzk I L Nigeria Mambila
+mzl I L Mazatlán Mixe
+mzm I L Mumuye
+mzn I L Mazanderani
+mzo I E Matipuhy
+mzp I L Movima
+mzq I L Mori Atas
+mzr I L Marúbo
+mzs I L Macanese
+mzt I L Mintil
+mzu I L Inapang
+mzv I L Manza
+mzw I L Deg
+mzx I L Mawayana
+mzy I L Mozambican Sign Language
+mzz I L Maiadomu
+naa I L Namla
+nab I L Southern Nambikuára
+nac I L Narak
+nad I L Nijadali
+nae I L Naka'ela
+naf I L Nabak
+nag I L Naga Pidgin
+naj I L Nalu
+nak I L Nakanai
+nal I L Nalik
+nam I L Ngan'gityemerri
+nan I L Min Nan Chinese
+nao I L Naaba
+nap nap nap I L Neapolitan
+naq I L Nama (Namibia)
+nar I L Iguta
+nas I L Naasioi
+nat I L Hungworo
+nau nau nau na I L Nauru
+nav nav nav nv I L Navajo
+naw I L Nawuri
+nax I L Nakwi
+nay I E Narrinyeri
+naz I L Coatepec Nahuatl
+nba I L Nyemba
+nbb I L Ndoe
+nbc I L Chang Naga
+nbd I L Ngbinda
+nbe I L Konyak Naga
+nbg I L Nagarchal
+nbh I L Ngamo
+nbi I L Mao Naga
+nbj I L Ngarinman
+nbk I L Nake
+nbl nbl nbl nr I L South Ndebele
+nbm I L Ngbaka Ma'bo
+nbn I L Kuri
+nbo I L Nkukoli
+nbp I L Nnam
+nbq I L Nggem
+nbr I L Numana-Nunku-Gbantu-Numbu
+nbs I L Namibian Sign Language
+nbt I L Na
+nbu I L Rongmei Naga
+nbv I L Ngamambo
+nbw I L Southern Ngbandi
+nby I L Ningera
+nca I L Iyo
+ncb I L Central Nicobarese
+ncc I L Ponam
+ncd I L Nachering
+nce I L Yale
+ncf I L Notsi
+ncg I L Nisga'a
+nch I L Central Huasteca Nahuatl
+nci I H Classical Nahuatl
+ncj I L Northern Puebla Nahuatl
+nck I L Nakara
+ncl I L Michoacán Nahuatl
+ncm I L Nambo
+ncn I L Nauna
+nco I L Sibe
+ncp I L Ndaktup
+ncr I L Ncane
+ncs I L Nicaraguan Sign Language
+nct I L Chothe Naga
+ncu I L Chumburung
+ncx I L Central Puebla Nahuatl
+ncz I E Natchez
+nda I L Ndasa
+ndb I L Kenswei Nsei
+ndc I L Ndau
+ndd I L Nde-Nsele-Nta
+nde nde nde nd I L North Ndebele
+ndf I E Nadruvian
+ndg I L Ndengereko
+ndh I L Ndali
+ndi I L Samba Leko
+ndj I L Ndamba
+ndk I L Ndaka
+ndl I L Ndolo
+ndm I L Ndam
+ndn I L Ngundi
+ndo ndo ndo ng I L Ndonga
+ndp I L Ndo
+ndq I L Ndombe
+ndr I L Ndoola
+nds nds nds I L Low German
+ndt I L Ndunga
+ndu I L Dugun
+ndv I L Ndut
+ndw I L Ndobo
+ndx I L Nduga
+ndy I L Lutos
+ndz I L Ndogo
+nea I L Eastern Ngad'a
+neb I L Toura (Côte d'Ivoire)
+nec I L Nedebang
+ned I L Nde-Gbite
+nee I L Nêlêmwa-Nixumwak
+nef I L Nefamese
+neg I L Negidal
+neh I L Nyenkha
+nei I A Neo-Hittite
+nej I L Neko
+nek I L Neku
+nem I L Nemi
+nen I L Nengone
+neo I L Ná-Meo
+nep nep nep ne M L Nepali (macrolanguage)
+neq I L North Central Mixe
+ner I L Yahadian
+nes I L Bhoti Kinnauri
+net I L Nete
+neu I C Neo
+nev I L Nyaheun
+new new new I L Newari
+nex I L Neme
+ney I L Neyo
+nez I L Nez Perce
+nfa I L Dhao
+nfd I L Ahwai
+nfl I L Ayiwo
+nfr I L Nafaanra
+nfu I L Mfumte
+nga I L Ngbaka
+ngb I L Northern Ngbandi
+ngc I L Ngombe (Democratic Republic of Congo)
+ngd I L Ngando (Central African Republic)
+nge I L Ngemba
+ngg I L Ngbaka Manza
+ngh I L N/u
+ngi I L Ngizim
+ngj I L Ngie
+ngk I L Dalabon
+ngl I L Lomwe
+ngm I L Ngatik Men's Creole
+ngn I L Ngwo
+ngo I L Ngoni
+ngp I L Ngulu
+ngq I L Ngurimi
+ngr I L Engdewu
+ngs I L Gvoko
+ngt I L Ngeq
+ngu I L Guerrero Nahuatl
+ngv I E Nagumi
+ngw I L Ngwaba
+ngx I L Nggwahyi
+ngy I L Tibea
+ngz I L Ngungwel
+nha I L Nhanda
+nhb I L Beng
+nhc I E Tabasco Nahuatl
+nhd I L Chiripá
+nhe I L Eastern Huasteca Nahuatl
+nhf I L Nhuwala
+nhg I L Tetelcingo Nahuatl
+nhh I L Nahari
+nhi I L Zacatlán-Ahuacatlán-Tepetzintla Nahuatl
+nhk I L Isthmus-Cosoleacaque Nahuatl
+nhm I L Morelos Nahuatl
+nhn I L Central Nahuatl
+nho I L Takuu
+nhp I L Isthmus-Pajapan Nahuatl
+nhq I L Huaxcaleca Nahuatl
+nhr I L Naro
+nht I L Ometepec Nahuatl
+nhu I L Noone
+nhv I L Temascaltepec Nahuatl
+nhw I L Western Huasteca Nahuatl
+nhx I L Isthmus-Mecayapan Nahuatl
+nhy I L Northern Oaxaca Nahuatl
+nhz I L Santa María La Alta Nahuatl
+nia nia nia I L Nias
+nib I L Nakame
+nid I E Ngandi
+nie I L Niellim
+nif I L Nek
+nig I E Ngalakan
+nih I L Nyiha (Tanzania)
+nii I L Nii
+nij I L Ngaju
+nik I L Southern Nicobarese
+nil I L Nila
+nim I L Nilamba
+nin I L Ninzo
+nio I L Nganasan
+niq I L Nandi
+nir I L Nimboran
+nis I L Nimi
+nit I L Southeastern Kolami
+niu niu niu I L Niuean
+niv I L Gilyak
+niw I L Nimo
+nix I L Hema
+niy I L Ngiti
+niz I L Ningil
+nja I L Nzanyi
+njb I L Nocte Naga
+njd I L Ndonde Hamba
+njh I L Lotha Naga
+nji I L Gudanji
+njj I L Njen
+njl I L Njalgulgule
+njm I L Angami Naga
+njn I L Liangmai Naga
+njo I L Ao Naga
+njr I L Njerep
+njs I L Nisa
+njt I L Ndyuka-Trio Pidgin
+nju I L Ngadjunmaya
+njx I L Kunyi
+njy I L Njyem
+njz I L Nyishi
+nka I L Nkoya
+nkb I L Khoibu Naga
+nkc I L Nkongho
+nkd I L Koireng
+nke I L Duke
+nkf I L Inpui Naga
+nkg I L Nekgini
+nkh I L Khezha Naga
+nki I L Thangal Naga
+nkj I L Nakai
+nkk I L Nokuku
+nkm I L Namat
+nkn I L Nkangala
+nko I L Nkonya
+nkp I E Niuatoputapu
+nkq I L Nkami
+nkr I L Nukuoro
+nks I L North Asmat
+nkt I L Nyika (Tanzania)
+nku I L Bouna Kulango
+nkv I L Nyika (Malawi and Zambia)
+nkw I L Nkutu
+nkx I L Nkoroo
+nkz I L Nkari
+nla I L Ngombale
+nlc I L Nalca
+nld dut nld nl I L Dutch
+nle I L East Nyala
+nlg I L Gela
+nli I L Grangali
+nlj I L Nyali
+nlk I L Ninia Yali
+nll I L Nihali
+nlo I L Ngul
+nlq I L Lao Naga
+nlu I L Nchumbulu
+nlv I L Orizaba Nahuatl
+nlw I E Walangama
+nlx I L Nahali
+nly I L Nyamal
+nlz I L Nalögo
+nma I L Maram Naga
+nmb I L Big Nambas
+nmc I L Ngam
+nmd I L Ndumu
+nme I L Mzieme Naga
+nmf I L Tangkhul Naga (India)
+nmg I L Kwasio
+nmh I L Monsang Naga
+nmi I L Nyam
+nmj I L Ngombe (Central African Republic)
+nmk I L Namakura
+nml I L Ndemli
+nmm I L Manangba
+nmn I L !Xóõ
+nmo I L Moyon Naga
+nmp I E Nimanbur
+nmq I L Nambya
+nmr I E Nimbari
+nms I L Letemboi
+nmt I L Namonuito
+nmu I L Northeast Maidu
+nmv I E Ngamini
+nmw I L Nimoa
+nmx I L Nama (Papua New Guinea)
+nmy I L Namuyi
+nmz I L Nawdm
+nna I L Nyangumarta
+nnb I L Nande
+nnc I L Nancere
+nnd I L West Ambae
+nne I L Ngandyera
+nnf I L Ngaing
+nng I L Maring Naga
+nnh I L Ngiemboon
+nni I L North Nuaulu
+nnj I L Nyangatom
+nnk I L Nankina
+nnl I L Northern Rengma Naga
+nnm I L Namia
+nnn I L Ngete
+nno nno nno nn I L Norwegian Nynorsk
+nnp I L Wancho Naga
+nnq I L Ngindo
+nnr I E Narungga
+nns I L Ningye
+nnt I E Nanticoke
+nnu I L Dwang
+nnv I E Nugunu (Australia)
+nnw I L Southern Nuni
+nnx I L Ngong
+nny I E Nyangga
+nnz I L Nda'nda'
+noa I L Woun Meu
+nob nob nob nb I L Norwegian Bokmål
+noc I L Nuk
+nod I L Northern Thai
+noe I L Nimadi
+nof I L Nomane
+nog nog nog I L Nogai
+noh I L Nomu
+noi I L Noiri
+noj I L Nonuya
+nok I E Nooksack
+nol I E Nomlaki
+nom I E Nocamán
+non non non I H Old Norse
+nop I L Numanggang
+noq I L Ngongo
+nor nor nor no M L Norwegian
+nos I L Eastern Nisu
+not I L Nomatsiguenga
+nou I L Ewage-Notu
+nov I C Novial
+now I L Nyambo
+noy I L Noy
+noz I L Nayi
+npa I L Nar Phu
+npb I L Nupbikha
+npg I L Ponyo-Gongwang Naga
+nph I L Phom Naga
+npi I L Nepali (individual language)
+npl I L Southeastern Puebla Nahuatl
+npn I L Mondropolon
+npo I L Pochuri Naga
+nps I L Nipsan
+npu I L Puimei Naga
+npy I L Napu
+nqg I L Southern Nago
+nqk I L Kura Ede Nago
+nqm I L Ndom
+nqn I L Nen
+nqo nqo nqo I L N'Ko
+nqq I L Kyan-Karyaw Naga
+nqy I L Akyaung Ari Naga
+nra I L Ngom
+nrb I L Nara
+nrc I A Noric
+nre I L Southern Rengma Naga
+nrg I L Narango
+nri I L Chokri Naga
+nrk I L Ngarla
+nrl I L Ngarluma
+nrm I L Narom
+nrn I E Norn
+nrp I A North Picene
+nrr I E Norra
+nrt I E Northern Kalapuya
+nru I L Narua
+nrx I E Ngurmbur
+nrz I L Lala
+nsa I L Sangtam Naga
+nsc I L Nshi
+nsd I L Southern Nisu
+nse I L Nsenga
+nsf I L Northwestern Nisu
+nsg I L Ngasa
+nsh I L Ngoshie
+nsi I L Nigerian Sign Language
+nsk I L Naskapi
+nsl I L Norwegian Sign Language
+nsm I L Sumi Naga
+nsn I L Nehan
+nso nso nso I L Pedi
+nsp I L Nepalese Sign Language
+nsq I L Northern Sierra Miwok
+nsr I L Maritime Sign Language
+nss I L Nali
+nst I L Tase Naga
+nsu I L Sierra Negra Nahuatl
+nsv I L Southwestern Nisu
+nsw I L Navut
+nsx I L Nsongo
+nsy I L Nasal
+nsz I L Nisenan
+nte I L Nathembo
+ntg I E Ngantangarra
+nti I L Natioro
+ntj I L Ngaanyatjarra
+ntk I L Ikoma-Nata-Isenye
+ntm I L Nateni
+nto I L Ntomba
+ntp I L Northern Tepehuan
+ntr I L Delo
+nts I E Natagaimas
+ntu I L Natügu
+ntw I E Nottoway
+ntx I L Tangkhul Naga (Myanmar)
+nty I L Mantsi
+ntz I L Natanzi
+nua I L Yuanga
+nuc I E Nukuini
+nud I L Ngala
+nue I L Ngundu
+nuf I L Nusu
+nug I E Nungali
+nuh I L Ndunda
+nui I L Ngumbi
+nuj I L Nyole
+nuk I L Nuu-chah-nulth
+nul I L Nusa Laut
+num I L Niuafo'ou
+nun I L Anong
+nuo I L Nguôn
+nup I L Nupe-Nupe-Tako
+nuq I L Nukumanu
+nur I L Nukuria
+nus I L Nuer
+nut I L Nung (Viet Nam)
+nuu I L Ngbundu
+nuv I L Northern Nuni
+nuw I L Nguluwan
+nux I L Mehek
+nuy I L Nunggubuyu
+nuz I L Tlamacazapa Nahuatl
+nvh I L Nasarian
+nvm I L Namiae
+nvo I L Nyokon
+nwa I E Nawathinehena
+nwb I L Nyabwa
+nwc nwc nwc I H Classical Newari
+nwe I L Ngwe
+nwg I E Ngayawung
+nwi I L Southwest Tanna
+nwm I L Nyamusa-Molo
+nwo I E Nauo
+nwr I L Nawaru
+nwx I H Middle Newar
+nwy I E Nottoway-Meherrin
+nxa I L Nauete
+nxd I L Ngando (Democratic Republic of Congo)
+nxe I L Nage
+nxg I L Ngad'a
+nxi I L Nindi
+nxk I L Koki Naga
+nxl I L South Nuaulu
+nxm I A Numidian
+nxn I E Ngawun
+nxq I L Naxi
+nxr I L Ninggerum
+nxu I E Narau
+nxx I L Nafri
+nya nya nya ny I L Nyanja
+nyb I L Nyangbo
+nyc I L Nyanga-li
+nyd I L Nyore
+nye I L Nyengo
+nyf I L Giryama
+nyg I L Nyindu
+nyh I L Nyigina
+nyi I L Ama (Sudan)
+nyj I L Nyanga
+nyk I L Nyaneka
+nyl I L Nyeu
+nym nym nym I L Nyamwezi
+nyn nyn nyn I L Nyankole
+nyo nyo nyo I L Nyoro
+nyp I E Nyang'i
+nyq I L Nayini
+nyr I L Nyiha (Malawi)
+nys I L Nyunga
+nyt I E Nyawaygi
+nyu I L Nyungwe
+nyv I E Nyulnyul
+nyw I L Nyaw
+nyx I E Nganyaywana
+nyy I L Nyakyusa-Ngonde
+nza I L Tigon Mbembe
+nzb I L Njebi
+nzi nzi nzi I L Nzima
+nzk I L Nzakara
+nzm I L Zeme Naga
+nzs I L New Zealand Sign Language
+nzu I L Teke-Nzikou
+nzy I L Nzakambay
+nzz I L Nanga Dama Dogon
+oaa I L Orok
+oac I L Oroch
+oar I A Old Aramaic (up to 700 BCE)
+oav I H Old Avar
+obi I E Obispeño
+obk I L Southern Bontok
+obl I L Oblo
+obm I A Moabite
+obo I L Obo Manobo
+obr I H Old Burmese
+obt I H Old Breton
+obu I L Obulom
+oca I L Ocaina
+och I A Old Chinese
+oci oci oci oc I L Occitan (post 1500)
+oco I H Old Cornish
+ocu I L Atzingo Matlatzinca
+oda I L Odut
+odk I L Od
+odt I H Old Dutch
+odu I L Odual
+ofo I E Ofo
+ofs I H Old Frisian
+ofu I L Efutop
+ogb I L Ogbia
+ogc I L Ogbah
+oge I H Old Georgian
+ogg I L Ogbogolo
+ogo I L Khana
+ogu I L Ogbronuagum
+oht I A Old Hittite
+ohu I H Old Hungarian
+oia I L Oirata
+oin I L Inebu One
+ojb I L Northwestern Ojibwa
+ojc I L Central Ojibwa
+ojg I L Eastern Ojibwa
+oji oji oji oj M L Ojibwa
+ojp I H Old Japanese
+ojs I L Severn Ojibwa
+ojv I L Ontong Java
+ojw I L Western Ojibwa
+oka I L Okanagan
+okb I L Okobo
+okd I L Okodia
+oke I L Okpe (Southwestern Edo)
+okg I E Koko Babangk
+okh I L Koresh-e Rostam
+oki I L Okiek
+okj I E Oko-Juwoi
+okk I L Kwamtim One
+okl I E Old Kentish Sign Language
+okm I H Middle Korean (10th-16th cent.)
+okn I L Oki-No-Erabu
+oko I H Old Korean (3rd-9th cent.)
+okr I L Kirike
+oks I L Oko-Eni-Osayen
+oku I L Oku
+okv I L Orokaiva
+okx I L Okpe (Northwestern Edo)
+ola I L Walungge
+old I L Mochi
+ole I L Olekha
+olk I E Olkol
+olm I L Oloma
+olo I L Livvi
+olr I L Olrat
+oma I L Omaha-Ponca
+omb I L East Ambae
+omc I E Mochica
+ome I E Omejes
+omg I L Omagua
+omi I L Omi
+omk I E Omok
+oml I L Ombo
+omn I A Minoan
+omo I L Utarmbung
+omp I H Old Manipuri
+omr I H Old Marathi
+omt I L Omotik
+omu I E Omurano
+omw I L South Tairora
+omx I H Old Mon
+ona I L Ona
+onb I L Lingao
+one I L Oneida
+ong I L Olo
+oni I L Onin
+onj I L Onjob
+onk I L Kabore One
+onn I L Onobasulu
+ono I L Onondaga
+onp I L Sartang
+onr I L Northern One
+ons I L Ono
+ont I L Ontenu
+onu I L Unua
+onw I H Old Nubian
+onx I L Onin Based Pidgin
+ood I L Tohono O'odham
+oog I L Ong
+oon I L Önge
+oor I L Oorlams
+oos I A Old Ossetic
+opa I L Okpamheri
+opk I L Kopkaka
+opm I L Oksapmin
+opo I L Opao
+opt I E Opata
+opy I L Ofayé
+ora I L Oroha
+orc I L Orma
+ore I L Orejón
+org I L Oring
+orh I L Oroqen
+ori ori ori or M L Oriya (macrolanguage)
+orm orm orm om M L Oromo
+orn I L Orang Kanaq
+oro I L Orokolo
+orr I L Oruma
+ors I L Orang Seletar
+ort I L Adivasi Oriya
+oru I L Ormuri
+orv I H Old Russian
+orw I L Oro Win
+orx I L Oro
+ory I L Oriya (individual language)
+orz I L Ormu
+osa osa osa I L Osage
+osc I A Oscan
+osi I L Osing
+oso I L Ososo
+osp I H Old Spanish
+oss oss oss os I L Ossetian
+ost I L Osatu
+osu I L Southern One
+osx I H Old Saxon
+ota ota ota I H Ottoman Turkish (1500-1928)
+otb I H Old Tibetan
+otd I L Ot Danum
+ote I L Mezquital Otomi
+oti I E Oti
+otk I H Old Turkish
+otl I L Tilapa Otomi
+otm I L Eastern Highland Otomi
+otn I L Tenango Otomi
+otq I L Querétaro Otomi
+otr I L Otoro
+ots I L Estado de México Otomi
+ott I L Temoaya Otomi
+otu I E Otuke
+otw I L Ottawa
+otx I L Texcatepec Otomi
+oty I A Old Tamil
+otz I L Ixtenco Otomi
+oua I L Tagargrent
+oub I L Glio-Oubi
+oue I L Oune
+oui I H Old Uighur
+oum I E Ouma
+oun I L !O!ung
+owi I L Owiniga
+owl I H Old Welsh
+oyb I L Oy
+oyd I L Oyda
+oym I L Wayampi
+oyy I L Oya'oya
+ozm I L Koonzime
+pab I L Parecís
+pac I L Pacoh
+pad I L Paumarí
+pae I L Pagibete
+paf I E Paranawát
+pag pag pag I L Pangasinan
+pah I L Tenharim
+pai I L Pe
+pak I L Parakanã
+pal pal pal I A Pahlavi
+pam pam pam I L Pampanga
+pan pan pan pa I L Panjabi
+pao I L Northern Paiute
+pap pap pap I L Papiamento
+paq I L Parya
+par I L Panamint
+pas I L Papasena
+pat I L Papitalai
+pau pau pau I L Palauan
+pav I L Pakaásnovos
+paw I L Pawnee
+pax I E Pankararé
+pay I L Pech
+paz I E Pankararú
+pbb I L Páez
+pbc I L Patamona
+pbe I L Mezontla Popoloca
+pbf I L Coyotepec Popoloca
+pbg I E Paraujano
+pbh I L E'ñapa Woromaipu
+pbi I L Parkwa
+pbl I L Mak (Nigeria)
+pbn I L Kpasam
+pbo I L Papel
+pbp I L Badyara
+pbr I L Pangwa
+pbs I L Central Pame
+pbt I L Southern Pashto
+pbu I L Northern Pashto
+pbv I L Pnar
+pby I L Pyu
+pca I L Santa Inés Ahuatempan Popoloca
+pcb I L Pear
+pcc I L Bouyei
+pcd I L Picard
+pce I L Ruching Palaung
+pcf I L Paliyan
+pcg I L Paniya
+pch I L Pardhan
+pci I L Duruwa
+pcj I L Parenga
+pck I L Paite Chin
+pcl I L Pardhi
+pcm I L Nigerian Pidgin
+pcn I L Piti
+pcp I L Pacahuara
+pcw I L Pyapun
+pda I L Anam
+pdc I L Pennsylvania German
+pdi I L Pa Di
+pdn I L Podena
+pdo I L Padoe
+pdt I L Plautdietsch
+pdu I L Kayan
+pea I L Peranakan Indonesian
+peb I E Eastern Pomo
+ped I L Mala (Papua New Guinea)
+pee I L Taje
+pef I E Northeastern Pomo
+peg I L Pengo
+peh I L Bonan
+pei I L Chichimeca-Jonaz
+pej I E Northern Pomo
+pek I L Penchal
+pel I L Pekal
+pem I L Phende
+peo peo peo I H Old Persian (ca. 600-400 B.C.)
+pep I L Kunja
+peq I L Southern Pomo
+pes I L Iranian Persian
+pev I L Pémono
+pex I L Petats
+pey I L Petjo
+pez I L Eastern Penan
+pfa I L Pááfang
+pfe I L Peere
+pfl I L Pfaelzisch
+pga I L Sudanese Creole Arabic
+pgg I L Pangwali
+pgi I L Pagi
+pgk I L Rerep
+pgl I A Primitive Irish
+pgn I A Paelignian
+pgs I L Pangseng
+pgu I L Pagu
+pha I L Pa-Hng
+phd I L Phudagi
+phg I L Phuong
+phh I L Phukha
+phk I L Phake
+phl I L Phalura
+phm I L Phimbi
+phn phn phn I A Phoenician
+pho I L Phunoi
+phq I L Phana'
+phr I L Pahari-Potwari
+pht I L Phu Thai
+phu I L Phuan
+phv I L Pahlavani
+phw I L Phangduwali
+pia I L Pima Bajo
+pib I L Yine
+pic I L Pinji
+pid I L Piaroa
+pie I E Piro
+pif I L Pingelapese
+pig I L Pisabo
+pih I L Pitcairn-Norfolk
+pii I L Pini
+pij I E Pijao
+pil I L Yom
+pim I E Powhatan
+pin I L Piame
+pio I L Piapoco
+pip I L Pero
+pir I L Piratapuyo
+pis I L Pijin
+pit I E Pitta Pitta
+piu I L Pintupi-Luritja
+piv I L Pileni
+piw I L Pimbwe
+pix I L Piu
+piy I L Piya-Kwonci
+piz I L Pije
+pjt I L Pitjantjatjara
+pka I H Ardhamāgadhī Prākrit
+pkb I L Pokomo
+pkc I E Paekche
+pkg I L Pak-Tong
+pkh I L Pankhu
+pkn I L Pakanha
+pko I L Pökoot
+pkp I L Pukapuka
+pkr I L Attapady Kurumba
+pks I L Pakistan Sign Language
+pkt I L Maleng
+pku I L Paku
+pla I L Miani
+plb I L Polonombauk
+plc I L Central Palawano
+pld I L Polari
+ple I L Palu'e
+plg I L Pilagá
+plh I L Paulohi
+pli pli pli pi I A Pali
+plj I L Polci
+plk I L Kohistani Shina
+pll I L Shwe Palaung
+pln I L Palenquero
+plo I L Oluta Popoluca
+plp I L Palpa
+plq I A Palaic
+plr I L Palaka Senoufo
+pls I L San Marcos Tlalcoyalco Popoloca
+plt I L Plateau Malagasy
+plu I L Palikúr
+plv I L Southwest Palawano
+plw I L Brooke's Point Palawano
+ply I L Bolyu
+plz I L Paluan
+pma I L Paama
+pmb I L Pambia
+pmc I E Palumata
+pmd I E Pallanganmiddang
+pme I L Pwaamei
+pmf I L Pamona
+pmh I H Māhārāṣṭri Prākrit
+pmi I L Northern Pumi
+pmj I L Southern Pumi
+pmk I E Pamlico
+pml I E Lingua Franca
+pmm I L Pomo
+pmn I L Pam
+pmo I L Pom
+pmq I L Northern Pame
+pmr I L Paynamar
+pms I L Piemontese
+pmt I L Tuamotuan
+pmu I L Mirpur Panjabi
+pmw I L Plains Miwok
+pmx I L Poumei Naga
+pmy I L Papuan Malay
+pmz I E Southern Pame
+pna I L Punan Bah-Biau
+pnb I L Western Panjabi
+pnc I L Pannei
+pne I L Western Penan
+png I L Pongu
+pnh I L Penrhyn
+pni I L Aoheng
+pnj I E Pinjarup
+pnk I L Paunaka
+pnl I L Paleni
+pnm I L Punan Batu 1
+pnn I L Pinai-Hagahai
+pno I E Panobo
+pnp I L Pancana
+pnq I L Pana (Burkina Faso)
+pnr I L Panim
+pns I L Ponosakan
+pnt I L Pontic
+pnu I L Jiongnai Bunu
+pnv I L Pinigura
+pnw I L Panytyima
+pnx I L Phong-Kniang
+pny I L Pinyin
+pnz I L Pana (Central African Republic)
+poc I L Poqomam
+pod I E Ponares
+poe I L San Juan Atzingo Popoloca
+pof I L Poke
+pog I E Potiguára
+poh I L Poqomchi'
+poi I L Highland Popoluca
+pok I L Pokangá
+pol pol pol pl I L Polish
+pom I L Southeastern Pomo
+pon pon pon I L Pohnpeian
+poo I L Central Pomo
+pop I L Pwapwâ
+poq I L Texistepec Popoluca
+por por por pt I L Portuguese
+pos I L Sayula Popoluca
+pot I L Potawatomi
+pov I L Upper Guinea Crioulo
+pow I L San Felipe Otlaltepec Popoloca
+pox I E Polabian
+poy I L Pogolo
+ppa I L Pao
+ppe I L Papi
+ppi I L Paipai
+ppk I L Uma
+ppl I L Pipil
+ppm I L Papuma
+ppn I L Papapana
+ppo I L Folopa
+ppp I L Pelende
+ppq I L Pei
+pps I L San Luís Temalacayuca Popoloca
+ppt I L Pare
+ppu I E Papora
+pqa I L Pa'a
+pqm I L Malecite-Passamaquoddy
+prb I L Lua'
+prc I L Parachi
+prd I L Parsi-Dari
+pre I L Principense
+prf I L Paranan
+prg I L Prussian
+prh I L Porohanon
+pri I L Paicî
+prk I L Parauk
+prl I L Peruvian Sign Language
+prm I L Kibiri
+prn I L Prasuni
+pro pro pro I H Old Provençal (to 1500)
+prp I L Parsi
+prq I L Ashéninka Perené
+prr I E Puri
+prs I L Dari
+prt I L Phai
+pru I L Puragi
+prw I L Parawen
+prx I L Purik
+pry I L Pray 3
+prz I L Providencia Sign Language
+psa I L Asue Awyu
+psc I L Persian Sign Language
+psd I L Plains Indian Sign Language
+pse I L Central Malay
+psg I L Penang Sign Language
+psh I L Southwest Pashayi
+psi I L Southeast Pashayi
+psl I L Puerto Rican Sign Language
+psm I E Pauserna
+psn I L Panasuan
+pso I L Polish Sign Language
+psp I L Philippine Sign Language
+psq I L Pasi
+psr I L Portuguese Sign Language
+pss I L Kaulong
+pst I L Central Pashto
+psu I H Sauraseni Prākrit
+psw I L Port Sandwich
+psy I E Piscataway
+pta I L Pai Tavytera
+pth I E Pataxó Hã-Ha-Hãe
+pti I L Pintiini
+ptn I L Patani
+pto I L Zo'é
+ptp I L Patep
+ptr I L Piamatsina
+ptt I L Enrekang
+ptu I L Bambam
+ptv I L Port Vato
+ptw I E Pentlatch
+pty I L Pathiya
+pua I L Western Highland Purepecha
+pub I L Purum
+puc I L Punan Merap
+pud I L Punan Aput
+pue I L Puelche
+puf I L Punan Merah
+pug I L Phuie
+pui I L Puinave
+puj I L Punan Tubu
+puk I L Pu Ko
+pum I L Puma
+puo I L Puoc
+pup I L Pulabu
+puq I E Puquina
+pur I L Puruborá
+pus pus pus ps M L Pushto
+put I L Putoh
+puu I L Punu
+puw I L Puluwatese
+pux I L Puare
+puy I E Purisimeño
+puz I L Purum Naga
+pwa I L Pawaia
+pwb I L Panawa
+pwg I L Gapapaiwa
+pwi I E Patwin
+pwm I L Molbog
+pwn I L Paiwan
+pwo I L Pwo Western Karen
+pwr I L Powari
+pww I L Pwo Northern Karen
+pxm I L Quetzaltepec Mixe
+pye I L Pye Krumen
+pym I L Fyam
+pyn I L Poyanáwa
+pys I L Paraguayan Sign Language
+pyu I L Puyuma
+pyx I A Pyu (Myanmar)
+pyy I L Pyen
+pzn I L Para Naga
+qua I L Quapaw
+qub I L Huallaga Huánuco Quechua
+quc I L K'iche'
+qud I L Calderón Highland Quichua
+que que que qu M L Quechua
+quf I L Lambayeque Quechua
+qug I L Chimborazo Highland Quichua
+quh I L South Bolivian Quechua
+qui I L Quileute
+quk I L Chachapoyas Quechua
+qul I L North Bolivian Quechua
+qum I L Sipacapense
+qun I E Quinault
+qup I L Southern Pastaza Quechua
+quq I L Quinqui
+qur I L Yanahuanca Pasco Quechua
+qus I L Santiago del Estero Quichua
+quv I L Sacapulteco
+quw I L Tena Lowland Quichua
+qux I L Yauyos Quechua
+quy I L Ayacucho Quechua
+quz I L Cusco Quechua
+qva I L Ambo-Pasco Quechua
+qvc I L Cajamarca Quechua
+qve I L Eastern Apurímac Quechua
+qvh I L Huamalíes-Dos de Mayo Huánuco Quechua
+qvi I L Imbabura Highland Quichua
+qvj I L Loja Highland Quichua
+qvl I L Cajatambo North Lima Quechua
+qvm I L Margos-Yarowilca-Lauricocha Quechua
+qvn I L North Junín Quechua
+qvo I L Napo Lowland Quechua
+qvp I L Pacaraos Quechua
+qvs I L San Martín Quechua
+qvw I L Huaylla Wanca Quechua
+qvy I L Queyu
+qvz I L Northern Pastaza Quichua
+qwa I L Corongo Ancash Quechua
+qwc I H Classical Quechua
+qwh I L Huaylas Ancash Quechua
+qwm I E Kuman (Russia)
+qws I L Sihuas Ancash Quechua
+qwt I E Kwalhioqua-Tlatskanai
+qxa I L Chiquián Ancash Quechua
+qxc I L Chincha Quechua
+qxh I L Panao Huánuco Quechua
+qxl I L Salasaca Highland Quichua
+qxn I L Northern Conchucos Ancash Quechua
+qxo I L Southern Conchucos Ancash Quechua
+qxp I L Puno Quechua
+qxq I L Qashqa'i
+qxr I L Cañar Highland Quichua
+qxs I L Southern Qiang
+qxt I L Santa Ana de Tusi Pasco Quechua
+qxu I L Arequipa-La Unión Quechua
+qxw I L Jauja Wanca Quechua
+qya I C Quenya
+qyp I E Quiripi
+raa I L Dungmali
+rab I L Camling
+rac I L Rasawa
+rad I L Rade
+raf I L Western Meohang
+rag I L Logooli
+rah I L Rabha
+rai I L Ramoaaina
+raj raj raj M L Rajasthani
+rak I L Tulu-Bohuai
+ral I L Ralte
+ram I L Canela
+ran I L Riantana
+rao I L Rao
+rap rap rap I L Rapanui
+raq I L Saam
+rar rar rar I L Rarotongan
+ras I L Tegali
+rat I L Razajerdi
+rau I L Raute
+rav I L Sampang
+raw I L Rawang
+rax I L Rang
+ray I L Rapa
+raz I L Rahambuu
+rbb I L Rumai Palaung
+rbk I L Northern Bontok
+rbl I L Miraya Bikol
+rbp I E Barababaraba
+rcf I L Réunion Creole French
+rdb I L Rudbari
+rea I L Rerau
+reb I L Rembong
+ree I L Rejang Kayan
+reg I L Kara (Tanzania)
+rei I L Reli
+rej I L Rejang
+rel I L Rendille
+rem I E Remo
+ren I L Rengao
+rer I E Rer Bare
+res I L Reshe
+ret I L Retta
+rey I L Reyesano
+rga I L Roria
+rge I L Romano-Greek
+rgk I E Rangkas
+rgn I L Romagnol
+rgr I L Resígaro
+rgs I L Southern Roglai
+rgu I L Ringgou
+rhg I L Rohingya
+rhp I L Yahang
+ria I L Riang (India)
+rie I L Rien
+rif I L Tarifit
+ril I L Riang (Myanmar)
+rim I L Nyaturu
+rin I L Nungu
+rir I L Ribun
+rit I L Ritarungo
+riu I L Riung
+rjg I L Rajong
+rji I L Raji
+rjs I L Rajbanshi
+rka I L Kraol
+rkb I L Rikbaktsa
+rkh I L Rakahanga-Manihiki
+rki I L Rakhine
+rkm I L Marka
+rkt I L Rangpuri
+rkw I E Arakwal
+rma I L Rama
+rmb I L Rembarunga
+rmc I L Carpathian Romani
+rmd I E Traveller Danish
+rme I L Angloromani
+rmf I L Kalo Finnish Romani
+rmg I L Traveller Norwegian
+rmh I L Murkim
+rmi I L Lomavren
+rmk I L Romkun
+rml I L Baltic Romani
+rmm I L Roma
+rmn I L Balkan Romani
+rmo I L Sinte Romani
+rmp I L Rempi
+rmq I L Caló
+rms I L Romanian Sign Language
+rmt I L Domari
+rmu I L Tavringer Romani
+rmv I C Romanova
+rmw I L Welsh Romani
+rmx I L Romam
+rmy I L Vlax Romani
+rmz I L Marma
+rna I E Runa
+rnd I L Ruund
+rng I L Ronga
+rnl I L Ranglong
+rnn I L Roon
+rnp I L Rongpo
+rnr I E Nari Nari
+rnw I L Rungwa
+rob I L Tae'
+roc I L Cacgia Roglai
+rod I L Rogo
+roe I L Ronji
+rof I L Rombo
+rog I L Northern Roglai
+roh roh roh rm I L Romansh
+rol I L Romblomanon
+rom rom rom M L Romany
+ron rum ron ro I L Romanian
+roo I L Rotokas
+rop I L Kriol
+ror I L Rongga
+rou I L Runga
+row I L Dela-Oenale
+rpn I L Repanbitip
+rpt I L Rapting
+rri I L Ririo
+rro I L Waima
+rrt I E Arritinngithigh
+rsb I L Romano-Serbian
+rsi I L Rennellese Sign Language
+rsl I L Russian Sign Language
+rtc I L Rungtu Chin
+rth I L Ratahan
+rtm I L Rotuman
+rtw I L Rathawi
+rub I L Gungu
+ruc I L Ruuli
+rue I L Rusyn
+ruf I L Luguru
+rug I L Roviana
+ruh I L Ruga
+rui I L Rufiji
+ruk I L Che
+run run run rn I L Rundi
+ruo I L Istro Romanian
+rup rup rup I L Macedo-Romanian
+ruq I L Megleno Romanian
+rus rus rus ru I L Russian
+rut I L Rutul
+ruu I L Lanas Lobu
+ruy I L Mala (Nigeria)
+ruz I L Ruma
+rwa I L Rawo
+rwk I L Rwa
+rwm I L Amba (Uganda)
+rwo I L Rawa
+rwr I L Marwari (India)
+rxd I L Ngardi
+rxw I E Karuwali
+ryn I L Northern Amami-Oshima
+rys I L Yaeyama
+ryu I L Central Okinawan
+saa I L Saba
+sab I L Buglere
+sac I L Meskwaki
+sad sad sad I L Sandawe
+sae I L Sabanê
+saf I L Safaliba
+sag sag sag sg I L Sango
+sah sah sah I L Yakut
+saj I L Sahu
+sak I L Sake
+sam sam sam I E Samaritan Aramaic
+san san san sa I A Sanskrit
+sao I L Sause
+sap I L Sanapaná
+saq I L Samburu
+sar I E Saraveca
+sas sas sas I L Sasak
+sat sat sat I L Santali
+sau I L Saleman
+sav I L Saafi-Saafi
+saw I L Sawi
+sax I L Sa
+say I L Saya
+saz I L Saurashtra
+sba I L Ngambay
+sbb I L Simbo
+sbc I L Kele (Papua New Guinea)
+sbd I L Southern Samo
+sbe I L Saliba
+sbf I L Shabo
+sbg I L Seget
+sbh I L Sori-Harengan
+sbi I L Seti
+sbj I L Surbakhal
+sbk I L Safwa
+sbl I L Botolan Sambal
+sbm I L Sagala
+sbn I L Sindhi Bhil
+sbo I L Sabüm
+sbp I L Sangu (Tanzania)
+sbq I L Sileibi
+sbr I L Sembakung Murut
+sbs I L Subiya
+sbt I L Kimki
+sbu I L Stod Bhoti
+sbv I A Sabine
+sbw I L Simba
+sbx I L Seberuang
+sby I L Soli
+sbz I L Sara Kaba
+scb I L Chut
+sce I L Dongxiang
+scf I L San Miguel Creole French
+scg I L Sanggau
+sch I L Sakachep
+sci I L Sri Lankan Creole Malay
+sck I L Sadri
+scl I L Shina
+scn scn scn I L Sicilian
+sco sco sco I L Scots
+scp I L Helambu Sherpa
+scq I L Sa'och
+scs I L North Slavey
+scu I L Shumcho
+scv I L Sheni
+scw I L Sha
+scx I A Sicel
+sda I L Toraja-Sa'dan
+sdb I L Shabak
+sdc I L Sassarese Sardinian
+sde I L Surubu
+sdf I L Sarli
+sdg I L Savi
+sdh I L Southern Kurdish
+sdj I L Suundi
+sdk I L Sos Kundi
+sdl I L Saudi Arabian Sign Language
+sdm I L Semandang
+sdn I L Gallurese Sardinian
+sdo I L Bukar-Sadung Bidayuh
+sdp I L Sherdukpen
+sdr I L Oraon Sadri
+sds I E Sened
+sdt I E Shuadit
+sdu I L Sarudu
+sdx I L Sibu Melanau
+sdz I L Sallands
+sea I L Semai
+seb I L Shempire Senoufo
+sec I L Sechelt
+sed I L Sedang
+see I L Seneca
+sef I L Cebaara Senoufo
+seg I L Segeju
+seh I L Sena
+sei I L Seri
+sej I L Sene
+sek I L Sekani
+sel sel sel I L Selkup
+sen I L Nanerigé Sénoufo
+seo I L Suarmin
+sep I L Sìcìté Sénoufo
+seq I L Senara Sénoufo
+ser I L Serrano
+ses I L Koyraboro Senni Songhai
+set I L Sentani
+seu I L Serui-Laut
+sev I L Nyarafolo Senoufo
+sew I L Sewa Bay
+sey I L Secoya
+sez I L Senthang Chin
+sfb I L Langue des signes de Belgique Francophone
+sfe I L Eastern Subanen
+sfm I L Small Flowery Miao
+sfs I L South African Sign Language
+sfw I L Sehwi
+sga sga sga I H Old Irish (to 900)
+sgb I L Mag-antsi Ayta
+sgc I L Kipsigis
+sgd I L Surigaonon
+sge I L Segai
+sgg I L Swiss-German Sign Language
+sgh I L Shughni
+sgi I L Suga
+sgj I L Surgujia
+sgk I L Sangkong
+sgm I E Singa
+sgo I L Songa
+sgp I L Singpho
+sgr I L Sangisari
+sgs I L Samogitian
+sgt I L Brokpake
+sgu I L Salas
+sgw I L Sebat Bet Gurage
+sgx I L Sierra Leone Sign Language
+sgy I L Sanglechi
+sgz I L Sursurunga
+sha I L Shall-Zwall
+shb I L Ninam
+shc I L Sonde
+shd I L Kundal Shahi
+she I L Sheko
+shg I L Shua
+shh I L Shoshoni
+shi I L Tachelhit
+shj I L Shatt
+shk I L Shilluk
+shl I L Shendu
+shm I L Shahrudi
+shn shn shn I L Shan
+sho I L Shanga
+shp I L Shipibo-Conibo
+shq I L Sala
+shr I L Shi
+shs I L Shuswap
+sht I E Shasta
+shu I L Chadian Arabic
+shv I L Shehri
+shw I L Shwai
+shx I L She
+shy I L Tachawit
+shz I L Syenara Senoufo
+sia I E Akkala Sami
+sib I L Sebop
+sid sid sid I L Sidamo
+sie I L Simaa
+sif I L Siamou
+sig I L Paasaal
+sih I L Zire
+sii I L Shom Peng
+sij I L Numbami
+sik I L Sikiana
+sil I L Tumulung Sisaala
+sim I L Mende (Papua New Guinea)
+sin sin sin si I L Sinhala
+sip I L Sikkimese
+siq I L Sonia
+sir I L Siri
+sis I E Siuslaw
+siu I L Sinagen
+siv I L Sumariup
+siw I L Siwai
+six I L Sumau
+siy I L Sivandi
+siz I L Siwi
+sja I L Epena
+sjb I L Sajau Basap
+sjd I L Kildin Sami
+sje I L Pite Sami
+sjg I L Assangori
+sjk I E Kemi Sami
+sjl I L Sajalong
+sjm I L Mapun
+sjn I C Sindarin
+sjo I L Xibe
+sjp I L Surjapuri
+sjr I L Siar-Lak
+sjs I E Senhaja De Srair
+sjt I L Ter Sami
+sju I L Ume Sami
+sjw I L Shawnee
+ska I L Skagit
+skb I L Saek
+skc I L Ma Manda
+skd I L Southern Sierra Miwok
+ske I L Seke (Vanuatu)
+skf I L Sakirabiá
+skg I L Sakalava Malagasy
+skh I L Sikule
+ski I L Sika
+skj I L Seke (Nepal)
+skk I L Sok
+skm I L Kutong
+skn I L Kolibugan Subanon
+sko I L Seko Tengah
+skp I L Sekapan
+skq I L Sininkere
+skr I L Seraiki
+sks I L Maia
+skt I L Sakata
+sku I L Sakao
+skv I L Skou
+skw I E Skepi Creole Dutch
+skx I L Seko Padang
+sky I L Sikaiana
+skz I L Sekar
+slc I L Sáliba
+sld I L Sissala
+sle I L Sholaga
+slf I L Swiss-Italian Sign Language
+slg I L Selungai Murut
+slh I L Southern Puget Sound Salish
+sli I L Lower Silesian
+slj I L Salumá
+slk slo slk sk I L Slovak
+sll I L Salt-Yui
+slm I L Pangutaran Sama
+sln I E Salinan
+slp I L Lamaholot
+slq I L Salchuq
+slr I L Salar
+sls I L Singapore Sign Language
+slt I L Sila
+slu I L Selaru
+slv slv slv sl I L Slovenian
+slw I L Sialum
+slx I L Salampasu
+sly I L Selayar
+slz I L Ma'ya
+sma sma sma I L Southern Sami
+smb I L Simbari
+smc I E Som
+smd I L Sama
+sme sme sme se I L Northern Sami
+smf I L Auwe
+smg I L Simbali
+smh I L Samei
+smj smj smj I L Lule Sami
+smk I L Bolinao
+sml I L Central Sama
+smm I L Musasa
+smn smn smn I L Inari Sami
+smo smo smo sm I L Samoan
+smp I E Samaritan
+smq I L Samo
+smr I L Simeulue
+sms sms sms I L Skolt Sami
+smt I L Simte
+smu I E Somray
+smv I L Samvedi
+smw I L Sumbawa
+smx I L Samba
+smy I L Semnani
+smz I L Simeku
+sna sna sna sn I L Shona
+snb I L Sebuyau
+snc I L Sinaugoro
+snd snd snd sd I L Sindhi
+sne I L Bau Bidayuh
+snf I L Noon
+sng I L Sanga (Democratic Republic of Congo)
+snh I E Shinabo
+sni I E Sensi
+snj I L Riverain Sango
+snk snk snk I L Soninke
+snl I L Sangil
+snm I L Southern Ma'di
+snn I L Siona
+sno I L Snohomish
+snp I L Siane
+snq I L Sangu (Gabon)
+snr I L Sihan
+sns I L South West Bay
+snu I L Senggi
+snv I L Sa'ban
+snw I L Selee
+snx I L Sam
+sny I L Saniyo-Hiyewe
+snz I L Sinsauru
+soa I L Thai Song
+sob I L Sobei
+soc I L So (Democratic Republic of Congo)
+sod I L Songoora
+soe I L Songomeno
+sog sog sog I A Sogdian
+soh I L Aka
+soi I L Sonha
+soj I L Soi
+sok I L Sokoro
+sol I L Solos
+som som som so I L Somali
+soo I L Songo
+sop I L Songe
+soq I L Kanasi
+sor I L Somrai
+sos I L Seeku
+sot sot sot st I L Southern Sotho
+sou I L Southern Thai
+sov I L Sonsorol
+sow I L Sowanda
+sox I L Swo
+soy I L Miyobe
+soz I L Temi
+spa spa spa es I L Spanish
+spb I L Sepa (Indonesia)
+spc I L Sapé
+spd I L Saep
+spe I L Sepa (Papua New Guinea)
+spg I L Sian
+spi I L Saponi
+spk I L Sengo
+spl I L Selepet
+spm I L Akukem
+spo I L Spokane
+spp I L Supyire Senoufo
+spq I L Loreto-Ucayali Spanish
+spr I L Saparua
+sps I L Saposa
+spt I L Spiti Bhoti
+spu I L Sapuan
+spv I L Sambalpuri
+spx I A South Picene
+spy I L Sabaot
+sqa I L Shama-Sambuga
+sqh I L Shau
+sqi alb sqi sq M L Albanian
+sqk I L Albanian Sign Language
+sqm I L Suma
+sqn I E Susquehannock
+sqo I L Sorkhei
+sqq I L Sou
+sqr I H Siculo Arabic
+sqs I L Sri Lankan Sign Language
+sqt I L Soqotri
+squ I L Squamish
+sra I L Saruga
+srb I L Sora
+src I L Logudorese Sardinian
+srd srd srd sc M L Sardinian
+sre I L Sara
+srf I L Nafi
+srg I L Sulod
+srh I L Sarikoli
+sri I L Siriano
+srk I L Serudung Murut
+srl I L Isirawa
+srm I L Saramaccan
+srn srn srn I L Sranan Tongo
+sro I L Campidanese Sardinian
+srp srp srp sr I L Serbian
+srq I L Sirionó
+srr srr srr I L Serer
+srs I L Sarsi
+srt I L Sauri
+sru I L Suruí
+srv I L Southern Sorsoganon
+srw I L Serua
+srx I L Sirmauri
+sry I L Sera
+srz I L Shahmirzadi
+ssb I L Southern Sama
+ssc I L Suba-Simbiti
+ssd I L Siroi
+sse I L Balangingi
+ssf I L Thao
+ssg I L Seimat
+ssh I L Shihhi Arabic
+ssi I L Sansi
+ssj I L Sausi
+ssk I L Sunam
+ssl I L Western Sisaala
+ssm I L Semnam
+ssn I L Waata
+sso I L Sissano
+ssp I L Spanish Sign Language
+ssq I L So'a
+ssr I L Swiss-French Sign Language
+sss I L Sô
+sst I L Sinasina
+ssu I L Susuami
+ssv I L Shark Bay
+ssw ssw ssw ss I L Swati
+ssx I L Samberigi
+ssy I L Saho
+ssz I L Sengseng
+sta I L Settla
+stb I L Northern Subanen
+std I L Sentinel
+ste I L Liana-Seti
+stf I L Seta
+stg I L Trieng
+sth I L Shelta
+sti I L Bulo Stieng
+stj I L Matya Samo
+stk I L Arammba
+stl I L Stellingwerfs
+stm I L Setaman
+stn I L Owa
+sto I L Stoney
+stp I L Southeastern Tepehuan
+stq I L Saterfriesisch
+str I L Straits Salish
+sts I L Shumashti
+stt I L Budeh Stieng
+stu I L Samtao
+stv I L Silt'e
+stw I L Satawalese
+sty I L Siberian Tatar
+sua I L Sulka
+sub I L Suku
+suc I L Western Subanon
+sue I L Suena
+sug I L Suganga
+sui I L Suki
+suj I L Shubi
+suk suk suk I L Sukuma
+sun sun sun su I L Sundanese
+suq I L Suri
+sur I L Mwaghavul
+sus sus sus I L Susu
+sut I E Subtiaba
+suv I L Puroik
+suw I L Sumbwa
+sux sux sux I A Sumerian
+suy I L Suyá
+suz I L Sunwar
+sva I L Svan
+svb I L Ulau-Suain
+svc I L Vincentian Creole English
+sve I L Serili
+svk I L Slovakian Sign Language
+svm I L Slavomolisano
+svr I L Savara
+svs I L Savosavo
+svx I E Skalvian
+swa swa swa sw M L Swahili (macrolanguage)
+swb I L Maore Comorian
+swc I L Congo Swahili
+swe swe swe sv I L Swedish
+swf I L Sere
+swg I L Swabian
+swh I L Swahili (individual language)
+swi I L Sui
+swj I L Sira
+swk I L Malawi Sena
+swl I L Swedish Sign Language
+swm I L Samosa
+swn I L Sawknah
+swo I L Shanenawa
+swp I L Suau
+swq I L Sharwa
+swr I L Saweru
+sws I L Seluwasan
+swt I L Sawila
+swu I L Suwawa
+swv I L Shekhawati
+sww I E Sowa
+swx I L Suruahá
+swy I L Sarua
+sxb I L Suba
+sxc I A Sicanian
+sxe I L Sighu
+sxg I L Shixing
+sxk I E Southern Kalapuya
+sxl I E Selian
+sxm I L Samre
+sxn I L Sangir
+sxo I A Sorothaptic
+sxr I L Saaroa
+sxs I L Sasaru
+sxu I L Upper Saxon
+sxw I L Saxwe Gbe
+sya I L Siang
+syb I L Central Subanen
+syc syc syc I H Classical Syriac
+syi I L Seki
+syk I L Sukur
+syl I L Sylheti
+sym I L Maya Samo
+syn I L Senaya
+syo I L Suoy
+syr syr syr M L Syriac
+sys I L Sinyar
+syw I L Kagate
+syy I L Al-Sayyid Bedouin Sign Language
+sza I L Semelai
+szb I L Ngalum
+szc I L Semaq Beri
+szd I E Seru
+sze I L Seze
+szg I L Sengele
+szl I L Silesian
+szn I L Sula
+szp I L Suabo
+szv I L Isu (Fako Division)
+szw I L Sawai
+taa I L Lower Tanana
+tab I L Tabassaran
+tac I L Lowland Tarahumara
+tad I L Tause
+tae I L Tariana
+taf I L Tapirapé
+tag I L Tagoi
+tah tah tah ty I L Tahitian
+taj I L Eastern Tamang
+tak I L Tala
+tal I L Tal
+tam tam tam ta I L Tamil
+tan I L Tangale
+tao I L Yami
+tap I L Taabwa
+taq I L Tamasheq
+tar I L Central Tarahumara
+tas I E Tay Boi
+tat tat tat tt I L Tatar
+tau I L Upper Tanana
+tav I L Tatuyo
+taw I L Tai
+tax I L Tamki
+tay I L Atayal
+taz I L Tocho
+tba I L Aikanã
+tbb I E Tapeba
+tbc I L Takia
+tbd I L Kaki Ae
+tbe I L Tanimbili
+tbf I L Mandara
+tbg I L North Tairora
+tbh I E Thurawal
+tbi I L Gaam
+tbj I L Tiang
+tbk I L Calamian Tagbanwa
+tbl I L Tboli
+tbm I L Tagbu
+tbn I L Barro Negro Tunebo
+tbo I L Tawala
+tbp I L Taworta
+tbr I L Tumtum
+tbs I L Tanguat
+tbt I L Tembo (Kitembo)
+tbu I E Tubar
+tbv I L Tobo
+tbw I L Tagbanwa
+tbx I L Kapin
+tby I L Tabaru
+tbz I L Ditammari
+tca I L Ticuna
+tcb I L Tanacross
+tcc I L Datooga
+tcd I L Tafi
+tce I L Southern Tutchone
+tcf I L Malinaltepec Me'phaa
+tcg I L Tamagario
+tch I L Turks And Caicos Creole English
+tci I L Wára
+tck I L Tchitchege
+tcl I E Taman (Myanmar)
+tcm I L Tanahmerah
+tcn I L Tichurong
+tco I L Taungyo
+tcp I L Tawr Chin
+tcq I L Kaiy
+tcs I L Torres Strait Creole
+tct I L T'en
+tcu I L Southeastern Tarahumara
+tcw I L Tecpatlán Totonac
+tcx I L Toda
+tcy I L Tulu
+tcz I L Thado Chin
+tda I L Tagdal
+tdb I L Panchpargania
+tdc I L Emberá-Tadó
+tdd I L Tai Nüa
+tde I L Tiranige Diga Dogon
+tdf I L Talieng
+tdg I L Western Tamang
+tdh I L Thulung
+tdi I L Tomadino
+tdj I L Tajio
+tdk I L Tambas
+tdl I L Sur
+tdn I L Tondano
+tdo I L Teme
+tdq I L Tita
+tdr I L Todrah
+tds I L Doutai
+tdt I L Tetun Dili
+tdu I L Tempasuk Dusun
+tdv I L Toro
+tdx I L Tandroy-Mahafaly Malagasy
+tdy I L Tadyawan
+tea I L Temiar
+teb I E Tetete
+tec I L Terik
+ted I L Tepo Krumen
+tee I L Huehuetla Tepehua
+tef I L Teressa
+teg I L Teke-Tege
+teh I L Tehuelche
+tei I L Torricelli
+tek I L Ibali Teke
+tel tel tel te I L Telugu
+tem tem tem I L Timne
+ten I E Tama (Colombia)
+teo I L Teso
+tep I E Tepecano
+teq I L Temein
+ter ter ter I L Tereno
+tes I L Tengger
+tet tet tet I L Tetum
+teu I L Soo
+tev I L Teor
+tew I L Tewa (USA)
+tex I L Tennet
+tey I L Tulishi
+tfi I L Tofin Gbe
+tfn I L Tanaina
+tfo I L Tefaro
+tfr I L Teribe
+tft I L Ternate
+tga I L Sagalla
+tgb I L Tobilung
+tgc I L Tigak
+tgd I L Ciwogai
+tge I L Eastern Gorkha Tamang
+tgf I L Chalikha
+tgh I L Tobagonian Creole English
+tgi I L Lawunuia
+tgj I L Tagin
+tgk tgk tgk tg I L Tajik
+tgl tgl tgl tl I L Tagalog
+tgn I L Tandaganon
+tgo I L Sudest
+tgp I L Tangoa
+tgq I L Tring
+tgr I L Tareng
+tgs I L Nume
+tgt I L Central Tagbanwa
+tgu I L Tanggu
+tgv I E Tingui-Boto
+tgw I L Tagwana Senoufo
+tgx I L Tagish
+tgy I E Togoyo
+tgz I E Tagalaka
+tha tha tha th I L Thai
+thc I L Tai Hang Tong
+thd I L Thayore
+the I L Chitwania Tharu
+thf I L Thangmi
+thh I L Northern Tarahumara
+thi I L Tai Long
+thk I L Tharaka
+thl I L Dangaura Tharu
+thm I L Aheu
+thn I L Thachanadan
+thp I L Thompson
+thq I L Kochila Tharu
+thr I L Rana Tharu
+ths I L Thakali
+tht I L Tahltan
+thu I L Thuri
+thv I L Tahaggart Tamahaq
+thw I L Thudam
+thx I L The
+thy I L Tha
+thz I L Tayart Tamajeq
+tia I L Tidikelt Tamazight
+tic I L Tira
+tid I L Tidong
+tif I L Tifal
+tig tig tig I L Tigre
+tih I L Timugon Murut
+tii I L Tiene
+tij I L Tilung
+tik I L Tikar
+til I E Tillamook
+tim I L Timbe
+tin I L Tindi
+tio I L Teop
+tip I L Trimuris
+tiq I L Tiéfo
+tir tir tir ti I L Tigrinya
+tis I L Masadiit Itneg
+tit I L Tinigua
+tiu I L Adasen
+tiv tiv tiv I L Tiv
+tiw I L Tiwi
+tix I L Southern Tiwa
+tiy I L Tiruray
+tiz I L Tai Hongjin
+tja I L Tajuasohn
+tjg I L Tunjung
+tji I L Northern Tujia
+tjl I L Tai Laing
+tjm I E Timucua
+tjn I E Tonjon
+tjo I L Temacine Tamazight
+tjs I L Southern Tujia
+tju I E Tjurruru
+tjw I L Djabwurrung
+tka I E Truká
+tkb I L Buksa
+tkd I L Tukudede
+tke I L Takwane
+tkf I E Tukumanféd
+tkg I L Tesaka Malagasy
+tkl tkl tkl I L Tokelau
+tkm I E Takelma
+tkn I L Toku-No-Shima
+tkp I L Tikopia
+tkq I L Tee
+tkr I L Tsakhur
+tks I L Takestani
+tkt I L Kathoriya Tharu
+tku I L Upper Necaxa Totonac
+tkw I L Teanu
+tkx I L Tangko
+tkz I L Takua
+tla I L Southwestern Tepehuan
+tlb I L Tobelo
+tlc I L Yecuatla Totonac
+tld I L Talaud
+tlf I L Telefol
+tlg I L Tofanma
+tlh tlh tlh I C Klingon
+tli tli tli I L Tlingit
+tlj I L Talinga-Bwisi
+tlk I L Taloki
+tll I L Tetela
+tlm I L Tolomako
+tln I L Talondo'
+tlo I L Talodi
+tlp I L Filomena Mata-Coahuitlán Totonac
+tlq I L Tai Loi
+tlr I L Talise
+tls I L Tambotalo
+tlt I L Teluti
+tlu I L Tulehu
+tlv I L Taliabu
+tlx I L Khehek
+tly I L Talysh
+tma I L Tama (Chad)
+tmb I L Katbol
+tmc I L Tumak
+tmd I L Haruai
+tme I E Tremembé
+tmf I L Toba-Maskoy
+tmg I E Ternateño
+tmh tmh tmh M L Tamashek
+tmi I L Tutuba
+tmj I L Samarokena
+tmk I L Northwestern Tamang
+tml I L Tamnim Citak
+tmm I L Tai Thanh
+tmn I L Taman (Indonesia)
+tmo I L Temoq
+tmp I L Tai Mène
+tmq I L Tumleo
+tmr I E Jewish Babylonian Aramaic (ca. 200-1200 CE)
+tms I L Tima
+tmt I L Tasmate
+tmu I L Iau
+tmv I L Tembo (Motembo)
+tmw I L Temuan
+tmy I L Tami
+tmz I E Tamanaku
+tna I L Tacana
+tnb I L Western Tunebo
+tnc I L Tanimuca-Retuarã
+tnd I L Angosturas Tunebo
+tne I L Tinoc Kallahan
+tng I L Tobanga
+tnh I L Maiani
+tni I L Tandia
+tnk I L Kwamera
+tnl I L Lenakel
+tnm I L Tabla
+tnn I L North Tanna
+tno I L Toromono
+tnp I L Whitesands
+tnq I E Taino
+tnr I L Ménik
+tns I L Tenis
+tnt I L Tontemboan
+tnu I L Tay Khang
+tnv I L Tangchangya
+tnw I L Tonsawang
+tnx I L Tanema
+tny I L Tongwe
+tnz I L Tonga (Thailand)
+tob I L Toba
+toc I L Coyutla Totonac
+tod I L Toma
+toe I E Tomedes
+tof I L Gizrra
+tog tog tog I L Tonga (Nyasa)
+toh I L Gitonga
+toi I L Tonga (Zambia)
+toj I L Tojolabal
+tol I L Tolowa
+tom I L Tombulu
+ton ton ton to I L Tonga (Tonga Islands)
+too I L Xicotepec De Juárez Totonac
+top I L Papantla Totonac
+toq I L Toposa
+tor I L Togbo-Vara Banda
+tos I L Highland Totonac
+tou I L Tho
+tov I L Upper Taromi
+tow I L Jemez
+tox I L Tobian
+toy I L Topoiyo
+toz I L To
+tpa I L Taupota
+tpc I L Azoyú Me'phaa
+tpe I L Tippera
+tpf I L Tarpia
+tpg I L Kula
+tpi tpi tpi I L Tok Pisin
+tpj I L Tapieté
+tpk I E Tupinikin
+tpl I L Tlacoapa Me'phaa
+tpm I L Tampulma
+tpn I E Tupinambá
+tpo I L Tai Pao
+tpp I L Pisaflores Tepehua
+tpq I L Tukpa
+tpr I L Tuparí
+tpt I L Tlachichilco Tepehua
+tpu I L Tampuan
+tpv I L Tanapag
+tpw I E Tupí
+tpx I L Acatepec Me'phaa
+tpy I L Trumai
+tpz I L Tinputz
+tqb I L Tembé
+tql I L Lehali
+tqm I L Turumsa
+tqn I L Tenino
+tqo I L Toaripi
+tqp I L Tomoip
+tqq I L Tunni
+tqr I E Torona
+tqt I L Western Totonac
+tqu I L Touo
+tqw I E Tonkawa
+tra I L Tirahi
+trb I L Terebu
+trc I L Copala Triqui
+trd I L Turi
+tre I L East Tarangan
+trf I L Trinidadian Creole English
+trg I L Lishán Didán
+trh I L Turaka
+tri I L Trió
+trj I L Toram
+trl I L Traveller Scottish
+trm I L Tregami
+trn I L Trinitario
+tro I L Tarao Naga
+trp I L Kok Borok
+trq I L San Martín Itunyoso Triqui
+trr I L Taushiro
+trs I L Chicahuaxtla Triqui
+trt I L Tunggare
+tru I L Turoyo
+trv I L Taroko
+trw I L Torwali
+trx I L Tringgus-Sembaan Bidayuh
+try I E Turung
+trz I E Torá
+tsa I L Tsaangi
+tsb I L Tsamai
+tsc I L Tswa
+tsd I L Tsakonian
+tse I L Tunisian Sign Language
+tsf I L Southwestern Tamang
+tsg I L Tausug
+tsh I L Tsuvan
+tsi tsi tsi I L Tsimshian
+tsj I L Tshangla
+tsk I L Tseku
+tsl I L Ts'ün-Lao
+tsm I L Turkish Sign Language
+tsn tsn tsn tn I L Tswana
+tso tso tso ts I L Tsonga
+tsp I L Northern Toussian
+tsq I L Thai Sign Language
+tsr I L Akei
+tss I L Taiwan Sign Language
+tst I L Tondi Songway Kiini
+tsu I L Tsou
+tsv I L Tsogo
+tsw I L Tsishingini
+tsx I L Mubami
+tsy I L Tebul Sign Language
+tsz I L Purepecha
+tta I E Tutelo
+ttb I L Gaa
+ttc I L Tektiteko
+ttd I L Tauade
+tte I L Bwanabwana
+ttf I L Tuotomb
+ttg I L Tutong
+tth I L Upper Ta'oih
+tti I L Tobati
+ttj I L Tooro
+ttk I L Totoro
+ttl I L Totela
+ttm I L Northern Tutchone
+ttn I L Towei
+tto I L Lower Ta'oih
+ttp I L Tombelala
+ttq I L Tawallammat Tamajaq
+ttr I L Tera
+tts I L Northeastern Thai
+ttt I L Muslim Tat
+ttu I L Torau
+ttv I L Titan
+ttw I L Long Wat
+tty I L Sikaritai
+ttz I L Tsum
+tua I L Wiarumus
+tub I L Tübatulabal
+tuc I L Mutu
+tud I E Tuxá
+tue I L Tuyuca
+tuf I L Central Tunebo
+tug I L Tunia
+tuh I L Taulil
+tui I L Tupuri
+tuj I L Tugutil
+tuk tuk tuk tk I L Turkmen
+tul I L Tula
+tum tum tum I L Tumbuka
+tun I E Tunica
+tuo I L Tucano
+tuq I L Tedaga
+tur tur tur tr I L Turkish
+tus I L Tuscarora
+tuu I L Tututni
+tuv I L Turkana
+tux I E Tuxináwa
+tuy I L Tugen
+tuz I L Turka
+tva I L Vaghua
+tvd I L Tsuvadi
+tve I L Te'un
+tvk I L Southeast Ambrym
+tvl tvl tvl I L Tuvalu
+tvm I L Tela-Masbuar
+tvn I L Tavoyan
+tvo I L Tidore
+tvs I L Taveta
+tvt I L Tutsa Naga
+tvu I L Tunen
+tvw I L Sedoa
+tvy I E Timor Pidgin
+twa I E Twana
+twb I L Western Tawbuid
+twc I E Teshenawa
+twd I L Twents
+twe I L Tewa (Indonesia)
+twf I L Northern Tiwa
+twg I L Tereweng
+twh I L Tai Dón
+twi twi twi tw I L Twi
+twl I L Tawara
+twm I L Tawang Monpa
+twn I L Twendi
+two I L Tswapong
+twp I L Ere
+twq I L Tasawaq
+twr I L Southwestern Tarahumara
+twt I E Turiwára
+twu I L Termanu
+tww I L Tuwari
+twx I L Tewe
+twy I L Tawoyan
+txa I L Tombonuo
+txb I A Tokharian B
+txc I E Tsetsaut
+txe I L Totoli
+txg I A Tangut
+txh I A Thracian
+txi I L Ikpeng
+txm I L Tomini
+txn I L West Tarangan
+txo I L Toto
+txq I L Tii
+txr I A Tartessian
+txs I L Tonsea
+txt I L Citak
+txu I L Kayapó
+txx I L Tatana
+txy I L Tanosy Malagasy
+tya I L Tauya
+tye I L Kyanga
+tyh I L O'du
+tyi I L Teke-Tsaayi
+tyj I L Tai Do
+tyl I L Thu Lao
+tyn I L Kombai
+typ I E Thaypan
+tyr I L Tai Daeng
+tys I L Tày Sa Pa
+tyt I L Tày Tac
+tyu I L Kua
+tyv tyv tyv I L Tuvinian
+tyx I L Teke-Tyee
+tyz I L Tày
+tza I L Tanzanian Sign Language
+tzh I L Tzeltal
+tzj I L Tz'utujil
+tzl I C Talossan
+tzm I L Central Atlas Tamazight
+tzn I L Tugun
+tzo I L Tzotzil
+tzx I L Tabriak
+uam I E Uamué
+uan I L Kuan
+uar I L Tairuma
+uba I L Ubang
+ubi I L Ubi
+ubl I L Buhi'non Bikol
+ubr I L Ubir
+ubu I L Umbu-Ungu
+uby I E Ubykh
+uda I L Uda
+ude I L Udihe
+udg I L Muduga
+udi I L Udi
+udj I L Ujir
+udl I L Wuzlam
+udm udm udm I L Udmurt
+udu I L Uduk
+ues I L Kioko
+ufi I L Ufim
+uga uga uga I A Ugaritic
+ugb I E Kuku-Ugbanh
+uge I L Ughele
+ugn I L Ugandan Sign Language
+ugo I L Ugong
+ugy I L Uruguayan Sign Language
+uha I L Uhami
+uhn I L Damal
+uig uig uig ug I L Uighur
+uis I L Uisai
+uiv I L Iyive
+uji I L Tanjijili
+uka I L Kaburi
+ukg I L Ukuriguma
+ukh I L Ukhwejo
+ukl I L Ukrainian Sign Language
+ukp I L Ukpe-Bayobiri
+ukq I L Ukwa
+ukr ukr ukr uk I L Ukrainian
+uks I L Urubú-Kaapor Sign Language
+uku I L Ukue
+ukw I L Ukwuani-Aboh-Ndoni
+uky I E Kuuk-Yak
+ula I L Fungwa
+ulb I L Ulukwumi
+ulc I L Ulch
+ule I E Lule
+ulf I L Usku
+uli I L Ulithian
+ulk I L Meriam
+ull I L Ullatan
+ulm I L Ulumanda'
+uln I L Unserdeutsch
+ulu I L Uma' Lung
+ulw I L Ulwa
+uma I L Umatilla
+umb umb umb I L Umbundu
+umc I A Marrucinian
+umd I E Umbindhamu
+umg I E Umbuygamu
+umi I L Ukit
+umm I L Umon
+umn I L Makyan Naga
+umo I E Umotína
+ump I L Umpila
+umr I E Umbugarla
+ums I L Pendau
+umu I L Munsee
+una I L North Watut
+und und und S S Undetermined
+une I L Uneme
+ung I L Ngarinyin
+unk I L Enawené-Nawé
+unm I E Unami
+unn I L Kurnai
+unr I L Mundari
+unu I L Unubahe
+unx I L Munda
+unz I L Unde Kaili
+uok I L Uokha
+upi I L Umeda
+upv I L Uripiv-Wala-Rano-Atchin
+ura I L Urarina
+urb I L Urubú-Kaapor
+urc I E Urningangg
+urd urd urd ur I L Urdu
+ure I L Uru
+urf I E Uradhi
+urg I L Urigina
+urh I L Urhobo
+uri I L Urim
+urk I L Urak Lawoi'
+url I L Urali
+urm I L Urapmin
+urn I L Uruangnirin
+uro I L Ura (Papua New Guinea)
+urp I L Uru-Pa-In
+urr I L Lehalurup
+urt I L Urat
+uru I E Urumi
+urv I E Uruava
+urw I L Sop
+urx I L Urimo
+ury I L Orya
+urz I L Uru-Eu-Wau-Wau
+usa I L Usarufa
+ush I L Ushojo
+usi I L Usui
+usk I L Usaghade
+usp I L Uspanteco
+usu I L Uya
+uta I L Otank
+ute I L Ute-Southern Paiute
+utp I L Amba (Solomon Islands)
+utr I L Etulo
+utu I L Utu
+uum I L Urum
+uun I L Kulon-Pazeh
+uur I L Ura (Vanuatu)
+uuu I L U
+uve I L West Uvean
+uvh I L Uri
+uvl I L Lote
+uwa I L Kuku-Uwanh
+uya I L Doko-Uyanga
+uzb uzb uzb uz M L Uzbek
+uzn I L Northern Uzbek
+uzs I L Southern Uzbek
+vaa I L Vaagri Booli
+vae I L Vale
+vaf I L Vafsi
+vag I L Vagla
+vah I L Varhadi-Nagpuri
+vai vai vai I L Vai
+vaj I L Vasekela Bushman
+val I L Vehes
+vam I L Vanimo
+van I L Valman
+vao I L Vao
+vap I L Vaiphei
+var I L Huarijio
+vas I L Vasavi
+vau I L Vanuma
+vav I L Varli
+vay I L Wayu
+vbb I L Southeast Babar
+vbk I L Southwestern Bontok
+vec I L Venetian
+ved I L Veddah
+vel I L Veluws
+vem I L Vemgo-Mabas
+ven ven ven ve I L Venda
+veo I E Ventureño
+vep I L Veps
+ver I L Mom Jango
+vgr I L Vaghri
+vgt I L Vlaamse Gebarentaal
+vic I L Virgin Islands Creole English
+vid I L Vidunda
+vie vie vie vi I L Vietnamese
+vif I L Vili
+vig I L Viemo
+vil I L Vilela
+vin I L Vinza
+vis I L Vishavan
+vit I L Viti
+viv I L Iduna
+vka I E Kariyarra
+vki I L Ija-Zuba
+vkj I L Kujarge
+vkk I L Kaur
+vkl I L Kulisusu
+vkm I E Kamakan
+vko I L Kodeoha
+vkp I L Korlai Creole Portuguese
+vkt I L Tenggarong Kutai Malay
+vku I L Kurrama
+vlp I L Valpei
+vls I L Vlaams
+vma I L Martuyhunira
+vmb I E Barbaram
+vmc I L Juxtlahuaca Mixtec
+vmd I L Mudu Koraga
+vme I L East Masela
+vmf I L Mainfränkisch
+vmg I L Lungalunga
+vmh I L Maraghei
+vmi I E Miwa
+vmj I L Ixtayutla Mixtec
+vmk I L Makhuwa-Shirima
+vml I E Malgana
+vmm I L Mitlatongo Mixtec
+vmp I L Soyaltepec Mazatec
+vmq I L Soyaltepec Mixtec
+vmr I L Marenje
+vms I E Moksela
+vmu I E Muluridyi
+vmv I E Valley Maidu
+vmw I L Makhuwa
+vmx I L Tamazola Mixtec
+vmy I L Ayautla Mazatec
+vmz I L Mazatlán Mazatec
+vnk I L Vano
+vnm I L Vinmavis
+vnp I L Vunapu
+vol vol vol vo I C Volapük
+vor I L Voro
+vot vot vot I L Votic
+vra I L Vera'a
+vro I L Võro
+vrs I L Varisi
+vrt I L Burmbar
+vsi I L Moldova Sign Language
+vsl I L Venezuelan Sign Language
+vsv I L Valencian Sign Language
+vto I L Vitou
+vum I L Vumbu
+vun I L Vunjo
+vut I L Vute
+vwa I L Awa (China)
+waa I L Walla Walla
+wab I L Wab
+wac I L Wasco-Wishram
+wad I L Wandamen
+wae I L Walser
+waf I E Wakoná
+wag I L Wa'ema
+wah I L Watubela
+wai I L Wares
+waj I L Waffa
+wal wal wal I L Wolaytta
+wam I E Wampanoag
+wan I L Wan
+wao I E Wappo
+wap I L Wapishana
+waq I L Wageman
+war war war I L Waray (Philippines)
+was was was I L Washo
+wat I L Kaninuwa
+wau I L Waurá
+wav I L Waka
+waw I L Waiwai
+wax I L Watam
+way I L Wayana
+waz I L Wampur
+wba I L Warao
+wbb I L Wabo
+wbe I L Waritai
+wbf I L Wara
+wbh I L Wanda
+wbi I L Vwanji
+wbj I L Alagwa
+wbk I L Waigali
+wbl I L Wakhi
+wbm I L Wa
+wbp I L Warlpiri
+wbq I L Waddar
+wbr I L Wagdi
+wbt I L Wanman
+wbv I L Wajarri
+wbw I L Woi
+wca I L Yanomámi
+wci I L Waci Gbe
+wdd I L Wandji
+wdg I L Wadaginam
+wdj I L Wadjiginy
+wdk I E Wadikali
+wdu I E Wadjigu
+wdy I E Wadjabangayi
+wea I E Wewaw
+wec I L Wè Western
+wed I L Wedau
+weg I L Wergaia
+weh I L Weh
+wei I L Kiunum
+wem I L Weme Gbe
+weo I L Wemale
+wep I L Westphalien
+wer I L Weri
+wes I L Cameroon Pidgin
+wet I L Perai
+weu I L Rawngtu Chin
+wew I L Wejewa
+wfg I L Yafi
+wga I E Wagaya
+wgb I L Wagawaga
+wgg I E Wangganguru
+wgi I L Wahgi
+wgo I L Waigeo
+wgu I E Wirangu
+wgy I L Warrgamay
+wha I L Manusela
+whg I L North Wahgi
+whk I L Wahau Kenyah
+whu I L Wahau Kayan
+wib I L Southern Toussian
+wic I L Wichita
+wie I E Wik-Epa
+wif I E Wik-Keyangan
+wig I L Wik-Ngathana
+wih I L Wik-Me'anha
+wii I L Minidien
+wij I L Wik-Iiyanh
+wik I L Wikalkan
+wil I E Wilawila
+wim I L Wik-Mungkan
+win I L Ho-Chunk
+wir I E Wiraféd
+wiu I L Wiru
+wiv I L Vitu
+wiy I E Wiyot
+wja I L Waja
+wji I L Warji
+wka I E Kw'adza
+wkb I L Kumbaran
+wkd I L Wakde
+wkl I L Kalanadi
+wku I L Kunduvadi
+wkw I E Wakawaka
+wky I E Wangkayutyuru
+wla I L Walio
+wlc I L Mwali Comorian
+wle I L Wolane
+wlg I L Kunbarlang
+wli I L Waioli
+wlk I E Wailaki
+wll I L Wali (Sudan)
+wlm I H Middle Welsh
+wln wln wln wa I L Walloon
+wlo I L Wolio
+wlr I L Wailapa
+wls I L Wallisian
+wlu I E Wuliwuli
+wlv I L Wichí Lhamtés Vejoz
+wlw I L Walak
+wlx I L Wali (Ghana)
+wly I E Waling
+wma I E Mawa (Nigeria)
+wmb I L Wambaya
+wmc I L Wamas
+wmd I L Mamaindé
+wme I L Wambule
+wmh I L Waima'a
+wmi I E Wamin
+wmm I L Maiwa (Indonesia)
+wmn I E Waamwang
+wmo I L Wom (Papua New Guinea)
+wms I L Wambon
+wmt I L Walmajarri
+wmw I L Mwani
+wmx I L Womo
+wnb I L Wanambre
+wnc I L Wantoat
+wnd I E Wandarang
+wne I L Waneci
+wng I L Wanggom
+wni I L Ndzwani Comorian
+wnk I L Wanukaka
+wnm I E Wanggamala
+wnn I E Wunumara
+wno I L Wano
+wnp I L Wanap
+wnu I L Usan
+wnw I L Wintu
+wny I L Wanyi
+woa I L Tyaraity
+wob I L Wè Northern
+woc I L Wogeo
+wod I L Wolani
+woe I L Woleaian
+wof I L Gambian Wolof
+wog I L Wogamusin
+woi I L Kamang
+wok I L Longto
+wol wol wol wo I L Wolof
+wom I L Wom (Nigeria)
+won I L Wongo
+woo I L Manombai
+wor I L Woria
+wos I L Hanga Hundi
+wow I L Wawonii
+woy I E Weyto
+wpc I L Maco
+wra I L Warapu
+wrb I E Warluwara
+wrd I L Warduji
+wrg I E Warungu
+wrh I E Wiradhuri
+wri I E Wariyangga
+wrk I L Garrwa
+wrl I L Warlmanpa
+wrm I L Warumungu
+wrn I L Warnang
+wro I E Worrorra
+wrp I L Waropen
+wrr I L Wardaman
+wrs I L Waris
+wru I L Waru
+wrv I L Waruna
+wrw I E Gugu Warra
+wrx I L Wae Rana
+wry I L Merwari
+wrz I E Waray (Australia)
+wsa I L Warembori
+wsi I L Wusi
+wsk I L Waskia
+wsr I L Owenia
+wss I L Wasa
+wsu I E Wasu
+wsv I E Wotapuri-Katarqalai
+wtf I L Watiwa
+wth I E Wathawurrung
+wti I L Berta
+wtk I L Watakataui
+wtm I L Mewati
+wtw I L Wotu
+wua I L Wikngenchera
+wub I L Wunambal
+wud I L Wudu
+wuh I L Wutunhua
+wul I L Silimo
+wum I L Wumbvu
+wun I L Bungu
+wur I E Wurrugu
+wut I L Wutung
+wuu I L Wu Chinese
+wuv I L Wuvulu-Aua
+wux I L Wulna
+wuy I L Wauyai
+wwa I L Waama
+wwb I E Wakabunga
+wwo I L Wetamut
+wwr I E Warrwa
+www I L Wawa
+wxa I L Waxianghua
+wxw I E Wardandi
+wya I L Wyandot
+wyb I L Wangaaybuwan-Ngiyambaa
+wyi I E Woiwurrung
+wym I L Wymysorys
+wyr I L Wayoró
+wyy I L Western Fijian
+xaa I H Andalusian Arabic
+xab I L Sambe
+xac I L Kachari
+xad I E Adai
+xae I A Aequian
+xag I E Aghwan
+xai I E Kaimbé
+xal xal xal I L Kalmyk
+xam I E /Xam
+xan I L Xamtanga
+xao I L Khao
+xap I E Apalachee
+xaq I A Aquitanian
+xar I E Karami
+xas I E Kamas
+xat I L Katawixi
+xau I L Kauwera
+xav I L Xavánte
+xaw I L Kawaiisu
+xay I L Kayan Mahakam
+xba I E Kamba (Brazil)
+xbb I E Lower Burdekin
+xbc I A Bactrian
+xbd I E Bindal
+xbe I E Bigambal
+xbg I E Bunganditj
+xbi I L Kombio
+xbj I E Birrpayi
+xbm I H Middle Breton
+xbn I E Kenaboi
+xbo I E Bolgarian
+xbp I E Bibbulman
+xbr I L Kambera
+xbw I E Kambiwá
+xbx I E Kabixí
+xby I L Batyala
+xcb I E Cumbric
+xcc I A Camunic
+xce I A Celtiberian
+xcg I A Cisalpine Gaulish
+xch I E Chemakum
+xcl I H Classical Armenian
+xcm I E Comecrudo
+xcn I E Cotoname
+xco I A Chorasmian
+xcr I A Carian
+xct I H Classical Tibetan
+xcu I E Curonian
+xcv I E Chuvantsy
+xcw I E Coahuilteco
+xcy I E Cayuse
+xda I L Darkinyung
+xdc I A Dacian
+xdk I E Dharuk
+xdm I A Edomite
+xdy I L Malayic Dayak
+xeb I A Eblan
+xed I L Hdi
+xeg I E //Xegwi
+xel I L Kelo
+xem I L Kembayan
+xep I A Epi-Olmec
+xer I L Xerénte
+xes I L Kesawai
+xet I L Xetá
+xeu I L Keoru-Ahia
+xfa I A Faliscan
+xga I A Galatian
+xgb I E Gbin
+xgd I E Gudang
+xgf I E Gabrielino-Fernandeño
+xgg I E Goreng
+xgi I E Garingbal
+xgl I E Galindan
+xgm I E Guwinmal
+xgr I E Garza
+xgu I L Unggumi
+xgw I E Guwa
+xha I A Harami
+xhc I E Hunnic
+xhd I A Hadrami
+xhe I L Khetrani
+xho xho xho xh I L Xhosa
+xhr I A Hernican
+xht I A Hattic
+xhu I A Hurrian
+xhv I L Khua
+xib I A Iberian
+xii I L Xiri
+xil I A Illyrian
+xin I E Xinca
+xip I E Xipináwa
+xir I E Xiriâna
+xiv I A Indus Valley Language
+xiy I L Xipaya
+xjb I E Minjungbal
+xjt I E Jaitmatang
+xka I L Kalkoti
+xkb I L Northern Nago
+xkc I L Kho'ini
+xkd I L Mendalam Kayan
+xke I L Kereho
+xkf I L Khengkha
+xkg I L Kagoro
+xkh I L Karahawyana
+xki I L Kenyan Sign Language
+xkj I L Kajali
+xkk I L Kaco'
+xkl I L Mainstream Kenyah
+xkn I L Kayan River Kayan
+xko I L Kiorr
+xkp I L Kabatei
+xkq I L Koroni
+xkr I E Xakriabá
+xks I L Kumbewaha
+xkt I L Kantosi
+xku I L Kaamba
+xkv I L Kgalagadi
+xkw I L Kembra
+xkx I L Karore
+xky I L Uma' Lasan
+xkz I L Kurtokha
+xla I L Kamula
+xlb I E Loup B
+xlc I A Lycian
+xld I A Lydian
+xle I A Lemnian
+xlg I A Ligurian (Ancient)
+xli I A Liburnian
+xln I A Alanic
+xlo I E Loup A
+xlp I A Lepontic
+xls I A Lusitanian
+xlu I A Cuneiform Luwian
+xly I A Elymian
+xma I L Mushungulu
+xmb I L Mbonga
+xmc I L Makhuwa-Marrevone
+xmd I L Mbudum
+xme I A Median
+xmf I L Mingrelian
+xmg I L Mengaka
+xmh I L Kuku-Muminh
+xmj I L Majera
+xmk I A Ancient Macedonian
+xml I L Malaysian Sign Language
+xmm I L Manado Malay
+xmn I H Manichaean Middle Persian
+xmo I L Morerebi
+xmp I E Kuku-Mu'inh
+xmq I E Kuku-Mangk
+xmr I A Meroitic
+xms I L Moroccan Sign Language
+xmt I L Matbat
+xmu I E Kamu
+xmv I L Antankarana Malagasy
+xmw I L Tsimihety Malagasy
+xmx I L Maden
+xmy I L Mayaguduna
+xmz I L Mori Bawah
+xna I A Ancient North Arabian
+xnb I L Kanakanabu
+xng I H Middle Mongolian
+xnh I L Kuanhua
+xni I E Ngarigu
+xnk I E Nganakarti
+xnn I L Northern Kankanay
+xno I H Anglo-Norman
+xnr I L Kangri
+xns I L Kanashi
+xnt I E Narragansett
+xnu I E Nukunul
+xny I L Nyiyaparli
+xnz I L Kenzi
+xoc I E O'chi'chi'
+xod I L Kokoda
+xog I L Soga
+xoi I L Kominimung
+xok I L Xokleng
+xom I L Komo (Sudan)
+xon I L Konkomba
+xoo I E Xukurú
+xop I L Kopar
+xor I L Korubo
+xow I L Kowaki
+xpa I E Pirriya
+xpc I E Pecheneg
+xpe I L Liberia Kpelle
+xpg I A Phrygian
+xpi I E Pictish
+xpj I E Mpalitjanh
+xpk I L Kulina Pano
+xpm I E Pumpokol
+xpn I E Kapinawá
+xpo I E Pochutec
+xpp I E Puyo-Paekche
+xpq I E Mohegan-Pequot
+xpr I A Parthian
+xps I E Pisidian
+xpt I E Punthamara
+xpu I A Punic
+xpy I E Puyo
+xqa I H Karakhanid
+xqt I A Qatabanian
+xra I L Krahô
+xrb I L Eastern Karaboro
+xrd I E Gundungurra
+xre I L Kreye
+xrg I E Minang
+xri I L Krikati-Timbira
+xrm I E Armazic
+xrn I E Arin
+xrq I E Karranga
+xrr I A Raetic
+xrt I E Aranama-Tamique
+xru I L Marriammu
+xrw I L Karawa
+xsa I A Sabaean
+xsb I L Sambal
+xsc I A Scythian
+xsd I A Sidetic
+xse I L Sempan
+xsh I L Shamang
+xsi I L Sio
+xsj I L Subi
+xsl I L South Slavey
+xsm I L Kasem
+xsn I L Sanga (Nigeria)
+xso I E Solano
+xsp I L Silopi
+xsq I L Makhuwa-Saka
+xsr I L Sherpa
+xss I E Assan
+xsu I L Sanumá
+xsv I E Sudovian
+xsy I L Saisiyat
+xta I L Alcozauca Mixtec
+xtb I L Chazumba Mixtec
+xtc I L Katcha-Kadugli-Miri
+xtd I L Diuxi-Tilantongo Mixtec
+xte I L Ketengban
+xtg I A Transalpine Gaulish
+xth I E Yitha Yitha
+xti I L Sinicahua Mixtec
+xtj I L San Juan Teita Mixtec
+xtl I L Tijaltepec Mixtec
+xtm I L Magdalena Peñasco Mixtec
+xtn I L Northern Tlaxiaco Mixtec
+xto I A Tokharian A
+xtp I L San Miguel Piedras Mixtec
+xtq I H Tumshuqese
+xtr I A Early Tripuri
+xts I L Sindihui Mixtec
+xtt I L Tacahua Mixtec
+xtu I L Cuyamecalco Mixtec
+xtv I E Thawa
+xtw I L Tawandê
+xty I L Yoloxochitl Mixtec
+xtz I E Tasmanian
+xua I L Alu Kurumba
+xub I L Betta Kurumba
+xud I E Umiida
+xug I L Kunigami
+xuj I L Jennu Kurumba
+xul I E Ngunawal
+xum I A Umbrian
+xun I E Unggaranggu
+xuo I L Kuo
+xup I E Upper Umpqua
+xur I A Urartian
+xut I E Kuthant
+xuu I L Kxoe
+xve I A Venetic
+xvi I L Kamviri
+xvn I A Vandalic
+xvo I A Volscian
+xvs I A Vestinian
+xwa I L Kwaza
+xwc I E Woccon
+xwd I E Wadi Wadi
+xwe I L Xwela Gbe
+xwg I L Kwegu
+xwj I E Wajuk
+xwk I E Wangkumara
+xwl I L Western Xwla Gbe
+xwo I E Written Oirat
+xwr I L Kwerba Mamberamo
+xwt I E Wotjobaluk
+xww I E Wemba Wemba
+xxb I E Boro (Ghana)
+xxk I L Ke'o
+xxm I E Minkin
+xxr I E Koropó
+xxt I E Tambora
+xya I E Yaygir
+xyb I E Yandjibara
+xyj I E Mayi-Yapi
+xyk I E Mayi-Kulan
+xyl I E Yalakalore
+xyt I E Mayi-Thakurti
+xyy I L Yorta Yorta
+xzh I A Zhang-Zhung
+xzm I E Zemgalian
+xzp I H Ancient Zapotec
+yaa I L Yaminahua
+yab I L Yuhup
+yac I L Pass Valley Yali
+yad I L Yagua
+yae I L Pumé
+yaf I L Yaka (Democratic Republic of Congo)
+yag I L Yámana
+yah I L Yazgulyam
+yai I L Yagnobi
+yaj I L Banda-Yangere
+yak I L Yakama
+yal I L Yalunka
+yam I L Yamba
+yan I L Mayangna
+yao yao yao I L Yao
+yap yap yap I L Yapese
+yaq I L Yaqui
+yar I L Yabarana
+yas I L Nugunu (Cameroon)
+yat I L Yambeta
+yau I L Yuwana
+yav I L Yangben
+yaw I L Yawalapití
+yax I L Yauma
+yay I L Agwagwune
+yaz I L Lokaa
+yba I L Yala
+ybb I L Yemba
+ybe I L West Yugur
+ybh I L Yakha
+ybi I L Yamphu
+ybj I L Hasha
+ybk I L Bokha
+ybl I L Yukuben
+ybm I L Yaben
+ybn I E Yabaâna
+ybo I L Yabong
+ybx I L Yawiyo
+yby I L Yaweyuha
+ych I L Chesu
+ycl I L Lolopo
+ycn I L Yucuna
+ycp I L Chepya
+yda I E Yanda
+ydd I L Eastern Yiddish
+yde I L Yangum Dey
+ydg I L Yidgha
+ydk I L Yoidik
+yds I L Yiddish Sign Language
+yea I L Ravula
+yec I L Yeniche
+yee I L Yimas
+yei I E Yeni
+yej I L Yevanic
+yel I L Yela
+yer I L Tarok
+yes I L Nyankpa
+yet I L Yetfa
+yeu I L Yerukula
+yev I L Yapunda
+yey I L Yeyi
+yga I E Malyangapa
+ygi I E Yiningayi
+ygl I L Yangum Gel
+ygm I L Yagomi
+ygp I L Gepo
+ygr I L Yagaria
+ygu I L Yugul
+ygw I L Yagwoia
+yha I L Baha Buyang
+yhd I L Judeo-Iraqi Arabic
+yhl I L Hlepho Phowa
+yia I L Yinggarda
+yid yid yid yi M L Yiddish
+yif I L Ache
+yig I L Wusa Nasu
+yih I L Western Yiddish
+yii I L Yidiny
+yij I L Yindjibarndi
+yik I L Dongshanba Lalo
+yil I E Yindjilandji
+yim I L Yimchungru Naga
+yin I L Yinchia
+yip I L Pholo
+yiq I L Miqie
+yir I L North Awyu
+yis I L Yis
+yit I L Eastern Lalu
+yiu I L Awu
+yiv I L Northern Nisu
+yix I L Axi Yi
+yiz I L Azhe
+yka I L Yakan
+ykg I L Northern Yukaghir
+yki I L Yoke
+ykk I L Yakaikeke
+ykl I L Khlula
+ykm I L Kap
+ykn I L Kua-nsi
+yko I L Yasa
+ykr I L Yekora
+ykt I L Kathu
+yku I L Kuamasi
+yky I L Yakoma
+yla I L Yaul
+ylb I L Yaleba
+yle I L Yele
+ylg I L Yelogu
+yli I L Angguruk Yali
+yll I L Yil
+ylm I L Limi
+yln I L Langnian Buyang
+ylo I L Naluo Yi
+ylr I E Yalarnnga
+ylu I L Aribwaung
+yly I L Nyâlayu
+ymb I L Yambes
+ymc I L Southern Muji
+ymd I L Muda
+yme I E Yameo
+ymg I L Yamongeri
+ymh I L Mili
+ymi I L Moji
+ymk I L Makwe
+yml I L Iamalele
+ymm I L Maay
+ymn I L Yamna
+ymo I L Yangum Mon
+ymp I L Yamap
+ymq I L Qila Muji
+ymr I L Malasar
+yms I A Mysian
+ymt I E Mator-Taygi-Karagas
+ymx I L Northern Muji
+ymz I L Muzi
+yna I L Aluo
+ynd I E Yandruwandha
+yne I L Lang'e
+yng I L Yango
+ynh I L Yangho
+ynk I L Naukan Yupik
+ynl I L Yangulam
+ynn I E Yana
+yno I L Yong
+ynq I L Yendang
+yns I L Yansi
+ynu I E Yahuna
+yob I E Yoba
+yog I L Yogad
+yoi I L Yonaguni
+yok I L Yokuts
+yol I E Yola
+yom I L Yombe
+yon I L Yongkom
+yor yor yor yo I L Yoruba
+yot I L Yotti
+yox I L Yoron
+yoy I L Yoy
+ypa I L Phala
+ypb I L Labo Phowa
+ypg I L Phola
+yph I L Phupha
+ypm I L Phuma
+ypn I L Ani Phowa
+ypo I L Alo Phola
+ypp I L Phupa
+ypz I L Phuza
+yra I L Yerakai
+yrb I L Yareba
+yre I L Yaouré
+yri I L Yarí
+yrk I L Nenets
+yrl I L Nhengatu
+yrm I L Yirrk-Mel
+yrn I L Yerong
+yrs I L Yarsun
+yrw I L Yarawata
+yry I L Yarluyandi
+ysc I E Yassic
+ysd I L Samatao
+ysg I L Sonaga
+ysl I L Yugoslavian Sign Language
+ysn I L Sani
+yso I L Nisi (China)
+ysp I L Southern Lolopo
+ysr I E Sirenik Yupik
+yss I L Yessan-Mayo
+ysy I L Sanie
+yta I L Talu
+ytl I L Tanglang
+ytp I L Thopho
+ytw I L Yout Wam
+yty I E Yatay
+yua I L Yucateco
+yub I E Yugambal
+yuc I L Yuchi
+yud I L Judeo-Tripolitanian Arabic
+yue I L Yue Chinese
+yuf I L Havasupai-Walapai-Yavapai
+yug I E Yug
+yui I L Yurutí
+yuj I L Karkar-Yuri
+yuk I E Yuki
+yul I L Yulu
+yum I L Quechan
+yun I L Bena (Nigeria)
+yup I L Yukpa
+yuq I L Yuqui
+yur I L Yurok
+yut I L Yopno
+yuu I L Yugh
+yuw I L Yau (Morobe Province)
+yux I L Southern Yukaghir
+yuy I L East Yugur
+yuz I L Yuracare
+yva I L Yawa
+yvt I E Yavitero
+ywa I L Kalou
+ywg I L Yinhawangka
+ywl I L Western Lalu
+ywn I L Yawanawa
+ywq I L Wuding-Luquan Yi
+ywr I L Yawuru
+ywt I L Xishanba Lalo
+ywu I L Wumeng Nasu
+yww I E Yawarawarga
+yxa I E Mayawali
+yxg I E Yagara
+yxl I E Yardliyawarra
+yxm I E Yinwum
+yxu I E Yuyu
+yxy I E Yabula Yabula
+yyr I E Yir Yoront
+yyu I L Yau (Sandaun Province)
+yyz I L Ayizi
+yzg I L E'ma Buyang
+yzk I L Zokhuo
+zaa I L Sierra de Juárez Zapotec
+zab I L San Juan Guelavía Zapotec
+zac I L Ocotlán Zapotec
+zad I L Cajonos Zapotec
+zae I L Yareni Zapotec
+zaf I L Ayoquesco Zapotec
+zag I L Zaghawa
+zah I L Zangwal
+zai I L Isthmus Zapotec
+zaj I L Zaramo
+zak I L Zanaki
+zal I L Zauzou
+zam I L Miahuatlán Zapotec
+zao I L Ozolotepec Zapotec
+zap zap zap M L Zapotec
+zaq I L Aloápam Zapotec
+zar I L Rincón Zapotec
+zas I L Santo Domingo Albarradas Zapotec
+zat I L Tabaa Zapotec
+zau I L Zangskari
+zav I L Yatzachi Zapotec
+zaw I L Mitla Zapotec
+zax I L Xadani Zapotec
+zay I L Zayse-Zergulla
+zaz I L Zari
+zbc I L Central Berawan
+zbe I L East Berawan
+zbl zbl zbl I C Blissymbols
+zbt I L Batui
+zbw I L West Berawan
+zca I L Coatecas Altas Zapotec
+zch I L Central Hongshuihe Zhuang
+zdj I L Ngazidja Comorian
+zea I L Zeeuws
+zeg I L Zenag
+zeh I L Eastern Hongshuihe Zhuang
+zen zen zen I L Zenaga
+zga I L Kinga
+zgb I L Guibei Zhuang
+zgh I L Standard Moroccan Tamazight
+zgm I L Minz Zhuang
+zgn I L Guibian Zhuang
+zgr I L Magori
+zha zha zha za M L Zhuang
+zhb I L Zhaba
+zhd I L Dai Zhuang
+zhi I L Zhire
+zhn I L Nong Zhuang
+zho chi zho zh M L Chinese
+zhw I L Zhoa
+zia I L Zia
+zib I L Zimbabwe Sign Language
+zik I L Zimakani
+zil I L Zialo
+zim I L Mesme
+zin I L Zinza
+zir I E Ziriya
+ziw I L Zigula
+ziz I L Zizilivakan
+zka I L Kaimbulawa
+zkb I E Koibal
+zkd I L Kadu
+zkg I E Koguryo
+zkh I E Khorezmian
+zkk I E Karankawa
+zkn I L Kanan
+zko I E Kott
+zkp I E São Paulo Kaingáng
+zkr I L Zakhring
+zkt I E Kitan
+zku I E Kaurna
+zkv I E Krevinian
+zkz I E Khazar
+zlj I L Liujiang Zhuang
+zlm I L Malay (individual language)
+zln I L Lianshan Zhuang
+zlq I L Liuqian Zhuang
+zma I L Manda (Australia)
+zmb I L Zimba
+zmc I E Margany
+zmd I L Maridan
+zme I E Mangerr
+zmf I L Mfinu
+zmg I L Marti Ke
+zmh I E Makolkol
+zmi I L Negeri Sembilan Malay
+zmj I L Maridjabin
+zmk I E Mandandanyi
+zml I L Madngele
+zmm I L Marimanindji
+zmn I L Mbangwe
+zmo I L Molo
+zmp I L Mpuono
+zmq I L Mituku
+zmr I L Maranunggu
+zms I L Mbesa
+zmt I L Maringarr
+zmu I E Muruwari
+zmv I E Mbariman-Gudhinma
+zmw I L Mbo (Democratic Republic of Congo)
+zmx I L Bomitaba
+zmy I L Mariyedi
+zmz I L Mbandja
+zna I L Zan Gula
+zne I L Zande (individual language)
+zng I L Mang
+znk I E Manangkari
+zns I L Mangas
+zoc I L Copainalá Zoque
+zoh I L Chimalapa Zoque
+zom I L Zou
+zoo I L Asunción Mixtepec Zapotec
+zoq I L Tabasco Zoque
+zor I L Rayón Zoque
+zos I L Francisco León Zoque
+zpa I L Lachiguiri Zapotec
+zpb I L Yautepec Zapotec
+zpc I L Choapan Zapotec
+zpd I L Southeastern Ixtlán Zapotec
+zpe I L Petapa Zapotec
+zpf I L San Pedro Quiatoni Zapotec
+zpg I L Guevea De Humboldt Zapotec
+zph I L Totomachapan Zapotec
+zpi I L Santa María Quiegolani Zapotec
+zpj I L Quiavicuzas Zapotec
+zpk I L Tlacolulita Zapotec
+zpl I L Lachixío Zapotec
+zpm I L Mixtepec Zapotec
+zpn I L Santa Inés Yatzechi Zapotec
+zpo I L Amatlán Zapotec
+zpp I L El Alto Zapotec
+zpq I L Zoogocho Zapotec
+zpr I L Santiago Xanica Zapotec
+zps I L Coatlán Zapotec
+zpt I L San Vicente Coatlán Zapotec
+zpu I L Yalálag Zapotec
+zpv I L Chichicapan Zapotec
+zpw I L Zaniza Zapotec
+zpx I L San Baltazar Loxicha Zapotec
+zpy I L Mazaltepec Zapotec
+zpz I L Texmelucan Zapotec
+zqe I L Qiubei Zhuang
+zra I E Kara (Korea)
+zrg I L Mirgan
+zrn I L Zerenkel
+zro I L Záparo
+zrp I E Zarphatic
+zrs I L Mairasi
+zsa I L Sarasira
+zsk I A Kaskean
+zsl I L Zambian Sign Language
+zsm I L Standard Malay
+zsr I L Southern Rincon Zapotec
+zsu I L Sukurum
+zte I L Elotepec Zapotec
+ztg I L Xanaguía Zapotec
+ztl I L Lapaguía-Guivini Zapotec
+ztm I L San Agustín Mixtepec Zapotec
+ztn I L Santa Catarina Albarradas Zapotec
+ztp I L Loxicha Zapotec
+ztq I L Quioquitani-Quierí Zapotec
+zts I L Tilquiapan Zapotec
+ztt I L Tejalapan Zapotec
+ztu I L Güilá Zapotec
+ztx I L Zaachila Zapotec
+zty I L Yatee Zapotec
+zua I L Zeem
+zuh I L Tokano
+zul zul zul zu I L Zulu
+zum I L Kumzari
+zun zun zun I L Zuni
+zuy I L Zumaya
+zwa I L Zay
+zxx zxx zxx S S No linguistic content
+zyb I L Yongbei Zhuang
+zyg I L Yang Zhuang
+zyj I L Youjiang Zhuang
+zyn I L Yongnan Zhuang
+zyp I L Zyphe Chin
+zza zza zza M L Zaza
+zzj I L Zuojiang Zhuang
\ No newline at end of file
diff --git a/lib/babelfish/data/iso15924-utf8-20131012.txt b/lib/babelfish/data/iso15924-utf8-20131012.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4b6ff47146973d7a86177207b4f84d6eb1f6cd0b
--- /dev/null
+++ b/lib/babelfish/data/iso15924-utf8-20131012.txt
@@ -0,0 +1,176 @@
+#
+# ISO 15924 - Codes for the representation of names of scripts
+# Codes pour la représentation des noms d’écritures
+# Format:
+# Code;N°;English Name;Nom français;PVA;Date
+#
+
+Afak;439;Afaka;afaka;;2010-12-21
+Aghb;239;Caucasian Albanian;aghbanien;;2012-10-16
+Ahom;338;Ahom, Tai Ahom;âhom;;2012-11-01
+Arab;160;Arabic;arabe;Arabic;2004-05-01
+Armi;124;Imperial Aramaic;araméen impérial;Imperial_Aramaic;2009-06-01
+Armn;230;Armenian;arménien;Armenian;2004-05-01
+Avst;134;Avestan;avestique;Avestan;2009-06-01
+Bali;360;Balinese;balinais;Balinese;2006-10-10
+Bamu;435;Bamum;bamoum;Bamum;2009-06-01
+Bass;259;Bassa Vah;bassa;;2010-03-26
+Batk;365;Batak;batik;Batak;2010-07-23
+Beng;325;Bengali;bengalî;Bengali;2004-05-01
+Blis;550;Blissymbols;symboles Bliss;;2004-05-01
+Bopo;285;Bopomofo;bopomofo;Bopomofo;2004-05-01
+Brah;300;Brahmi;brahma;Brahmi;2010-07-23
+Brai;570;Braille;braille;Braille;2004-05-01
+Bugi;367;Buginese;bouguis;Buginese;2006-06-21
+Buhd;372;Buhid;bouhide;Buhid;2004-05-01
+Cakm;349;Chakma;chakma;Chakma;2012-02-06
+Cans;440;Unified Canadian Aboriginal Syllabics;syllabaire autochtone canadien unifié;Canadian_Aboriginal;2004-05-29
+Cari;201;Carian;carien;Carian;2007-07-02
+Cham;358;Cham;cham (čam, tcham);Cham;2009-11-11
+Cher;445;Cherokee;tchérokî;Cherokee;2004-05-01
+Cirt;291;Cirth;cirth;;2004-05-01
+Copt;204;Coptic;copte;Coptic;2006-06-21
+Cprt;403;Cypriot;syllabaire chypriote;Cypriot;2004-05-01
+Cyrl;220;Cyrillic;cyrillique;Cyrillic;2004-05-01
+Cyrs;221;Cyrillic (Old Church Slavonic variant);cyrillique (variante slavonne);;2004-05-01
+Deva;315;Devanagari (Nagari);dévanâgarî;Devanagari;2004-05-01
+Dsrt;250;Deseret (Mormon);déseret (mormon);Deseret;2004-05-01
+Dupl;755;Duployan shorthand, Duployan stenography;sténographie Duployé;;2010-07-18
+Egyd;070;Egyptian demotic;démotique égyptien;;2004-05-01
+Egyh;060;Egyptian hieratic;hiératique égyptien;;2004-05-01
+Egyp;050;Egyptian hieroglyphs;hiéroglyphes égyptiens;Egyptian_Hieroglyphs;2009-06-01
+Elba;226;Elbasan;elbasan;;2010-07-18
+Ethi;430;Ethiopic (Geʻez);éthiopien (geʻez, guèze);Ethiopic;2004-10-25
+Geor;240;Georgian (Mkhedruli);géorgien (mkhédrouli);Georgian;2004-05-29
+Geok;241;Khutsuri (Asomtavruli and Nuskhuri);khoutsouri (assomtavrouli et nouskhouri);Georgian;2012-10-16
+Glag;225;Glagolitic;glagolitique;Glagolitic;2006-06-21
+Goth;206;Gothic;gotique;Gothic;2004-05-01
+Gran;343;Grantha;grantha;;2009-11-11
+Grek;200;Greek;grec;Greek;2004-05-01
+Gujr;320;Gujarati;goudjarâtî (gujrâtî);Gujarati;2004-05-01
+Guru;310;Gurmukhi;gourmoukhî;Gurmukhi;2004-05-01
+Hang;286;Hangul (Hangŭl, Hangeul);hangûl (hangŭl, hangeul);Hangul;2004-05-29
+Hani;500;Han (Hanzi, Kanji, Hanja);idéogrammes han (sinogrammes);Han;2009-02-23
+Hano;371;Hanunoo (Hanunóo);hanounóo;Hanunoo;2004-05-29
+Hans;501;Han (Simplified variant);idéogrammes han (variante simplifiée);;2004-05-29
+Hant;502;Han (Traditional variant);idéogrammes han (variante traditionnelle);;2004-05-29
+Hatr;127;Hatran;hatrénien;;2012-11-01
+Hebr;125;Hebrew;hébreu;Hebrew;2004-05-01
+Hira;410;Hiragana;hiragana;Hiragana;2004-05-01
+Hluw;080;Anatolian Hieroglyphs (Luwian Hieroglyphs, Hittite Hieroglyphs);hiéroglyphes anatoliens (hiéroglyphes louvites, hiéroglyphes hittites);;2011-12-09
+Hmng;450;Pahawh Hmong;pahawh hmong;;2004-05-01
+Hrkt;412;Japanese syllabaries (alias for Hiragana + Katakana);syllabaires japonais (alias pour hiragana + katakana);Katakana_Or_Hiragana;2011-06-21
+Hung;176;Old Hungarian (Hungarian Runic);runes hongroises (ancien hongrois);;2012-10-16
+Inds;610;Indus (Harappan);indus;;2004-05-01
+Ital;210;Old Italic (Etruscan, Oscan, etc.);ancien italique (étrusque, osque, etc.);Old_Italic;2004-05-29
+Java;361;Javanese;javanais;Javanese;2009-06-01
+Jpan;413;Japanese (alias for Han + Hiragana + Katakana);japonais (alias pour han + hiragana + katakana);;2006-06-21
+Jurc;510;Jurchen;jurchen;;2010-12-21
+Kali;357;Kayah Li;kayah li;Kayah_Li;2007-07-02
+Kana;411;Katakana;katakana;Katakana;2004-05-01
+Khar;305;Kharoshthi;kharochthî;Kharoshthi;2006-06-21
+Khmr;355;Khmer;khmer;Khmer;2004-05-29
+Khoj;322;Khojki;khojkî;;2011-06-21
+Knda;345;Kannada;kannara (canara);Kannada;2004-05-29
+Kore;287;Korean (alias for Hangul + Han);coréen (alias pour hangûl + han);;2007-06-13
+Kpel;436;Kpelle;kpèllé;;2010-03-26
+Kthi;317;Kaithi;kaithî;Kaithi;2009-06-01
+Lana;351;Tai Tham (Lanna);taï tham (lanna);Tai_Tham;2009-06-01
+Laoo;356;Lao;laotien;Lao;2004-05-01
+Latf;217;Latin (Fraktur variant);latin (variante brisée);;2004-05-01
+Latg;216;Latin (Gaelic variant);latin (variante gaélique);;2004-05-01
+Latn;215;Latin;latin;Latin;2004-05-01
+Lepc;335;Lepcha (Róng);lepcha (róng);Lepcha;2007-07-02
+Limb;336;Limbu;limbou;Limbu;2004-05-29
+Lina;400;Linear A;linéaire A;;2004-05-01
+Linb;401;Linear B;linéaire B;Linear_B;2004-05-29
+Lisu;399;Lisu (Fraser);lisu (Fraser);Lisu;2009-06-01
+Loma;437;Loma;loma;;2010-03-26
+Lyci;202;Lycian;lycien;Lycian;2007-07-02
+Lydi;116;Lydian;lydien;Lydian;2007-07-02
+Mahj;314;Mahajani;mahâjanî;;2012-10-16
+Mand;140;Mandaic, Mandaean;mandéen;Mandaic;2010-07-23
+Mani;139;Manichaean;manichéen;;2007-07-15
+Maya;090;Mayan hieroglyphs;hiéroglyphes mayas;;2004-05-01
+Mend;438;Mende Kikakui;mendé kikakui;;2013-10-12
+Merc;101;Meroitic Cursive;cursif méroïtique;Meroitic_Cursive;2012-02-06
+Mero;100;Meroitic Hieroglyphs;hiéroglyphes méroïtiques;Meroitic_Hieroglyphs;2012-02-06
+Mlym;347;Malayalam;malayâlam;Malayalam;2004-05-01
+Modi;323;Modi, Moḍī;modî;;2013-10-12
+Moon;218;Moon (Moon code, Moon script, Moon type);écriture Moon;;2006-12-11
+Mong;145;Mongolian;mongol;Mongolian;2004-05-01
+Mroo;199;Mro, Mru;mro;;2010-12-21
+Mtei;337;Meitei Mayek (Meithei, Meetei);meitei mayek;Meetei_Mayek;2009-06-01
+Mult;323; Multani;multanî;;2012-11-01
+Mymr;350;Myanmar (Burmese);birman;Myanmar;2004-05-01
+Narb;106;Old North Arabian (Ancient North Arabian);nord-arabique;;2010-03-26
+Nbat;159;Nabataean;nabatéen;;2010-03-26
+Nkgb;420;Nakhi Geba ('Na-'Khi ²Ggŏ-¹baw, Naxi Geba);nakhi géba;;2009-02-23
+Nkoo;165;N’Ko;n’ko;Nko;2006-10-10
+Nshu;499;Nüshu;nüshu;;2010-12-21
+Ogam;212;Ogham;ogam;Ogham;2004-05-01
+Olck;261;Ol Chiki (Ol Cemet’, Ol, Santali);ol tchiki;Ol_Chiki;2007-07-02
+Orkh;175;Old Turkic, Orkhon Runic;orkhon;Old_Turkic;2009-06-01
+Orya;327;Oriya;oriyâ;Oriya;2004-05-01
+Osma;260;Osmanya;osmanais;Osmanya;2004-05-01
+Palm;126;Palmyrene;palmyrénien;;2010-03-26
+Pauc;263;Pau Cin Hau;paou chin haou;;2013-10-12
+Perm;227;Old Permic;ancien permien;;2004-05-01
+Phag;331;Phags-pa;’phags pa;Phags_Pa;2006-10-10
+Phli;131;Inscriptional Pahlavi;pehlevi des inscriptions;Inscriptional_Pahlavi;2009-06-01
+Phlp;132;Psalter Pahlavi;pehlevi des psautiers;;2007-11-26
+Phlv;133;Book Pahlavi;pehlevi des livres;;2007-07-15
+Phnx;115;Phoenician;phénicien;Phoenician;2006-10-10
+Plrd;282;Miao (Pollard);miao (Pollard);Miao;2012-02-06
+Prti;130;Inscriptional Parthian;parthe des inscriptions;Inscriptional_Parthian;2009-06-01
+Qaaa;900;Reserved for private use (start);réservé à l’usage privé (début);;2004-05-29
+Qabx;949;Reserved for private use (end);réservé à l’usage privé (fin);;2004-05-29
+Rjng;363;Rejang (Redjang, Kaganga);redjang (kaganga);Rejang;2009-02-23
+Roro;620;Rongorongo;rongorongo;;2004-05-01
+Runr;211;Runic;runique;Runic;2004-05-01
+Samr;123;Samaritan;samaritain;Samaritan;2009-06-01
+Sara;292;Sarati;sarati;;2004-05-29
+Sarb;105;Old South Arabian;sud-arabique, himyarite;Old_South_Arabian;2009-06-01
+Saur;344;Saurashtra;saurachtra;Saurashtra;2007-07-02
+Sgnw;095;SignWriting;SignÉcriture, SignWriting;;2006-10-10
+Shaw;281;Shavian (Shaw);shavien (Shaw);Shavian;2004-05-01
+Shrd;319;Sharada, Śāradā;charada, shard;Sharada;2012-02-06
+Sidd;302;Siddham, Siddhaṃ, Siddhamātṛkā;siddham;;2013-10-12
+Sind;318;Khudawadi, Sindhi;khoudawadî, sindhî;;2010-12-21
+Sinh;348;Sinhala;singhalais;Sinhala;2004-05-01
+Sora;398;Sora Sompeng;sora sompeng;Sora_Sompeng;2012-02-06
+Sund;362;Sundanese;sundanais;Sundanese;2007-07-02
+Sylo;316;Syloti Nagri;sylotî nâgrî;Syloti_Nagri;2006-06-21
+Syrc;135;Syriac;syriaque;Syriac;2004-05-01
+Syre;138;Syriac (Estrangelo variant);syriaque (variante estranghélo);;2004-05-01
+Syrj;137;Syriac (Western variant);syriaque (variante occidentale);;2004-05-01
+Syrn;136;Syriac (Eastern variant);syriaque (variante orientale);;2004-05-01
+Tagb;373;Tagbanwa;tagbanoua;Tagbanwa;2004-05-01
+Takr;321;Takri, Ṭākrī, Ṭāṅkrī;tâkrî;Takri;2012-02-06
+Tale;353;Tai Le;taï-le;Tai_Le;2004-10-25
+Talu;354;New Tai Lue;nouveau taï-lue;New_Tai_Lue;2006-06-21
+Taml;346;Tamil;tamoul;Tamil;2004-05-01
+Tang;520;Tangut;tangoute;;2010-12-21
+Tavt;359;Tai Viet;taï viêt;Tai_Viet;2009-06-01
+Telu;340;Telugu;télougou;Telugu;2004-05-01
+Teng;290;Tengwar;tengwar;;2004-05-01
+Tfng;120;Tifinagh (Berber);tifinagh (berbère);Tifinagh;2006-06-21
+Tglg;370;Tagalog (Baybayin, Alibata);tagal (baybayin, alibata);Tagalog;2009-02-23
+Thaa;170;Thaana;thâna;Thaana;2004-05-01
+Thai;352;Thai;thaï;Thai;2004-05-01
+Tibt;330;Tibetan;tibétain;Tibetan;2004-05-01
+Tirh;326;Tirhuta;tirhouta;;2011-12-09
+Ugar;040;Ugaritic;ougaritique;Ugaritic;2004-05-01
+Vaii;470;Vai;vaï;Vai;2007-07-02
+Visp;280;Visible Speech;parole visible;;2004-05-01
+Wara;262;Warang Citi (Varang Kshiti);warang citi;;2009-11-11
+Wole;480;Woleai;woléaï;;2010-12-21
+Xpeo;030;Old Persian;cunéiforme persépolitain;Old_Persian;2006-06-21
+Xsux;020;Cuneiform, Sumero-Akkadian;cunéiforme suméro-akkadien;Cuneiform;2006-10-10
+Yiii;460;Yi;yi;Yi;2004-05-01
+Zinh;994;Code for inherited script;codet pour écriture héritée;Inherited;2009-02-23
+Zmth;995;Mathematical notation;notation mathématique;;2007-11-26
+Zsym;996;Symbols;symboles;;2007-11-26
+Zxxx;997;Code for unwritten documents;codet pour les documents non écrits;;2011-06-21
+Zyyy;998;Code for undetermined script;codet pour écriture indéterminée;Common;2004-05-29
+Zzzz;999;Code for uncoded script;codet pour écriture non codée;Unknown;2006-10-10
diff --git a/lib/babelfish/data/opensubtitles_languages.txt b/lib/babelfish/data/opensubtitles_languages.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1bd35063b040d650339c037c5a6657dd960853cd
--- /dev/null
+++ b/lib/babelfish/data/opensubtitles_languages.txt
@@ -0,0 +1,474 @@
+IdSubLanguage ISO639 LanguageName UploadEnabled WebEnabled
+aar aa Afar, afar 0 0
+abk ab Abkhazian 0 0
+ace Achinese 0 0
+ach Acoli 0 0
+ada Adangme 0 0
+ady adyghé 0 0
+afa Afro-Asiatic (Other) 0 0
+afh Afrihili 0 0
+afr af Afrikaans 1 0
+ain Ainu 0 0
+aka ak Akan 0 0
+akk Akkadian 0 0
+alb sq Albanian 1 1
+ale Aleut 0 0
+alg Algonquian languages 0 0
+alt Southern Altai 0 0
+amh am Amharic 0 0
+ang English, Old (ca.450-1100) 0 0
+apa Apache languages 0 0
+ara ar Arabic 1 1
+arc Aramaic 0 0
+arg an Aragonese 0 0
+arm hy Armenian 1 0
+arn Araucanian 0 0
+arp Arapaho 0 0
+art Artificial (Other) 0 0
+arw Arawak 0 0
+asm as Assamese 0 0
+ast Asturian, Bable 0 0
+ath Athapascan languages 0 0
+aus Australian languages 0 0
+ava av Avaric 0 0
+ave ae Avestan 0 0
+awa Awadhi 0 0
+aym ay Aymara 0 0
+aze az Azerbaijani 0 0
+bad Banda 0 0
+bai Bamileke languages 0 0
+bak ba Bashkir 0 0
+bal Baluchi 0 0
+bam bm Bambara 0 0
+ban Balinese 0 0
+baq eu Basque 1 1
+bas Basa 0 0
+bat Baltic (Other) 0 0
+bej Beja 0 0
+bel be Belarusian 0 0
+bem Bemba 0 0
+ben bn Bengali 1 0
+ber Berber (Other) 0 0
+bho Bhojpuri 0 0
+bih bh Bihari 0 0
+bik Bikol 0 0
+bin Bini 0 0
+bis bi Bislama 0 0
+bla Siksika 0 0
+bnt Bantu (Other) 0 0
+bos bs Bosnian 1 0
+bra Braj 0 0
+bre br Breton 1 0
+btk Batak (Indonesia) 0 0
+bua Buriat 0 0
+bug Buginese 0 0
+bul bg Bulgarian 1 1
+bur my Burmese 1 0
+byn Blin 0 0
+cad Caddo 0 0
+cai Central American Indian (Other) 0 0
+car Carib 0 0
+cat ca Catalan 1 1
+cau Caucasian (Other) 0 0
+ceb Cebuano 0 0
+cel Celtic (Other) 0 0
+cha ch Chamorro 0 0
+chb Chibcha 0 0
+che ce Chechen 0 0
+chg Chagatai 0 0
+chi zh Chinese 1 1
+chk Chuukese 0 0
+chm Mari 0 0
+chn Chinook jargon 0 0
+cho Choctaw 0 0
+chp Chipewyan 0 0
+chr Cherokee 0 0
+chu cu Church Slavic 0 0
+chv cv Chuvash 0 0
+chy Cheyenne 0 0
+cmc Chamic languages 0 0
+cop Coptic 0 0
+cor kw Cornish 0 0
+cos co Corsican 0 0
+cpe Creoles and pidgins, English based (Other) 0 0
+cpf Creoles and pidgins, French-based (Other) 0 0
+cpp Creoles and pidgins, Portuguese-based (Other) 0 0
+cre cr Cree 0 0
+crh Crimean Tatar 0 0
+crp Creoles and pidgins (Other) 0 0
+csb Kashubian 0 0
+cus Cushitic (Other)' couchitiques, autres langues 0 0
+cze cs Czech 1 1
+dak Dakota 0 0
+dan da Danish 1 1
+dar Dargwa 0 0
+day Dayak 0 0
+del Delaware 0 0
+den Slave (Athapascan) 0 0
+dgr Dogrib 0 0
+din Dinka 0 0
+div dv Divehi 0 0
+doi Dogri 0 0
+dra Dravidian (Other) 0 0
+dua Duala 0 0
+dum Dutch, Middle (ca.1050-1350) 0 0
+dut nl Dutch 1 1
+dyu Dyula 0 0
+dzo dz Dzongkha 0 0
+efi Efik 0 0
+egy Egyptian (Ancient) 0 0
+eka Ekajuk 0 0
+elx Elamite 0 0
+eng en English 1 1
+enm English, Middle (1100-1500) 0 0
+epo eo Esperanto 1 0
+est et Estonian 1 1
+ewe ee Ewe 0 0
+ewo Ewondo 0 0
+fan Fang 0 0
+fao fo Faroese 0 0
+fat Fanti 0 0
+fij fj Fijian 0 0
+fil Filipino 0 0
+fin fi Finnish 1 1
+fiu Finno-Ugrian (Other) 0 0
+fon Fon 0 0
+fre fr French 1 1
+frm French, Middle (ca.1400-1600) 0 0
+fro French, Old (842-ca.1400) 0 0
+fry fy Frisian 0 0
+ful ff Fulah 0 0
+fur Friulian 0 0
+gaa Ga 0 0
+gay Gayo 0 0
+gba Gbaya 0 0
+gem Germanic (Other) 0 0
+geo ka Georgian 1 1
+ger de German 1 1
+gez Geez 0 0
+gil Gilbertese 0 0
+gla gd Gaelic 0 0
+gle ga Irish 0 0
+glg gl Galician 1 1
+glv gv Manx 0 0
+gmh German, Middle High (ca.1050-1500) 0 0
+goh German, Old High (ca.750-1050) 0 0
+gon Gondi 0 0
+gor Gorontalo 0 0
+got Gothic 0 0
+grb Grebo 0 0
+grc Greek, Ancient (to 1453) 0 0
+ell el Greek 1 1
+grn gn Guarani 0 0
+guj gu Gujarati 0 0
+gwi Gwich´in 0 0
+hai Haida 0 0
+hat ht Haitian 0 0
+hau ha Hausa 0 0
+haw Hawaiian 0 0
+heb he Hebrew 1 1
+her hz Herero 0 0
+hil Hiligaynon 0 0
+him Himachali 0 0
+hin hi Hindi 1 1
+hit Hittite 0 0
+hmn Hmong 0 0
+hmo ho Hiri Motu 0 0
+hrv hr Croatian 1 1
+hun hu Hungarian 1 1
+hup Hupa 0 0
+iba Iban 0 0
+ibo ig Igbo 0 0
+ice is Icelandic 1 1
+ido io Ido 0 0
+iii ii Sichuan Yi 0 0
+ijo Ijo 0 0
+iku iu Inuktitut 0 0
+ile ie Interlingue 0 0
+ilo Iloko 0 0
+ina ia Interlingua (International Auxiliary Language Asso 0 0
+inc Indic (Other) 0 0
+ind id Indonesian 1 1
+ine Indo-European (Other) 0 0
+inh Ingush 0 0
+ipk ik Inupiaq 0 0
+ira Iranian (Other) 0 0
+iro Iroquoian languages 0 0
+ita it Italian 1 1
+jav jv Javanese 0 0
+jpn ja Japanese 1 1
+jpr Judeo-Persian 0 0
+jrb Judeo-Arabic 0 0
+kaa Kara-Kalpak 0 0
+kab Kabyle 0 0
+kac Kachin 0 0
+kal kl Kalaallisut 0 0
+kam Kamba 0 0
+kan kn Kannada 0 0
+kar Karen 0 0
+kas ks Kashmiri 0 0
+kau kr Kanuri 0 0
+kaw Kawi 0 0
+kaz kk Kazakh 1 0
+kbd Kabardian 0 0
+kha Khasi 0 0
+khi Khoisan (Other) 0 0
+khm km Khmer 1 1
+kho Khotanese 0 0
+kik ki Kikuyu 0 0
+kin rw Kinyarwanda 0 0
+kir ky Kirghiz 0 0
+kmb Kimbundu 0 0
+kok Konkani 0 0
+kom kv Komi 0 0
+kon kg Kongo 0 0
+kor ko Korean 1 1
+kos Kosraean 0 0
+kpe Kpelle 0 0
+krc Karachay-Balkar 0 0
+kro Kru 0 0
+kru Kurukh 0 0
+kua kj Kuanyama 0 0
+kum Kumyk 0 0
+kur ku Kurdish 0 0
+kut Kutenai 0 0
+lad Ladino 0 0
+lah Lahnda 0 0
+lam Lamba 0 0
+lao lo Lao 0 0
+lat la Latin 0 0
+lav lv Latvian 1 0
+lez Lezghian 0 0
+lim li Limburgan 0 0
+lin ln Lingala 0 0
+lit lt Lithuanian 1 0
+lol Mongo 0 0
+loz Lozi 0 0
+ltz lb Luxembourgish 1 0
+lua Luba-Lulua 0 0
+lub lu Luba-Katanga 0 0
+lug lg Ganda 0 0
+lui Luiseno 0 0
+lun Lunda 0 0
+luo Luo (Kenya and Tanzania) 0 0
+lus lushai 0 0
+mac mk Macedonian 1 1
+mad Madurese 0 0
+mag Magahi 0 0
+mah mh Marshallese 0 0
+mai Maithili 0 0
+mak Makasar 0 0
+mal ml Malayalam 1 0
+man Mandingo 0 0
+mao mi Maori 0 0
+map Austronesian (Other) 0 0
+mar mr Marathi 0 0
+mas Masai 0 0
+may ms Malay 1 1
+mdf Moksha 0 0
+mdr Mandar 0 0
+men Mende 0 0
+mga Irish, Middle (900-1200) 0 0
+mic Mi'kmaq 0 0
+min Minangkabau 0 0
+mis Miscellaneous languages 0 0
+mkh Mon-Khmer (Other) 0 0
+mlg mg Malagasy 0 0
+mlt mt Maltese 0 0
+mnc Manchu 0 0
+mni Manipuri 0 0
+mno Manobo languages 0 0
+moh Mohawk 0 0
+mol mo Moldavian 0 0
+mon mn Mongolian 1 0
+mos Mossi 0 0
+mwl Mirandese 0 0
+mul Multiple languages 0 0
+mun Munda languages 0 0
+mus Creek 0 0
+mwr Marwari 0 0
+myn Mayan languages 0 0
+myv Erzya 0 0
+nah Nahuatl 0 0
+nai North American Indian 0 0
+nap Neapolitan 0 0
+nau na Nauru 0 0
+nav nv Navajo 0 0
+nbl nr Ndebele, South 0 0
+nde nd Ndebele, North 0 0
+ndo ng Ndonga 0 0
+nds Low German 0 0
+nep ne Nepali 0 0
+new Nepal Bhasa 0 0
+nia Nias 0 0
+nic Niger-Kordofanian (Other) 0 0
+niu Niuean 0 0
+nno nn Norwegian Nynorsk 0 0
+nob nb Norwegian Bokmal 0 0
+nog Nogai 0 0
+non Norse, Old 0 0
+nor no Norwegian 1 1
+nso Northern Sotho 0 0
+nub Nubian languages 0 0
+nwc Classical Newari 0 0
+nya ny Chichewa 0 0
+nym Nyamwezi 0 0
+nyn Nyankole 0 0
+nyo Nyoro 0 0
+nzi Nzima 0 0
+oci oc Occitan 1 1
+oji oj Ojibwa 0 0
+ori or Oriya 0 0
+orm om Oromo 0 0
+osa Osage 0 0
+oss os Ossetian 0 0
+ota Turkish, Ottoman (1500-1928) 0 0
+oto Otomian languages 0 0
+paa Papuan (Other) 0 0
+pag Pangasinan 0 0
+pal Pahlavi 0 0
+pam Pampanga 0 0
+pan pa Panjabi 0 0
+pap Papiamento 0 0
+pau Palauan 0 0
+peo Persian, Old (ca.600-400 B.C.) 0 0
+per fa Persian 1 1
+phi Philippine (Other) 0 0
+phn Phoenician 0 0
+pli pi Pali 0 0
+pol pl Polish 1 1
+pon Pohnpeian 0 0
+por pt Portuguese 1 1
+pra Prakrit languages 0 0
+pro Provençal, Old (to 1500) 0 0
+pus ps Pushto 0 0
+que qu Quechua 0 0
+raj Rajasthani 0 0
+rap Rapanui 0 0
+rar Rarotongan 0 0
+roa Romance (Other) 0 0
+roh rm Raeto-Romance 0 0
+rom Romany 0 0
+run rn Rundi 0 0
+rup Aromanian 0 0
+rus ru Russian 1 1
+sad Sandawe 0 0
+sag sg Sango 0 0
+sah Yakut 0 0
+sai South American Indian (Other) 0 0
+sal Salishan languages 0 0
+sam Samaritan Aramaic 0 0
+san sa Sanskrit 0 0
+sas Sasak 0 0
+sat Santali 0 0
+scc sr Serbian 1 1
+scn Sicilian 0 0
+sco Scots 0 0
+sel Selkup 0 0
+sem Semitic (Other) 0 0
+sga Irish, Old (to 900) 0 0
+sgn Sign Languages 0 0
+shn Shan 0 0
+sid Sidamo 0 0
+sin si Sinhalese 1 1
+sio Siouan languages 0 0
+sit Sino-Tibetan (Other) 0 0
+sla Slavic (Other) 0 0
+slo sk Slovak 1 1
+slv sl Slovenian 1 1
+sma Southern Sami 0 0
+sme se Northern Sami 0 0
+smi Sami languages (Other) 0 0
+smj Lule Sami 0 0
+smn Inari Sami 0 0
+smo sm Samoan 0 0
+sms Skolt Sami 0 0
+sna sn Shona 0 0
+snd sd Sindhi 0 0
+snk Soninke 0 0
+sog Sogdian 0 0
+som so Somali 0 0
+son Songhai 0 0
+sot st Sotho, Southern 0 0
+spa es Spanish 1 1
+srd sc Sardinian 0 0
+srr Serer 0 0
+ssa Nilo-Saharan (Other) 0 0
+ssw ss Swati 0 0
+suk Sukuma 0 0
+sun su Sundanese 0 0
+sus Susu 0 0
+sux Sumerian 0 0
+swa sw Swahili 1 0
+swe sv Swedish 1 1
+syr Syriac 1 0
+tah ty Tahitian 0 0
+tai Tai (Other) 0 0
+tam ta Tamil 1 0
+tat tt Tatar 0 0
+tel te Telugu 1 0
+tem Timne 0 0
+ter Tereno 0 0
+tet Tetum 0 0
+tgk tg Tajik 0 0
+tgl tl Tagalog 1 1
+tha th Thai 1 1
+tib bo Tibetan 0 0
+tig Tigre 0 0
+tir ti Tigrinya 0 0
+tiv Tiv 0 0
+tkl Tokelau 0 0
+tlh Klingon 0 0
+tli Tlingit 0 0
+tmh Tamashek 0 0
+tog Tonga (Nyasa) 0 0
+ton to Tonga (Tonga Islands) 0 0
+tpi Tok Pisin 0 0
+tsi Tsimshian 0 0
+tsn tn Tswana 0 0
+tso ts Tsonga 0 0
+tuk tk Turkmen 0 0
+tum Tumbuka 0 0
+tup Tupi languages 0 0
+tur tr Turkish 1 1
+tut Altaic (Other) 0 0
+tvl Tuvalu 0 0
+twi tw Twi 0 0
+tyv Tuvinian 0 0
+udm Udmurt 0 0
+uga Ugaritic 0 0
+uig ug Uighur 0 0
+ukr uk Ukrainian 1 1
+umb Umbundu 0 0
+und Undetermined 0 0
+urd ur Urdu 1 0
+uzb uz Uzbek 0 0
+vai Vai 0 0
+ven ve Venda 0 0
+vie vi Vietnamese 1 1
+vol vo Volapük 0 0
+vot Votic 0 0
+wak Wakashan languages 0 0
+wal Walamo 0 0
+war Waray 0 0
+was Washo 0 0
+wel cy Welsh 0 0
+wen Sorbian languages 0 0
+wln wa Walloon 0 0
+wol wo Wolof 0 0
+xal Kalmyk 0 0
+xho xh Xhosa 0 0
+yao Yao 0 0
+yap Yapese 0 0
+yid yi Yiddish 0 0
+yor yo Yoruba 0 0
+ypk Yupik languages 0 0
+zap Zapotec 0 0
+zen Zenaga 0 0
+zha za Zhuang 0 0
+znd Zande 0 0
+zul zu Zulu 0 0
+zun Zuni 0 0
+rum ro Romanian 1 1
+pob pb Brazilian 1 1
+mne Montenegrin 1 0
diff --git a/lib/babelfish/exceptions.py b/lib/babelfish/exceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..bbc6efe3a6ab1e639d3bd73e4bea4d6ce6b5a7ef
--- /dev/null
+++ b/lib/babelfish/exceptions.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+
+
+class Error(Exception):
+ """Base class for all exceptions in babelfish"""
+ pass
+
+
+class LanguageError(Error, AttributeError):
+ """Base class for all language exceptions in babelfish"""
+ pass
+
+
+class LanguageConvertError(LanguageError):
+ """Exception raised by converters when :meth:`~babelfish.converters.LanguageConverter.convert` fails
+
+ :param string alpha3: alpha3 code that failed conversion
+ :param country: country code that failed conversion, if any
+ :type country: string or None
+ :param script: script code that failed conversion, if any
+ :type script: string or None
+
+ """
+ def __init__(self, alpha3, country=None, script=None):
+ self.alpha3 = alpha3
+ self.country = country
+ self.script = script
+
+ def __str__(self):
+ s = self.alpha3
+ if self.country is not None:
+ s += '-' + self.country
+ if self.script is not None:
+ s += '-' + self.script
+ return s
+
+
+class LanguageReverseError(LanguageError):
+ """Exception raised by converters when :meth:`~babelfish.converters.LanguageReverseConverter.reverse` fails
+
+ :param string code: code that failed reverse conversion
+
+ """
+ def __init__(self, code):
+ self.code = code
+
+ def __str__(self):
+ return repr(self.code)
+
+
+class CountryError(Error, AttributeError):
+ """Base class for all country exceptions in babelfish"""
+ pass
+
+
+class CountryConvertError(CountryError):
+ """Exception raised by converters when :meth:`~babelfish.converters.CountryConverter.convert` fails
+
+ :param string alpha2: alpha2 code that failed conversion
+
+ """
+ def __init__(self, alpha2):
+ self.alpha2 = alpha2
+
+ def __str__(self):
+ return self.alpha2
+
+
+class CountryReverseError(CountryError):
+ """Exception raised by converters when :meth:`~babelfish.converters.CountryReverseConverter.reverse` fails
+
+ :param string code: code that failed reverse conversion
+
+ """
+ def __init__(self, code):
+ self.code = code
+
+ def __str__(self):
+ return repr(self.code)
diff --git a/lib/babelfish/language.py b/lib/babelfish/language.py
new file mode 100644
index 0000000000000000000000000000000000000000..b4b251937646ce2826cd4c64632b14288e43c34e
--- /dev/null
+++ b/lib/babelfish/language.py
@@ -0,0 +1,185 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+from collections import namedtuple
+from functools import partial
+from pkg_resources import resource_stream # @UnresolvedImport
+from .converters import ConverterManager
+from .country import Country
+from .exceptions import LanguageConvertError
+from .script import Script
+from . import basestr
+
+
+LANGUAGES = set()
+LANGUAGE_MATRIX = []
+
+#: The namedtuple used in the :data:`LANGUAGE_MATRIX`
+IsoLanguage = namedtuple('IsoLanguage', ['alpha3', 'alpha3b', 'alpha3t', 'alpha2', 'scope', 'type', 'name', 'comment'])
+
+f = resource_stream('babelfish', 'data/iso-639-3.tab')
+f.readline()
+for l in f:
+ iso_language = IsoLanguage(*l.decode('utf-8').split('\t'))
+ LANGUAGES.add(iso_language.alpha3)
+ LANGUAGE_MATRIX.append(iso_language)
+f.close()
+
+
+class LanguageConverterManager(ConverterManager):
+ """:class:`~babelfish.converters.ConverterManager` for language converters"""
+ entry_point = 'babelfish.language_converters'
+ internal_converters = ['alpha2 = babelfish.converters.alpha2:Alpha2Converter',
+ 'alpha3b = babelfish.converters.alpha3b:Alpha3BConverter',
+ 'alpha3t = babelfish.converters.alpha3t:Alpha3TConverter',
+ 'name = babelfish.converters.name:NameConverter',
+ 'scope = babelfish.converters.scope:ScopeConverter',
+ 'type = babelfish.converters.type:LanguageTypeConverter',
+ 'opensubtitles = babelfish.converters.opensubtitles:OpenSubtitlesConverter']
+
+language_converters = LanguageConverterManager()
+
+
+class LanguageMeta(type):
+ """The :class:`Language` metaclass
+
+ Dynamically redirect :meth:`Language.frommycode` to :meth:`Language.fromcode` with the ``mycode`` `converter`
+
+ """
+ def __getattr__(cls, name):
+ if name.startswith('from'):
+ return partial(cls.fromcode, converter=name[4:])
+ return type.__getattribute__(cls, name)
+
+
+class Language(LanguageMeta(str('LanguageBase'), (object,), {})):
+ """A human language
+
+ A human language is composed of a language part following the ISO-639
+ standard and can be country-specific when a :class:`~babelfish.country.Country`
+ is specified.
+
+ The :class:`Language` is extensible with custom converters (see :ref:`custom_converters`)
+
+ :param string language: the language as a 3-letter ISO-639-3 code
+ :param country: the country (if any) as a 2-letter ISO-3166 code or :class:`~babelfish.country.Country` instance
+ :type country: string or :class:`~babelfish.country.Country` or None
+ :param script: the script (if any) as a 4-letter ISO-15924 code or :class:`~babelfish.script.Script` instance
+ :type script: string or :class:`~babelfish.script.Script` or None
+ :param unknown: the unknown language as a three-letters ISO-639-3 code to use as fallback
+ :type unknown: string or None
+ :raise: ValueError if the language could not be recognized and `unknown` is ``None``
+
+ """
+ def __init__(self, language, country=None, script=None, unknown=None):
+ if unknown is not None and language not in LANGUAGES:
+ language = unknown
+ if language not in LANGUAGES:
+ raise ValueError('%r is not a valid language' % language)
+ self.alpha3 = language
+ self.country = None
+ if isinstance(country, Country):
+ self.country = country
+ elif country is None:
+ self.country = None
+ else:
+ self.country = Country(country)
+ self.script = None
+ if isinstance(script, Script):
+ self.script = script
+ elif script is None:
+ self.script = None
+ else:
+ self.script = Script(script)
+
+ @classmethod
+ def fromcode(cls, code, converter):
+ """Create a :class:`Language` by its `code` using `converter` to
+ :meth:`~babelfish.converters.LanguageReverseConverter.reverse` it
+
+ :param string code: the code to reverse
+ :param string converter: name of the :class:`~babelfish.converters.LanguageReverseConverter` to use
+ :return: the corresponding :class:`Language` instance
+ :rtype: :class:`Language`
+
+ """
+ return cls(*language_converters[converter].reverse(code))
+
+ @classmethod
+ def fromietf(cls, ietf):
+ """Create a :class:`Language` by from an IETF language code
+
+ :param string ietf: the ietf code
+ :return: the corresponding :class:`Language` instance
+ :rtype: :class:`Language`
+
+ """
+ subtags = ietf.split('-')
+ language_subtag = subtags.pop(0).lower()
+ if len(language_subtag) == 2:
+ language = cls.fromalpha2(language_subtag)
+ else:
+ language = cls(language_subtag)
+ while subtags:
+ subtag = subtags.pop(0)
+ if len(subtag) == 2:
+ language.country = Country(subtag.upper())
+ else:
+ language.script = Script(subtag.capitalize())
+ if language.script is not None:
+ if subtags:
+ raise ValueError('Wrong IETF format. Unmatched subtags: %r' % subtags)
+ break
+ return language
+
+ def __getstate__(self):
+ return self.alpha3, self.country, self.script
+
+ def __setstate__(self, state):
+ self.alpha3, self.country, self.script = state
+
+ def __getattr__(self, name):
+ alpha3 = self.alpha3
+ country = self.country.alpha2 if self.country is not None else None
+ script = self.script.code if self.script is not None else None
+ try:
+ return language_converters[name].convert(alpha3, country, script)
+ except KeyError:
+ raise AttributeError(name)
+
+ def __hash__(self):
+ return hash(str(self))
+
+ def __eq__(self, other):
+ if isinstance(other, basestr):
+ return str(self) == other
+ if not isinstance(other, Language):
+ return False
+ return (self.alpha3 == other.alpha3 and
+ self.country == other.country and
+ self.script == other.script)
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __bool__(self):
+ return self.alpha3 != 'und'
+ __nonzero__ = __bool__
+
+ def __repr__(self):
+ return '<Language [%s]>' % self
+
+ def __str__(self):
+ try:
+ s = self.alpha2
+ except LanguageConvertError:
+ s = self.alpha3
+ if self.country is not None:
+ s += '-' + str(self.country)
+ if self.script is not None:
+ s += '-' + str(self.script)
+ return s
diff --git a/lib/babelfish/script.py b/lib/babelfish/script.py
new file mode 100644
index 0000000000000000000000000000000000000000..4b59ce016e0378c8c4a7d353018fa4ad54827832
--- /dev/null
+++ b/lib/babelfish/script.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+from collections import namedtuple
+from pkg_resources import resource_stream # @UnresolvedImport
+from . import basestr
+
+#: Script code to script name mapping
+SCRIPTS = {}
+
+#: List of countries in the ISO-15924 as namedtuple of code, number, name, french_name, pva and date
+SCRIPT_MATRIX = []
+
+#: The namedtuple used in the :data:`SCRIPT_MATRIX`
+IsoScript = namedtuple('IsoScript', ['code', 'number', 'name', 'french_name', 'pva', 'date'])
+
+f = resource_stream('babelfish', 'data/iso15924-utf8-20131012.txt')
+f.readline()
+for l in f:
+ l = l.decode('utf-8').strip()
+ if not l or l.startswith('#'):
+ continue
+ script = IsoScript._make(l.split(';'))
+ SCRIPT_MATRIX.append(script)
+ SCRIPTS[script.code] = script.name
+f.close()
+
+
+class Script(object):
+ """A human writing system
+
+ A script is represented by a 4-letter code from the ISO-15924 standard
+
+ :param string script: 4-letter ISO-15924 script code
+
+ """
+ def __init__(self, script):
+ if script not in SCRIPTS:
+ raise ValueError('%r is not a valid script' % script)
+
+ #: ISO-15924 4-letter script code
+ self.code = script
+
+ @property
+ def name(self):
+ """English name of the script"""
+ return SCRIPTS[self.code]
+
+ def __getstate__(self):
+ return self.code
+
+ def __setstate__(self, state):
+ self.code = state
+
+ def __hash__(self):
+ return hash(self.code)
+
+ def __eq__(self, other):
+ if isinstance(other, basestr):
+ return self.code == other
+ if not isinstance(other, Script):
+ return False
+ return self.code == other.code
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __repr__(self):
+ return '<Script [%s]>' % self
+
+ def __str__(self):
+ return self.code
diff --git a/lib/babelfish/tests.py b/lib/babelfish/tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..cf688af929e221587573c5e119ffbeb85442fc8e
--- /dev/null
+++ b/lib/babelfish/tests.py
@@ -0,0 +1,368 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2013 the BabelFish authors. All rights reserved.
+# Use of this source code is governed by the 3-clause BSD license
+# that can be found in the LICENSE file.
+#
+from __future__ import unicode_literals
+import re
+import sys
+import pickle
+from unittest import TestCase, TestSuite, TestLoader, TextTestRunner
+from pkg_resources import resource_stream # @UnresolvedImport
+from babelfish import (LANGUAGES, Language, Country, Script, language_converters, country_converters,
+ LanguageReverseConverter, LanguageConvertError, LanguageReverseError, CountryReverseError)
+
+
+if sys.version_info[:2] <= (2, 6):
+ _MAX_LENGTH = 80
+
+ def safe_repr(obj, short=False):
+ try:
+ result = repr(obj)
+ except Exception:
+ result = object.__repr__(obj)
+ if not short or len(result) < _MAX_LENGTH:
+ return result
+ return result[:_MAX_LENGTH] + ' [truncated]...'
+
+ class _AssertRaisesContext(object):
+ """A context manager used to implement TestCase.assertRaises* methods."""
+
+ def __init__(self, expected, test_case, expected_regexp=None):
+ self.expected = expected
+ self.failureException = test_case.failureException
+ self.expected_regexp = expected_regexp
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, tb):
+ if exc_type is None:
+ try:
+ exc_name = self.expected.__name__
+ except AttributeError:
+ exc_name = str(self.expected)
+ raise self.failureException(
+ "{0} not raised".format(exc_name))
+ if not issubclass(exc_type, self.expected):
+ # let unexpected exceptions pass through
+ return False
+ self.exception = exc_value # store for later retrieval
+ if self.expected_regexp is None:
+ return True
+
+ expected_regexp = self.expected_regexp
+ if isinstance(expected_regexp, basestring):
+ expected_regexp = re.compile(expected_regexp)
+ if not expected_regexp.search(str(exc_value)):
+ raise self.failureException('"%s" does not match "%s"' %
+ (expected_regexp.pattern, str(exc_value)))
+ return True
+
+ class _Py26FixTestCase(object):
+ def assertIsNone(self, obj, msg=None):
+ """Same as self.assertTrue(obj is None), with a nicer default message."""
+ if obj is not None:
+ standardMsg = '%s is not None' % (safe_repr(obj),)
+ self.fail(self._formatMessage(msg, standardMsg))
+
+ def assertIsNotNone(self, obj, msg=None):
+ """Included for symmetry with assertIsNone."""
+ if obj is None:
+ standardMsg = 'unexpectedly None'
+ self.fail(self._formatMessage(msg, standardMsg))
+
+ def assertIn(self, member, container, msg=None):
+ """Just like self.assertTrue(a in b), but with a nicer default message."""
+ if member not in container:
+ standardMsg = '%s not found in %s' % (safe_repr(member),
+ safe_repr(container))
+ self.fail(self._formatMessage(msg, standardMsg))
+
+ def assertNotIn(self, member, container, msg=None):
+ """Just like self.assertTrue(a not in b), but with a nicer default message."""
+ if member in container:
+ standardMsg = '%s unexpectedly found in %s' % (safe_repr(member),
+ safe_repr(container))
+ self.fail(self._formatMessage(msg, standardMsg))
+
+ def assertIs(self, expr1, expr2, msg=None):
+ """Just like self.assertTrue(a is b), but with a nicer default message."""
+ if expr1 is not expr2:
+ standardMsg = '%s is not %s' % (safe_repr(expr1),
+ safe_repr(expr2))
+ self.fail(self._formatMessage(msg, standardMsg))
+
+ def assertIsNot(self, expr1, expr2, msg=None):
+ """Just like self.assertTrue(a is not b), but with a nicer default message."""
+ if expr1 is expr2:
+ standardMsg = 'unexpectedly identical: %s' % (safe_repr(expr1),)
+ self.fail(self._formatMessage(msg, standardMsg))
+
+else:
+ class _Py26FixTestCase(object):
+ pass
+
+
+class TestScript(TestCase, _Py26FixTestCase):
+ def test_wrong_script(self):
+ self.assertRaises(ValueError, lambda: Script('Azer'))
+
+ def test_eq(self):
+ self.assertEqual(Script('Latn'), Script('Latn'))
+
+ def test_ne(self):
+ self.assertNotEqual(Script('Cyrl'), Script('Latn'))
+
+ def test_hash(self):
+ self.assertEqual(hash(Script('Hira')), hash('Hira'))
+
+ def test_pickle(self):
+ self.assertEqual(pickle.loads(pickle.dumps(Script('Latn'))), Script('Latn'))
+
+
+class TestCountry(TestCase, _Py26FixTestCase):
+ def test_wrong_country(self):
+ self.assertRaises(ValueError, lambda: Country('ZZ'))
+
+ def test_eq(self):
+ self.assertEqual(Country('US'), Country('US'))
+
+ def test_ne(self):
+ self.assertNotEqual(Country('GB'), Country('US'))
+ self.assertIsNotNone(Country('US'))
+
+ def test_hash(self):
+ self.assertEqual(hash(Country('US')), hash('US'))
+
+ def test_pickle(self):
+ for country in [Country('GB'), Country('US')]:
+ self.assertEqual(pickle.loads(pickle.dumps(country)), country)
+
+ def test_converter_name(self):
+ self.assertEqual(Country('US').name, 'UNITED STATES')
+ self.assertEqual(Country.fromname('UNITED STATES'), Country('US'))
+ self.assertEqual(Country.fromcode('UNITED STATES', 'name'), Country('US'))
+ self.assertRaises(CountryReverseError, lambda: Country.fromname('ZZZZZ'))
+ self.assertEqual(len(country_converters['name'].codes), 249)
+
+
+class TestLanguage(TestCase, _Py26FixTestCase):
+ def test_languages(self):
+ self.assertEqual(len(LANGUAGES), 7874)
+
+ def test_wrong_language(self):
+ self.assertRaises(ValueError, lambda: Language('zzz'))
+
+ def test_unknown_language(self):
+ self.assertEqual(Language('zzzz', unknown='und'), Language('und'))
+
+ def test_converter_alpha2(self):
+ self.assertEqual(Language('eng').alpha2, 'en')
+ self.assertEqual(Language.fromalpha2('en'), Language('eng'))
+ self.assertEqual(Language.fromcode('en', 'alpha2'), Language('eng'))
+ self.assertRaises(LanguageReverseError, lambda: Language.fromalpha2('zz'))
+ self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha2)
+ self.assertEqual(len(language_converters['alpha2'].codes), 184)
+
+ def test_converter_alpha3b(self):
+ self.assertEqual(Language('fra').alpha3b, 'fre')
+ self.assertEqual(Language.fromalpha3b('fre'), Language('fra'))
+ self.assertEqual(Language.fromcode('fre', 'alpha3b'), Language('fra'))
+ self.assertRaises(LanguageReverseError, lambda: Language.fromalpha3b('zzz'))
+ self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha3b)
+ self.assertEqual(len(language_converters['alpha3b'].codes), 418)
+
+ def test_converter_alpha3t(self):
+ self.assertEqual(Language('fra').alpha3t, 'fra')
+ self.assertEqual(Language.fromalpha3t('fra'), Language('fra'))
+ self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra'))
+ self.assertRaises(LanguageReverseError, lambda: Language.fromalpha3t('zzz'))
+ self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha3t)
+ self.assertEqual(len(language_converters['alpha3t'].codes), 418)
+
+ def test_converter_name(self):
+ self.assertEqual(Language('eng').name, 'English')
+ self.assertEqual(Language.fromname('English'), Language('eng'))
+ self.assertEqual(Language.fromcode('English', 'name'), Language('eng'))
+ self.assertRaises(LanguageReverseError, lambda: Language.fromname('Zzzzzzzzz'))
+ self.assertEqual(len(language_converters['name'].codes), 7874)
+
+ def test_converter_scope(self):
+ self.assertEqual(language_converters['scope'].codes, set(['I', 'S', 'M']))
+ self.assertEqual(Language('eng').scope, 'individual')
+ self.assertEqual(Language('und').scope, 'special')
+
+ def test_converter_type(self):
+ self.assertEqual(language_converters['type'].codes, set(['A', 'C', 'E', 'H', 'L', 'S']))
+ self.assertEqual(Language('eng').type, 'living')
+ self.assertEqual(Language('und').type, 'special')
+
+ def test_converter_opensubtitles(self):
+ self.assertEqual(Language('fra').opensubtitles, Language('fra').alpha3b)
+ self.assertEqual(Language('por', 'BR').opensubtitles, 'pob')
+ self.assertEqual(Language.fromopensubtitles('fre'), Language('fra'))
+ self.assertEqual(Language.fromopensubtitles('pob'), Language('por', 'BR'))
+ self.assertEqual(Language.fromopensubtitles('pb'), Language('por', 'BR'))
+ # Montenegrin is not recognized as an ISO language (yet?) but for now it is
+ # unofficially accepted as Serbian from Montenegro
+ self.assertEqual(Language.fromopensubtitles('mne'), Language('srp', 'ME'))
+ self.assertEqual(Language.fromcode('pob', 'opensubtitles'), Language('por', 'BR'))
+ self.assertRaises(LanguageReverseError, lambda: Language.fromopensubtitles('zzz'))
+ self.assertRaises(LanguageConvertError, lambda: Language('aaa').opensubtitles)
+ self.assertEqual(len(language_converters['opensubtitles'].codes), 606)
+
+ # test with all the LANGUAGES from the opensubtitles api
+ # downloaded from: http://www.opensubtitles.org/addons/export_languages.php
+ f = resource_stream('babelfish', 'data/opensubtitles_languages.txt')
+ f.readline()
+ for l in f:
+ idlang, alpha2, _, upload_enabled, web_enabled = l.decode('utf-8').strip().split('\t')
+ if not int(upload_enabled) and not int(web_enabled):
+ # do not test LANGUAGES that are too esoteric / not widely available
+ continue
+ self.assertEqual(Language.fromopensubtitles(idlang).opensubtitles, idlang)
+ if alpha2:
+ self.assertEqual(Language.fromopensubtitles(idlang), Language.fromopensubtitles(alpha2))
+ f.close()
+
+ def test_fromietf_country_script(self):
+ language = Language.fromietf('fra-FR-Latn')
+ self.assertEqual(language.alpha3, 'fra')
+ self.assertEqual(language.country, Country('FR'))
+ self.assertEqual(language.script, Script('Latn'))
+
+ def test_fromietf_country_no_script(self):
+ language = Language.fromietf('fra-FR')
+ self.assertEqual(language.alpha3, 'fra')
+ self.assertEqual(language.country, Country('FR'))
+ self.assertIsNone(language.script)
+
+ def test_fromietf_no_country_no_script(self):
+ language = Language.fromietf('fra-FR')
+ self.assertEqual(language.alpha3, 'fra')
+ self.assertEqual(language.country, Country('FR'))
+ self.assertIsNone(language.script)
+
+ def test_fromietf_no_country_script(self):
+ language = Language.fromietf('fra-Latn')
+ self.assertEqual(language.alpha3, 'fra')
+ self.assertIsNone(language.country)
+ self.assertEqual(language.script, Script('Latn'))
+
+ def test_fromietf_alpha2_language(self):
+ language = Language.fromietf('fr-Latn')
+ self.assertEqual(language.alpha3, 'fra')
+ self.assertIsNone(language.country)
+ self.assertEqual(language.script, Script('Latn'))
+
+ def test_fromietf_wrong_language(self):
+ self.assertRaises(ValueError, lambda: Language.fromietf('xyz-FR'))
+
+ def test_fromietf_wrong_country(self):
+ self.assertRaises(ValueError, lambda: Language.fromietf('fra-YZ'))
+
+ def test_fromietf_wrong_script(self):
+ self.assertRaises(ValueError, lambda: Language.fromietf('fra-FR-Wxyz'))
+
+ def test_eq(self):
+ self.assertEqual(Language('eng'), Language('eng'))
+
+ def test_ne(self):
+ self.assertNotEqual(Language('fra'), Language('eng'))
+ self.assertIsNotNone(Language('fra'))
+
+ def test_nonzero(self):
+ self.assertFalse(bool(Language('und')))
+ self.assertTrue(bool(Language('eng')))
+
+ def test_language_hasattr(self):
+ self.assertTrue(hasattr(Language('fra'), 'alpha3'))
+ self.assertTrue(hasattr(Language('fra'), 'alpha2'))
+ self.assertFalse(hasattr(Language('bej'), 'alpha2'))
+
+ def test_country(self):
+ self.assertEqual(Language('por', 'BR').country, Country('BR'))
+ self.assertEqual(Language('eng', Country('US')).country, Country('US'))
+
+ def test_eq_with_country(self):
+ self.assertEqual(Language('eng', 'US'), Language('eng', Country('US')))
+
+ def test_ne_with_country(self):
+ self.assertNotEqual(Language('eng', 'US'), Language('eng', Country('GB')))
+
+ def test_script(self):
+ self.assertEqual(Language('srp', script='Latn').script, Script('Latn'))
+ self.assertEqual(Language('srp', script=Script('Cyrl')).script, Script('Cyrl'))
+
+ def test_eq_with_script(self):
+ self.assertEqual(Language('srp', script='Latn'), Language('srp', script=Script('Latn')))
+
+ def test_ne_with_script(self):
+ self.assertNotEqual(Language('srp', script='Latn'), Language('srp', script=Script('Cyrl')))
+
+ def test_eq_with_country_and_script(self):
+ self.assertEqual(Language('srp', 'SR', 'Latn'), Language('srp', Country('SR'), Script('Latn')))
+
+ def test_ne_with_country_and_script(self):
+ self.assertNotEqual(Language('srp', 'SR', 'Latn'), Language('srp', Country('SR'), Script('Cyrl')))
+
+ def test_hash(self):
+ self.assertEqual(hash(Language('fra')), hash('fr'))
+ self.assertEqual(hash(Language('ace')), hash('ace'))
+ self.assertEqual(hash(Language('por', 'BR')), hash('pt-BR'))
+ self.assertEqual(hash(Language('srp', script='Cyrl')), hash('sr-Cyrl'))
+ self.assertEqual(hash(Language('eng', 'US', 'Latn')), hash('en-US-Latn'))
+
+ def test_pickle(self):
+ for lang in [Language('fra'),
+ Language('eng', 'US'),
+ Language('srp', script='Latn'),
+ Language('eng', 'US', 'Latn')]:
+ self.assertEqual(pickle.loads(pickle.dumps(lang)), lang)
+
+ def test_str(self):
+ self.assertEqual(Language.fromietf(str(Language('eng', 'US', 'Latn'))), Language('eng', 'US', 'Latn'))
+ self.assertEqual(Language.fromietf(str(Language('fra', 'FR'))), Language('fra', 'FR'))
+ self.assertEqual(Language.fromietf(str(Language('bel'))), Language('bel'))
+
+ def test_register_converter(self):
+ class TestConverter(LanguageReverseConverter):
+ def __init__(self):
+ self.to_test = {'fra': 'test1', 'eng': 'test2'}
+ self.from_test = {'test1': 'fra', 'test2': 'eng'}
+
+ def convert(self, alpha3, country=None, script=None):
+ if alpha3 not in self.to_test:
+ raise LanguageConvertError(alpha3, country, script)
+ return self.to_test[alpha3]
+
+ def reverse(self, test):
+ if test not in self.from_test:
+ raise LanguageReverseError(test)
+ return (self.from_test[test], None)
+ language = Language('fra')
+ self.assertFalse(hasattr(language, 'test'))
+ language_converters['test'] = TestConverter()
+ self.assertTrue(hasattr(language, 'test'))
+ self.assertIn('test', language_converters)
+ self.assertEqual(Language('fra').test, 'test1')
+ self.assertEqual(Language.fromtest('test2').alpha3, 'eng')
+ del language_converters['test']
+ self.assertNotIn('test', language_converters)
+ self.assertRaises(KeyError, lambda: Language.fromtest('test1'))
+ self.assertRaises(AttributeError, lambda: Language('fra').test)
+
+
+def suite():
+ suite = TestSuite()
+ suite.addTest(TestLoader().loadTestsFromTestCase(TestScript))
+ suite.addTest(TestLoader().loadTestsFromTestCase(TestCountry))
+ suite.addTest(TestLoader().loadTestsFromTestCase(TestLanguage))
+ return suite
+
+
+if __name__ == '__main__':
+ TextTestRunner().run(suite())
diff --git a/lib/cachecontrol/__init__.py b/lib/cachecontrol/__init__.py
index fe9875d9fcc07223b4f95a2973591085e281fa09..d6af9b934a92e2fa5646d2e8a42d0228c23c8329 100644
--- a/lib/cachecontrol/__init__.py
+++ b/lib/cachecontrol/__init__.py
@@ -2,9 +2,10 @@
Make it easy to import from cachecontrol without long namespaces.
"""
+__author__ = 'Eric Larson'
+__email__ = 'eric@ionrock.org'
+__version__ = '0.11.5'
+
from .wrapper import CacheControl
from .adapter import CacheControlAdapter
from .controller import CacheController
-
-from lib.requests.packages import urllib3
-urllib3.disable_warnings()
diff --git a/lib/cachecontrol/adapter.py b/lib/cachecontrol/adapter.py
index ef302aceb533401baf19cd1dc883fa94adbd2b66..54f1b51215a56ca1e51e18399b383e6058ab7120 100644
--- a/lib/cachecontrol/adapter.py
+++ b/lib/cachecontrol/adapter.py
@@ -1,15 +1,24 @@
-from lib.requests.adapters import HTTPAdapter
+import functools
+
+from requests.adapters import HTTPAdapter
from .controller import CacheController
from .cache import DictCache
+from .filewrapper import CallbackFileWrapper
+
class CacheControlAdapter(HTTPAdapter):
invalidating_methods = set(['PUT', 'DELETE'])
- def __init__(self, cache=None, cache_etags=True, controller_class=None,
- serializer=None, *args, **kw):
+ def __init__(self, cache=None,
+ cache_etags=True,
+ controller_class=None,
+ serializer=None,
+ heuristic=None,
+ *args, **kw):
super(CacheControlAdapter, self).__init__(*args, **kw)
self.cache = cache or DictCache()
+ self.heuristic = heuristic
controller_factory = controller_class or CacheController
self.controller = controller_factory(
@@ -26,10 +35,13 @@ class CacheControlAdapter(HTTPAdapter):
if request.method == 'GET':
cached_response = self.controller.cached_request(request)
if cached_response:
- return self.build_response(request, cached_response, from_cache=True)
+ return self.build_response(request, cached_response,
+ from_cache=True)
# check for etags and add headers if appropriate
- request.headers.update(self.controller.conditional_headers(request))
+ request.headers.update(
+ self.controller.conditional_headers(request)
+ )
resp = super(CacheControlAdapter, self).send(request, **kw)
@@ -43,6 +55,8 @@ class CacheControlAdapter(HTTPAdapter):
cached response
"""
if not from_cache and request.method == 'GET':
+
+ # apply any expiration heuristics
if response.status == 304:
# We must have sent an ETag request. This could mean
# that we've been expired already or that we simply
@@ -55,14 +69,34 @@ class CacheControlAdapter(HTTPAdapter):
if cached_response is not response:
from_cache = True
+ # We are done with the server response, read a
+ # possible response body (compliant servers will
+ # not return one, but we cannot be 100% sure) and
+ # release the connection back to the pool.
+ response.read(decode_content=False)
+ response.release_conn()
+
response = cached_response
+
+ # We always cache the 301 responses
+ elif response.status == 301:
+ self.controller.cache_response(request, response)
else:
- # try to cache the response
- try:
- self.controller.cache_response(request, response)
- except Exception as e:
- # Failed to cache the results
- pass
+ # Check for any heuristics that might update headers
+ # before trying to cache.
+ if self.heuristic:
+ response = self.heuristic.apply(response)
+
+ # Wrap the response file with a wrapper that will cache the
+ # response when the stream has been consumed.
+ response._fp = CallbackFileWrapper(
+ response._fp,
+ functools.partial(
+ self.controller.cache_response,
+ request,
+ response,
+ )
+ )
resp = super(CacheControlAdapter, self).build_response(
request, response
@@ -77,3 +111,7 @@ class CacheControlAdapter(HTTPAdapter):
resp.from_cache = from_cache
return resp
+
+ def close(self):
+ self.cache.close()
+ super(CacheControlAdapter, self).close()
diff --git a/lib/cachecontrol/cache.py b/lib/cachecontrol/cache.py
index b8a0098c9f5b28c12b13800921c67e99a3067f92..7389a73f8c5dac9ab07b005b115168eb025b735b 100644
--- a/lib/cachecontrol/cache.py
+++ b/lib/cachecontrol/cache.py
@@ -1,9 +1,10 @@
"""
-The cache object API for implementing caches. The default is just a
-dictionary, which in turns means it is not threadsafe for writing.
+The cache object API for implementing caches. The default is a thread
+safe in-memory dictionary.
"""
from threading import Lock
+
class BaseCache(object):
def get(self, key):
@@ -15,6 +16,10 @@ class BaseCache(object):
def delete(self, key):
raise NotImplemented()
+ def close(self):
+ pass
+
+
class DictCache(BaseCache):
def __init__(self, init_dict=None):
diff --git a/lib/cachecontrol/caches/file_cache.py b/lib/cachecontrol/caches/file_cache.py
index 711687ca1c25dc3472fec6a2baefb88e5a756182..504ad1e62966163c41ca9eab541db722a134a7cb 100644
--- a/lib/cachecontrol/caches/file_cache.py
+++ b/lib/cachecontrol/caches/file_cache.py
@@ -1,7 +1,11 @@
import hashlib
import os
-from lockfile import FileLock
+from lockfile import LockFile
+from lockfile.mkdirlockfile import MkdirLockFile
+
+from ..cache import BaseCache
+from ..controller import CacheController
def _secure_open_write(filename, fmode):
@@ -44,22 +48,36 @@ def _secure_open_write(filename, fmode):
raise
-class FileCache(object):
+class FileCache(BaseCache):
def __init__(self, directory, forever=False, filemode=0o0600,
- dirmode=0o0700):
+ dirmode=0o0700, use_dir_lock=None, lock_class=None):
+
+ if use_dir_lock is not None and lock_class is not None:
+ raise ValueError("Cannot use use_dir_lock and lock_class together")
+
+ if use_dir_lock:
+ lock_class = MkdirLockFile
+
+ if lock_class is None:
+ lock_class = LockFile
+
self.directory = directory
self.forever = forever
self.filemode = filemode
+ self.dirmode = dirmode
+ self.lock_class = lock_class
- if not os.path.isdir(self.directory):
- os.makedirs(self.directory, dirmode)
@staticmethod
def encode(x):
return hashlib.sha224(x.encode()).hexdigest()
def _fn(self, name):
- return os.path.join(self.directory, self.encode(name))
+ # NOTE: This method should not change as some may depend on it.
+ # See: https://github.com/ionrock/cachecontrol/issues/63
+ hashed = self.encode(name)
+ parts = list(hashed[:5]) + [hashed]
+ return os.path.join(self.directory, *parts)
def get(self, key):
name = self._fn(key)
@@ -71,7 +89,15 @@ class FileCache(object):
def set(self, key, value):
name = self._fn(key)
- with FileLock(name) as lock:
+
+ # Make sure the directory exists
+ try:
+ os.makedirs(os.path.dirname(name), self.dirmode)
+ except (IOError, OSError):
+ pass
+
+ with self.lock_class(name) as lock:
+ # Write our actual file
with _secure_open_write(lock.path, self.filemode) as fh:
fh.write(value)
@@ -79,3 +105,12 @@ class FileCache(object):
name = self._fn(key)
if not self.forever:
os.remove(name)
+
+
+def url_to_file_path(url, filecache):
+ """Return the file cache path based on the URL.
+
+ This does not ensure the file exists!
+ """
+ key = CacheController.cache_url(url)
+ return filecache._fn(key)
diff --git a/lib/cachecontrol/caches/redis_cache.py b/lib/cachecontrol/caches/redis_cache.py
index 72b8ca3170f8e6e9b6765cbee45aa4e97748c6bd..9f5d55fd98c989c17039fcc8024fee3932646b57 100644
--- a/lib/cachecontrol/caches/redis_cache.py
+++ b/lib/cachecontrol/caches/redis_cache.py
@@ -36,3 +36,6 @@ class RedisCache(object):
caution!"""
for key in self.conn.keys():
self.conn.delete(key)
+
+ def close(self):
+ self.conn.disconnect()
diff --git a/lib/cachecontrol/compat.py b/lib/cachecontrol/compat.py
index aa117d02b3778213d65035563beb430df10a6b63..489eb8687847044a0ed8513334c75465879aaf87 100644
--- a/lib/cachecontrol/compat.py
+++ b/lib/cachecontrol/compat.py
@@ -4,23 +4,20 @@ except ImportError:
from urlparse import urljoin
-try:
- import email.utils
- parsedate_tz = email.utils.parsedate_tz
-except ImportError:
- import email.Utils
- parsedate_tz = email.Utils.parsedate_tz
-
-
try:
import cPickle as pickle
except ImportError:
import pickle
-# Handle the case where the requests has been patched to not have urllib3
-# bundled as part of it's source.
+# Handle the case where the requests module has been patched to not have
+# urllib3 bundled as part of its source.
try:
- from lib.requests.packages.urllib3.response import HTTPResponse
+ from requests.packages.urllib3.response import HTTPResponse
except ImportError:
from urllib3.response import HTTPResponse
+
+try:
+ from requests.packages.urllib3.util import is_fp_closed
+except ImportError:
+ from urllib3.util import is_fp_closed
diff --git a/lib/cachecontrol/controller.py b/lib/cachecontrol/controller.py
index cab3dd2d61fc873a930725c40c510304f886be15..f0380747779e478302e0d2fafae87097a4e87dd3 100644
--- a/lib/cachecontrol/controller.py
+++ b/lib/cachecontrol/controller.py
@@ -4,14 +4,14 @@ The httplib2 algorithms ported for use with requests.
import re
import calendar
import time
-import datetime
+from email.utils import parsedate_tz
-from lib.requests.structures import CaseInsensitiveDict
+from requests.structures import CaseInsensitiveDict
from .cache import DictCache
-from .compat import parsedate_tz
from .serialize import Serializer
+
URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
@@ -32,26 +32,29 @@ class CacheController(object):
self.cache_etags = cache_etags
self.serializer = serializer or Serializer()
- def _urlnorm(self, uri):
+ @classmethod
+ def _urlnorm(cls, uri):
"""Normalize the URL to create a safe key for the cache"""
(scheme, authority, path, query, fragment) = parse_uri(uri)
if not scheme or not authority:
raise Exception("Only absolute URIs are allowed. uri = %s" % uri)
- authority = authority.lower()
+
scheme = scheme.lower()
+ authority = authority.lower()
+
if not path:
path = "/"
# Could do syntax based normalization of the URI before
# computing the digest. See Section 6.2.2 of Std 66.
request_uri = query and "?".join([path, query]) or path
- scheme = scheme.lower()
defrag_uri = scheme + "://" + authority + request_uri
return defrag_uri
- def cache_url(self, uri):
- return self._urlnorm(uri)
+ @classmethod
+ def cache_url(cls, uri):
+ return cls._urlnorm(uri)
def parse_cache_control(self, headers):
"""
@@ -68,13 +71,20 @@ class CacheController(object):
parts = headers[cc_header].split(',')
parts_with_args = [
tuple([x.strip().lower() for x in part.split("=", 1)])
- for part in parts if -1 != part.find("=")]
- parts_wo_args = [(name.strip().lower(), 1)
- for name in parts if -1 == name.find("=")]
+ for part in parts if -1 != part.find("=")
+ ]
+ parts_wo_args = [
+ (name.strip().lower(), 1)
+ for name in parts if -1 == name.find("=")
+ ]
retval = dict(parts_with_args + parts_wo_args)
return retval
def cached_request(self, request):
+ """
+ Return a cached response if it exists in the cache, otherwise
+ return False.
+ """
cache_url = self.cache_url(request.url)
cc = self.parse_cache_control(request.headers)
@@ -95,7 +105,24 @@ class CacheController(object):
if not resp:
return False
+ # If we have a cached 301, return it immediately. We don't
+ # need to test our response for other headers b/c it is
+ # intrinsically "cacheable" as it is Permanent.
+ # See:
+ # https://tools.ietf.org/html/rfc7231#section-6.4.2
+ #
+ # Client can try to refresh the value by repeating the request
+ # with cache busting headers as usual (ie no-cache).
+ if resp.status == 301:
+ return resp
+
headers = CaseInsensitiveDict(resp.headers)
+ if not headers or 'date' not in headers:
+ # With date or etag, the cached response can never be used
+ # and should be deleted.
+ if 'etag' not in headers:
+ self.cache.delete(cache_url)
+ return False
now = time.time()
date = calendar.timegm(
@@ -104,15 +131,19 @@ class CacheController(object):
current_age = max(0, now - date)
# TODO: There is an assumption that the result will be a
- # urllib3 response object. This may not be best since we
- # could probably avoid instantiating or constructing the
- # response until we know we need it.
+ # urllib3 response object. This may not be best since we
+ # could probably avoid instantiating or constructing the
+ # response until we know we need it.
resp_cc = self.parse_cache_control(headers)
# determine freshness
freshness_lifetime = 0
+
+ # Check the max-age pragma in the cache control header
if 'max-age' in resp_cc and resp_cc['max-age'].isdigit():
freshness_lifetime = int(resp_cc['max-age'])
+
+ # If there isn't a max-age, check for an expires header
elif 'expires' in headers:
expires = parsedate_tz(headers['expires'])
if expires is not None:
@@ -163,32 +194,24 @@ class CacheController(object):
return new_headers
- def cache_response(self, request, response):
+ def cache_response(self, request, response, body=None):
"""
Algorithm for caching requests.
This assumes a requests Response object.
"""
# From httplib2: Don't cache 206's since we aren't going to
- # handle byte range requests
- if response.status not in [200, 203]:
+ # handle byte range requests
+ if response.status not in [200, 203, 300, 301]:
return
- # Cache Session Params
- cache_auto = getattr(request, 'cache_auto', False)
- cache_urls = getattr(request, 'cache_urls', [])
- cache_max_age = getattr(request, 'cache_max_age', None)
-
response_headers = CaseInsensitiveDict(response.headers)
- # Check if we are wanting to cache responses from specific urls only
- cache_url = self.cache_url(request.url)
- if len(cache_urls) > 0 and not any(s in cache_url for s in cache_urls):
- return
-
cc_req = self.parse_cache_control(request.headers)
cc = self.parse_cache_control(response_headers)
+ cache_url = self.cache_url(request.url)
+
# Delete it from the cache if we happen to have it stored there
no_store = cc.get('no-store') or cc_req.get('no-store')
if no_store and self.cache.get(cache_url):
@@ -196,21 +219,18 @@ class CacheController(object):
# If we've been given an etag, then keep the response
if self.cache_etags and 'etag' in response_headers:
- self.cache.set(cache_url, self.serializer.dumps(request, response))
-
- # If we want to cache sites not setup with cache headers then add the proper headers and keep the response
- elif cache_auto and not cc and response_headers:
- headers = {'Cache-Control': 'public,max-age=%d' % int(cache_max_age or 900)}
- response.headers.update(headers)
-
- if 'expires' not in response_headers:
- if getattr(response_headers, 'expires', None) is None:
- expires = datetime.datetime.utcnow() + datetime.timedelta(days=1)
- expires = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
- headers = {'Expires': expires}
- response.headers.update(headers)
-
- self.cache.set(cache_url, self.serializer.dumps(request, response))
+ self.cache.set(
+ cache_url,
+ self.serializer.dumps(request, response, body=body),
+ )
+
+ # Add to the cache any 301s. We do this before looking that
+ # the Date headers.
+ elif response.status == 301:
+ self.cache.set(
+ cache_url,
+ self.serializer.dumps(request, response)
+ )
# Add to the cache if the response headers demand it. If there
# is no date header then we can't do anything about expiring
@@ -219,10 +239,10 @@ class CacheController(object):
# cache when there is a max-age > 0
if cc and cc.get('max-age'):
if int(cc['max-age']) > 0:
- if isinstance(cache_max_age, int):
- cc['max-age'] = int(cache_max_age)
- response.headers['cache-control'] = ''.join(['%s=%s' % (key, value) for (key, value) in cc.items()])
- self.cache.set(cache_url, self.serializer.dumps(request, response))
+ self.cache.set(
+ cache_url,
+ self.serializer.dumps(request, response, body=body),
+ )
# If the request can expire, it means we should cache it
# in the meantime.
@@ -230,7 +250,7 @@ class CacheController(object):
if response_headers['expires']:
self.cache.set(
cache_url,
- self.serializer.dumps(request, response),
+ self.serializer.dumps(request, response, body=body),
)
def update_cached_response(self, request, response):
@@ -242,14 +262,30 @@ class CacheController(object):
"""
cache_url = self.cache_url(request.url)
- cached_response = self.serializer.loads(request, self.cache.get(cache_url))
+ cached_response = self.serializer.loads(
+ request,
+ self.cache.get(cache_url)
+ )
if not cached_response:
# we didn't have a cached response
return response
- # did so lets update our headers
- cached_response.headers.update(response.headers)
+ # Lets update our headers with the headers from the new request:
+ # http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-26#section-4.1
+ #
+ # The server isn't supposed to send headers that would make
+ # the cached body invalid. But... just in case, we'll be sure
+ # to strip out ones we know that might be problmatic due to
+ # typical assumptions.
+ excluded_headers = [
+ "content-length",
+ ]
+
+ cached_response.headers.update(
+ dict((k, v) for k, v in response.headers.items()
+ if k.lower() not in excluded_headers)
+ )
# we want a 200 b/c we have content via the cache
cached_response.status = 200
diff --git a/lib/cachecontrol/filewrapper.py b/lib/cachecontrol/filewrapper.py
new file mode 100644
index 0000000000000000000000000000000000000000..4b91bce04b08147adc36d5f84909482160d39726
--- /dev/null
+++ b/lib/cachecontrol/filewrapper.py
@@ -0,0 +1,63 @@
+from io import BytesIO
+
+
+class CallbackFileWrapper(object):
+ """
+ Small wrapper around a fp object which will tee everything read into a
+ buffer, and when that file is closed it will execute a callback with the
+ contents of that buffer.
+
+ All attributes are proxied to the underlying file object.
+
+ This class uses members with a double underscore (__) leading prefix so as
+ not to accidentally shadow an attribute.
+ """
+
+ def __init__(self, fp, callback):
+ self.__buf = BytesIO()
+ self.__fp = fp
+ self.__callback = callback
+
+ def __getattr__(self, name):
+ # The vaguaries of garbage collection means that self.__fp is
+ # not always set. By using __getattribute__ and the private
+ # name[0] allows looking up the attribute value and raising an
+ # AttributeError when it doesn't exist. This stop thigns from
+ # infinitely recursing calls to getattr in the case where
+ # self.__fp hasn't been set.
+ #
+ # [0] https://docs.python.org/2/reference/expressions.html#atom-identifiers
+ fp = self.__getattribute__('_CallbackFileWrapper__fp')
+ return getattr(fp, name)
+
+ def __is_fp_closed(self):
+ try:
+ return self.__fp.fp is None
+ except AttributeError:
+ pass
+
+ try:
+ return self.__fp.closed
+ except AttributeError:
+ pass
+
+ # We just don't cache it then.
+ # TODO: Add some logging here...
+ return False
+
+ def read(self, amt=None):
+ data = self.__fp.read(amt)
+ self.__buf.write(data)
+
+ if self.__is_fp_closed():
+ if self.__callback:
+ self.__callback(self.__buf.getvalue())
+
+ # We assign this to None here, because otherwise we can get into
+ # really tricky problems where the CPython interpreter dead locks
+ # because the callback is holding a reference to something which
+ # has a __del__ method. Setting this to None breaks the cycle
+ # and allows the garbage collector to do it's thing normally.
+ self.__callback = None
+
+ return data
diff --git a/lib/cachecontrol/heuristics.py b/lib/cachecontrol/heuristics.py
new file mode 100644
index 0000000000000000000000000000000000000000..01b63141624dbf30d7834c8531ced91f7826667f
--- /dev/null
+++ b/lib/cachecontrol/heuristics.py
@@ -0,0 +1,134 @@
+import calendar
+import time
+
+from email.utils import formatdate, parsedate, parsedate_tz
+
+from datetime import datetime, timedelta
+
+TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT"
+
+
+def expire_after(delta, date=None):
+ date = date or datetime.now()
+ return date + delta
+
+
+def datetime_to_header(dt):
+ return formatdate(calendar.timegm(dt.timetuple()))
+
+
+class BaseHeuristic(object):
+
+ def warning(self, response):
+ """
+ Return a valid 1xx warning header value describing the cache
+ adjustments.
+
+ The response is provided too allow warnings like 113
+ http://tools.ietf.org/html/rfc7234#section-5.5.4 where we need
+ to explicitly say response is over 24 hours old.
+ """
+ return '110 - "Response is Stale"'
+
+ def update_headers(self, response):
+ """Update the response headers with any new headers.
+
+ NOTE: This SHOULD always include some Warning header to
+ signify that the response was cached by the client, not
+ by way of the provided headers.
+ """
+ return {}
+
+ def apply(self, response):
+ warning_header_value = self.warning(response)
+ response.headers.update(self.update_headers(response))
+ if warning_header_value is not None:
+ response.headers.update({'Warning': warning_header_value})
+ return response
+
+
+class OneDayCache(BaseHeuristic):
+ """
+ Cache the response by providing an expires 1 day in the
+ future.
+ """
+ def update_headers(self, response):
+ headers = {}
+
+ if 'expires' not in response.headers:
+ date = parsedate(response.headers['date'])
+ expires = expire_after(timedelta(days=1),
+ date=datetime(*date[:6]))
+ headers['expires'] = datetime_to_header(expires)
+ headers['cache-control'] = 'public'
+ return headers
+
+
+class ExpiresAfter(BaseHeuristic):
+ """
+ Cache **all** requests for a defined time period.
+ """
+
+ def __init__(self, **kw):
+ self.delta = timedelta(**kw)
+
+ def update_headers(self, response):
+ expires = expire_after(self.delta)
+ return {
+ 'expires': datetime_to_header(expires),
+ 'cache-control': 'public',
+ }
+
+ def warning(self, response):
+ tmpl = '110 - Automatically cached for %s. Response might be stale'
+ return tmpl % self.delta
+
+
+class LastModified(BaseHeuristic):
+ """
+ If there is no Expires header already, fall back on Last-Modified
+ using the heuristic from
+ http://tools.ietf.org/html/rfc7234#section-4.2.2
+ to calculate a reasonable value.
+
+ Firefox also does something like this per
+ https://developer.mozilla.org/en-US/docs/Web/HTTP/Caching_FAQ
+ http://lxr.mozilla.org/mozilla-release/source/netwerk/protocol/http/nsHttpResponseHead.cpp#397
+ Unlike mozilla we limit this to 24-hr.
+ """
+ cacheable_by_default_statuses = set([
+ 200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501
+ ])
+
+ def update_headers(self, resp):
+ headers = resp.headers
+
+ if 'expires' in headers:
+ return {}
+
+ if 'cache-control' in headers and headers['cache-control'] != 'public':
+ return {}
+
+ if resp.status not in self.cacheable_by_default_statuses:
+ return {}
+
+ if 'date' not in headers or 'last-modified' not in headers:
+ return {}
+
+ date = calendar.timegm(parsedate_tz(headers['date']))
+ last_modified = parsedate(headers['last-modified'])
+ if date is None or last_modified is None:
+ return {}
+
+ now = time.time()
+ current_age = max(0, now - date)
+ delta = date - calendar.timegm(last_modified)
+ freshness_lifetime = max(0, min(delta / 10, 24 * 3600))
+ if freshness_lifetime <= current_age:
+ return {}
+
+ expires = date + freshness_lifetime
+ return {'expires': time.strftime(TIME_FMT, time.gmtime(expires))}
+
+ def warning(self, resp):
+ return None
diff --git a/lib/cachecontrol/patch_requests.py b/lib/cachecontrol/patch_requests.py
deleted file mode 100644
index a5e02d10d612936b101a47c918e09ae7ce2ab531..0000000000000000000000000000000000000000
--- a/lib/cachecontrol/patch_requests.py
+++ /dev/null
@@ -1,56 +0,0 @@
-from lib import requests
-
-from lib.requests import models
-from lib.requests.packages.urllib3.response import HTTPResponse
-
-__attrs__ = [
- '_content',
- 'status_code',
- 'headers',
- 'url',
- 'history',
- 'encoding',
- 'reason',
- 'cookies',
- 'elapsed',
-]
-
-
-def response_getstate(self):
- # consume everything
- if not self._content_consumed:
- self.content
-
- state = dict(
- (attr, getattr(self, attr, None))
- for attr in __attrs__
- )
-
- # deal with our raw content b/c we need it for our cookie jar
- state['raw_original_response'] = self.raw._original_response
- return state
-
-
-def response_setstate(self, state):
- for name, value in state.items():
- if name != 'raw_original_response':
- setattr(self, name, value)
-
- setattr(self, 'raw', HTTPResponse())
- self.raw._original_response = state['raw_original_response']
-
-
-def make_responses_pickleable():
- try:
- version_parts = [int(part) for part in requests.__version__.split('.')]
-
- # must be >= 2.2.x
- if not version_parts[0] >= 2 or not version_parts[1] >= 2:
- models.Response.__getstate__ = response_getstate
- models.Response.__setstate__ = response_setstate
- except:
- raise
- pass
-
-
-make_responses_pickleable()
diff --git a/lib/cachecontrol/serialize.py b/lib/cachecontrol/serialize.py
index fd49a42f377b71493434e11b93b039b21f5029ad..6b17d80ebbf2374d5d091dc0635e66b57b8aa1e2 100644
--- a/lib/cachecontrol/serialize.py
+++ b/lib/cachecontrol/serialize.py
@@ -1,27 +1,59 @@
+import base64
import io
+import json
+import zlib
-from lib.requests.structures import CaseInsensitiveDict
+from requests.structures import CaseInsensitiveDict
from .compat import HTTPResponse, pickle
+def _b64_encode_bytes(b):
+ return base64.b64encode(b).decode("ascii")
+
+
+def _b64_encode_str(s):
+ return _b64_encode_bytes(s.encode("utf8"))
+
+
+def _b64_decode_bytes(b):
+ return base64.b64decode(b.encode("ascii"))
+
+
+def _b64_decode_str(s):
+ return _b64_decode_bytes(s).decode("utf8")
+
+
class Serializer(object):
+
def dumps(self, request, response, body=None):
response_headers = CaseInsensitiveDict(response.headers)
if body is None:
- # TODO: Figure out a way to handle this which doesn't break
- # streaming
body = response.read(decode_content=False)
+
+ # NOTE: 99% sure this is dead code. I'm only leaving it
+ # here b/c I don't have a test yet to prove
+ # it. Basically, before using
+ # `cachecontrol.filewrapper.CallbackFileWrapper`,
+ # this made an effort to reset the file handle. The
+ # `CallbackFileWrapper` short circuits this code by
+ # setting the body as the content is consumed, the
+ # result being a `body` argument is *always* passed
+ # into cache_response, and in turn,
+ # `Serializer.dump`.
response._fp = io.BytesIO(body)
data = {
"response": {
- "body": body,
- "headers": response.headers,
+ "body": _b64_encode_bytes(body),
+ "headers": dict(
+ (_b64_encode_str(k), _b64_encode_str(v))
+ for k, v in response.headers.items()
+ ),
"status": response.status,
"version": response.version,
- "reason": response.reason,
+ "reason": _b64_encode_str(response.reason),
"strict": response.strict,
"decode_content": response.decode_content,
},
@@ -35,7 +67,20 @@ class Serializer(object):
header = header.strip()
data["vary"][header] = request.headers.get(header, None)
- return b"cc=1," + pickle.dumps(data, pickle.HIGHEST_PROTOCOL)
+ # Encode our Vary headers to ensure they can be serialized as JSON
+ data["vary"] = dict(
+ (_b64_encode_str(k), _b64_encode_str(v) if v is not None else v)
+ for k, v in data["vary"].items()
+ )
+
+ return b",".join([
+ b"cc=2",
+ zlib.compress(
+ json.dumps(
+ data, separators=(",", ":"), sort_keys=True,
+ ).encode("utf8"),
+ ),
+ ])
def loads(self, request, data):
# Short circuit if we've been given an empty set of data
@@ -66,18 +111,10 @@ class Serializer(object):
# just treat it as a miss and return None
return
- def _loads_v0(self, request, data):
- # The original legacy cache data. This doesn't contain enough
- # information to construct everything we need, so we'll treat this as
- # a miss.
- return
-
- def _loads_v1(self, request, data):
- try:
- cached = pickle.loads(data)
- except ValueError:
- return
-
+ def prepare_response(self, request, cached):
+ """Verify our vary headers match and construct a real urllib3
+ HTTPResponse object.
+ """
# Special case the '*' Vary value as it means we cannot actually
# determine if the cached response is suitable for this request.
if "*" in cached.get("vary", {}):
@@ -89,9 +126,59 @@ class Serializer(object):
if request.headers.get(header, None) != value:
return
- body = io.BytesIO(cached["response"].pop("body"))
+ body_raw = cached["response"].pop("body")
+
+ try:
+ body = io.BytesIO(body_raw)
+ except TypeError:
+ # This can happen if cachecontrol serialized to v1 format (pickle)
+ # using Python 2. A Python 2 str(byte string) will be unpickled as
+ # a Python 3 str (unicode string), which will cause the above to
+ # fail with:
+ #
+ # TypeError: 'str' does not support the buffer interface
+ body = io.BytesIO(body_raw.encode('utf8'))
+
return HTTPResponse(
body=body,
preload_content=False,
**cached["response"]
)
+
+ def _loads_v0(self, request, data):
+ # The original legacy cache data. This doesn't contain enough
+ # information to construct everything we need, so we'll treat this as
+ # a miss.
+ return
+
+ def _loads_v1(self, request, data):
+ try:
+ cached = pickle.loads(data)
+ except ValueError:
+ return
+
+ return self.prepare_response(request, cached)
+
+ def _loads_v2(self, request, data):
+ try:
+ cached = json.loads(zlib.decompress(data).decode("utf8"))
+ except ValueError:
+ return
+
+ # We need to decode the items that we've base64 encoded
+ cached["response"]["body"] = _b64_decode_bytes(
+ cached["response"]["body"]
+ )
+ cached["response"]["headers"] = dict(
+ (_b64_decode_str(k), _b64_decode_str(v))
+ for k, v in cached["response"]["headers"].items()
+ )
+ cached["response"]["reason"] = _b64_decode_str(
+ cached["response"]["reason"],
+ )
+ cached["vary"] = dict(
+ (_b64_decode_str(k), _b64_decode_str(v) if v is not None else v)
+ for k, v in cached["vary"].items()
+ )
+
+ return self.prepare_response(request, cached)
diff --git a/lib/cachecontrol/session.py b/lib/cachecontrol/session.py
deleted file mode 100644
index 15211fa5a68807c5cdf4ca2790902b5e808a3501..0000000000000000000000000000000000000000
--- a/lib/cachecontrol/session.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from lib.requests.sessions import Session
-
-class CacheControlSession(Session):
- def __init__(self):
- super(CacheControlSession, self).__init__()
-
- def get(self, *args, **kw):
- # auto-cache response
- self.cache_auto = False
- if kw.get('cache_auto'):
- self.cache_auto = kw.pop('cache_auto')
-
- # urls allowed to cache
- self.cache_urls = []
- if kw.get('cache_urls'):
- self.cache_urls = [str(args[0])] + kw.pop('cache_urls')
-
- # timeout for cached responses
- self.cache_max_age = None
- if kw.get('cache_max_age'):
- self.cache_max_age = int(kw.pop('cache_max_age'))
-
- return super(CacheControlSession, self).get(*args, **kw)
-
- def prepare_request(self, *args, **kw):
- # get response
- req = super(CacheControlSession, self).prepare_request(*args, **kw)
-
- # attach params to request
- req.cache_auto = self.cache_auto
- req.cache_urls = self.cache_urls
- req.cache_max_age = self.cache_max_age
-
- return req
diff --git a/lib/cachecontrol/wrapper.py b/lib/cachecontrol/wrapper.py
index 0dc608a04c20756ff99dc7e9a82b2001abfb54fb..ea421aa7e712b3950fc30dff914b5240b747ce21 100644
--- a/lib/cachecontrol/wrapper.py
+++ b/lib/cachecontrol/wrapper.py
@@ -1,14 +1,19 @@
from .adapter import CacheControlAdapter
from .cache import DictCache
-from .session import CacheControlSession
-def CacheControl(sess=None, cache=None, cache_etags=True, serializer=None):
- sess = sess or CacheControlSession()
+
+def CacheControl(sess,
+ cache=None,
+ cache_etags=True,
+ serializer=None,
+ heuristic=None):
+
cache = cache or DictCache()
adapter = CacheControlAdapter(
cache,
cache_etags=cache_etags,
serializer=serializer,
+ heuristic=heuristic,
)
sess.mount('http://', adapter)
sess.mount('https://', adapter)
diff --git a/lib/dogpile/__init__.py b/lib/dogpile/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..f48ad10528712b2b8960f1863d156b88ed1ce311
--- /dev/null
+++ b/lib/dogpile/__init__.py
@@ -0,0 +1,6 @@
+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+ __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+ from pkgutil import extend_path
+ __path__ = extend_path(__path__, __name__)
diff --git a/lib/dogpile/cache/__init__.py b/lib/dogpile/cache/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..13e6fdad1353ede969d6047fb15acc3f9cd11091
--- /dev/null
+++ b/lib/dogpile/cache/__init__.py
@@ -0,0 +1,3 @@
+__version__ = '0.5.7'
+
+from .region import CacheRegion, register_backend, make_region # noqa
diff --git a/lib/dogpile/cache/api.py b/lib/dogpile/cache/api.py
new file mode 100644
index 0000000000000000000000000000000000000000..85b6de181c2058f31161c3e399fda2d5a3c0b31d
--- /dev/null
+++ b/lib/dogpile/cache/api.py
@@ -0,0 +1,195 @@
+import operator
+from .compat import py3k
+
+
+class NoValue(object):
+ """Describe a missing cache value.
+
+ The :attr:`.NO_VALUE` module global
+ should be used.
+
+ """
+ @property
+ def payload(self):
+ return self
+
+ if py3k:
+ def __bool__(self): # pragma NO COVERAGE
+ return False
+ else:
+ def __nonzero__(self): # pragma NO COVERAGE
+ return False
+
+NO_VALUE = NoValue()
+"""Value returned from ``get()`` that describes
+a key not present."""
+
+
+class CachedValue(tuple):
+ """Represent a value stored in the cache.
+
+ :class:`.CachedValue` is a two-tuple of
+ ``(payload, metadata)``, where ``metadata``
+ is dogpile.cache's tracking information (
+ currently the creation time). The metadata
+ and tuple structure is pickleable, if
+ the backend requires serialization.
+
+ """
+ payload = property(operator.itemgetter(0))
+ """Named accessor for the payload."""
+
+ metadata = property(operator.itemgetter(1))
+ """Named accessor for the dogpile.cache metadata dictionary."""
+
+ def __new__(cls, payload, metadata):
+ return tuple.__new__(cls, (payload, metadata))
+
+ def __reduce__(self):
+ return CachedValue, (self.payload, self.metadata)
+
+
+class CacheBackend(object):
+ """Base class for backend implementations."""
+
+ key_mangler = None
+ """Key mangling function.
+
+ May be None, or otherwise declared
+ as an ordinary instance method.
+
+ """
+
+ def __init__(self, arguments): # pragma NO COVERAGE
+ """Construct a new :class:`.CacheBackend`.
+
+ Subclasses should override this to
+ handle the given arguments.
+
+ :param arguments: The ``arguments`` parameter
+ passed to :func:`.make_registry`.
+
+ """
+ raise NotImplementedError()
+
+ @classmethod
+ def from_config_dict(cls, config_dict, prefix):
+ prefix_len = len(prefix)
+ return cls(
+ dict(
+ (key[prefix_len:], config_dict[key])
+ for key in config_dict
+ if key.startswith(prefix)
+ )
+ )
+
+ def get_mutex(self, key):
+ """Return an optional mutexing object for the given key.
+
+ This object need only provide an ``acquire()``
+ and ``release()`` method.
+
+ May return ``None``, in which case the dogpile
+ lock will use a regular ``threading.Lock``
+ object to mutex concurrent threads for
+ value creation. The default implementation
+ returns ``None``.
+
+ Different backends may want to provide various
+ kinds of "mutex" objects, such as those which
+ link to lock files, distributed mutexes,
+ memcached semaphores, etc. Whatever
+ kind of system is best suited for the scope
+ and behavior of the caching backend.
+
+ A mutex that takes the key into account will
+ allow multiple regenerate operations across
+ keys to proceed simultaneously, while a mutex
+ that does not will serialize regenerate operations
+ to just one at a time across all keys in the region.
+ The latter approach, or a variant that involves
+ a modulus of the given key's hash value,
+ can be used as a means of throttling the total
+ number of value recreation operations that may
+ proceed at one time.
+
+ """
+ return None
+
+ def get(self, key): # pragma NO COVERAGE
+ """Retrieve a value from the cache.
+
+ The returned value should be an instance of
+ :class:`.CachedValue`, or ``NO_VALUE`` if
+ not present.
+
+ """
+ raise NotImplementedError()
+
+ def get_multi(self, keys): # pragma NO COVERAGE
+ """Retrieve multiple values from the cache.
+
+ The returned value should be a list, corresponding
+ to the list of keys given.
+
+ .. versionadded:: 0.5.0
+
+ """
+ raise NotImplementedError()
+
+ def set(self, key, value): # pragma NO COVERAGE
+ """Set a value in the cache.
+
+ The key will be whatever was passed
+ to the registry, processed by the
+ "key mangling" function, if any.
+ The value will always be an instance
+ of :class:`.CachedValue`.
+
+ """
+ raise NotImplementedError()
+
+ def set_multi(self, mapping): # pragma NO COVERAGE
+ """Set multiple values in the cache.
+
+ The key will be whatever was passed
+ to the registry, processed by the
+ "key mangling" function, if any.
+ The value will always be an instance
+ of :class:`.CachedValue`.
+
+ .. versionadded:: 0.5.0
+
+ """
+ raise NotImplementedError()
+
+ def delete(self, key): # pragma NO COVERAGE
+ """Delete a value from the cache.
+
+ The key will be whatever was passed
+ to the registry, processed by the
+ "key mangling" function, if any.
+
+ The behavior here should be idempotent,
+ that is, can be called any number of times
+ regardless of whether or not the
+ key exists.
+ """
+ raise NotImplementedError()
+
+ def delete_multi(self, keys): # pragma NO COVERAGE
+ """Delete multiple values from the cache.
+
+ The key will be whatever was passed
+ to the registry, processed by the
+ "key mangling" function, if any.
+
+ The behavior here should be idempotent,
+ that is, can be called any number of times
+ regardless of whether or not the
+ key exists.
+
+ .. versionadded:: 0.5.0
+
+ """
+ raise NotImplementedError()
diff --git a/lib/dogpile/cache/backends/__init__.py b/lib/dogpile/cache/backends/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..041f05a3ed0b626281f9fd2bc65f70d206c30023
--- /dev/null
+++ b/lib/dogpile/cache/backends/__init__.py
@@ -0,0 +1,22 @@
+from dogpile.cache.region import register_backend
+
+register_backend(
+ "dogpile.cache.null", "dogpile.cache.backends.null", "NullBackend")
+register_backend(
+ "dogpile.cache.dbm", "dogpile.cache.backends.file", "DBMBackend")
+register_backend(
+ "dogpile.cache.pylibmc", "dogpile.cache.backends.memcached",
+ "PylibmcBackend")
+register_backend(
+ "dogpile.cache.bmemcached", "dogpile.cache.backends.memcached",
+ "BMemcachedBackend")
+register_backend(
+ "dogpile.cache.memcached", "dogpile.cache.backends.memcached",
+ "MemcachedBackend")
+register_backend(
+ "dogpile.cache.memory", "dogpile.cache.backends.memory", "MemoryBackend")
+register_backend(
+ "dogpile.cache.memory_pickle", "dogpile.cache.backends.memory",
+ "MemoryPickleBackend")
+register_backend(
+ "dogpile.cache.redis", "dogpile.cache.backends.redis", "RedisBackend")
diff --git a/lib/dogpile/cache/backends/file.py b/lib/dogpile/cache/backends/file.py
new file mode 100644
index 0000000000000000000000000000000000000000..42d749299cc437c4d166676ad26c56e6906d31bf
--- /dev/null
+++ b/lib/dogpile/cache/backends/file.py
@@ -0,0 +1,447 @@
+"""
+File Backends
+------------------
+
+Provides backends that deal with local filesystem access.
+
+"""
+
+from __future__ import with_statement
+from dogpile.cache.api import CacheBackend, NO_VALUE
+from contextlib import contextmanager
+from dogpile.cache import compat
+from dogpile.cache import util
+import os
+
+__all__ = 'DBMBackend', 'FileLock', 'AbstractFileLock'
+
+
+class DBMBackend(CacheBackend):
+ """A file-backend using a dbm file to store keys.
+
+ Basic usage::
+
+ from dogpile.cache import make_region
+
+ region = make_region().configure(
+ 'dogpile.cache.dbm',
+ expiration_time = 3600,
+ arguments = {
+ "filename":"/path/to/cachefile.dbm"
+ }
+ )
+
+ DBM access is provided using the Python ``anydbm`` module,
+ which selects a platform-specific dbm module to use.
+ This may be made to be more configurable in a future
+ release.
+
+ Note that different dbm modules have different behaviors.
+ Some dbm implementations handle their own locking, while
+ others don't. The :class:`.DBMBackend` uses a read/write
+ lockfile by default, which is compatible even with those
+ DBM implementations for which this is unnecessary,
+ though the behavior can be disabled.
+
+ The DBM backend by default makes use of two lockfiles.
+ One is in order to protect the DBM file itself from
+ concurrent writes, the other is to coordinate
+ value creation (i.e. the dogpile lock). By default,
+ these lockfiles use the ``flock()`` system call
+ for locking; this is **only available on Unix
+ platforms**. An alternative lock implementation, such as one
+ which is based on threads or uses a third-party system
+ such as `portalocker <https://pypi.python.org/pypi/portalocker>`_,
+ can be dropped in using the ``lock_factory`` argument
+ in conjunction with the :class:`.AbstractFileLock` base class.
+
+ Currently, the dogpile lock is against the entire
+ DBM file, not per key. This means there can
+ only be one "creator" job running at a time
+ per dbm file.
+
+ A future improvement might be to have the dogpile lock
+ using a filename that's based on a modulus of the key.
+ Locking on a filename that uniquely corresponds to the
+ key is problematic, since it's not generally safe to
+ delete lockfiles as the application runs, implying an
+ unlimited number of key-based files would need to be
+ created and never deleted.
+
+ Parameters to the ``arguments`` dictionary are
+ below.
+
+ :param filename: path of the filename in which to
+ create the DBM file. Note that some dbm backends
+ will change this name to have additional suffixes.
+ :param rw_lockfile: the name of the file to use for
+ read/write locking. If omitted, a default name
+ is used by appending the suffix ".rw.lock" to the
+ DBM filename. If False, then no lock is used.
+ :param dogpile_lockfile: the name of the file to use
+ for value creation, i.e. the dogpile lock. If
+ omitted, a default name is used by appending the
+ suffix ".dogpile.lock" to the DBM filename. If
+ False, then dogpile.cache uses the default dogpile
+ lock, a plain thread-based mutex.
+ :param lock_factory: a function or class which provides
+ for a read/write lock. Defaults to :class:`.FileLock`.
+ Custom implementations need to implement context-manager
+ based ``read()`` and ``write()`` functions - the
+ :class:`.AbstractFileLock` class is provided as a base class
+ which provides these methods based on individual read/write lock
+ functions. E.g. to replace the lock with the dogpile.core
+ :class:`.ReadWriteMutex`::
+
+ from dogpile.core.readwrite_lock import ReadWriteMutex
+ from dogpile.cache.backends.file import AbstractFileLock
+
+ class MutexLock(AbstractFileLock):
+ def __init__(self, filename):
+ self.mutex = ReadWriteMutex()
+
+ def acquire_read_lock(self, wait):
+ ret = self.mutex.acquire_read_lock(wait)
+ return wait or ret
+
+ def acquire_write_lock(self, wait):
+ ret = self.mutex.acquire_write_lock(wait)
+ return wait or ret
+
+ def release_read_lock(self):
+ return self.mutex.release_read_lock()
+
+ def release_write_lock(self):
+ return self.mutex.release_write_lock()
+
+ from dogpile.cache import make_region
+
+ region = make_region().configure(
+ "dogpile.cache.dbm",
+ expiration_time=300,
+ arguments={
+ "filename": "file.dbm",
+ "lock_factory": MutexLock
+ }
+ )
+
+ While the included :class:`.FileLock` uses ``os.flock()``, a
+ windows-compatible implementation can be built using a library
+ such as `portalocker <https://pypi.python.org/pypi/portalocker>`_.
+
+ .. versionadded:: 0.5.2
+
+
+
+ """
+ def __init__(self, arguments):
+ self.filename = os.path.abspath(
+ os.path.normpath(arguments['filename'])
+ )
+ dir_, filename = os.path.split(self.filename)
+
+ self.lock_factory = arguments.get("lock_factory", FileLock)
+ self._rw_lock = self._init_lock(
+ arguments.get('rw_lockfile'),
+ ".rw.lock", dir_, filename)
+ self._dogpile_lock = self._init_lock(
+ arguments.get('dogpile_lockfile'),
+ ".dogpile.lock",
+ dir_, filename,
+ util.KeyReentrantMutex.factory)
+
+ # TODO: make this configurable
+ if compat.py3k:
+ import dbm
+ else:
+ import anydbm as dbm
+ self.dbmmodule = dbm
+ self._init_dbm_file()
+
+ def _init_lock(self, argument, suffix, basedir, basefile, wrapper=None):
+ if argument is None:
+ lock = self.lock_factory(os.path.join(basedir, basefile + suffix))
+ elif argument is not False:
+ lock = self.lock_factory(
+ os.path.abspath(
+ os.path.normpath(argument)
+ ))
+ else:
+ return None
+ if wrapper:
+ lock = wrapper(lock)
+ return lock
+
+ def _init_dbm_file(self):
+ exists = os.access(self.filename, os.F_OK)
+ if not exists:
+ for ext in ('db', 'dat', 'pag', 'dir'):
+ if os.access(self.filename + os.extsep + ext, os.F_OK):
+ exists = True
+ break
+ if not exists:
+ fh = self.dbmmodule.open(self.filename, 'c')
+ fh.close()
+
+ def get_mutex(self, key):
+ # using one dogpile for the whole file. Other ways
+ # to do this might be using a set of files keyed to a
+ # hash/modulus of the key. the issue is it's never
+ # really safe to delete a lockfile as this can
+ # break other processes trying to get at the file
+ # at the same time - so handling unlimited keys
+ # can't imply unlimited filenames
+ if self._dogpile_lock:
+ return self._dogpile_lock(key)
+ else:
+ return None
+
+ @contextmanager
+ def _use_rw_lock(self, write):
+ if self._rw_lock is None:
+ yield
+ elif write:
+ with self._rw_lock.write():
+ yield
+ else:
+ with self._rw_lock.read():
+ yield
+
+ @contextmanager
+ def _dbm_file(self, write):
+ with self._use_rw_lock(write):
+ dbm = self.dbmmodule.open(
+ self.filename,
+ "w" if write else "r")
+ yield dbm
+ dbm.close()
+
+ def get(self, key):
+ with self._dbm_file(False) as dbm:
+ if hasattr(dbm, 'get'):
+ value = dbm.get(key, NO_VALUE)
+ else:
+ # gdbm objects lack a .get method
+ try:
+ value = dbm[key]
+ except KeyError:
+ value = NO_VALUE
+ if value is not NO_VALUE:
+ value = compat.pickle.loads(value)
+ return value
+
+ def get_multi(self, keys):
+ return [self.get(key) for key in keys]
+
+ def set(self, key, value):
+ with self._dbm_file(True) as dbm:
+ dbm[key] = compat.pickle.dumps(value,
+ compat.pickle.HIGHEST_PROTOCOL)
+
+ def set_multi(self, mapping):
+ with self._dbm_file(True) as dbm:
+ for key, value in mapping.items():
+ dbm[key] = compat.pickle.dumps(value,
+ compat.pickle.HIGHEST_PROTOCOL)
+
+ def delete(self, key):
+ with self._dbm_file(True) as dbm:
+ try:
+ del dbm[key]
+ except KeyError:
+ pass
+
+ def delete_multi(self, keys):
+ with self._dbm_file(True) as dbm:
+ for key in keys:
+ try:
+ del dbm[key]
+ except KeyError:
+ pass
+
+
+class AbstractFileLock(object):
+ """Coordinate read/write access to a file.
+
+ typically is a file-based lock but doesn't necessarily have to be.
+
+ The default implementation here is :class:`.FileLock`.
+
+ Implementations should provide the following methods::
+
+ * __init__()
+ * acquire_read_lock()
+ * acquire_write_lock()
+ * release_read_lock()
+ * release_write_lock()
+
+ The ``__init__()`` method accepts a single argument "filename", which
+ may be used as the "lock file", for those implementations that use a lock
+ file.
+
+ Note that multithreaded environments must provide a thread-safe
+ version of this lock. The recommended approach for file-
+ descriptor-based locks is to use a Python ``threading.local()`` so
+ that a unique file descriptor is held per thread. See the source
+ code of :class:`.FileLock` for an implementation example.
+
+
+ """
+
+ def __init__(self, filename):
+ """Constructor, is given the filename of a potential lockfile.
+
+ The usage of this filename is optional and no file is
+ created by default.
+
+ Raises ``NotImplementedError`` by default, must be
+ implemented by subclasses.
+ """
+ raise NotImplementedError()
+
+ def acquire(self, wait=True):
+ """Acquire the "write" lock.
+
+ This is a direct call to :meth:`.AbstractFileLock.acquire_write_lock`.
+
+ """
+ return self.acquire_write_lock(wait)
+
+ def release(self):
+ """Release the "write" lock.
+
+ This is a direct call to :meth:`.AbstractFileLock.release_write_lock`.
+
+ """
+ self.release_write_lock()
+
+ @contextmanager
+ def read(self):
+ """Provide a context manager for the "read" lock.
+
+ This method makes use of :meth:`.AbstractFileLock.acquire_read_lock`
+ and :meth:`.AbstractFileLock.release_read_lock`
+
+ """
+
+ self.acquire_read_lock(True)
+ try:
+ yield
+ finally:
+ self.release_read_lock()
+
+ @contextmanager
+ def write(self):
+ """Provide a context manager for the "write" lock.
+
+ This method makes use of :meth:`.AbstractFileLock.acquire_write_lock`
+ and :meth:`.AbstractFileLock.release_write_lock`
+
+ """
+
+ self.acquire_write_lock(True)
+ try:
+ yield
+ finally:
+ self.release_write_lock()
+
+ @property
+ def is_open(self):
+ """optional method."""
+ raise NotImplementedError()
+
+ def acquire_read_lock(self, wait):
+ """Acquire a 'reader' lock.
+
+ Raises ``NotImplementedError`` by default, must be
+ implemented by subclasses.
+ """
+ raise NotImplementedError()
+
+ def acquire_write_lock(self, wait):
+ """Acquire a 'write' lock.
+
+ Raises ``NotImplementedError`` by default, must be
+ implemented by subclasses.
+ """
+ raise NotImplementedError()
+
+ def release_read_lock(self):
+ """Release a 'reader' lock.
+
+ Raises ``NotImplementedError`` by default, must be
+ implemented by subclasses.
+ """
+ raise NotImplementedError()
+
+ def release_write_lock(self):
+ """Release a 'writer' lock.
+
+ Raises ``NotImplementedError`` by default, must be
+ implemented by subclasses.
+ """
+ raise NotImplementedError()
+
+
+class FileLock(AbstractFileLock):
+ """Use lockfiles to coordinate read/write access to a file.
+
+ Only works on Unix systems, using
+ `fcntl.flock() <http://docs.python.org/library/fcntl.html>`_.
+
+ """
+
+ def __init__(self, filename):
+ self._filedescriptor = compat.threading.local()
+ self.filename = filename
+
+ @util.memoized_property
+ def _module(self):
+ import fcntl
+ return fcntl
+
+ @property
+ def is_open(self):
+ return hasattr(self._filedescriptor, 'fileno')
+
+ def acquire_read_lock(self, wait):
+ return self._acquire(wait, os.O_RDONLY, self._module.LOCK_SH)
+
+ def acquire_write_lock(self, wait):
+ return self._acquire(wait, os.O_WRONLY, self._module.LOCK_EX)
+
+ def release_read_lock(self):
+ self._release()
+
+ def release_write_lock(self):
+ self._release()
+
+ def _acquire(self, wait, wrflag, lockflag):
+ wrflag |= os.O_CREAT
+ fileno = os.open(self.filename, wrflag)
+ try:
+ if not wait:
+ lockflag |= self._module.LOCK_NB
+ self._module.flock(fileno, lockflag)
+ except IOError:
+ os.close(fileno)
+ if not wait:
+ # this is typically
+ # "[Errno 35] Resource temporarily unavailable",
+ # because of LOCK_NB
+ return False
+ else:
+ raise
+ else:
+ self._filedescriptor.fileno = fileno
+ return True
+
+ def _release(self):
+ try:
+ fileno = self._filedescriptor.fileno
+ except AttributeError:
+ return
+ else:
+ self._module.flock(fileno, self._module.LOCK_UN)
+ os.close(fileno)
+ del self._filedescriptor.fileno
diff --git a/lib/dogpile/cache/backends/memcached.py b/lib/dogpile/cache/backends/memcached.py
new file mode 100644
index 0000000000000000000000000000000000000000..80acc77b42ad7da9754773d2b9d5a2fae8155869
--- /dev/null
+++ b/lib/dogpile/cache/backends/memcached.py
@@ -0,0 +1,351 @@
+"""
+Memcached Backends
+------------------
+
+Provides backends for talking to `memcached <http://memcached.org>`_.
+
+"""
+
+from dogpile.cache.api import CacheBackend, NO_VALUE
+from dogpile.cache import compat
+from dogpile.cache import util
+import random
+import time
+
+__all__ = 'GenericMemcachedBackend', 'MemcachedBackend',\
+ 'PylibmcBackend', 'BMemcachedBackend', 'MemcachedLock'
+
+
+class MemcachedLock(object):
+ """Simple distributed lock using memcached.
+
+ This is an adaptation of the lock featured at
+ http://amix.dk/blog/post/19386
+
+ """
+
+ def __init__(self, client_fn, key):
+ self.client_fn = client_fn
+ self.key = "_lock" + key
+
+ def acquire(self, wait=True):
+ client = self.client_fn()
+ i = 0
+ while True:
+ if client.add(self.key, 1):
+ return True
+ elif not wait:
+ return False
+ else:
+ sleep_time = (((i + 1) * random.random()) + 2 ** i) / 2.5
+ time.sleep(sleep_time)
+ if i < 15:
+ i += 1
+
+ def release(self):
+ client = self.client_fn()
+ client.delete(self.key)
+
+
+class GenericMemcachedBackend(CacheBackend):
+ """Base class for memcached backends.
+
+ This base class accepts a number of paramters
+ common to all backends.
+
+ :param url: the string URL to connect to. Can be a single
+ string or a list of strings. This is the only argument
+ that's required.
+ :param distributed_lock: boolean, when True, will use a
+ memcached-lock as the dogpile lock (see :class:`.MemcachedLock`).
+ Use this when multiple
+ processes will be talking to the same memcached instance.
+ When left at False, dogpile will coordinate on a regular
+ threading mutex.
+ :param memcached_expire_time: integer, when present will
+ be passed as the ``time`` parameter to ``pylibmc.Client.set``.
+ This is used to set the memcached expiry time for a value.
+
+ .. note::
+
+ This parameter is **different** from Dogpile's own
+ ``expiration_time``, which is the number of seconds after
+ which Dogpile will consider the value to be expired.
+ When Dogpile considers a value to be expired,
+ it **continues to use the value** until generation
+ of a new value is complete, when using
+ :meth:`.CacheRegion.get_or_create`.
+ Therefore, if you are setting ``memcached_expire_time``, you'll
+ want to make sure it is greater than ``expiration_time``
+ by at least enough seconds for new values to be generated,
+ else the value won't be available during a regeneration,
+ forcing all threads to wait for a regeneration each time
+ a value expires.
+
+ The :class:`.GenericMemachedBackend` uses a ``threading.local()``
+ object to store individual client objects per thread,
+ as most modern memcached clients do not appear to be inherently
+ threadsafe.
+
+ In particular, ``threading.local()`` has the advantage over pylibmc's
+ built-in thread pool in that it automatically discards objects
+ associated with a particular thread when that thread ends.
+
+ """
+
+ set_arguments = {}
+ """Additional arguments which will be passed
+ to the :meth:`set` method."""
+
+ def __init__(self, arguments):
+ self._imports()
+ # using a plain threading.local here. threading.local
+ # automatically deletes the __dict__ when a thread ends,
+ # so the idea is that this is superior to pylibmc's
+ # own ThreadMappedPool which doesn't handle this
+ # automatically.
+ self.url = util.to_list(arguments['url'])
+ self.distributed_lock = arguments.get('distributed_lock', False)
+ self.memcached_expire_time = arguments.get(
+ 'memcached_expire_time', 0)
+
+ def _imports(self):
+ """client library imports go here."""
+ raise NotImplementedError()
+
+ def _create_client(self):
+ """Creation of a Client instance goes here."""
+ raise NotImplementedError()
+
+ @util.memoized_property
+ def _clients(self):
+ backend = self
+
+ class ClientPool(compat.threading.local):
+ def __init__(self):
+ self.memcached = backend._create_client()
+
+ return ClientPool()
+
+ @property
+ def client(self):
+ """Return the memcached client.
+
+ This uses a threading.local by
+ default as it appears most modern
+ memcached libs aren't inherently
+ threadsafe.
+
+ """
+ return self._clients.memcached
+
+ def get_mutex(self, key):
+ if self.distributed_lock:
+ return MemcachedLock(lambda: self.client, key)
+ else:
+ return None
+
+ def get(self, key):
+ value = self.client.get(key)
+ if value is None:
+ return NO_VALUE
+ else:
+ return value
+
+ def get_multi(self, keys):
+ values = self.client.get_multi(keys)
+ return [
+ NO_VALUE if key not in values
+ else values[key] for key in keys
+ ]
+
+ def set(self, key, value):
+ self.client.set(
+ key,
+ value,
+ **self.set_arguments
+ )
+
+ def set_multi(self, mapping):
+ self.client.set_multi(
+ mapping,
+ **self.set_arguments
+ )
+
+ def delete(self, key):
+ self.client.delete(key)
+
+ def delete_multi(self, keys):
+ self.client.delete_multi(keys)
+
+
+class MemcacheArgs(object):
+ """Mixin which provides support for the 'time' argument to set(),
+ 'min_compress_len' to other methods.
+
+ """
+ def __init__(self, arguments):
+ self.min_compress_len = arguments.get('min_compress_len', 0)
+
+ self.set_arguments = {}
+ if "memcached_expire_time" in arguments:
+ self.set_arguments["time"] = arguments["memcached_expire_time"]
+ if "min_compress_len" in arguments:
+ self.set_arguments["min_compress_len"] = \
+ arguments["min_compress_len"]
+ super(MemcacheArgs, self).__init__(arguments)
+
+pylibmc = None
+
+
+class PylibmcBackend(MemcacheArgs, GenericMemcachedBackend):
+ """A backend for the
+ `pylibmc <http://sendapatch.se/projects/pylibmc/index.html>`_
+ memcached client.
+
+ A configuration illustrating several of the optional
+ arguments described in the pylibmc documentation::
+
+ from dogpile.cache import make_region
+
+ region = make_region().configure(
+ 'dogpile.cache.pylibmc',
+ expiration_time = 3600,
+ arguments = {
+ 'url':["127.0.0.1"],
+ 'binary':True,
+ 'behaviors':{"tcp_nodelay": True,"ketama":True}
+ }
+ )
+
+ Arguments accepted here include those of
+ :class:`.GenericMemcachedBackend`, as well as
+ those below.
+
+ :param binary: sets the ``binary`` flag understood by
+ ``pylibmc.Client``.
+ :param behaviors: a dictionary which will be passed to
+ ``pylibmc.Client`` as the ``behaviors`` parameter.
+ :param min_compress_len: Integer, will be passed as the
+ ``min_compress_len`` parameter to the ``pylibmc.Client.set``
+ method.
+
+ """
+
+ def __init__(self, arguments):
+ self.binary = arguments.get('binary', False)
+ self.behaviors = arguments.get('behaviors', {})
+ super(PylibmcBackend, self).__init__(arguments)
+
+ def _imports(self):
+ global pylibmc
+ import pylibmc # noqa
+
+ def _create_client(self):
+ return pylibmc.Client(
+ self.url,
+ binary=self.binary,
+ behaviors=self.behaviors
+ )
+
+memcache = None
+
+
+class MemcachedBackend(MemcacheArgs, GenericMemcachedBackend):
+ """A backend using the standard
+ `Python-memcached <http://www.tummy.com/Community/software/\
+ python-memcached/>`_
+ library.
+
+ Example::
+
+ from dogpile.cache import make_region
+
+ region = make_region().configure(
+ 'dogpile.cache.memcached',
+ expiration_time = 3600,
+ arguments = {
+ 'url':"127.0.0.1:11211"
+ }
+ )
+
+ """
+ def _imports(self):
+ global memcache
+ import memcache # noqa
+
+ def _create_client(self):
+ return memcache.Client(self.url)
+
+
+bmemcached = None
+
+
+class BMemcachedBackend(GenericMemcachedBackend):
+ """A backend for the
+ `python-binary-memcached <https://github.com/jaysonsantos/\
+ python-binary-memcached>`_
+ memcached client.
+
+ This is a pure Python memcached client which
+ includes the ability to authenticate with a memcached
+ server using SASL.
+
+ A typical configuration using username/password::
+
+ from dogpile.cache import make_region
+
+ region = make_region().configure(
+ 'dogpile.cache.bmemcached',
+ expiration_time = 3600,
+ arguments = {
+ 'url':["127.0.0.1"],
+ 'username':'scott',
+ 'password':'tiger'
+ }
+ )
+
+ Arguments which can be passed to the ``arguments``
+ dictionary include:
+
+ :param username: optional username, will be used for
+ SASL authentication.
+ :param password: optional password, will be used for
+ SASL authentication.
+
+ """
+ def __init__(self, arguments):
+ self.username = arguments.get('username', None)
+ self.password = arguments.get('password', None)
+ super(BMemcachedBackend, self).__init__(arguments)
+
+ def _imports(self):
+ global bmemcached
+ import bmemcached
+
+ class RepairBMemcachedAPI(bmemcached.Client):
+ """Repairs BMemcached's non-standard method
+ signatures, which was fixed in BMemcached
+ ef206ed4473fec3b639e.
+
+ """
+
+ def add(self, key, value):
+ try:
+ return super(RepairBMemcachedAPI, self).add(key, value)
+ except ValueError:
+ return False
+
+ self.Client = RepairBMemcachedAPI
+
+ def _create_client(self):
+ return self.Client(
+ self.url,
+ username=self.username,
+ password=self.password
+ )
+
+ def delete_multi(self, keys):
+ """python-binary-memcached api does not implements delete_multi"""
+ for key in keys:
+ self.delete(key)
diff --git a/lib/dogpile/cache/backends/memory.py b/lib/dogpile/cache/backends/memory.py
new file mode 100644
index 0000000000000000000000000000000000000000..2f9bd3a4ec099ea7616344dd9ecd7d171f8d06f1
--- /dev/null
+++ b/lib/dogpile/cache/backends/memory.py
@@ -0,0 +1,124 @@
+"""
+Memory Backends
+---------------
+
+Provides simple dictionary-based backends.
+
+The two backends are :class:`.MemoryBackend` and :class:`.MemoryPickleBackend`;
+the latter applies a serialization step to cached values while the former
+places the value as given into the dictionary.
+
+"""
+
+from dogpile.cache.api import CacheBackend, NO_VALUE
+from dogpile.cache.compat import pickle
+
+
+class MemoryBackend(CacheBackend):
+ """A backend that uses a plain dictionary.
+
+ There is no size management, and values which
+ are placed into the dictionary will remain
+ until explicitly removed. Note that
+ Dogpile's expiration of items is based on
+ timestamps and does not remove them from
+ the cache.
+
+ E.g.::
+
+ from dogpile.cache import make_region
+
+ region = make_region().configure(
+ 'dogpile.cache.memory'
+ )
+
+
+ To use a Python dictionary of your choosing,
+ it can be passed in with the ``cache_dict``
+ argument::
+
+ my_dictionary = {}
+ region = make_region().configure(
+ 'dogpile.cache.memory',
+ arguments={
+ "cache_dict":my_dictionary
+ }
+ )
+
+
+ """
+ pickle_values = False
+
+ def __init__(self, arguments):
+ self._cache = arguments.pop("cache_dict", {})
+
+ def get(self, key):
+ value = self._cache.get(key, NO_VALUE)
+ if value is not NO_VALUE and self.pickle_values:
+ value = pickle.loads(value)
+ return value
+
+ def get_multi(self, keys):
+ ret = [
+ self._cache.get(key, NO_VALUE)
+ for key in keys]
+ if self.pickle_values:
+ ret = [
+ pickle.loads(value)
+ if value is not NO_VALUE else value
+ for value in ret
+ ]
+ return ret
+
+ def set(self, key, value):
+ if self.pickle_values:
+ value = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
+ self._cache[key] = value
+
+ def set_multi(self, mapping):
+ pickle_values = self.pickle_values
+ for key, value in mapping.items():
+ if pickle_values:
+ value = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
+ self._cache[key] = value
+
+ def delete(self, key):
+ self._cache.pop(key, None)
+
+ def delete_multi(self, keys):
+ for key in keys:
+ self._cache.pop(key, None)
+
+
+class MemoryPickleBackend(MemoryBackend):
+ """A backend that uses a plain dictionary, but serializes objects on
+ :meth:`.MemoryBackend.set` and deserializes :meth:`.MemoryBackend.get`.
+
+ E.g.::
+
+ from dogpile.cache import make_region
+
+ region = make_region().configure(
+ 'dogpile.cache.memory_pickle'
+ )
+
+ The usage of pickle to serialize cached values allows an object
+ as placed in the cache to be a copy of the original given object, so
+ that any subsequent changes to the given object aren't reflected
+ in the cached value, thus making the backend behave the same way
+ as other backends which make use of serialization.
+
+ The serialization is performed via pickle, and incurs the same
+ performance hit in doing so as that of other backends; in this way
+ the :class:`.MemoryPickleBackend` performance is somewhere in between
+ that of the pure :class:`.MemoryBackend` and the remote server oriented
+ backends such as that of Memcached or Redis.
+
+ Pickle behavior here is the same as that of the Redis backend, using
+ either ``cPickle`` or ``pickle`` and specifying ``HIGHEST_PROTOCOL``
+ upon serialize.
+
+ .. versionadded:: 0.5.3
+
+ """
+ pickle_values = True
diff --git a/lib/dogpile/cache/backends/null.py b/lib/dogpile/cache/backends/null.py
new file mode 100644
index 0000000000000000000000000000000000000000..c1f46a9d6d14fe85cfbc1ce8ce63cc0a33b779b8
--- /dev/null
+++ b/lib/dogpile/cache/backends/null.py
@@ -0,0 +1,62 @@
+"""
+Null Backend
+-------------
+
+The Null backend does not do any caching at all. It can be
+used to test behavior without caching, or as a means of disabling
+caching for a region that is otherwise used normally.
+
+.. versionadded:: 0.5.4
+
+"""
+
+from dogpile.cache.api import CacheBackend, NO_VALUE
+
+
+__all__ = ['NullBackend']
+
+
+class NullLock(object):
+ def acquire(self):
+ pass
+
+ def release(self):
+ pass
+
+
+class NullBackend(CacheBackend):
+ """A "null" backend that effectively disables all cache operations.
+
+ Basic usage::
+
+ from dogpile.cache import make_region
+
+ region = make_region().configure(
+ 'dogpile.cache.null'
+ )
+
+ """
+
+ def __init__(self, arguments):
+ pass
+
+ def get_mutex(self, key):
+ return NullLock()
+
+ def get(self, key):
+ return NO_VALUE
+
+ def get_multi(self, keys):
+ return [NO_VALUE for k in keys]
+
+ def set(self, key, value):
+ pass
+
+ def set_multi(self, mapping):
+ pass
+
+ def delete(self, key):
+ pass
+
+ def delete_multi(self, keys):
+ pass
diff --git a/lib/dogpile/cache/backends/redis.py b/lib/dogpile/cache/backends/redis.py
new file mode 100644
index 0000000000000000000000000000000000000000..b4d93e8b520f557d1dcd9095375a170c64f0a5d6
--- /dev/null
+++ b/lib/dogpile/cache/backends/redis.py
@@ -0,0 +1,182 @@
+"""
+Redis Backends
+------------------
+
+Provides backends for talking to `Redis <http://redis.io>`_.
+
+"""
+
+from __future__ import absolute_import
+from dogpile.cache.api import CacheBackend, NO_VALUE
+from dogpile.cache.compat import pickle, u
+
+redis = None
+
+__all__ = 'RedisBackend',
+
+
+class RedisBackend(CacheBackend):
+ """A `Redis <http://redis.io/>`_ backend, using the
+ `redis-py <http://pypi.python.org/pypi/redis/>`_ backend.
+
+ Example configuration::
+
+ from dogpile.cache import make_region
+
+ region = make_region().configure(
+ 'dogpile.cache.redis',
+ arguments = {
+ 'host': 'localhost',
+ 'port': 6379,
+ 'db': 0,
+ 'redis_expiration_time': 60*60*2, # 2 hours
+ 'distributed_lock': True
+ }
+ )
+
+ Arguments accepted in the arguments dictionary:
+
+ :param url: string. If provided, will override separate host/port/db
+ params. The format is that accepted by ``StrictRedis.from_url()``.
+
+ .. versionadded:: 0.4.1
+
+ :param host: string, default is ``localhost``.
+
+ :param password: string, default is no password.
+
+ .. versionadded:: 0.4.1
+
+ :param port: integer, default is ``6379``.
+
+ :param db: integer, default is ``0``.
+
+ :param redis_expiration_time: integer, number of seconds after setting
+ a value that Redis should expire it. This should be larger than dogpile's
+ cache expiration. By default no expiration is set.
+
+ :param distributed_lock: boolean, when True, will use a
+ redis-lock as the dogpile lock.
+ Use this when multiple
+ processes will be talking to the same redis instance.
+ When left at False, dogpile will coordinate on a regular
+ threading mutex.
+
+ :param lock_timeout: integer, number of seconds after acquiring a lock that
+ Redis should expire it. This argument is only valid when
+ ``distributed_lock`` is ``True``.
+
+ .. versionadded:: 0.5.0
+
+ :param socket_timeout: float, seconds for socket timeout.
+ Default is None (no timeout).
+
+ .. versionadded:: 0.5.4
+
+ :param lock_sleep: integer, number of seconds to sleep when failed to
+ acquire a lock. This argument is only valid when
+ ``distributed_lock`` is ``True``.
+
+ .. versionadded:: 0.5.0
+
+ :param connection_pool: ``redis.ConnectionPool`` object. If provided,
+ this object supersedes other connection arguments passed to the
+ ``redis.StrictRedis`` instance, including url and/or host as well as
+ socket_timeout, and will be passed to ``redis.StrictRedis`` as the
+ source of connectivity.
+
+ .. versionadded:: 0.5.4
+
+
+ """
+
+ def __init__(self, arguments):
+ self._imports()
+ self.url = arguments.pop('url', None)
+ self.host = arguments.pop('host', 'localhost')
+ self.password = arguments.pop('password', None)
+ self.port = arguments.pop('port', 6379)
+ self.db = arguments.pop('db', 0)
+ self.distributed_lock = arguments.get('distributed_lock', False)
+ self.socket_timeout = arguments.pop('socket_timeout', None)
+
+ self.lock_timeout = arguments.get('lock_timeout', None)
+ self.lock_sleep = arguments.get('lock_sleep', 0.1)
+
+ self.redis_expiration_time = arguments.pop('redis_expiration_time', 0)
+ self.connection_pool = arguments.get('connection_pool', None)
+ self.client = self._create_client()
+
+ def _imports(self):
+ # defer imports until backend is used
+ global redis
+ import redis # noqa
+
+ def _create_client(self):
+ if self.connection_pool is not None:
+ # the connection pool already has all other connection
+ # options present within, so here we disregard socket_timeout
+ # and others.
+ return redis.StrictRedis(connection_pool=self.connection_pool)
+
+ args = {}
+ if self.socket_timeout:
+ args['socket_timeout'] = self.socket_timeout
+
+ if self.url is not None:
+ args.update(url=self.url)
+ return redis.StrictRedis.from_url(**args)
+ else:
+ args.update(
+ host=self.host, password=self.password,
+ port=self.port, db=self.db
+ )
+ return redis.StrictRedis(**args)
+
+ def get_mutex(self, key):
+ if self.distributed_lock:
+ return self.client.lock(u('_lock{0}').format(key),
+ self.lock_timeout, self.lock_sleep)
+ else:
+ return None
+
+ def get(self, key):
+ value = self.client.get(key)
+ if value is None:
+ return NO_VALUE
+ return pickle.loads(value)
+
+ def get_multi(self, keys):
+ if not keys:
+ return []
+ values = self.client.mget(keys)
+ return [
+ pickle.loads(v) if v is not None else NO_VALUE
+ for v in values]
+
+ def set(self, key, value):
+ if self.redis_expiration_time:
+ self.client.setex(key, self.redis_expiration_time,
+ pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
+ else:
+ self.client.set(key, pickle.dumps(value, pickle.HIGHEST_PROTOCOL))
+
+ def set_multi(self, mapping):
+ mapping = dict(
+ (k, pickle.dumps(v, pickle.HIGHEST_PROTOCOL))
+ for k, v in mapping.items()
+ )
+
+ if not self.redis_expiration_time:
+ self.client.mset(mapping)
+ else:
+ pipe = self.client.pipeline()
+ for key, value in mapping.items():
+ pipe.setex(key, self.redis_expiration_time, value)
+ pipe.execute()
+
+ def delete(self, key):
+ self.client.delete(key)
+
+ def delete_multi(self, keys):
+ self.client.delete(*keys)
diff --git a/lib/dogpile/cache/compat.py b/lib/dogpile/cache/compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..d29bb1dacf80dd3b3fe4f5df0284f55c1fa4db3f
--- /dev/null
+++ b/lib/dogpile/cache/compat.py
@@ -0,0 +1,65 @@
+import sys
+
+py2k = sys.version_info < (3, 0)
+py3k = sys.version_info >= (3, 0)
+py32 = sys.version_info >= (3, 2)
+py27 = sys.version_info >= (2, 7)
+jython = sys.platform.startswith('java')
+win32 = sys.platform.startswith('win')
+
+try:
+ import threading
+except ImportError:
+ import dummy_threading as threading # noqa
+
+
+if py3k: # pragma: no cover
+ string_types = str,
+ text_type = str
+ string_type = str
+
+ if py32:
+ callable = callable
+ else:
+ def callable(fn):
+ return hasattr(fn, '__call__')
+
+ def u(s):
+ return s
+
+ def ue(s):
+ return s
+
+ import configparser
+ import io
+ import _thread as thread
+else:
+ string_types = basestring,
+ text_type = unicode
+ string_type = str
+
+ def u(s):
+ return unicode(s, "utf-8")
+
+ def ue(s):
+ return unicode(s, "unicode_escape")
+
+ import ConfigParser as configparser # noqa
+ import StringIO as io # noqa
+
+ callable = callable # noqa
+ import thread # noqa
+
+
+if py3k or jython:
+ import pickle
+else:
+ import cPickle as pickle # noqa
+
+
+def timedelta_total_seconds(td):
+ if py27:
+ return td.total_seconds()
+ else:
+ return (td.microseconds + (
+ td.seconds + td.days * 24 * 3600) * 1e6) / 1e6
diff --git a/lib/dogpile/cache/exception.py b/lib/dogpile/cache/exception.py
new file mode 100644
index 0000000000000000000000000000000000000000..58068f8bc161f285f211281d6eaa7f81cfbc7b81
--- /dev/null
+++ b/lib/dogpile/cache/exception.py
@@ -0,0 +1,17 @@
+"""Exception classes for dogpile.cache."""
+
+
+class DogpileCacheException(Exception):
+ """Base Exception for dogpile.cache exceptions to inherit from."""
+
+
+class RegionAlreadyConfigured(DogpileCacheException):
+ """CacheRegion instance is already configured."""
+
+
+class RegionNotConfigured(DogpileCacheException):
+ """CacheRegion instance has not been configured."""
+
+
+class ValidationError(DogpileCacheException):
+ """Error validating a value or option."""
diff --git a/lib/dogpile/cache/plugins/__init__.py b/lib/dogpile/cache/plugins/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lib/dogpile/cache/plugins/mako_cache.py b/lib/dogpile/cache/plugins/mako_cache.py
new file mode 100644
index 0000000000000000000000000000000000000000..61f4ffaf327b21b11c6d0e0e9c9368dd85e90f72
--- /dev/null
+++ b/lib/dogpile/cache/plugins/mako_cache.py
@@ -0,0 +1,90 @@
+"""
+Mako Integration
+----------------
+
+dogpile.cache includes a `Mako <http://www.makotemplates.org>`_ plugin
+that replaces `Beaker <http://beaker.groovie.org>`_
+as the cache backend.
+Setup a Mako template lookup using the "dogpile.cache" cache implementation
+and a region dictionary::
+
+ from dogpile.cache import make_region
+ from mako.lookup import TemplateLookup
+
+ my_regions = {
+ "local":make_region().configure(
+ "dogpile.cache.dbm",
+ expiration_time=360,
+ arguments={"filename":"file.dbm"}
+ ),
+ "memcached":make_region().configure(
+ "dogpile.cache.pylibmc",
+ expiration_time=3600,
+ arguments={"url":["127.0.0.1"]}
+ )
+ }
+
+ mako_lookup = TemplateLookup(
+ directories=["/myapp/templates"],
+ cache_impl="dogpile.cache",
+ cache_args={
+ 'regions':my_regions
+ }
+ )
+
+To use the above configuration in a template, use the ``cached=True``
+argument on any Mako tag which accepts it, in conjunction with the
+name of the desired region as the ``cache_region`` argument::
+
+ <%def name="mysection()" cached="True" cache_region="memcached">
+ some content that's cached
+ </%def>
+
+
+"""
+from mako.cache import CacheImpl
+
+
+class MakoPlugin(CacheImpl):
+ """A Mako ``CacheImpl`` which talks to dogpile.cache."""
+
+ def __init__(self, cache):
+ super(MakoPlugin, self).__init__(cache)
+ try:
+ self.regions = self.cache.template.cache_args['regions']
+ except KeyError:
+ raise KeyError(
+ "'cache_regions' argument is required on the "
+ "Mako Lookup or Template object for usage "
+ "with the dogpile.cache plugin.")
+
+ def _get_region(self, **kw):
+ try:
+ region = kw['region']
+ except KeyError:
+ raise KeyError(
+ "'cache_region' argument must be specified with 'cache=True'"
+ "within templates for usage with the dogpile.cache plugin.")
+ try:
+ return self.regions[region]
+ except KeyError:
+ raise KeyError("No such region '%s'" % region)
+
+ def get_and_replace(self, key, creation_function, **kw):
+ expiration_time = kw.pop("timeout", None)
+ return self._get_region(**kw).get_or_create(
+ key, creation_function,
+ expiration_time=expiration_time)
+
+ def get_or_create(self, key, creation_function, **kw):
+ return self.get_and_replace(key, creation_function, **kw)
+
+ def put(self, key, value, **kw):
+ self._get_region(**kw).put(key, value)
+
+ def get(self, key, **kw):
+ expiration_time = kw.pop("timeout", None)
+ return self._get_region(**kw).get(key, expiration_time=expiration_time)
+
+ def invalidate(self, key, **kw):
+ self._get_region(**kw).delete(key)
diff --git a/lib/dogpile/cache/proxy.py b/lib/dogpile/cache/proxy.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fe49d6e5dcb049acd03a9914f4552fd09490646
--- /dev/null
+++ b/lib/dogpile/cache/proxy.py
@@ -0,0 +1,95 @@
+"""
+Proxy Backends
+------------------
+
+Provides a utility and a decorator class that allow for modifying the behavior
+of different backends without altering the class itself or having to extend the
+base backend.
+
+.. versionadded:: 0.5.0 Added support for the :class:`.ProxyBackend` class.
+
+"""
+
+from .api import CacheBackend
+
+
+class ProxyBackend(CacheBackend):
+ """A decorator class for altering the functionality of backends.
+
+ Basic usage::
+
+ from dogpile.cache import make_region
+ from dogpile.cache.proxy import ProxyBackend
+
+ class MyFirstProxy(ProxyBackend):
+ def get(self, key):
+ # ... custom code goes here ...
+ return self.proxied.get(key)
+
+ def set(self, key, value):
+ # ... custom code goes here ...
+ self.proxied.set(key)
+
+ class MySecondProxy(ProxyBackend):
+ def get(self, key):
+ # ... custom code goes here ...
+ return self.proxied.get(key)
+
+
+ region = make_region().configure(
+ 'dogpile.cache.dbm',
+ expiration_time = 3600,
+ arguments = {
+ "filename":"/path/to/cachefile.dbm"
+ },
+ wrap = [ MyFirstProxy, MySecondProxy ]
+ )
+
+ Classes that extend :class:`.ProxyBackend` can be stacked
+ together. The ``.proxied`` property will always
+ point to either the concrete backend instance or
+ the next proxy in the chain that a method can be
+ delegated towards.
+
+ .. versionadded:: 0.5.0
+
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.proxied = None
+
+ def wrap(self, backend):
+ ''' Take a backend as an argument and setup the self.proxied property.
+ Return an object that be used as a backend by a :class:`.CacheRegion`
+ object.
+ '''
+ assert(
+ isinstance(backend, CacheBackend) or
+ isinstance(backend, ProxyBackend))
+ self.proxied = backend
+ return self
+
+ #
+ # Delegate any functions that are not already overridden to
+ # the proxies backend
+ #
+ def get(self, key):
+ return self.proxied.get(key)
+
+ def set(self, key, value):
+ self.proxied.set(key, value)
+
+ def delete(self, key):
+ self.proxied.delete(key)
+
+ def get_multi(self, keys):
+ return self.proxied.get_multi(keys)
+
+ def set_multi(self, keys):
+ self.proxied.set_multi(keys)
+
+ def delete_multi(self, keys):
+ self.proxied.delete_multi(keys)
+
+ def get_mutex(self, key):
+ return self.proxied.get_mutex(key)
diff --git a/lib/dogpile/cache/region.py b/lib/dogpile/cache/region.py
new file mode 100644
index 0000000000000000000000000000000000000000..afa2b547c155f91ea7fd2fbf5eed0f4dc20a9bb5
--- /dev/null
+++ b/lib/dogpile/cache/region.py
@@ -0,0 +1,1287 @@
+from __future__ import with_statement
+from dogpile.core import Lock, NeedRegenerationException
+from dogpile.core.nameregistry import NameRegistry
+from . import exception
+from .util import function_key_generator, PluginLoader, \
+ memoized_property, coerce_string_conf, function_multi_key_generator
+from .api import NO_VALUE, CachedValue
+from .proxy import ProxyBackend
+from . import compat
+import time
+import datetime
+from numbers import Number
+from functools import wraps
+import threading
+
+_backend_loader = PluginLoader("dogpile.cache")
+register_backend = _backend_loader.register
+from . import backends # noqa
+
+value_version = 1
+"""An integer placed in the :class:`.CachedValue`
+so that new versions of dogpile.cache can detect cached
+values from a previous, backwards-incompatible version.
+
+"""
+
+
+class CacheRegion(object):
+ """A front end to a particular cache backend.
+
+ :param name: Optional, a string name for the region.
+ This isn't used internally
+ but can be accessed via the ``.name`` parameter, helpful
+ for configuring a region from a config file.
+ :param function_key_generator: Optional. A
+ function that will produce a "cache key" given
+ a data creation function and arguments, when using
+ the :meth:`.CacheRegion.cache_on_arguments` method.
+ The structure of this function
+ should be two levels: given the data creation function,
+ return a new function that generates the key based on
+ the given arguments. Such as::
+
+ def my_key_generator(namespace, fn, **kw):
+ fname = fn.__name__
+ def generate_key(*arg):
+ return namespace + "_" + fname + "_".join(str(s) for s in arg)
+ return generate_key
+
+
+ region = make_region(
+ function_key_generator = my_key_generator
+ ).configure(
+ "dogpile.cache.dbm",
+ expiration_time=300,
+ arguments={
+ "filename":"file.dbm"
+ }
+ )
+
+ The ``namespace`` is that passed to
+ :meth:`.CacheRegion.cache_on_arguments`. It's not consulted
+ outside this function, so in fact can be of any form.
+ For example, it can be passed as a tuple, used to specify
+ arguments to pluck from \**kw::
+
+ def my_key_generator(namespace, fn):
+ def generate_key(*arg, **kw):
+ return ":".join(
+ [kw[k] for k in namespace] +
+ [str(x) for x in arg]
+ )
+ return generate_key
+
+
+ Where the decorator might be used as::
+
+ @my_region.cache_on_arguments(namespace=('x', 'y'))
+ def my_function(a, b, **kw):
+ return my_data()
+
+ :param function_multi_key_generator: Optional.
+ Similar to ``function_key_generator`` parameter, but it's used in
+ :meth:`.CacheRegion.cache_multi_on_arguments`. Generated function
+ should return list of keys. For example::
+
+ def my_multi_key_generator(namespace, fn, **kw):
+ namespace = fn.__name__ + (namespace or '')
+
+ def generate_keys(*args):
+ return [namespace + ':' + str(a) for a in args]
+
+ return generate_keys
+
+ :param key_mangler: Function which will be used on all incoming
+ keys before passing to the backend. Defaults to ``None``,
+ in which case the key mangling function recommended by
+ the cache backend will be used. A typical mangler
+ is the SHA1 mangler found at :func:`.sha1_mangle_key`
+ which coerces keys into a SHA1
+ hash, so that the string length is fixed. To
+ disable all key mangling, set to ``False``. Another typical
+ mangler is the built-in Python function ``str``, which can be used
+ to convert non-string or Unicode keys to bytestrings, which is
+ needed when using a backend such as bsddb or dbm under Python 2.x
+ in conjunction with Unicode keys.
+ :param async_creation_runner: A callable that, when specified,
+ will be passed to and called by dogpile.lock when
+ there is a stale value present in the cache. It will be passed the
+ mutex and is responsible releasing that mutex when finished.
+ This can be used to defer the computation of expensive creator
+ functions to later points in the future by way of, for example, a
+ background thread, a long-running queue, or a task manager system
+ like Celery.
+
+ For a specific example using async_creation_runner, new values can
+ be created in a background thread like so::
+
+ import threading
+
+ def async_creation_runner(cache, somekey, creator, mutex):
+ ''' Used by dogpile.core:Lock when appropriate '''
+ def runner():
+ try:
+ value = creator()
+ cache.set(somekey, value)
+ finally:
+ mutex.release()
+
+ thread = threading.Thread(target=runner)
+ thread.start()
+
+
+ region = make_region(
+ async_creation_runner=async_creation_runner,
+ ).configure(
+ 'dogpile.cache.memcached',
+ expiration_time=5,
+ arguments={
+ 'url': '127.0.0.1:11211',
+ 'distributed_lock': True,
+ }
+ )
+
+ Remember that the first request for a key with no associated
+ value will always block; async_creator will not be invoked.
+ However, subsequent requests for cached-but-expired values will
+ still return promptly. They will be refreshed by whatever
+ asynchronous means the provided async_creation_runner callable
+ implements.
+
+ By default the async_creation_runner is disabled and is set
+ to ``None``.
+
+ .. versionadded:: 0.4.2 added the async_creation_runner
+ feature.
+
+ """
+
+ def __init__(
+ self,
+ name=None,
+ function_key_generator=function_key_generator,
+ function_multi_key_generator=function_multi_key_generator,
+ key_mangler=None,
+ async_creation_runner=None,
+ ):
+ """Construct a new :class:`.CacheRegion`."""
+ self.name = name
+ self.function_key_generator = function_key_generator
+ self.function_multi_key_generator = function_multi_key_generator
+ if key_mangler:
+ self.key_mangler = key_mangler
+ else:
+ self.key_mangler = None
+ self._hard_invalidated = None
+ self._soft_invalidated = None
+ self.async_creation_runner = async_creation_runner
+
+ def configure(
+ self, backend,
+ expiration_time=None,
+ arguments=None,
+ _config_argument_dict=None,
+ _config_prefix=None,
+ wrap=None
+ ):
+ """Configure a :class:`.CacheRegion`.
+
+ The :class:`.CacheRegion` itself
+ is returned.
+
+ :param backend: Required. This is the name of the
+ :class:`.CacheBackend` to use, and is resolved by loading
+ the class from the ``dogpile.cache`` entrypoint.
+
+ :param expiration_time: Optional. The expiration time passed
+ to the dogpile system. May be passed as an integer number
+ of seconds, or as a ``datetime.timedelta`` value.
+
+ .. versionadded 0.5.0
+ ``expiration_time`` may be optionally passed as a
+ ``datetime.timedelta`` value.
+
+ The :meth:`.CacheRegion.get_or_create`
+ method as well as the :meth:`.CacheRegion.cache_on_arguments`
+ decorator (though note: **not** the :meth:`.CacheRegion.get`
+ method) will call upon the value creation function after this
+ time period has passed since the last generation.
+
+ :param arguments: Optional. The structure here is passed
+ directly to the constructor of the :class:`.CacheBackend`
+ in use, though is typically a dictionary.
+
+ :param wrap: Optional. A list of :class:`.ProxyBackend`
+ classes and/or instances, each of which will be applied
+ in a chain to ultimately wrap the original backend,
+ so that custom functionality augmentation can be applied.
+
+ .. versionadded:: 0.5.0
+
+ .. seealso::
+
+ :ref:`changing_backend_behavior`
+
+ """
+
+ if "backend" in self.__dict__:
+ raise exception.RegionAlreadyConfigured(
+ "This region is already "
+ "configured with backend: %s"
+ % self.backend)
+ backend_cls = _backend_loader.load(backend)
+ if _config_argument_dict:
+ self.backend = backend_cls.from_config_dict(
+ _config_argument_dict,
+ _config_prefix
+ )
+ else:
+ self.backend = backend_cls(arguments or {})
+
+ if not expiration_time or isinstance(expiration_time, Number):
+ self.expiration_time = expiration_time
+ elif isinstance(expiration_time, datetime.timedelta):
+ self.expiration_time = int(
+ compat.timedelta_total_seconds(expiration_time))
+ else:
+ raise exception.ValidationError(
+ 'expiration_time is not a number or timedelta.')
+
+ if self.key_mangler is None:
+ self.key_mangler = self.backend.key_mangler
+
+ self._lock_registry = NameRegistry(self._create_mutex)
+
+ if getattr(wrap, '__iter__', False):
+ for wrapper in reversed(wrap):
+ self.wrap(wrapper)
+
+ return self
+
+ def wrap(self, proxy):
+ ''' Takes a ProxyBackend instance or class and wraps the
+ attached backend. '''
+
+ # if we were passed a type rather than an instance then
+ # initialize it.
+ if type(proxy) == type:
+ proxy = proxy()
+
+ if not issubclass(type(proxy), ProxyBackend):
+ raise TypeError("Type %s is not a valid ProxyBackend"
+ % type(proxy))
+
+ self.backend = proxy.wrap(self.backend)
+
+ def _mutex(self, key):
+ return self._lock_registry.get(key)
+
+ class _LockWrapper(object):
+ """weakref-capable wrapper for threading.Lock"""
+ def __init__(self):
+ self.lock = threading.Lock()
+
+ def acquire(self, wait=True):
+ return self.lock.acquire(wait)
+
+ def release(self):
+ self.lock.release()
+
+ def _create_mutex(self, key):
+ mutex = self.backend.get_mutex(key)
+ if mutex is not None:
+ return mutex
+ else:
+ return self._LockWrapper()
+
+ def invalidate(self, hard=True):
+ """Invalidate this :class:`.CacheRegion`.
+
+ Invalidation works by setting a current timestamp
+ (using ``time.time()``)
+ representing the "minimum creation time" for
+ a value. Any retrieved value whose creation
+ time is prior to this timestamp
+ is considered to be stale. It does not
+ affect the data in the cache in any way, and is also
+ local to this instance of :class:`.CacheRegion`.
+
+ Once set, the invalidation time is honored by
+ the :meth:`.CacheRegion.get_or_create`,
+ :meth:`.CacheRegion.get_or_create_multi` and
+ :meth:`.CacheRegion.get` methods.
+
+ The method supports both "hard" and "soft" invalidation
+ options. With "hard" invalidation,
+ :meth:`.CacheRegion.get_or_create` will force an immediate
+ regeneration of the value which all getters will wait for.
+ With "soft" invalidation, subsequent getters will return the
+ "old" value until the new one is available.
+
+ Usage of "soft" invalidation requires that the region or the method
+ is given a non-None expiration time.
+
+ .. versionadded:: 0.3.0
+
+ :param hard: if True, cache values will all require immediate
+ regeneration; dogpile logic won't be used. If False, the
+ creation time of existing values will be pushed back before
+ the expiration time so that a return+regen will be invoked.
+
+ .. versionadded:: 0.5.1
+
+ """
+ if hard:
+ self._hard_invalidated = time.time()
+ self._soft_invalidated = None
+ else:
+ self._hard_invalidated = None
+ self._soft_invalidated = time.time()
+
+ def configure_from_config(self, config_dict, prefix):
+ """Configure from a configuration dictionary
+ and a prefix.
+
+ Example::
+
+ local_region = make_region()
+ memcached_region = make_region()
+
+ # regions are ready to use for function
+ # decorators, but not yet for actual caching
+
+ # later, when config is available
+ myconfig = {
+ "cache.local.backend":"dogpile.cache.dbm",
+ "cache.local.arguments.filename":"/path/to/dbmfile.dbm",
+ "cache.memcached.backend":"dogpile.cache.pylibmc",
+ "cache.memcached.arguments.url":"127.0.0.1, 10.0.0.1",
+ }
+ local_region.configure_from_config(myconfig, "cache.local.")
+ memcached_region.configure_from_config(myconfig,
+ "cache.memcached.")
+
+ """
+ config_dict = coerce_string_conf(config_dict)
+ return self.configure(
+ config_dict["%sbackend" % prefix],
+ expiration_time=config_dict.get(
+ "%sexpiration_time" % prefix, None),
+ _config_argument_dict=config_dict,
+ _config_prefix="%sarguments." % prefix,
+ wrap=config_dict.get(
+ "%swrap" % prefix, None),
+ )
+
+ @memoized_property
+ def backend(self):
+ raise exception.RegionNotConfigured(
+ "No backend is configured on this region.")
+
+ @property
+ def is_configured(self):
+ """Return True if the backend has been configured via the
+ :meth:`.CacheRegion.configure` method already.
+
+ .. versionadded:: 0.5.1
+
+ """
+ return 'backend' in self.__dict__
+
+ def get(self, key, expiration_time=None, ignore_expiration=False):
+ """Return a value from the cache, based on the given key.
+
+ If the value is not present, the method returns the token
+ ``NO_VALUE``. ``NO_VALUE`` evaluates to False, but is separate from
+ ``None`` to distinguish between a cached value of ``None``.
+
+ By default, the configured expiration time of the
+ :class:`.CacheRegion`, or alternatively the expiration
+ time supplied by the ``expiration_time`` argument,
+ is tested against the creation time of the retrieved
+ value versus the current time (as reported by ``time.time()``).
+ If stale, the cached value is ignored and the ``NO_VALUE``
+ token is returned. Passing the flag ``ignore_expiration=True``
+ bypasses the expiration time check.
+
+ .. versionchanged:: 0.3.0
+ :meth:`.CacheRegion.get` now checks the value's creation time
+ against the expiration time, rather than returning
+ the value unconditionally.
+
+ The method also interprets the cached value in terms
+ of the current "invalidation" time as set by
+ the :meth:`.invalidate` method. If a value is present,
+ but its creation time is older than the current
+ invalidation time, the ``NO_VALUE`` token is returned.
+ Passing the flag ``ignore_expiration=True`` bypasses
+ the invalidation time check.
+
+ .. versionadded:: 0.3.0
+ Support for the :meth:`.CacheRegion.invalidate`
+ method.
+
+ :param key: Key to be retrieved. While it's typical for a key to be a
+ string, it is ultimately passed directly down to the cache backend,
+ before being optionally processed by the key_mangler function, so can
+ be of any type recognized by the backend or by the key_mangler
+ function, if present.
+
+ :param expiration_time: Optional expiration time value
+ which will supersede that configured on the :class:`.CacheRegion`
+ itself.
+
+ .. versionadded:: 0.3.0
+
+ :param ignore_expiration: if ``True``, the value is returned
+ from the cache if present, regardless of configured
+ expiration times or whether or not :meth:`.invalidate`
+ was called.
+
+ .. versionadded:: 0.3.0
+
+ """
+
+ if self.key_mangler:
+ key = self.key_mangler(key)
+ value = self.backend.get(key)
+ value = self._unexpired_value_fn(
+ expiration_time, ignore_expiration)(value)
+
+ return value.payload
+
+ def _unexpired_value_fn(self, expiration_time, ignore_expiration):
+ if ignore_expiration:
+ return lambda value: value
+ else:
+ if expiration_time is None:
+ expiration_time = self.expiration_time
+
+ current_time = time.time()
+
+ invalidated = self._hard_invalidated or self._soft_invalidated
+
+ def value_fn(value):
+ if value is NO_VALUE:
+ return value
+ elif expiration_time is not None and \
+ current_time - value.metadata["ct"] > expiration_time:
+ return NO_VALUE
+ elif invalidated and \
+ value.metadata["ct"] < invalidated:
+ return NO_VALUE
+ else:
+ return value
+
+ return value_fn
+
+ def get_multi(self, keys, expiration_time=None, ignore_expiration=False):
+ """Return multiple values from the cache, based on the given keys.
+
+ Returns values as a list matching the keys given.
+
+ E.g.::
+
+ values = region.get_multi(["one", "two", "three"])
+
+ To convert values to a dictionary, use ``zip()``::
+
+ keys = ["one", "two", "three"]
+ values = region.get_multi(keys)
+ dictionary = dict(zip(keys, values))
+
+ Keys which aren't present in the list are returned as
+ the ``NO_VALUE`` token. ``NO_VALUE`` evaluates to False,
+ but is separate from
+ ``None`` to distinguish between a cached value of ``None``.
+
+ By default, the configured expiration time of the
+ :class:`.CacheRegion`, or alternatively the expiration
+ time supplied by the ``expiration_time`` argument,
+ is tested against the creation time of the retrieved
+ value versus the current time (as reported by ``time.time()``).
+ If stale, the cached value is ignored and the ``NO_VALUE``
+ token is returned. Passing the flag ``ignore_expiration=True``
+ bypasses the expiration time check.
+
+ .. versionadded:: 0.5.0
+
+ """
+ if not keys:
+ return []
+
+ if self.key_mangler:
+ keys = list(map(lambda key: self.key_mangler(key), keys))
+
+ backend_values = self.backend.get_multi(keys)
+
+ _unexpired_value_fn = self._unexpired_value_fn(
+ expiration_time, ignore_expiration)
+ return [
+ value.payload if value is not NO_VALUE else value
+ for value in
+ (
+ _unexpired_value_fn(value) for value in
+ backend_values
+ )
+ ]
+
+ def get_or_create(
+ self, key, creator, expiration_time=None, should_cache_fn=None):
+ """Return a cached value based on the given key.
+
+ If the value does not exist or is considered to be expired
+ based on its creation time, the given
+ creation function may or may not be used to recreate the value
+ and persist the newly generated value in the cache.
+
+ Whether or not the function is used depends on if the
+ *dogpile lock* can be acquired or not. If it can't, it means
+ a different thread or process is already running a creation
+ function for this key against the cache. When the dogpile
+ lock cannot be acquired, the method will block if no
+ previous value is available, until the lock is released and
+ a new value available. If a previous value
+ is available, that value is returned immediately without blocking.
+
+ If the :meth:`.invalidate` method has been called, and
+ the retrieved value's timestamp is older than the invalidation
+ timestamp, the value is unconditionally prevented from
+ being returned. The method will attempt to acquire the dogpile
+ lock to generate a new value, or will wait
+ until the lock is released to return the new value.
+
+ .. versionchanged:: 0.3.0
+ The value is unconditionally regenerated if the creation
+ time is older than the last call to :meth:`.invalidate`.
+
+ :param key: Key to be retrieved. While it's typical for a key to be a
+ string, it is ultimately passed directly down to the cache backend,
+ before being optionally processed by the key_mangler function, so can
+ be of any type recognized by the backend or by the key_mangler
+ function, if present.
+
+ :param creator: function which creates a new value.
+
+ :param expiration_time: optional expiration time which will overide
+ the expiration time already configured on this :class:`.CacheRegion`
+ if not None. To set no expiration, use the value -1.
+
+ :param should_cache_fn: optional callable function which will receive
+ the value returned by the "creator", and will then return True or
+ False, indicating if the value should actually be cached or not. If
+ it returns False, the value is still returned, but isn't cached.
+ E.g.::
+
+ def dont_cache_none(value):
+ return value is not None
+
+ value = region.get_or_create("some key",
+ create_value,
+ should_cache_fn=dont_cache_none)
+
+ Above, the function returns the value of create_value() if
+ the cache is invalid, however if the return value is None,
+ it won't be cached.
+
+ .. versionadded:: 0.4.3
+
+ .. seealso::
+
+ :meth:`.CacheRegion.cache_on_arguments` - applies
+ :meth:`.get_or_create` to any function using a decorator.
+
+ :meth:`.CacheRegion.get_or_create_multi` - multiple key/value
+ version
+
+ """
+ orig_key = key
+ if self.key_mangler:
+ key = self.key_mangler(key)
+
+ def get_value():
+ value = self.backend.get(key)
+ if value is NO_VALUE or \
+ value.metadata['v'] != value_version or \
+ (
+ self._hard_invalidated and
+ value.metadata["ct"] < self._hard_invalidated):
+ raise NeedRegenerationException()
+ ct = value.metadata["ct"]
+ if self._soft_invalidated:
+ if ct < self._soft_invalidated:
+ ct = time.time() - expiration_time - .0001
+
+ return value.payload, ct
+
+ def gen_value():
+ created_value = creator()
+ value = self._value(created_value)
+
+ if not should_cache_fn or \
+ should_cache_fn(created_value):
+ self.backend.set(key, value)
+
+ return value.payload, value.metadata["ct"]
+
+ if expiration_time is None:
+ expiration_time = self.expiration_time
+
+ if expiration_time is None and self._soft_invalidated:
+ raise exception.DogpileCacheException(
+ "Non-None expiration time required "
+ "for soft invalidation")
+
+ if expiration_time == -1:
+ expiration_time = None
+
+ if self.async_creation_runner:
+ def async_creator(mutex):
+ return self.async_creation_runner(
+ self, orig_key, creator, mutex)
+ else:
+ async_creator = None
+
+ with Lock(
+ self._mutex(key),
+ gen_value,
+ get_value,
+ expiration_time,
+ async_creator) as value:
+ return value
+
+ def get_or_create_multi(
+ self, keys, creator, expiration_time=None, should_cache_fn=None):
+ """Return a sequence of cached values based on a sequence of keys.
+
+ The behavior for generation of values based on keys corresponds
+ to that of :meth:`.Region.get_or_create`, with the exception that
+ the ``creator()`` function may be asked to generate any subset of
+ the given keys. The list of keys to be generated is passed to
+ ``creator()``, and ``creator()`` should return the generated values
+ as a sequence corresponding to the order of the keys.
+
+ The method uses the same approach as :meth:`.Region.get_multi`
+ and :meth:`.Region.set_multi` to get and set values from the
+ backend.
+
+ :param keys: Sequence of keys to be retrieved.
+
+ :param creator: function which accepts a sequence of keys and
+ returns a sequence of new values.
+
+ :param expiration_time: optional expiration time which will overide
+ the expiration time already configured on this :class:`.CacheRegion`
+ if not None. To set no expiration, use the value -1.
+
+ :param should_cache_fn: optional callable function which will receive
+ each value returned by the "creator", and will then return True or
+ False, indicating if the value should actually be cached or not. If
+ it returns False, the value is still returned, but isn't cached.
+
+ .. versionadded:: 0.5.0
+
+ .. seealso::
+
+
+ :meth:`.CacheRegion.cache_multi_on_arguments`
+
+ :meth:`.CacheRegion.get_or_create`
+
+ """
+
+ def get_value(key):
+ value = values.get(key, NO_VALUE)
+
+ if value is NO_VALUE or \
+ value.metadata['v'] != value_version or \
+ (self._hard_invalidated and
+ value.metadata["ct"] < self._hard_invalidated):
+ # dogpile.core understands a 0 here as
+ # "the value is not available", e.g.
+ # _has_value() will return False.
+ return value.payload, 0
+ else:
+ ct = value.metadata["ct"]
+ if self._soft_invalidated:
+ if ct < self._soft_invalidated:
+ ct = time.time() - expiration_time - .0001
+
+ return value.payload, ct
+
+ def gen_value():
+ raise NotImplementedError()
+
+ def async_creator(key, mutex):
+ mutexes[key] = mutex
+
+ if expiration_time is None:
+ expiration_time = self.expiration_time
+
+ if expiration_time is None and self._soft_invalidated:
+ raise exception.DogpileCacheException(
+ "Non-None expiration time required "
+ "for soft invalidation")
+
+ if expiration_time == -1:
+ expiration_time = None
+
+ mutexes = {}
+
+ sorted_unique_keys = sorted(set(keys))
+
+ if self.key_mangler:
+ mangled_keys = [self.key_mangler(k) for k in sorted_unique_keys]
+ else:
+ mangled_keys = sorted_unique_keys
+
+ orig_to_mangled = dict(zip(sorted_unique_keys, mangled_keys))
+
+ values = dict(zip(mangled_keys, self.backend.get_multi(mangled_keys)))
+
+ for orig_key, mangled_key in orig_to_mangled.items():
+ with Lock(
+ self._mutex(mangled_key),
+ gen_value,
+ lambda: get_value(mangled_key),
+ expiration_time,
+ async_creator=lambda mutex: async_creator(orig_key, mutex)
+ ):
+ pass
+ try:
+ if mutexes:
+ # sort the keys, the idea is to prevent deadlocks.
+ # though haven't been able to simulate one anyway.
+ keys_to_get = sorted(mutexes)
+ new_values = creator(*keys_to_get)
+
+ values_w_created = dict(
+ (orig_to_mangled[k], self._value(v))
+ for k, v in zip(keys_to_get, new_values)
+ )
+
+ if not should_cache_fn:
+ self.backend.set_multi(values_w_created)
+ else:
+ self.backend.set_multi(dict(
+ (k, v)
+ for k, v in values_w_created.items()
+ if should_cache_fn(v[0])
+ ))
+
+ values.update(values_w_created)
+ return [values[orig_to_mangled[k]].payload for k in keys]
+ finally:
+ for mutex in mutexes.values():
+ mutex.release()
+
+ def _value(self, value):
+ """Return a :class:`.CachedValue` given a value."""
+ return CachedValue(
+ value,
+ {
+ "ct": time.time(),
+ "v": value_version
+ })
+
+ def set(self, key, value):
+ """Place a new value in the cache under the given key."""
+
+ if self.key_mangler:
+ key = self.key_mangler(key)
+ self.backend.set(key, self._value(value))
+
+ def set_multi(self, mapping):
+ """Place new values in the cache under the given keys.
+
+ .. versionadded:: 0.5.0
+
+ """
+ if not mapping:
+ return
+
+ if self.key_mangler:
+ mapping = dict((
+ self.key_mangler(k), self._value(v))
+ for k, v in mapping.items())
+ else:
+ mapping = dict((k, self._value(v)) for k, v in mapping.items())
+ self.backend.set_multi(mapping)
+
+ def delete(self, key):
+ """Remove a value from the cache.
+
+ This operation is idempotent (can be called multiple times, or on a
+ non-existent key, safely)
+ """
+
+ if self.key_mangler:
+ key = self.key_mangler(key)
+
+ self.backend.delete(key)
+
+ def delete_multi(self, keys):
+ """Remove multiple values from the cache.
+
+ This operation is idempotent (can be called multiple times, or on a
+ non-existent key, safely)
+
+ .. versionadded:: 0.5.0
+
+ """
+
+ if self.key_mangler:
+ keys = list(map(lambda key: self.key_mangler(key), keys))
+
+ self.backend.delete_multi(keys)
+
+ def cache_on_arguments(
+ self, namespace=None,
+ expiration_time=None,
+ should_cache_fn=None,
+ to_str=compat.string_type,
+ function_key_generator=None):
+ """A function decorator that will cache the return
+ value of the function using a key derived from the
+ function itself and its arguments.
+
+ The decorator internally makes use of the
+ :meth:`.CacheRegion.get_or_create` method to access the
+ cache and conditionally call the function. See that
+ method for additional behavioral details.
+
+ E.g.::
+
+ @someregion.cache_on_arguments()
+ def generate_something(x, y):
+ return somedatabase.query(x, y)
+
+ The decorated function can then be called normally, where
+ data will be pulled from the cache region unless a new
+ value is needed::
+
+ result = generate_something(5, 6)
+
+ The function is also given an attribute ``invalidate()``, which
+ provides for invalidation of the value. Pass to ``invalidate()``
+ the same arguments you'd pass to the function itself to represent
+ a particular value::
+
+ generate_something.invalidate(5, 6)
+
+ Another attribute ``set()`` is added to provide extra caching
+ possibilities relative to the function. This is a convenience
+ method for :meth:`.CacheRegion.set` which will store a given
+ value directly without calling the decorated function.
+ The value to be cached is passed as the first argument, and the
+ arguments which would normally be passed to the function
+ should follow::
+
+ generate_something.set(3, 5, 6)
+
+ The above example is equivalent to calling
+ ``generate_something(5, 6)``, if the function were to produce
+ the value ``3`` as the value to be cached.
+
+ .. versionadded:: 0.4.1 Added ``set()`` method to decorated function.
+
+ Similar to ``set()`` is ``refresh()``. This attribute will
+ invoke the decorated function and populate a new value into
+ the cache with the new value, as well as returning that value::
+
+ newvalue = generate_something.refresh(5, 6)
+
+ .. versionadded:: 0.5.0 Added ``refresh()`` method to decorated
+ function.
+
+ Lastly, the ``get()`` method returns either the value cached
+ for the given key, or the token ``NO_VALUE`` if no such key
+ exists::
+
+ value = generate_something.get(5, 6)
+
+ .. versionadded:: 0.5.3 Added ``get()`` method to decorated
+ function.
+
+ The default key generation will use the name
+ of the function, the module name for the function,
+ the arguments passed, as well as an optional "namespace"
+ parameter in order to generate a cache key.
+
+ Given a function ``one`` inside the module
+ ``myapp.tools``::
+
+ @region.cache_on_arguments(namespace="foo")
+ def one(a, b):
+ return a + b
+
+ Above, calling ``one(3, 4)`` will produce a
+ cache key as follows::
+
+ myapp.tools:one|foo|3 4
+
+ The key generator will ignore an initial argument
+ of ``self`` or ``cls``, making the decorator suitable
+ (with caveats) for use with instance or class methods.
+ Given the example::
+
+ class MyClass(object):
+ @region.cache_on_arguments(namespace="foo")
+ def one(self, a, b):
+ return a + b
+
+ The cache key above for ``MyClass().one(3, 4)`` will
+ again produce the same cache key of ``myapp.tools:one|foo|3 4`` -
+ the name ``self`` is skipped.
+
+ The ``namespace`` parameter is optional, and is used
+ normally to disambiguate two functions of the same
+ name within the same module, as can occur when decorating
+ instance or class methods as below::
+
+ class MyClass(object):
+ @region.cache_on_arguments(namespace='MC')
+ def somemethod(self, x, y):
+ ""
+
+ class MyOtherClass(object):
+ @region.cache_on_arguments(namespace='MOC')
+ def somemethod(self, x, y):
+ ""
+
+ Above, the ``namespace`` parameter disambiguates
+ between ``somemethod`` on ``MyClass`` and ``MyOtherClass``.
+ Python class declaration mechanics otherwise prevent
+ the decorator from having awareness of the ``MyClass``
+ and ``MyOtherClass`` names, as the function is received
+ by the decorator before it becomes an instance method.
+
+ The function key generation can be entirely replaced
+ on a per-region basis using the ``function_key_generator``
+ argument present on :func:`.make_region` and
+ :class:`.CacheRegion`. If defaults to
+ :func:`.function_key_generator`.
+
+ :param namespace: optional string argument which will be
+ established as part of the cache key. This may be needed
+ to disambiguate functions of the same name within the same
+ source file, such as those
+ associated with classes - note that the decorator itself
+ can't see the parent class on a function as the class is
+ being declared.
+
+ :param expiration_time: if not None, will override the normal
+ expiration time.
+
+ May be specified as a callable, taking no arguments, that
+ returns a value to be used as the ``expiration_time``. This callable
+ will be called whenever the decorated function itself is called, in
+ caching or retrieving. Thus, this can be used to
+ determine a *dynamic* expiration time for the cached function
+ result. Example use cases include "cache the result until the
+ end of the day, week or time period" and "cache until a certain date
+ or time passes".
+
+ .. versionchanged:: 0.5.0
+ ``expiration_time`` may be passed as a callable to
+ :meth:`.CacheRegion.cache_on_arguments`.
+
+ :param should_cache_fn: passed to :meth:`.CacheRegion.get_or_create`.
+
+ .. versionadded:: 0.4.3
+
+ :param to_str: callable, will be called on each function argument
+ in order to convert to a string. Defaults to ``str()``. If the
+ function accepts non-ascii unicode arguments on Python 2.x, the
+ ``unicode()`` builtin can be substituted, but note this will
+ produce unicode cache keys which may require key mangling before
+ reaching the cache.
+
+ .. versionadded:: 0.5.0
+
+ :param function_key_generator: a function that will produce a
+ "cache key". This function will supersede the one configured on the
+ :class:`.CacheRegion` itself.
+
+ .. versionadded:: 0.5.5
+
+ .. seealso::
+
+ :meth:`.CacheRegion.cache_multi_on_arguments`
+
+ :meth:`.CacheRegion.get_or_create`
+
+ """
+ expiration_time_is_callable = compat.callable(expiration_time)
+
+ if function_key_generator is None:
+ function_key_generator = self.function_key_generator
+
+ def decorator(fn):
+ if to_str is compat.string_type:
+ # backwards compatible
+ key_generator = function_key_generator(namespace, fn)
+ else:
+ key_generator = function_key_generator(
+ namespace, fn,
+ to_str=to_str)
+
+ @wraps(fn)
+ def decorate(*arg, **kw):
+ key = key_generator(*arg, **kw)
+
+ @wraps(fn)
+ def creator():
+ return fn(*arg, **kw)
+ timeout = expiration_time() if expiration_time_is_callable \
+ else expiration_time
+ return self.get_or_create(key, creator, timeout,
+ should_cache_fn)
+
+ def invalidate(*arg, **kw):
+ key = key_generator(*arg, **kw)
+ self.delete(key)
+
+ def set_(value, *arg, **kw):
+ key = key_generator(*arg, **kw)
+ self.set(key, value)
+
+ def get(*arg, **kw):
+ key = key_generator(*arg, **kw)
+ return self.get(key)
+
+ def refresh(*arg, **kw):
+ key = key_generator(*arg, **kw)
+ value = fn(*arg, **kw)
+ self.set(key, value)
+ return value
+
+ decorate.set = set_
+ decorate.invalidate = invalidate
+ decorate.refresh = refresh
+ decorate.get = get
+
+ return decorate
+ return decorator
+
+ def cache_multi_on_arguments(
+ self, namespace=None, expiration_time=None,
+ should_cache_fn=None,
+ asdict=False, to_str=compat.string_type,
+ function_multi_key_generator=None):
+ """A function decorator that will cache multiple return
+ values from the function using a sequence of keys derived from the
+ function itself and the arguments passed to it.
+
+ This method is the "multiple key" analogue to the
+ :meth:`.CacheRegion.cache_on_arguments` method.
+
+ Example::
+
+ @someregion.cache_multi_on_arguments()
+ def generate_something(*keys):
+ return [
+ somedatabase.query(key)
+ for key in keys
+ ]
+
+ The decorated function can be called normally. The decorator
+ will produce a list of cache keys using a mechanism similar to
+ that of :meth:`.CacheRegion.cache_on_arguments`, combining the
+ name of the function with the optional namespace and with the
+ string form of each key. It will then consult the cache using
+ the same mechanism as that of :meth:`.CacheRegion.get_multi`
+ to retrieve all current values; the originally passed keys
+ corresponding to those values which aren't generated or need
+ regeneration will be assembled into a new argument list, and
+ the decorated function is then called with that subset of
+ arguments.
+
+ The returned result is a list::
+
+ result = generate_something("key1", "key2", "key3")
+
+ The decorator internally makes use of the
+ :meth:`.CacheRegion.get_or_create_multi` method to access the
+ cache and conditionally call the function. See that
+ method for additional behavioral details.
+
+ Unlike the :meth:`.CacheRegion.cache_on_arguments` method,
+ :meth:`.CacheRegion.cache_multi_on_arguments` works only with
+ a single function signature, one which takes a simple list of
+ keys as arguments.
+
+ Like :meth:`.CacheRegion.cache_on_arguments`, the decorated function
+ is also provided with a ``set()`` method, which here accepts a
+ mapping of keys and values to set in the cache::
+
+ generate_something.set({"k1": "value1",
+ "k2": "value2", "k3": "value3"})
+
+ ...an ``invalidate()`` method, which has the effect of deleting
+ the given sequence of keys using the same mechanism as that of
+ :meth:`.CacheRegion.delete_multi`::
+
+ generate_something.invalidate("k1", "k2", "k3")
+
+ ...a ``refresh()`` method, which will call the creation
+ function, cache the new values, and return them::
+
+ values = generate_something.refresh("k1", "k2", "k3")
+
+ ...and a ``get()`` method, which will return values
+ based on the given arguments::
+
+ values = generate_something.get("k1", "k2", "k3")
+
+ .. versionadded:: 0.5.3 Added ``get()`` method to decorated
+ function.
+
+ Parameters passed to :meth:`.CacheRegion.cache_multi_on_arguments`
+ have the same meaning as those passed to
+ :meth:`.CacheRegion.cache_on_arguments`.
+
+ :param namespace: optional string argument which will be
+ established as part of each cache key.
+
+ :param expiration_time: if not None, will override the normal
+ expiration time. May be passed as an integer or a
+ callable.
+
+ :param should_cache_fn: passed to
+ :meth:`.CacheRegion.get_or_create_multi`. This function is given a
+ value as returned by the creator, and only if it returns True will
+ that value be placed in the cache.
+
+ :param asdict: if ``True``, the decorated function should return
+ its result as a dictionary of keys->values, and the final result
+ of calling the decorated function will also be a dictionary.
+ If left at its default value of ``False``, the decorated function
+ should return its result as a list of values, and the final
+ result of calling the decorated function will also be a list.
+
+ When ``asdict==True`` if the dictionary returned by the decorated
+ function is missing keys, those keys will not be cached.
+
+ :param to_str: callable, will be called on each function argument
+ in order to convert to a string. Defaults to ``str()``. If the
+ function accepts non-ascii unicode arguments on Python 2.x, the
+ ``unicode()`` builtin can be substituted, but note this will
+ produce unicode cache keys which may require key mangling before
+ reaching the cache.
+
+ .. versionadded:: 0.5.0
+
+ :param function_multi_key_generator: a function that will produce a
+ list of keys. This function will supersede the one configured on the
+ :class:`.CacheRegion` itself.
+
+ .. versionadded:: 0.5.5
+
+ .. seealso::
+
+ :meth:`.CacheRegion.cache_on_arguments`
+
+ :meth:`.CacheRegion.get_or_create_multi`
+
+ """
+ expiration_time_is_callable = compat.callable(expiration_time)
+
+ if function_multi_key_generator is None:
+ function_multi_key_generator = self.function_multi_key_generator
+
+ def decorator(fn):
+ key_generator = function_multi_key_generator(
+ namespace, fn,
+ to_str=to_str)
+
+ @wraps(fn)
+ def decorate(*arg, **kw):
+ cache_keys = arg
+ keys = key_generator(*arg, **kw)
+ key_lookup = dict(zip(keys, cache_keys))
+
+ @wraps(fn)
+ def creator(*keys_to_create):
+ return fn(*[key_lookup[k] for k in keys_to_create])
+
+ timeout = expiration_time() if expiration_time_is_callable \
+ else expiration_time
+
+ if asdict:
+ def dict_create(*keys):
+ d_values = creator(*keys)
+ return [
+ d_values.get(key_lookup[k], NO_VALUE)
+ for k in keys]
+
+ def wrap_cache_fn(value):
+ if value is NO_VALUE:
+ return False
+ elif not should_cache_fn:
+ return True
+ else:
+ return should_cache_fn(value)
+
+ result = self.get_or_create_multi(
+ keys, dict_create, timeout, wrap_cache_fn)
+ result = dict(
+ (k, v) for k, v in zip(cache_keys, result)
+ if v is not NO_VALUE)
+ else:
+ result = self.get_or_create_multi(
+ keys, creator, timeout,
+ should_cache_fn)
+
+ return result
+
+ def invalidate(*arg):
+ keys = key_generator(*arg)
+ self.delete_multi(keys)
+
+ def set_(mapping):
+ keys = list(mapping)
+ gen_keys = key_generator(*keys)
+ self.set_multi(dict(
+ (gen_key, mapping[key])
+ for gen_key, key
+ in zip(gen_keys, keys))
+ )
+
+ def get(*arg):
+ keys = key_generator(*arg)
+ return self.get_multi(keys)
+
+ def refresh(*arg):
+ keys = key_generator(*arg)
+ values = fn(*arg)
+ if asdict:
+ self.set_multi(
+ dict(zip(keys, [values[a] for a in arg]))
+ )
+ return values
+ else:
+ self.set_multi(
+ dict(zip(keys, values))
+ )
+ return values
+
+ decorate.set = set_
+ decorate.invalidate = invalidate
+ decorate.refresh = refresh
+ decorate.get = get
+
+ return decorate
+ return decorator
+
+
+def make_region(*arg, **kw):
+ """Instantiate a new :class:`.CacheRegion`.
+
+ Currently, :func:`.make_region` is a passthrough
+ to :class:`.CacheRegion`. See that class for
+ constructor arguments.
+
+ """
+ return CacheRegion(*arg, **kw)
diff --git a/lib/dogpile/cache/util.py b/lib/dogpile/cache/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..51fe483fd6dbcb6e63c287a868de4f6578f63a99
--- /dev/null
+++ b/lib/dogpile/cache/util.py
@@ -0,0 +1,195 @@
+from hashlib import sha1
+import inspect
+import re
+import collections
+from . import compat
+
+
+def coerce_string_conf(d):
+ result = {}
+ for k, v in d.items():
+ if not isinstance(v, compat.string_types):
+ result[k] = v
+ continue
+
+ v = v.strip()
+ if re.match(r'^[-+]?\d+$', v):
+ result[k] = int(v)
+ elif re.match(r'^[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?$', v):
+ result[k] = float(v)
+ elif v.lower() in ('false', 'true'):
+ result[k] = v.lower() == 'true'
+ elif v == 'None':
+ result[k] = None
+ else:
+ result[k] = v
+ return result
+
+
+class PluginLoader(object):
+ def __init__(self, group):
+ self.group = group
+ self.impls = {}
+
+ def load(self, name):
+ if name in self.impls:
+ return self.impls[name]()
+ else: # pragma NO COVERAGE
+ import pkg_resources
+ for impl in pkg_resources.iter_entry_points(
+ self.group, name):
+ self.impls[name] = impl.load
+ return impl.load()
+ else:
+ raise Exception(
+ "Can't load plugin %s %s" %
+ (self.group, name))
+
+ def register(self, name, modulepath, objname):
+ def load():
+ mod = __import__(modulepath, fromlist=[objname])
+ return getattr(mod, objname)
+ self.impls[name] = load
+
+
+def function_key_generator(namespace, fn, to_str=compat.string_type):
+ """Return a function that generates a string
+ key, based on a given function as well as
+ arguments to the returned function itself.
+
+ This is used by :meth:`.CacheRegion.cache_on_arguments`
+ to generate a cache key from a decorated function.
+
+ It can be replaced using the ``function_key_generator``
+ argument passed to :func:`.make_region`.
+
+ """
+
+ if namespace is None:
+ namespace = '%s:%s' % (fn.__module__, fn.__name__)
+ else:
+ namespace = '%s:%s|%s' % (fn.__module__, fn.__name__, namespace)
+
+ args = inspect.getargspec(fn)
+ has_self = args[0] and args[0][0] in ('self', 'cls')
+
+ def generate_key(*args, **kw):
+ if kw:
+ raise ValueError(
+ "dogpile.cache's default key creation "
+ "function does not accept keyword arguments.")
+ if has_self:
+ args = args[1:]
+
+ return namespace + "|" + " ".join(map(to_str, args))
+ return generate_key
+
+
+def function_multi_key_generator(namespace, fn, to_str=compat.string_type):
+
+ if namespace is None:
+ namespace = '%s:%s' % (fn.__module__, fn.__name__)
+ else:
+ namespace = '%s:%s|%s' % (fn.__module__, fn.__name__, namespace)
+
+ args = inspect.getargspec(fn)
+ has_self = args[0] and args[0][0] in ('self', 'cls')
+
+ def generate_keys(*args, **kw):
+ if kw:
+ raise ValueError(
+ "dogpile.cache's default key creation "
+ "function does not accept keyword arguments.")
+ if has_self:
+ args = args[1:]
+ return [namespace + "|" + key for key in map(to_str, args)]
+ return generate_keys
+
+
+def sha1_mangle_key(key):
+ """a SHA1 key mangler."""
+
+ return sha1(key).hexdigest()
+
+
+def length_conditional_mangler(length, mangler):
+ """a key mangler that mangles if the length of the key is
+ past a certain threshold.
+
+ """
+ def mangle(key):
+ if len(key) >= length:
+ return mangler(key)
+ else:
+ return key
+ return mangle
+
+
+class memoized_property(object):
+ """A read-only @property that is only evaluated once."""
+ def __init__(self, fget, doc=None):
+ self.fget = fget
+ self.__doc__ = doc or fget.__doc__
+ self.__name__ = fget.__name__
+
+ def __get__(self, obj, cls):
+ if obj is None:
+ return self
+ obj.__dict__[self.__name__] = result = self.fget(obj)
+ return result
+
+
+def to_list(x, default=None):
+ """Coerce to a list."""
+ if x is None:
+ return default
+ if not isinstance(x, (list, tuple)):
+ return [x]
+ else:
+ return x
+
+
+class KeyReentrantMutex(object):
+
+ def __init__(self, key, mutex, keys):
+ self.key = key
+ self.mutex = mutex
+ self.keys = keys
+
+ @classmethod
+ def factory(cls, mutex):
+ # this collection holds zero or one
+ # thread idents as the key; a set of
+ # keynames held as the value.
+ keystore = collections.defaultdict(set)
+
+ def fac(key):
+ return KeyReentrantMutex(key, mutex, keystore)
+ return fac
+
+ def acquire(self, wait=True):
+ current_thread = compat.threading.current_thread().ident
+ keys = self.keys.get(current_thread)
+ if keys is not None and \
+ self.key not in keys:
+ # current lockholder, new key. add it in
+ keys.add(self.key)
+ return True
+ elif self.mutex.acquire(wait=wait):
+ # after acquire, create new set and add our key
+ self.keys[current_thread].add(self.key)
+ return True
+ else:
+ return False
+
+ def release(self):
+ current_thread = compat.threading.current_thread().ident
+ keys = self.keys.get(current_thread)
+ assert keys is not None, "this thread didn't do the acquire"
+ assert self.key in keys, "No acquire held for key '%s'" % self.key
+ keys.remove(self.key)
+ if not keys:
+ # when list of keys empty, remove
+ # the thread ident and unlock.
+ del self.keys[current_thread]
+ self.mutex.release()
diff --git a/lib/dogpile/core/__init__.py b/lib/dogpile/core/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..fb9d756d675fb0b39f44a202d75acfab084249b1
--- /dev/null
+++ b/lib/dogpile/core/__init__.py
@@ -0,0 +1,11 @@
+from .dogpile import NeedRegenerationException, Lock
+from .nameregistry import NameRegistry
+from .readwrite_lock import ReadWriteMutex
+from .legacy import Dogpile, SyncReaderDogpile
+
+__all__ = [
+ 'Dogpile', 'SyncReaderDogpile', 'NeedRegenerationException',
+ 'NameRegistry', 'ReadWriteMutex', 'Lock']
+
+__version__ = '0.4.1'
+
diff --git a/lib/dogpile/core/dogpile.py b/lib/dogpile/core/dogpile.py
new file mode 100644
index 0000000000000000000000000000000000000000..2e3ca0e9315fb66fa3637c43c7b9f3d360ad25b7
--- /dev/null
+++ b/lib/dogpile/core/dogpile.py
@@ -0,0 +1,162 @@
+import time
+import logging
+
+log = logging.getLogger(__name__)
+
+class NeedRegenerationException(Exception):
+ """An exception that when raised in the 'with' block,
+ forces the 'has_value' flag to False and incurs a
+ regeneration of the value.
+
+ """
+
+NOT_REGENERATED = object()
+
+class Lock(object):
+ """Dogpile lock class.
+
+ Provides an interface around an arbitrary mutex
+ that allows one thread/process to be elected as
+ the creator of a new value, while other threads/processes
+ continue to return the previous version
+ of that value.
+
+ .. versionadded:: 0.4.0
+ The :class:`.Lock` class was added as a single-use object
+ representing the dogpile API without dependence on
+ any shared state between multiple instances.
+
+ :param mutex: A mutex object that provides ``acquire()``
+ and ``release()`` methods.
+ :param creator: Callable which returns a tuple of the form
+ (new_value, creation_time). "new_value" should be a newly
+ generated value representing completed state. "creation_time"
+ should be a floating point time value which is relative
+ to Python's ``time.time()`` call, representing the time
+ at which the value was created. This time value should
+ be associated with the created value.
+ :param value_and_created_fn: Callable which returns
+ a tuple of the form (existing_value, creation_time). This
+ basically should return what the last local call to the ``creator()``
+ callable has returned, i.e. the value and the creation time,
+ which would be assumed here to be from a cache. If the
+ value is not available, the :class:`.NeedRegenerationException`
+ exception should be thrown.
+ :param expiretime: Expiration time in seconds. Set to
+ ``None`` for never expires. This timestamp is compared
+ to the creation_time result and ``time.time()`` to determine if
+ the value returned by value_and_created_fn is "expired".
+ :param async_creator: A callable. If specified, this callable will be
+ passed the mutex as an argument and is responsible for releasing the mutex
+ after it finishes some asynchronous value creation. The intent is for
+ this to be used to defer invocation of the creator callable until some
+ later time.
+
+ .. versionadded:: 0.4.1 added the async_creator argument.
+
+ """
+
+ def __init__(self,
+ mutex,
+ creator,
+ value_and_created_fn,
+ expiretime,
+ async_creator=None,
+ ):
+ self.mutex = mutex
+ self.creator = creator
+ self.value_and_created_fn = value_and_created_fn
+ self.expiretime = expiretime
+ self.async_creator = async_creator
+
+ def _is_expired(self, createdtime):
+ """Return true if the expiration time is reached, or no
+ value is available."""
+
+ return not self._has_value(createdtime) or \
+ (
+ self.expiretime is not None and
+ time.time() - createdtime > self.expiretime
+ )
+
+ def _has_value(self, createdtime):
+ """Return true if the creation function has proceeded
+ at least once."""
+ return createdtime > 0
+
+ def _enter(self):
+ value_fn = self.value_and_created_fn
+
+ try:
+ value = value_fn()
+ value, createdtime = value
+ except NeedRegenerationException:
+ log.debug("NeedRegenerationException")
+ value = NOT_REGENERATED
+ createdtime = -1
+
+ generated = self._enter_create(createdtime)
+
+ if generated is not NOT_REGENERATED:
+ generated, createdtime = generated
+ return generated
+ elif value is NOT_REGENERATED:
+ try:
+ value, createdtime = value_fn()
+ return value
+ except NeedRegenerationException:
+ raise Exception("Generation function should "
+ "have just been called by a concurrent "
+ "thread.")
+ else:
+ return value
+
+ def _enter_create(self, createdtime):
+
+ if not self._is_expired(createdtime):
+ return NOT_REGENERATED
+
+ async = False
+
+ if self._has_value(createdtime):
+ if not self.mutex.acquire(False):
+ log.debug("creation function in progress "
+ "elsewhere, returning")
+ return NOT_REGENERATED
+ else:
+ log.debug("no value, waiting for create lock")
+ self.mutex.acquire()
+
+ try:
+ log.debug("value creation lock %r acquired" % self.mutex)
+
+ # see if someone created the value already
+ try:
+ value, createdtime = self.value_and_created_fn()
+ except NeedRegenerationException:
+ pass
+ else:
+ if not self._is_expired(createdtime):
+ log.debug("value already present")
+ return value, createdtime
+ elif self.async_creator:
+ log.debug("Passing creation lock to async runner")
+ self.async_creator(self.mutex)
+ async = True
+ return value, createdtime
+
+ log.debug("Calling creation function")
+ created = self.creator()
+ return created
+ finally:
+ if not async:
+ self.mutex.release()
+ log.debug("Released creation lock")
+
+
+ def __enter__(self):
+ return self._enter()
+
+ def __exit__(self, type, value, traceback):
+ pass
+
diff --git a/lib/dogpile/core/legacy.py b/lib/dogpile/core/legacy.py
new file mode 100644
index 0000000000000000000000000000000000000000..dad4e1609d2434d206e05c6b55abca020c7f6bc0
--- /dev/null
+++ b/lib/dogpile/core/legacy.py
@@ -0,0 +1,154 @@
+from __future__ import with_statement
+
+from .util import threading
+from .readwrite_lock import ReadWriteMutex
+from .dogpile import Lock
+import time
+import contextlib
+
+class Dogpile(object):
+ """Dogpile lock class.
+
+ .. deprecated:: 0.4.0
+ The :class:`.Lock` object specifies the full
+ API of the :class:`.Dogpile` object in a single way,
+ rather than providing multiple modes of usage which
+ don't necessarily work in the majority of cases.
+ :class:`.Dogpile` is now a wrapper around the :class:`.Lock` object
+ which provides dogpile.core's original usage pattern.
+ This usage pattern began as something simple, but was
+ not of general use in real-world caching environments without
+ several extra complicating factors; the :class:`.Lock`
+ object presents the "real-world" API more succinctly,
+ and also fixes a cross-process concurrency issue.
+
+ :param expiretime: Expiration time in seconds. Set to
+ ``None`` for never expires.
+ :param init: if True, set the 'createdtime' to the
+ current time.
+ :param lock: a mutex object that provides
+ ``acquire()`` and ``release()`` methods.
+
+ """
+ def __init__(self, expiretime, init=False, lock=None):
+ """Construct a new :class:`.Dogpile`.
+
+ """
+ if lock:
+ self.dogpilelock = lock
+ else:
+ self.dogpilelock = threading.Lock()
+
+ self.expiretime = expiretime
+ if init:
+ self.createdtime = time.time()
+
+ createdtime = -1
+ """The last known 'creation time' of the value,
+ stored as an epoch (i.e. from ``time.time()``).
+
+ If the value here is -1, it is assumed the value
+ should recreate immediately.
+
+ """
+
+ def acquire(self, creator,
+ value_fn=None,
+ value_and_created_fn=None):
+ """Acquire the lock, returning a context manager.
+
+ :param creator: Creation function, used if this thread
+ is chosen to create a new value.
+
+ :param value_fn: Optional function that returns
+ the value from some datasource. Will be returned
+ if regeneration is not needed.
+
+ :param value_and_created_fn: Like value_fn, but returns a tuple
+ of (value, createdtime). The returned createdtime
+ will replace the "createdtime" value on this dogpile
+ lock. This option removes the need for the dogpile lock
+ itself to remain persistent across usages; another
+ dogpile can come along later and pick up where the
+ previous one left off.
+
+ """
+
+ if value_and_created_fn is None:
+ if value_fn is None:
+ def value_and_created_fn():
+ return None, self.createdtime
+ else:
+ def value_and_created_fn():
+ return value_fn(), self.createdtime
+
+ def creator_wrapper():
+ value = creator()
+ self.createdtime = time.time()
+ return value, self.createdtime
+ else:
+ def creator_wrapper():
+ value = creator()
+ self.createdtime = time.time()
+ return value
+
+ return Lock(
+ self.dogpilelock,
+ creator_wrapper,
+ value_and_created_fn,
+ self.expiretime
+ )
+
+ @property
+ def is_expired(self):
+ """Return true if the expiration time is reached, or no
+ value is available."""
+
+ return not self.has_value or \
+ (
+ self.expiretime is not None and
+ time.time() - self.createdtime > self.expiretime
+ )
+
+ @property
+ def has_value(self):
+ """Return true if the creation function has proceeded
+ at least once."""
+ return self.createdtime > 0
+
+
+class SyncReaderDogpile(Dogpile):
+ """Provide a read-write lock function on top of the :class:`.Dogpile`
+ class.
+
+ .. deprecated:: 0.4.0
+ The :class:`.ReadWriteMutex` object can be used directly.
+
+ """
+ def __init__(self, *args, **kw):
+ super(SyncReaderDogpile, self).__init__(*args, **kw)
+ self.readwritelock = ReadWriteMutex()
+
+ @contextlib.contextmanager
+ def acquire_write_lock(self):
+ """Return the "write" lock context manager.
+
+ This will provide a section that is mutexed against
+ all readers/writers for the dogpile-maintained value.
+
+ """
+
+ self.readwritelock.acquire_write_lock()
+ try:
+ yield
+ finally:
+ self.readwritelock.release_write_lock()
+
+ @contextlib.contextmanager
+ def acquire(self, *arg, **kw):
+ with super(SyncReaderDogpile, self).acquire(*arg, **kw) as value:
+ self.readwritelock.acquire_read_lock()
+ try:
+ yield value
+ finally:
+ self.readwritelock.release_read_lock()
diff --git a/lib/dogpile/core/nameregistry.py b/lib/dogpile/core/nameregistry.py
new file mode 100644
index 0000000000000000000000000000000000000000..a73f450c71b67d15097657bee2ead812f354b1ef
--- /dev/null
+++ b/lib/dogpile/core/nameregistry.py
@@ -0,0 +1,83 @@
+from .util import threading
+import weakref
+
+class NameRegistry(object):
+ """Generates and return an object, keeping it as a
+ singleton for a certain identifier for as long as its
+ strongly referenced.
+
+ e.g.::
+
+ class MyFoo(object):
+ "some important object."
+ def __init__(self, identifier):
+ self.identifier = identifier
+
+ registry = NameRegistry(MyFoo)
+
+ # thread 1:
+ my_foo = registry.get("foo1")
+
+ # thread 2
+ my_foo = registry.get("foo1")
+
+ Above, ``my_foo`` in both thread #1 and #2 will
+ be *the same object*. The constructor for
+ ``MyFoo`` will be called once, passing the
+ identifier ``foo1`` as the argument.
+
+ When thread 1 and thread 2 both complete or
+ otherwise delete references to ``my_foo``, the
+ object is *removed* from the :class:`.NameRegistry` as
+ a result of Python garbage collection.
+
+ :param creator: A function that will create a new
+ value, given the identifier passed to the :meth:`.NameRegistry.get`
+ method.
+
+ """
+ _locks = weakref.WeakValueDictionary()
+ _mutex = threading.RLock()
+
+ def __init__(self, creator):
+ """Create a new :class:`.NameRegistry`.
+
+
+ """
+ self._values = weakref.WeakValueDictionary()
+ self._mutex = threading.RLock()
+ self.creator = creator
+
+ def get(self, identifier, *args, **kw):
+ """Get and possibly create the value.
+
+ :param identifier: Hash key for the value.
+ If the creation function is called, this identifier
+ will also be passed to the creation function.
+ :param \*args, \**kw: Additional arguments which will
+ also be passed to the creation function if it is
+ called.
+
+ """
+ try:
+ if identifier in self._values:
+ return self._values[identifier]
+ else:
+ return self._sync_get(identifier, *args, **kw)
+ except KeyError:
+ return self._sync_get(identifier, *args, **kw)
+
+ def _sync_get(self, identifier, *args, **kw):
+ self._mutex.acquire()
+ try:
+ try:
+ if identifier in self._values:
+ return self._values[identifier]
+ else:
+ self._values[identifier] = value = self.creator(identifier, *args, **kw)
+ return value
+ except KeyError:
+ self._values[identifier] = value = self.creator(identifier, *args, **kw)
+ return value
+ finally:
+ self._mutex.release()
diff --git a/lib/dogpile/core/readwrite_lock.py b/lib/dogpile/core/readwrite_lock.py
new file mode 100644
index 0000000000000000000000000000000000000000..1ea25e47ab95fb0d3ee369c97c784d0aac9fe48f
--- /dev/null
+++ b/lib/dogpile/core/readwrite_lock.py
@@ -0,0 +1,130 @@
+from .util import threading
+
+import logging
+log = logging.getLogger(__name__)
+
+class LockError(Exception):
+ pass
+
+class ReadWriteMutex(object):
+ """A mutex which allows multiple readers, single writer.
+
+ :class:`.ReadWriteMutex` uses a Python ``threading.Condition``
+ to provide this functionality across threads within a process.
+
+ The Beaker package also contained a file-lock based version
+ of this concept, so that readers/writers could be synchronized
+ across processes with a common filesystem. A future Dogpile
+ release may include this additional class at some point.
+
+ """
+
+ def __init__(self):
+ # counts how many asynchronous methods are executing
+ self.async = 0
+
+ # pointer to thread that is the current sync operation
+ self.current_sync_operation = None
+
+ # condition object to lock on
+ self.condition = threading.Condition(threading.Lock())
+
+ def acquire_read_lock(self, wait = True):
+ """Acquire the 'read' lock."""
+ self.condition.acquire()
+ try:
+ # see if a synchronous operation is waiting to start
+ # or is already running, in which case we wait (or just
+ # give up and return)
+ if wait:
+ while self.current_sync_operation is not None:
+ self.condition.wait()
+ else:
+ if self.current_sync_operation is not None:
+ return False
+
+ self.async += 1
+ log.debug("%s acquired read lock", self)
+ finally:
+ self.condition.release()
+
+ if not wait:
+ return True
+
+ def release_read_lock(self):
+ """Release the 'read' lock."""
+ self.condition.acquire()
+ try:
+ self.async -= 1
+
+ # check if we are the last asynchronous reader thread
+ # out the door.
+ if self.async == 0:
+ # yes. so if a sync operation is waiting, notifyAll to wake
+ # it up
+ if self.current_sync_operation is not None:
+ self.condition.notifyAll()
+ elif self.async < 0:
+ raise LockError("Synchronizer error - too many "
+ "release_read_locks called")
+ log.debug("%s released read lock", self)
+ finally:
+ self.condition.release()
+
+ def acquire_write_lock(self, wait = True):
+ """Acquire the 'write' lock."""
+ self.condition.acquire()
+ try:
+ # here, we are not a synchronous reader, and after returning,
+ # assuming waiting or immediate availability, we will be.
+
+ if wait:
+ # if another sync is working, wait
+ while self.current_sync_operation is not None:
+ self.condition.wait()
+ else:
+ # if another sync is working,
+ # we dont want to wait, so forget it
+ if self.current_sync_operation is not None:
+ return False
+
+ # establish ourselves as the current sync
+ # this indicates to other read/write operations
+ # that they should wait until this is None again
+ self.current_sync_operation = threading.currentThread()
+
+ # now wait again for asyncs to finish
+ if self.async > 0:
+ if wait:
+ # wait
+ self.condition.wait()
+ else:
+ # we dont want to wait, so forget it
+ self.current_sync_operation = None
+ return False
+ log.debug("%s acquired write lock", self)
+ finally:
+ self.condition.release()
+
+ if not wait:
+ return True
+
+ def release_write_lock(self):
+ """Release the 'write' lock."""
+ self.condition.acquire()
+ try:
+ if self.current_sync_operation is not threading.currentThread():
+ raise LockError("Synchronizer error - current thread doesn't "
+ "have the write lock")
+
+ # reset the current sync operation so
+ # another can get it
+ self.current_sync_operation = None
+
+ # tell everyone to get ready
+ self.condition.notifyAll()
+
+ log.debug("%s released write lock", self)
+ finally:
+ # everyone go !!
+ self.condition.release()
diff --git a/lib/dogpile/core/util.py b/lib/dogpile/core/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..f53c6818c4a7c67839c0138de4e6802d88ed67fa
--- /dev/null
+++ b/lib/dogpile/core/util.py
@@ -0,0 +1,8 @@
+import sys
+py3k = sys.version_info >= (3, 0)
+
+try:
+ import threading
+except ImportError:
+ import dummy_threading as threading
+
diff --git a/lib/enzyme/__init__.py b/lib/enzyme/__init__.py
index b11c633fdb407a516a6c9305e3b54469d23170d1..4ed31f426823dd1cc09c62efb7fb815e8d2156f6 100644
--- a/lib/enzyme/__init__.py
+++ b/lib/enzyme/__init__.py
@@ -1,63 +1,16 @@
# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
-# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-import mimetypes
-import os
-import sys
-from exceptions import *
+__title__ = 'enzyme'
+__version__ = '0.4.2'
+__author__ = 'Antoine Bertin'
+__license__ = 'Apache 2.0'
+__copyright__ = 'Copyright 2013 Antoine Bertin'
+import logging
+from .exceptions import *
+from .mkv import *
-PARSERS = [('asf', ['video/asf'], ['asf', 'wmv', 'wma']),
- ('flv', ['video/flv'], ['flv']),
- ('mkv', ['video/x-matroska', 'application/mkv'], ['mkv', 'mka', 'webm']),
- ('mp4', ['video/quicktime', 'video/mp4'], ['mov', 'qt', 'mp4', 'mp4a', 'm4v', '3gp', '3gp2', '3g2', 'mk2']),
- ('mpeg', ['video/mpeg'], ['mpeg', 'mpg', 'mp4', 'ts']),
- ('ogm', ['application/ogg'], ['ogm', 'ogg', 'ogv']),
- ('real', ['video/real'], ['rm', 'ra', 'ram']),
- ('riff', ['video/avi'], ['wav', 'avi'])
-]
+class NullHandler(logging.Handler):
+ def emit(self, record):
+ pass
-
-def parse(path):
- """Parse metadata of the given video
-
- :param string path: path to the video file to parse
- :return: a parser corresponding to the video's mimetype or extension
- :rtype: :class:`~enzyme.core.AVContainer`
-
- """
- if not os.path.isfile(path):
- raise ValueError('Invalid path')
- extension = os.path.splitext(path)[1][1:]
- mimetype = mimetypes.guess_type(path)[0]
- parser_ext = None
- parser_mime = None
- for (parser_name, parser_mimetypes, parser_extensions) in PARSERS:
- if mimetype in parser_mimetypes:
- parser_mime = parser_name
- if extension in parser_extensions:
- parser_ext = parser_name
- parser = parser_mime or parser_ext
- if not parser:
- raise NoParserError()
- mod = __import__(parser, globals=globals(), locals=locals(), fromlist=[], level=-1)
- with open(path, 'rb') as f:
- p = mod.Parser(f)
- return p
+logging.getLogger(__name__).addHandler(NullHandler())
diff --git a/lib/enzyme/asf.py b/lib/enzyme/asf.py
deleted file mode 100644
index a623b5919d3d37a3b012e422fed7c1187500e3bd..0000000000000000000000000000000000000000
--- a/lib/enzyme/asf.py
+++ /dev/null
@@ -1,389 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
-# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-from exceptions import ParseError
-import core
-import logging
-import string
-import struct
-
-
-__all__ = ['Parser']
-
-
-# get logging object
-log = logging.getLogger(__name__)
-
-def _guid(input):
- # Remove any '-'
- s = string.join(string.split(input, '-'), '')
- r = ''
- if len(s) != 32:
- return ''
- for i in range(0, 16):
- r += chr(int(s[2 * i:2 * i + 2], 16))
- guid = struct.unpack('>IHHBB6s', r)
- return guid
-
-GUIDS = {
- 'ASF_Header_Object' : _guid('75B22630-668E-11CF-A6D9-00AA0062CE6C'),
- 'ASF_Data_Object' : _guid('75B22636-668E-11CF-A6D9-00AA0062CE6C'),
- 'ASF_Simple_Index_Object' : _guid('33000890-E5B1-11CF-89F4-00A0C90349CB'),
- 'ASF_Index_Object' : _guid('D6E229D3-35DA-11D1-9034-00A0C90349BE'),
- 'ASF_Media_Object_Index_Object' : _guid('FEB103F8-12AD-4C64-840F-2A1D2F7AD48C'),
- 'ASF_Timecode_Index_Object' : _guid('3CB73FD0-0C4A-4803-953D-EDF7B6228F0C'),
-
- 'ASF_File_Properties_Object' : _guid('8CABDCA1-A947-11CF-8EE4-00C00C205365'),
- 'ASF_Stream_Properties_Object' : _guid('B7DC0791-A9B7-11CF-8EE6-00C00C205365'),
- 'ASF_Header_Extension_Object' : _guid('5FBF03B5-A92E-11CF-8EE3-00C00C205365'),
- 'ASF_Codec_List_Object' : _guid('86D15240-311D-11D0-A3A4-00A0C90348F6'),
- 'ASF_Script_Command_Object' : _guid('1EFB1A30-0B62-11D0-A39B-00A0C90348F6'),
- 'ASF_Marker_Object' : _guid('F487CD01-A951-11CF-8EE6-00C00C205365'),
- 'ASF_Bitrate_Mutual_Exclusion_Object' : _guid('D6E229DC-35DA-11D1-9034-00A0C90349BE'),
- 'ASF_Error_Correction_Object' : _guid('75B22635-668E-11CF-A6D9-00AA0062CE6C'),
- 'ASF_Content_Description_Object' : _guid('75B22633-668E-11CF-A6D9-00AA0062CE6C'),
- 'ASF_Extended_Content_Description_Object' : _guid('D2D0A440-E307-11D2-97F0-00A0C95EA850'),
- 'ASF_Content_Branding_Object' : _guid('2211B3FA-BD23-11D2-B4B7-00A0C955FC6E'),
- 'ASF_Stream_Bitrate_Properties_Object' : _guid('7BF875CE-468D-11D1-8D82-006097C9A2B2'),
- 'ASF_Content_Encryption_Object' : _guid('2211B3FB-BD23-11D2-B4B7-00A0C955FC6E'),
- 'ASF_Extended_Content_Encryption_Object' : _guid('298AE614-2622-4C17-B935-DAE07EE9289C'),
- 'ASF_Alt_Extended_Content_Encryption_Obj' : _guid('FF889EF1-ADEE-40DA-9E71-98704BB928CE'),
- 'ASF_Digital_Signature_Object' : _guid('2211B3FC-BD23-11D2-B4B7-00A0C955FC6E'),
- 'ASF_Padding_Object' : _guid('1806D474-CADF-4509-A4BA-9AABCB96AAE8'),
-
- 'ASF_Extended_Stream_Properties_Object' : _guid('14E6A5CB-C672-4332-8399-A96952065B5A'),
- 'ASF_Advanced_Mutual_Exclusion_Object' : _guid('A08649CF-4775-4670-8A16-6E35357566CD'),
- 'ASF_Group_Mutual_Exclusion_Object' : _guid('D1465A40-5A79-4338-B71B-E36B8FD6C249'),
- 'ASF_Stream_Prioritization_Object' : _guid('D4FED15B-88D3-454F-81F0-ED5C45999E24'),
- 'ASF_Bandwidth_Sharing_Object' : _guid('A69609E6-517B-11D2-B6AF-00C04FD908E9'),
- 'ASF_Language_List_Object' : _guid('7C4346A9-EFE0-4BFC-B229-393EDE415C85'),
- 'ASF_Metadata_Object' : _guid('C5F8CBEA-5BAF-4877-8467-AA8C44FA4CCA'),
- 'ASF_Metadata_Library_Object' : _guid('44231C94-9498-49D1-A141-1D134E457054'),
- 'ASF_Index_Parameters_Object' : _guid('D6E229DF-35DA-11D1-9034-00A0C90349BE'),
- 'ASF_Media_Object_Index_Parameters_Obj' : _guid('6B203BAD-3F11-4E84-ACA8-D7613DE2CFA7'),
- 'ASF_Timecode_Index_Parameters_Object' : _guid('F55E496D-9797-4B5D-8C8B-604DFE9BFB24'),
-
- 'ASF_Audio_Media' : _guid('F8699E40-5B4D-11CF-A8FD-00805F5C442B'),
- 'ASF_Video_Media' : _guid('BC19EFC0-5B4D-11CF-A8FD-00805F5C442B'),
- 'ASF_Command_Media' : _guid('59DACFC0-59E6-11D0-A3AC-00A0C90348F6'),
- 'ASF_JFIF_Media' : _guid('B61BE100-5B4E-11CF-A8FD-00805F5C442B'),
- 'ASF_Degradable_JPEG_Media' : _guid('35907DE0-E415-11CF-A917-00805F5C442B'),
- 'ASF_File_Transfer_Media' : _guid('91BD222C-F21C-497A-8B6D-5AA86BFC0185'),
- 'ASF_Binary_Media' : _guid('3AFB65E2-47EF-40F2-AC2C-70A90D71D343'),
-
- 'ASF_Web_Stream_Media_Subtype' : _guid('776257D4-C627-41CB-8F81-7AC7FF1C40CC'),
- 'ASF_Web_Stream_Format' : _guid('DA1E6B13-8359-4050-B398-388E965BF00C'),
-
- 'ASF_No_Error_Correction' : _guid('20FB5700-5B55-11CF-A8FD-00805F5C442B'),
- 'ASF_Audio_Spread' : _guid('BFC3CD50-618F-11CF-8BB2-00AA00B4E220')}
-
-
-class Asf(core.AVContainer):
- """
- ASF video parser. The ASF format is also used for Microsft Windows
- Media files like wmv.
- """
- def __init__(self, file):
- core.AVContainer.__init__(self)
- self.mime = 'video/x-ms-asf'
- self.type = 'asf format'
- self._languages = []
- self._extinfo = {}
-
- h = file.read(30)
- if len(h) < 30:
- raise ParseError()
-
- (guidstr, objsize, objnum, reserved1, \
- reserved2) = struct.unpack('<16sQIBB', h)
- guid = self._parseguid(guidstr)
-
- if (guid != GUIDS['ASF_Header_Object']):
- raise ParseError()
- if reserved1 != 0x01 or reserved2 != 0x02:
- raise ParseError()
-
- log.debug(u'Header size: %d / %d objects' % (objsize, objnum))
- header = file.read(objsize - 30)
- for _ in range(0, objnum):
- h = self._getnextheader(header)
- header = header[h[1]:]
-
- del self._languages
- del self._extinfo
-
-
- def _findstream(self, id):
- for stream in self.video + self.audio:
- if stream.id == id:
- return stream
-
- def _apply_extinfo(self, streamid):
- stream = self._findstream(streamid)
- if not stream or streamid not in self._extinfo:
- return
- stream.bitrate, stream.fps, langid, metadata = self._extinfo[streamid]
- if langid is not None and langid >= 0 and langid < len(self._languages):
- stream.language = self._languages[langid]
- if metadata:
- stream._appendtable('ASFMETADATA', metadata)
-
-
- def _parseguid(self, string):
- return struct.unpack('<IHHBB6s', string[:16])
-
-
- def _parsekv(self, s):
- pos = 0
- (descriptorlen,) = struct.unpack('<H', s[pos:pos + 2])
- pos += 2
- descriptorname = s[pos:pos + descriptorlen]
- pos += descriptorlen
- descriptortype, valuelen = struct.unpack('<HH', s[pos:pos + 4])
- pos += 4
- descriptorvalue = s[pos:pos + valuelen]
- pos += valuelen
- value = None
- if descriptortype == 0x0000:
- # Unicode string
- value = descriptorvalue
- elif descriptortype == 0x0001:
- # Byte Array
- value = descriptorvalue
- elif descriptortype == 0x0002:
- # Bool (?)
- value = struct.unpack('<I', descriptorvalue)[0] != 0
- elif descriptortype == 0x0003:
- # DWORD
- value = struct.unpack('<I', descriptorvalue)[0]
- elif descriptortype == 0x0004:
- # QWORD
- value = struct.unpack('<Q', descriptorvalue)[0]
- elif descriptortype == 0x0005:
- # WORD
- value = struct.unpack('<H', descriptorvalue)[0]
- else:
- log.debug(u'Unknown Descriptor Type %d' % descriptortype)
- return (pos, descriptorname, value)
-
-
- def _parsekv2(self, s):
- pos = 0
- strno, descriptorlen, descriptortype, valuelen = struct.unpack('<2xHHHI', s[pos:pos + 12])
- pos += 12
- descriptorname = s[pos:pos + descriptorlen]
- pos += descriptorlen
- descriptorvalue = s[pos:pos + valuelen]
- pos += valuelen
- value = None
-
- if descriptortype == 0x0000:
- # Unicode string
- value = descriptorvalue
- elif descriptortype == 0x0001:
- # Byte Array
- value = descriptorvalue
- elif descriptortype == 0x0002:
- # Bool
- value = struct.unpack('<H', descriptorvalue)[0] != 0
- pass
- elif descriptortype == 0x0003:
- # DWORD
- value = struct.unpack('<I', descriptorvalue)[0]
- elif descriptortype == 0x0004:
- # QWORD
- value = struct.unpack('<Q', descriptorvalue)[0]
- elif descriptortype == 0x0005:
- # WORD
- value = struct.unpack('<H', descriptorvalue)[0]
- else:
- log.debug(u'Unknown Descriptor Type %d' % descriptortype)
- return (pos, descriptorname, value, strno)
-
-
- def _getnextheader(self, s):
- r = struct.unpack('<16sQ', s[:24])
- (guidstr, objsize) = r
- guid = self._parseguid(guidstr)
- if guid == GUIDS['ASF_File_Properties_Object']:
- log.debug(u'File Properties Object')
- val = struct.unpack('<16s6Q4I', s[24:24 + 80])
- (fileid, size, date, packetcount, duration, \
- senddur, preroll, flags, minpack, maxpack, maxbr) = \
- val
- # FIXME: parse date to timestamp
- self.length = duration / 10000000.0
-
- elif guid == GUIDS['ASF_Stream_Properties_Object']:
- log.debug(u'Stream Properties Object [%d]' % objsize)
- streamtype = self._parseguid(s[24:40])
- errortype = self._parseguid(s[40:56])
- offset, typelen, errorlen, flags = struct.unpack('<QIIH', s[56:74])
- strno = flags & 0x7f
- encrypted = flags >> 15
- if encrypted:
- self._set('encrypted', True)
- if streamtype == GUIDS['ASF_Video_Media']:
- vi = core.VideoStream()
- vi.width, vi.height, depth, codec, = struct.unpack('<4xII2xH4s', s[89:89 + 20])
- vi.codec = codec
- vi.id = strno
- self.video.append(vi)
- elif streamtype == GUIDS['ASF_Audio_Media']:
- ai = core.AudioStream()
- twocc, ai.channels, ai.samplerate, bitrate, block, \
- ai.samplebits, = struct.unpack('<HHIIHH', s[78:78 + 16])
- ai.bitrate = 8 * bitrate
- ai.codec = twocc
- ai.id = strno
- self.audio.append(ai)
-
- self._apply_extinfo(strno)
-
- elif guid == GUIDS['ASF_Extended_Stream_Properties_Object']:
- streamid, langid, frametime = struct.unpack('<HHQ', s[72:84])
- (bitrate,) = struct.unpack('<I', s[40:40 + 4])
- if streamid not in self._extinfo:
- self._extinfo[streamid] = [None, None, None, {}]
- if frametime == 0:
- # Problaby VFR, report as 1000fps (which is what MPlayer does)
- frametime = 10000.0
- self._extinfo[streamid][:3] = [bitrate, 10000000.0 / frametime, langid]
- self._apply_extinfo(streamid)
-
- elif guid == GUIDS['ASF_Header_Extension_Object']:
- log.debug(u'ASF_Header_Extension_Object %d' % objsize)
- size = struct.unpack('<I', s[42:46])[0]
- data = s[46:46 + size]
- while len(data):
- log.debug(u'Sub:')
- h = self._getnextheader(data)
- data = data[h[1]:]
-
- elif guid == GUIDS['ASF_Codec_List_Object']:
- log.debug(u'List Object')
- pass
-
- elif guid == GUIDS['ASF_Error_Correction_Object']:
- log.debug(u'Error Correction')
- pass
-
- elif guid == GUIDS['ASF_Content_Description_Object']:
- log.debug(u'Content Description Object')
- val = struct.unpack('<5H', s[24:24 + 10])
- pos = 34
- strings = []
- for i in val:
- ss = s[pos:pos + i].replace('\0', '').lstrip().rstrip()
- strings.append(ss)
- pos += i
-
- # Set empty strings to None
- strings = [x or None for x in strings]
- self.title, self.artist, self.copyright, self.caption, rating = strings
-
- elif guid == GUIDS['ASF_Extended_Content_Description_Object']:
- (count,) = struct.unpack('<H', s[24:26])
- pos = 26
- descriptor = {}
- for i in range(0, count):
- # Read additional content descriptors
- d = self._parsekv(s[pos:])
- pos += d[0]
- descriptor[d[1]] = d[2]
- self._appendtable('ASFDESCRIPTOR', descriptor)
-
- elif guid == GUIDS['ASF_Metadata_Object']:
- (count,) = struct.unpack('<H', s[24:26])
- pos = 26
- streams = {}
- for i in range(0, count):
- # Read additional content descriptors
- size, key, value, strno = self._parsekv2(s[pos:])
- if strno not in streams:
- streams[strno] = {}
- streams[strno][key] = value
- pos += size
-
- for strno, metadata in streams.items():
- if strno not in self._extinfo:
- self._extinfo[strno] = [None, None, None, {}]
- self._extinfo[strno][3].update(metadata)
- self._apply_extinfo(strno)
-
- elif guid == GUIDS['ASF_Language_List_Object']:
- count = struct.unpack('<H', s[24:26])[0]
- pos = 26
- for i in range(0, count):
- idlen = struct.unpack('<B', s[pos:pos + 1])[0]
- idstring = s[pos + 1:pos + 1 + idlen]
- idstring = unicode(idstring, 'utf-16').replace('\0', '')
- log.debug(u'Language: %d/%d: %r' % (i + 1, count, idstring))
- self._languages.append(idstring)
- pos += 1 + idlen
-
- elif guid == GUIDS['ASF_Stream_Bitrate_Properties_Object']:
- # This record contains stream bitrate with payload overhead. For
- # audio streams, we should have the average bitrate from
- # ASF_Stream_Properties_Object. For video streams, we get it from
- # ASF_Extended_Stream_Properties_Object. So this record is not
- # used.
- pass
-
- elif guid == GUIDS['ASF_Content_Encryption_Object'] or \
- guid == GUIDS['ASF_Extended_Content_Encryption_Object']:
- self._set('encrypted', True)
- else:
- # Just print the type:
- for h in GUIDS.keys():
- if GUIDS[h] == guid:
- log.debug(u'Unparsed %r [%d]' % (h, objsize))
- break
- else:
- u = "%.8X-%.4X-%.4X-%.2X%.2X-%s" % guid
- log.debug(u'unknown: len=%d [%d]' % (len(u), objsize))
- return r
-
-
-class AsfAudio(core.AudioStream):
- """
- ASF audio parser for wma files.
- """
- def __init__(self):
- core.AudioStream.__init__(self)
- self.mime = 'audio/x-ms-asf'
- self.type = 'asf format'
-
-
-def Parser(file):
- """
- Wrapper around audio and av content.
- """
- asf = Asf(file)
- if not len(asf.audio) or len(asf.video):
- # AV container
- return asf
- # No video but audio streams. Handle has audio core
- audio = AsfAudio()
- for key in audio._keys:
- if key in asf._keys:
- if not getattr(audio, key, None):
- setattr(audio, key, getattr(asf, key))
- return audio
diff --git a/lib/enzyme/compat.py b/lib/enzyme/compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..81e3a4c9de579e4746d64fe490ae32564ef9f013
--- /dev/null
+++ b/lib/enzyme/compat.py
@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+import sys
+
+
+_ver = sys.version_info
+is_py3 = _ver[0] == 3
+is_py2 = _ver[0] == 2
+
+
+if is_py2:
+ bytes = lambda x: chr(x[0]) # @ReservedAssignment
+elif is_py3:
+ bytes = bytes # @ReservedAssignment
diff --git a/lib/enzyme/core.py b/lib/enzyme/core.py
deleted file mode 100644
index 565cb4f674fca74351c4346b29dd0e9798bae8dc..0000000000000000000000000000000000000000
--- a/lib/enzyme/core.py
+++ /dev/null
@@ -1,450 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
-# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-import re
-import logging
-import fourcc
-import language
-from strutils import str_to_unicode, unicode_to_str
-
-UNPRINTABLE_KEYS = ['thumbnail', 'url', 'codec_private']
-MEDIACORE = ['title', 'caption', 'comment', 'size', 'type', 'subtype', 'timestamp',
- 'keywords', 'country', 'language', 'langcode', 'url', 'artist',
- 'mime', 'datetime', 'tags', 'hash']
-AUDIOCORE = ['channels', 'samplerate', 'length', 'encoder', 'codec', 'format',
- 'samplebits', 'bitrate', 'fourcc', 'trackno', 'id', 'userdate',
- 'enabled', 'default', 'codec_private']
-MUSICCORE = ['trackof', 'album', 'genre', 'discs', 'thumbnail']
-VIDEOCORE = ['length', 'encoder', 'bitrate', 'samplerate', 'codec', 'format',
- 'samplebits', 'width', 'height', 'fps', 'aspect', 'trackno',
- 'fourcc', 'id', 'enabled', 'default', 'codec_private']
-AVCORE = ['length', 'encoder', 'trackno', 'trackof', 'copyright', 'product',
- 'genre', 'writer', 'producer', 'studio', 'rating', 'actors', 'thumbnail',
- 'delay', 'image', 'video', 'audio', 'subtitles', 'chapters', 'software',
- 'summary', 'synopsis', 'season', 'episode', 'series']
-
-# get logging object
-log = logging.getLogger(__name__)
-
-
-class Media(object):
- """
- Media is the base class to all Media Metadata Containers. It defines
- the basic structures that handle metadata. Media and its derivates
- contain a common set of metadata attributes that is listed in keys.
- Specific derivates contain additional keys to the dublin core set that is
- defined in Media.
- """
- media = None
- _keys = MEDIACORE
- table_mapping = {}
-
- def __init__(self, hash=None):
- if hash is not None:
- # create Media based on dict
- for key, value in hash.items():
- if isinstance(value, list) and value and isinstance(value[0], dict):
- value = [Media(x) for x in value]
- self._set(key, value)
- return
-
- self._keys = self._keys[:]
- self.tables = {}
- # Tags, unlike tables, are more well-defined dicts whose values are
- # either Tag objects, other dicts (for nested tags), or lists of either
- # (for multiple instances of the tag, e.g. actor). Where possible,
- # parsers should transform tag names to conform to the Official
- # Matroska tags defined at http://www.matroska.org/technical/specs/tagging/index.html
- # All tag names will be lower-cased.
- self.tags = Tags()
- for key in set(self._keys) - set(['media', 'tags']):
- setattr(self, key, None)
-
- #
- # unicode and string convertion for debugging
- #
- #TODO: Fix that mess
- def __unicode__(self):
- result = u''
-
- # print normal attributes
- lists = []
- for key in self._keys:
- value = getattr(self, key, None)
- if value == None or key == 'url':
- continue
- if isinstance(value, list):
- if not value:
- continue
- elif isinstance(value[0], basestring):
- # Just a list of strings (keywords?), so don't treat it specially.
- value = u', '.join(value)
- else:
- lists.append((key, value))
- continue
- elif isinstance(value, dict):
- # Tables or tags treated separately.
- continue
- if key in UNPRINTABLE_KEYS:
- value = '<unprintable data, size=%d>' % len(value)
- result += u'| %10s: %s\n' % (unicode(key), unicode(value))
-
- # print tags (recursively, to support nested tags).
- def print_tags(tags, suffix, show_label):
- result = ''
- for n, (name, tag) in enumerate(tags.items()):
- result += u'| %12s%s%s = ' % (u'tags: ' if n == 0 and show_label else '', suffix, name)
- if isinstance(tag, list):
- # TODO: doesn't support lists/dicts within lists.
- result += u'%s\n' % ', '.join(subtag.value for subtag in tag)
- else:
- result += u'%s\n' % (tag.value or '')
- if isinstance(tag, dict):
- result += print_tags(tag, ' ', False)
- return result
- result += print_tags(self.tags, '', True)
-
- # print lists
- for key, l in lists:
- for n, item in enumerate(l):
- label = '+-- ' + key.rstrip('s').capitalize()
- if key not in ['tracks', 'subtitles', 'chapters']:
- label += ' Track'
- result += u'%s #%d\n' % (label, n + 1)
- result += '| ' + re.sub(r'\n(.)', r'\n| \1', unicode(item))
-
- # print tables
- #FIXME: WTH?
-# if log.level >= 10:
-# for name, table in self.tables.items():
-# result += '+-- Table %s\n' % str(name)
-# for key, value in table.items():
-# try:
-# value = unicode(value)
-# if len(value) > 50:
-# value = u'<unprintable data, size=%d>' % len(value)
-# except (UnicodeDecodeError, TypeError):
-# try:
-# value = u'<unprintable data, size=%d>' % len(value)
-# except AttributeError:
-# value = u'<unprintable data>'
-# result += u'| | %s: %s\n' % (unicode(key), value)
- return result
-
- def __str__(self):
- return unicode(self).encode()
-
- def __repr__(self):
- if hasattr(self, 'url'):
- return '<%s %s>' % (str(self.__class__)[8:-2], self.url)
- else:
- return '<%s>' % (str(self.__class__)[8:-2])
-
- #
- # internal functions
- #
- def _appendtable(self, name, hashmap):
- """
- Appends a tables of additional metadata to the Object.
- If such a table already exists, the given tables items are
- added to the existing one.
- """
- if name not in self.tables:
- self.tables[name] = hashmap
- else:
- # Append to the already existing table
- for k in hashmap.keys():
- self.tables[name][k] = hashmap[k]
-
- def _set(self, key, value):
- """
- Set key to value and add the key to the internal keys list if
- missing.
- """
- if value is None and getattr(self, key, None) is None:
- return
- if isinstance(value, str):
- value = str_to_unicode(value)
- setattr(self, key, value)
- if not key in self._keys:
- self._keys.append(key)
-
- def _set_url(self, url):
- """
- Set the URL of the source
- """
- self.url = url
-
- def _finalize(self):
- """
- Correct same data based on specific rules
- """
- # make sure all strings are unicode
- for key in self._keys:
- if key in UNPRINTABLE_KEYS:
- continue
- value = getattr(self, key)
- if value is None:
- continue
- if key == 'image':
- if isinstance(value, unicode):
- setattr(self, key, unicode_to_str(value))
- continue
- if isinstance(value, str):
- setattr(self, key, str_to_unicode(value))
- if isinstance(value, unicode):
- setattr(self, key, value.strip().rstrip().replace(u'\0', u''))
- if isinstance(value, list) and value and isinstance(value[0], Media):
- for submenu in value:
- submenu._finalize()
-
- # copy needed tags from tables
- for name, table in self.tables.items():
- mapping = self.table_mapping.get(name, {})
- for tag, attr in mapping.items():
- if self.get(attr):
- continue
- value = table.get(tag, None)
- if value is not None:
- if not isinstance(value, (str, unicode)):
- value = str_to_unicode(str(value))
- elif isinstance(value, str):
- value = str_to_unicode(value)
- value = value.strip().rstrip().replace(u'\0', u'')
- setattr(self, attr, value)
-
- if 'fourcc' in self._keys and 'codec' in self._keys and self.codec is not None:
- # Codec may be a fourcc, in which case we resolve it to its actual
- # name and set the fourcc attribute.
- self.fourcc, self.codec = fourcc.resolve(self.codec)
- if 'language' in self._keys:
- self.langcode, self.language = language.resolve(self.language)
-
- #
- # data access
- #
- def __contains__(self, key):
- """
- Test if key exists in the dict
- """
- return hasattr(self, key)
-
- def get(self, attr, default=None):
- """
- Returns the given attribute. If the attribute is not set by
- the parser return 'default'.
- """
- return getattr(self, attr, default)
-
- def __getitem__(self, attr):
- """
- Get the value of the given attribute
- """
- return getattr(self, attr, None)
-
- def __setitem__(self, key, value):
- """
- Set the value of 'key' to 'value'
- """
- setattr(self, key, value)
-
- def has_key(self, key):
- """
- Check if the object has an attribute 'key'
- """
- return hasattr(self, key)
-
- def convert(self):
- """
- Convert Media to dict.
- """
- result = {}
- for k in self._keys:
- value = getattr(self, k, None)
- if isinstance(value, list) and value and isinstance(value[0], Media):
- value = [x.convert() for x in value]
- result[k] = value
- return result
-
- def keys(self):
- """
- Return all keys for the attributes set by the parser.
- """
- return self._keys
-
-
-class Collection(Media):
- """
- Collection of Digial Media like CD, DVD, Directory, Playlist
- """
- _keys = Media._keys + ['id', 'tracks']
-
- def __init__(self):
- Media.__init__(self)
- self.tracks = []
-
-
-class Tag(object):
- """
- An individual tag, which will be a value stored in a Tags object.
-
- Tag values are strings (for binary data), unicode objects, or datetime
- objects for tags that represent dates or times.
- """
- def __init__(self, value=None, langcode='und', binary=False):
- super(Tag, self).__init__()
- self.value = value
- self.langcode = langcode
- self.binary = binary
-
- def __unicode__(self):
- return unicode(self.value)
-
- def __str__(self):
- return str(self.value)
-
- def __repr__(self):
- if not self.binary:
- return '<Tag object: %s>' % repr(self.value)
- else:
- return '<Binary Tag object: size=%d>' % len(self.value)
-
- @property
- def langcode(self):
- return self._langcode
-
- @langcode.setter
- def langcode(self, code):
- self._langcode, self.language = language.resolve(code)
-
-
-class Tags(dict, Tag):
- """
- A dictionary containing Tag objects. Values can be other Tags objects
- (for nested tags), lists, or Tag objects.
-
- A Tags object is more or less a dictionary but it also contains a value.
- This is necessary in order to represent this kind of tag specification
- (e.g. for Matroska)::
-
- <Simple>
- <Name>LAW_RATING</Name>
- <String>PG</String>
- <Simple>
- <Name>COUNTRY</Name>
- <String>US</String>
- </Simple>
- </Simple>
-
- The attribute RATING has a value (PG), but it also has a child tag
- COUNTRY that specifies the country code the rating belongs to.
- """
- def __init__(self, value=None, langcode='und', binary=False):
- super(Tags, self).__init__()
- self.value = value
- self.langcode = langcode
- self.binary = False
-
-
-class AudioStream(Media):
- """
- Audio Tracks in a Multiplexed Container.
- """
- _keys = Media._keys + AUDIOCORE
-
-
-class Music(AudioStream):
- """
- Digital Music.
- """
- _keys = AudioStream._keys + MUSICCORE
-
- def _finalize(self):
- """
- Correct same data based on specific rules
- """
- AudioStream._finalize(self)
- if self.trackof:
- try:
- # XXX Why is this needed anyway?
- if int(self.trackno) < 10:
- self.trackno = u'0%s' % int(self.trackno)
- except (AttributeError, ValueError):
- pass
-
-
-class VideoStream(Media):
- """
- Video Tracks in a Multiplexed Container.
- """
- _keys = Media._keys + VIDEOCORE
-
-
-class Chapter(Media):
- """
- Chapter in a Multiplexed Container.
- """
- _keys = ['enabled', 'name', 'pos', 'id']
-
- def __init__(self, name=None, pos=0):
- Media.__init__(self)
- self.name = name
- self.pos = pos
- self.enabled = True
-
-
-class Subtitle(Media):
- """
- Subtitle Tracks in a Multiplexed Container.
- """
- _keys = ['enabled', 'default', 'langcode', 'language', 'trackno', 'title',
- 'id', 'codec']
-
- def __init__(self, language=None):
- Media.__init__(self)
- self.language = language
-
-
-class AVContainer(Media):
- """
- Container for Audio and Video streams. This is the Container Type for
- all media, that contain more than one stream.
- """
- _keys = Media._keys + AVCORE
-
- def __init__(self):
- Media.__init__(self)
- self.audio = []
- self.video = []
- self.subtitles = []
- self.chapters = []
-
- def _finalize(self):
- """
- Correct same data based on specific rules
- """
- Media._finalize(self)
- if not self.length and len(self.video) and self.video[0].length:
- self.length = 0
- # Length not specified for container, so use the largest length
- # of its tracks as container length.
- for track in self.video + self.audio:
- if track.length:
- self.length = max(self.length, track.length)
diff --git a/lib/enzyme/exceptions.py b/lib/enzyme/exceptions.py
index 5bc4f7acccb30722e5313c6e3fae29bc8b458db9..b2252aa474abcaaec9e2d6dbfcdd329d7ba6448a 100644
--- a/lib/enzyme/exceptions.py
+++ b/lib/enzyme/exceptions.py
@@ -1,28 +1,27 @@
# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
+__all__ = ['Error', 'MalformedMKVError', 'ParserError', 'ReadError', 'SizeError']
+
+
class Error(Exception):
+ """Base class for enzyme exceptions"""
+ pass
+
+
+class MalformedMKVError(Error):
+ """Wrong or malformed element found"""
+ pass
+
+
+class ParserError(Error):
+ """Base class for exceptions in parsers"""
pass
-class NoParserError(Error):
+class ReadError(ParserError):
+ """Unable to correctly read"""
pass
-class ParseError(Error):
+class SizeError(ParserError):
+ """Mismatch between the type of the element and the size of its data"""
pass
diff --git a/lib/enzyme/flv.py b/lib/enzyme/flv.py
deleted file mode 100644
index 61f34a78e7b0f7143a88aabf87c955579f1f2390..0000000000000000000000000000000000000000
--- a/lib/enzyme/flv.py
+++ /dev/null
@@ -1,181 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-from exceptions import ParseError
-import core
-import logging
-import struct
-
-__all__ = ['Parser']
-
-
-# get logging object
-log = logging.getLogger(__name__)
-
-FLV_TAG_TYPE_AUDIO = 0x08
-FLV_TAG_TYPE_VIDEO = 0x09
-FLV_TAG_TYPE_META = 0x12
-
-# audio flags
-FLV_AUDIO_CHANNEL_MASK = 0x01
-FLV_AUDIO_SAMPLERATE_MASK = 0x0c
-FLV_AUDIO_CODECID_MASK = 0xf0
-
-FLV_AUDIO_SAMPLERATE_OFFSET = 2
-FLV_AUDIO_CODECID_OFFSET = 4
-FLV_AUDIO_CODECID = (0x0001, 0x0002, 0x0055, 0x0001)
-
-# video flags
-FLV_VIDEO_CODECID_MASK = 0x0f
-FLV_VIDEO_CODECID = ('FLV1', 'MSS1', 'VP60') # wild guess
-
-FLV_DATA_TYPE_NUMBER = 0x00
-FLV_DATA_TYPE_BOOL = 0x01
-FLV_DATA_TYPE_STRING = 0x02
-FLV_DATA_TYPE_OBJECT = 0x03
-FLC_DATA_TYPE_CLIP = 0x04
-FLV_DATA_TYPE_REFERENCE = 0x07
-FLV_DATA_TYPE_ECMARRAY = 0x08
-FLV_DATA_TYPE_ENDOBJECT = 0x09
-FLV_DATA_TYPE_ARRAY = 0x0a
-FLV_DATA_TYPE_DATE = 0x0b
-FLV_DATA_TYPE_LONGSTRING = 0x0c
-
-FLVINFO = {
- 'creator': 'copyright',
-}
-
-class FlashVideo(core.AVContainer):
- """
- Experimental parser for Flash videos. It requires certain flags to
- be set to report video resolutions and in most cases it does not
- provide that information.
- """
- table_mapping = { 'FLVINFO' : FLVINFO }
-
- def __init__(self, file):
- core.AVContainer.__init__(self)
- self.mime = 'video/flv'
- self.type = 'Flash Video'
- data = file.read(13)
- if len(data) < 13 or struct.unpack('>3sBBII', data)[0] != 'FLV':
- raise ParseError()
-
- for _ in range(10):
- if self.audio and self.video:
- break
- data = file.read(11)
- if len(data) < 11:
- break
- chunk = struct.unpack('>BH4BI', data)
- size = (chunk[1] << 8) + chunk[2]
-
- if chunk[0] == FLV_TAG_TYPE_AUDIO:
- flags = ord(file.read(1))
- if not self.audio:
- a = core.AudioStream()
- a.channels = (flags & FLV_AUDIO_CHANNEL_MASK) + 1
- srate = (flags & FLV_AUDIO_SAMPLERATE_MASK)
- a.samplerate = (44100 << (srate >> FLV_AUDIO_SAMPLERATE_OFFSET) >> 3)
- codec = (flags & FLV_AUDIO_CODECID_MASK) >> FLV_AUDIO_CODECID_OFFSET
- if codec < len(FLV_AUDIO_CODECID):
- a.codec = FLV_AUDIO_CODECID[codec]
- self.audio.append(a)
-
- file.seek(size - 1, 1)
-
- elif chunk[0] == FLV_TAG_TYPE_VIDEO:
- flags = ord(file.read(1))
- if not self.video:
- v = core.VideoStream()
- codec = (flags & FLV_VIDEO_CODECID_MASK) - 2
- if codec < len(FLV_VIDEO_CODECID):
- v.codec = FLV_VIDEO_CODECID[codec]
- # width and height are in the meta packet, but I have
- # no file with such a packet inside. So maybe we have
- # to decode some parts of the video.
- self.video.append(v)
-
- file.seek(size - 1, 1)
-
- elif chunk[0] == FLV_TAG_TYPE_META:
- log.info(u'metadata %r', str(chunk))
- metadata = file.read(size)
- try:
- while metadata:
- length, value = self._parse_value(metadata)
- if isinstance(value, dict):
- log.info(u'metadata: %r', value)
- if value.get('creator'):
- self.copyright = value.get('creator')
- if value.get('width'):
- self.width = value.get('width')
- if value.get('height'):
- self.height = value.get('height')
- if value.get('duration'):
- self.length = value.get('duration')
- self._appendtable('FLVINFO', value)
- if not length:
- # parse error
- break
- metadata = metadata[length:]
- except (IndexError, struct.error, TypeError):
- pass
- else:
- log.info(u'unkown %r', str(chunk))
- file.seek(size, 1)
-
- file.seek(4, 1)
-
- def _parse_value(self, data):
- """
- Parse the next metadata value.
- """
- if ord(data[0]) == FLV_DATA_TYPE_NUMBER:
- value = struct.unpack('>d', data[1:9])[0]
- return 9, value
-
- if ord(data[0]) == FLV_DATA_TYPE_BOOL:
- return 2, bool(data[1])
-
- if ord(data[0]) == FLV_DATA_TYPE_STRING:
- length = (ord(data[1]) << 8) + ord(data[2])
- return length + 3, data[3:length + 3]
-
- if ord(data[0]) == FLV_DATA_TYPE_ECMARRAY:
- init_length = len(data)
- num = struct.unpack('>I', data[1:5])[0]
- data = data[5:]
- result = {}
- for _ in range(num):
- length = (ord(data[0]) << 8) + ord(data[1])
- key = data[2:length + 2]
- data = data[length + 2:]
- length, value = self._parse_value(data)
- if not length:
- return 0, result
- result[key] = value
- data = data[length:]
- return init_length - len(data), result
-
- log.info(u'unknown code: %x. Stop metadata parser', ord(data[0]))
- return 0, None
-
-
-Parser = FlashVideo
diff --git a/lib/enzyme/fourcc.py b/lib/enzyme/fourcc.py
deleted file mode 100644
index ac15b0b2b90c4ee3ee073f888c0887093beab560..0000000000000000000000000000000000000000
--- a/lib/enzyme/fourcc.py
+++ /dev/null
@@ -1,850 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-import string
-import re
-import struct
-
-__all__ = ['resolve']
-
-
-def resolve(code):
- """
- Transform a twocc or fourcc code into a name. Returns a 2-tuple of (cc,
- codec) where both are strings and cc is a string in the form '0xXX' if it's
- a twocc, or 'ABCD' if it's a fourcc. If the given code is not a known
- twocc or fourcc, the return value will be (None, 'Unknown'), unless the
- code is otherwise a printable string in which case it will be returned as
- the codec.
- """
- if isinstance(code, basestring):
- codec = u'Unknown'
- # Check for twocc
- if re.match(r'^0x[\da-f]{1,4}$', code, re.I):
- # Twocc in hex form
- return code, TWOCC.get(int(code, 16), codec)
- elif code.isdigit() and 0 <= int(code) <= 0xff:
- # Twocc in decimal form
- return hex(int(code)), TWOCC.get(int(code), codec)
- elif len(code) == 2:
- code = struct.unpack('H', code)[0]
- return hex(code), TWOCC.get(code, codec)
- elif len(code) != 4 and len([x for x in code if x not in string.printable]) == 0:
- # Code is a printable string.
- codec = unicode(code)
-
- if code[:2] == 'MS' and code[2:].upper() in FOURCC:
- code = code[2:]
-
- if code.upper() in FOURCC:
- return code.upper(), unicode(FOURCC[code.upper()])
- return None, codec
- elif isinstance(code, (int, long)):
- return hex(code), TWOCC.get(code, u'Unknown')
-
- return None, u'Unknown'
-
-
-TWOCC = {
- 0x0000: 'Unknown Wave Format',
- 0x0001: 'PCM',
- 0x0002: 'Microsoft ADPCM',
- 0x0003: 'IEEE Float',
- 0x0004: 'Compaq Computer VSELP',
- 0x0005: 'IBM CVSD',
- 0x0006: 'A-Law',
- 0x0007: 'mu-Law',
- 0x0008: 'Microsoft DTS',
- 0x0009: 'Microsoft DRM',
- 0x0010: 'OKI ADPCM',
- 0x0011: 'Intel DVI/IMA ADPCM',
- 0x0012: 'Videologic MediaSpace ADPCM',
- 0x0013: 'Sierra Semiconductor ADPCM',
- 0x0014: 'Antex Electronics G.723 ADPCM',
- 0x0015: 'DSP Solutions DigiSTD',
- 0x0016: 'DSP Solutions DigiFIX',
- 0x0017: 'Dialogic OKI ADPCM',
- 0x0018: 'MediaVision ADPCM',
- 0x0019: 'Hewlett-Packard CU',
- 0x0020: 'Yamaha ADPCM',
- 0x0021: 'Speech Compression Sonarc',
- 0x0022: 'DSP Group TrueSpeech',
- 0x0023: 'Echo Speech EchoSC1',
- 0x0024: 'Audiofile AF36',
- 0x0025: 'Audio Processing Technology APTX',
- 0x0026: 'AudioFile AF10',
- 0x0027: 'Prosody 1612',
- 0x0028: 'LRC',
- 0x0030: 'Dolby AC2',
- 0x0031: 'Microsoft GSM 6.10',
- 0x0032: 'MSNAudio',
- 0x0033: 'Antex Electronics ADPCME',
- 0x0034: 'Control Resources VQLPC',
- 0x0035: 'DSP Solutions DigiREAL',
- 0x0036: 'DSP Solutions DigiADPCM',
- 0x0037: 'Control Resources CR10',
- 0x0038: 'Natural MicroSystems VBXADPCM',
- 0x0039: 'Crystal Semiconductor IMA ADPCM',
- 0x003A: 'EchoSC3',
- 0x003B: 'Rockwell ADPCM',
- 0x003C: 'Rockwell Digit LK',
- 0x003D: 'Xebec',
- 0x0040: 'Antex Electronics G.721 ADPCM',
- 0x0041: 'G.728 CELP',
- 0x0042: 'MSG723',
- 0x0043: 'IBM AVC ADPCM',
- 0x0045: 'ITU-T G.726 ADPCM',
- 0x0050: 'MPEG 1, Layer 1,2',
- 0x0052: 'RT24',
- 0x0053: 'PAC',
- 0x0055: 'MPEG Layer 3',
- 0x0059: 'Lucent G.723',
- 0x0060: 'Cirrus',
- 0x0061: 'ESPCM',
- 0x0062: 'Voxware',
- 0x0063: 'Canopus Atrac',
- 0x0064: 'G.726 ADPCM',
- 0x0065: 'G.722 ADPCM',
- 0x0066: 'DSAT',
- 0x0067: 'DSAT Display',
- 0x0069: 'Voxware Byte Aligned',
- 0x0070: 'Voxware AC8',
- 0x0071: 'Voxware AC10',
- 0x0072: 'Voxware AC16',
- 0x0073: 'Voxware AC20',
- 0x0074: 'Voxware MetaVoice',
- 0x0075: 'Voxware MetaSound',
- 0x0076: 'Voxware RT29HW',
- 0x0077: 'Voxware VR12',
- 0x0078: 'Voxware VR18',
- 0x0079: 'Voxware TQ40',
- 0x0080: 'Softsound',
- 0x0081: 'Voxware TQ60',
- 0x0082: 'MSRT24',
- 0x0083: 'G.729A',
- 0x0084: 'MVI MV12',
- 0x0085: 'DF G.726',
- 0x0086: 'DF GSM610',
- 0x0088: 'ISIAudio',
- 0x0089: 'Onlive',
- 0x0091: 'SBC24',
- 0x0092: 'Dolby AC3 SPDIF',
- 0x0093: 'MediaSonic G.723',
- 0x0094: 'Aculab PLC Prosody 8KBPS',
- 0x0097: 'ZyXEL ADPCM',
- 0x0098: 'Philips LPCBB',
- 0x0099: 'Packed',
- 0x00A0: 'Malden Electronics PHONYTALK',
- 0x00FF: 'AAC',
- 0x0100: 'Rhetorex ADPCM',
- 0x0101: 'IBM mu-law',
- 0x0102: 'IBM A-law',
- 0x0103: 'IBM AVC Adaptive Differential Pulse Code Modulation',
- 0x0111: 'Vivo G.723',
- 0x0112: 'Vivo Siren',
- 0x0123: 'Digital G.723',
- 0x0125: 'Sanyo LD ADPCM',
- 0x0130: 'Sipro Lab Telecom ACELP.net',
- 0x0131: 'Sipro Lab Telecom ACELP.4800',
- 0x0132: 'Sipro Lab Telecom ACELP.8V3',
- 0x0133: 'Sipro Lab Telecom ACELP.G.729',
- 0x0134: 'Sipro Lab Telecom ACELP.G.729A',
- 0x0135: 'Sipro Lab Telecom ACELP.KELVIN',
- 0x0140: 'Windows Media Video V8',
- 0x0150: 'Qualcomm PureVoice',
- 0x0151: 'Qualcomm HalfRate',
- 0x0155: 'Ring Zero Systems TUB GSM',
- 0x0160: 'Windows Media Audio V1 / DivX audio (WMA)',
- 0x0161: 'Windows Media Audio V7 / V8 / V9',
- 0x0162: 'Windows Media Audio Professional V9',
- 0x0163: 'Windows Media Audio Lossless V9',
- 0x0170: 'UNISYS NAP ADPCM',
- 0x0171: 'UNISYS NAP ULAW',
- 0x0172: 'UNISYS NAP ALAW',
- 0x0173: 'UNISYS NAP 16K',
- 0x0200: 'Creative Labs ADPCM',
- 0x0202: 'Creative Labs Fastspeech8',
- 0x0203: 'Creative Labs Fastspeech10',
- 0x0210: 'UHER Informatic ADPCM',
- 0x0215: 'Ulead DV ACM',
- 0x0216: 'Ulead DV ACM',
- 0x0220: 'Quarterdeck',
- 0x0230: 'I-link Worldwide ILINK VC',
- 0x0240: 'Aureal Semiconductor RAW SPORT',
- 0x0241: 'ESST AC3',
- 0x0250: 'Interactive Products HSX',
- 0x0251: 'Interactive Products RPELP',
- 0x0260: 'Consistent Software CS2',
- 0x0270: 'Sony ATRAC3 (SCX, same as MiniDisk LP2)',
- 0x0300: 'Fujitsu FM Towns Snd',
- 0x0400: 'BTV Digital',
- 0x0401: 'Intel Music Coder (IMC)',
- 0x0402: 'Ligos Indeo Audio',
- 0x0450: 'QDesign Music',
- 0x0680: 'VME VMPCM',
- 0x0681: 'AT&T Labs TPC',
- 0x0700: 'YMPEG Alpha',
- 0x08AE: 'ClearJump LiteWave',
- 0x1000: 'Olivetti GSM',
- 0x1001: 'Olivetti ADPCM',
- 0x1002: 'Olivetti CELP',
- 0x1003: 'Olivetti SBC',
- 0x1004: 'Olivetti OPR',
- 0x1100: 'Lernout & Hauspie LH Codec',
- 0x1101: 'Lernout & Hauspie CELP codec',
- 0x1102: 'Lernout & Hauspie SBC codec',
- 0x1103: 'Lernout & Hauspie SBC codec',
- 0x1104: 'Lernout & Hauspie SBC codec',
- 0x1400: 'Norris',
- 0x1401: 'AT&T ISIAudio',
- 0x1500: 'Soundspace Music Compression',
- 0x181C: 'VoxWare RT24 speech codec',
- 0x181E: 'Lucent elemedia AX24000P Music codec',
- 0x1C07: 'Lucent SX8300P speech codec',
- 0x1C0C: 'Lucent SX5363S G.723 compliant codec',
- 0x1F03: 'CUseeMe DigiTalk (ex-Rocwell)',
- 0x1FC4: 'NCT Soft ALF2CD ACM',
- 0x2000: 'AC3',
- 0x2001: 'Dolby DTS (Digital Theater System)',
- 0x2002: 'RealAudio 1 / 2 14.4',
- 0x2003: 'RealAudio 1 / 2 28.8',
- 0x2004: 'RealAudio G2 / 8 Cook (low bitrate)',
- 0x2005: 'RealAudio 3 / 4 / 5 Music (DNET)',
- 0x2006: 'RealAudio 10 AAC (RAAC)',
- 0x2007: 'RealAudio 10 AAC+ (RACP)',
- 0x3313: 'makeAVIS',
- 0x4143: 'Divio MPEG-4 AAC audio',
- 0x434C: 'LEAD Speech',
- 0x564C: 'LEAD Vorbis',
- 0x674F: 'Ogg Vorbis (mode 1)',
- 0x6750: 'Ogg Vorbis (mode 2)',
- 0x6751: 'Ogg Vorbis (mode 3)',
- 0x676F: 'Ogg Vorbis (mode 1+)',
- 0x6770: 'Ogg Vorbis (mode 2+)',
- 0x6771: 'Ogg Vorbis (mode 3+)',
- 0x7A21: 'GSM-AMR (CBR, no SID)',
- 0x7A22: 'GSM-AMR (VBR, including SID)',
- 0xDFAC: 'DebugMode SonicFoundry Vegas FrameServer ACM Codec',
- 0xF1AC: 'Free Lossless Audio Codec FLAC',
- 0xFFFE: 'Extensible wave format',
- 0xFFFF: 'development'
-}
-
-
-FOURCC = {
- '1978': 'A.M.Paredes predictor (LossLess)',
- '2VUY': 'Optibase VideoPump 8-bit 4:2:2 Component YCbCr',
- '3IV0': 'MPEG4-based codec 3ivx',
- '3IV1': '3ivx v1',
- '3IV2': '3ivx v2',
- '3IVD': 'FFmpeg DivX ;-) (MS MPEG-4 v3)',
- '3IVX': 'MPEG4-based codec 3ivx',
- '8BPS': 'Apple QuickTime Planar RGB with Alpha-channel',
- 'AAS4': 'Autodesk Animator codec (RLE)',
- 'AASC': 'Autodesk Animator',
- 'ABYR': 'Kensington ABYR',
- 'ACTL': 'Streambox ACT-L2',
- 'ADV1': 'Loronix WaveCodec',
- 'ADVJ': 'Avid M-JPEG Avid Technology Also known as AVRn',
- 'AEIK': 'Intel Indeo Video 3.2',
- 'AEMI': 'Array VideoONE MPEG1-I Capture',
- 'AFLC': 'Autodesk Animator FLC',
- 'AFLI': 'Autodesk Animator FLI',
- 'AHDV': 'CineForm 10-bit Visually Perfect HD',
- 'AJPG': '22fps JPEG-based codec for digital cameras',
- 'AMPG': 'Array VideoONE MPEG',
- 'ANIM': 'Intel RDX (ANIM)',
- 'AP41': 'AngelPotion Definitive',
- 'AP42': 'AngelPotion Definitive',
- 'ASLC': 'AlparySoft Lossless Codec',
- 'ASV1': 'Asus Video v1',
- 'ASV2': 'Asus Video v2',
- 'ASVX': 'Asus Video 2.0 (audio)',
- 'ATM4': 'Ahead Nero Digital MPEG-4 Codec',
- 'AUR2': 'Aura 2 Codec - YUV 4:2:2',
- 'AURA': 'Aura 1 Codec - YUV 4:1:1',
- 'AV1X': 'Avid 1:1x (Quick Time)',
- 'AVC1': 'H.264 AVC',
- 'AVD1': 'Avid DV (Quick Time)',
- 'AVDJ': 'Avid Meridien JFIF with Alpha-channel',
- 'AVDN': 'Avid DNxHD (Quick Time)',
- 'AVDV': 'Avid DV',
- 'AVI1': 'MainConcept Motion JPEG Codec',
- 'AVI2': 'MainConcept Motion JPEG Codec',
- 'AVID': 'Avid Motion JPEG',
- 'AVIS': 'Wrapper for AviSynth',
- 'AVMP': 'Avid IMX (Quick Time)',
- 'AVR ': 'Avid ABVB/NuVista MJPEG with Alpha-channel',
- 'AVRN': 'Avid Motion JPEG',
- 'AVUI': 'Avid Meridien Uncompressed with Alpha-channel',
- 'AVUP': 'Avid 10bit Packed (Quick Time)',
- 'AYUV': '4:4:4 YUV (AYUV)',
- 'AZPR': 'Quicktime Apple Video',
- 'AZRP': 'Quicktime Apple Video',
- 'BGR ': 'Uncompressed BGR32 8:8:8:8',
- 'BGR(15)': 'Uncompressed BGR15 5:5:5',
- 'BGR(16)': 'Uncompressed BGR16 5:6:5',
- 'BGR(24)': 'Uncompressed BGR24 8:8:8',
- 'BHIV': 'BeHere iVideo',
- 'BINK': 'RAD Game Tools Bink Video',
- 'BIT ': 'BI_BITFIELDS (Raw RGB)',
- 'BITM': 'Microsoft H.261',
- 'BLOX': 'Jan Jezabek BLOX MPEG Codec',
- 'BLZ0': 'DivX for Blizzard Decoder Filter',
- 'BT20': 'Conexant Prosumer Video',
- 'BTCV': 'Conexant Composite Video Codec',
- 'BTVC': 'Conexant Composite Video',
- 'BW00': 'BergWave (Wavelet)',
- 'BW10': 'Data Translation Broadway MPEG Capture',
- 'BXBG': 'BOXX BGR',
- 'BXRG': 'BOXX RGB',
- 'BXY2': 'BOXX 10-bit YUV',
- 'BXYV': 'BOXX YUV',
- 'CC12': 'Intel YUV12',
- 'CDV5': 'Canopus SD50/DVHD',
- 'CDVC': 'Canopus DV',
- 'CDVH': 'Canopus SD50/DVHD',
- 'CFCC': 'Digital Processing Systems DPS Perception',
- 'CFHD': 'CineForm 10-bit Visually Perfect HD',
- 'CGDI': 'Microsoft Office 97 Camcorder Video',
- 'CHAM': 'Winnov Caviara Champagne',
- 'CJPG': 'Creative WebCam JPEG',
- 'CLJR': 'Cirrus Logic YUV 4 pixels',
- 'CLLC': 'Canopus LossLess',
- 'CLPL': 'YV12',
- 'CMYK': 'Common Data Format in Printing',
- 'COL0': 'FFmpeg DivX ;-) (MS MPEG-4 v3)',
- 'COL1': 'FFmpeg DivX ;-) (MS MPEG-4 v3)',
- 'CPLA': 'Weitek 4:2:0 YUV Planar',
- 'CRAM': 'Microsoft Video 1 (CRAM)',
- 'CSCD': 'RenderSoft CamStudio lossless Codec',
- 'CTRX': 'Citrix Scalable Video Codec',
- 'CUVC': 'Canopus HQ',
- 'CVID': 'Radius Cinepak',
- 'CWLT': 'Microsoft Color WLT DIB',
- 'CYUV': 'Creative Labs YUV',
- 'CYUY': 'ATI YUV',
- 'D261': 'H.261',
- 'D263': 'H.263',
- 'DAVC': 'Dicas MPEGable H.264/MPEG-4 AVC base profile codec',
- 'DC25': 'MainConcept ProDV Codec',
- 'DCAP': 'Pinnacle DV25 Codec',
- 'DCL1': 'Data Connection Conferencing Codec',
- 'DCT0': 'WniWni Codec',
- 'DFSC': 'DebugMode FrameServer VFW Codec',
- 'DIB ': 'Full Frames (Uncompressed)',
- 'DIV1': 'FFmpeg-4 V1 (hacked MS MPEG-4 V1)',
- 'DIV2': 'MS MPEG-4 V2',
- 'DIV3': 'DivX v3 MPEG-4 Low-Motion',
- 'DIV4': 'DivX v3 MPEG-4 Fast-Motion',
- 'DIV5': 'DIV5',
- 'DIV6': 'DivX MPEG-4',
- 'DIVX': 'DivX',
- 'DM4V': 'Dicas MPEGable MPEG-4',
- 'DMB1': 'Matrox Rainbow Runner hardware MJPEG',
- 'DMB2': 'Paradigm MJPEG',
- 'DMK2': 'ViewSonic V36 PDA Video',
- 'DP02': 'DynaPel MPEG-4',
- 'DPS0': 'DPS Reality Motion JPEG',
- 'DPSC': 'DPS PAR Motion JPEG',
- 'DRWX': 'Pinnacle DV25 Codec',
- 'DSVD': 'DSVD',
- 'DTMT': 'Media-100 Codec',
- 'DTNT': 'Media-100 Codec',
- 'DUCK': 'Duck True Motion 1.0',
- 'DV10': 'BlueFish444 (lossless RGBA, YUV 10-bit)',
- 'DV25': 'Matrox DVCPRO codec',
- 'DV50': 'Matrox DVCPRO50 codec',
- 'DVAN': 'DVAN',
- 'DVC ': 'Apple QuickTime DV (DVCPRO NTSC)',
- 'DVCP': 'Apple QuickTime DV (DVCPRO PAL)',
- 'DVCS': 'MainConcept DV Codec',
- 'DVE2': 'InSoft DVE-2 Videoconferencing',
- 'DVH1': 'Pinnacle DVHD100',
- 'DVHD': 'DV 1125 lines at 30.00 Hz or 1250 lines at 25.00 Hz',
- 'DVIS': 'VSYNC DualMoon Iris DV codec',
- 'DVL ': 'Radius SoftDV 16:9 NTSC',
- 'DVLP': 'Radius SoftDV 16:9 PAL',
- 'DVMA': 'Darim Vision DVMPEG',
- 'DVOR': 'BlueFish444 (lossless RGBA, YUV 10-bit)',
- 'DVPN': 'Apple QuickTime DV (DV NTSC)',
- 'DVPP': 'Apple QuickTime DV (DV PAL)',
- 'DVR1': 'TARGA2000 Codec',
- 'DVRS': 'VSYNC DualMoon Iris DV codec',
- 'DVSD': 'DV',
- 'DVSL': 'DV compressed in SD (SDL)',
- 'DVX1': 'DVX1000SP Video Decoder',
- 'DVX2': 'DVX2000S Video Decoder',
- 'DVX3': 'DVX3000S Video Decoder',
- 'DX50': 'DivX v5',
- 'DXGM': 'Electronic Arts Game Video codec',
- 'DXSB': 'DivX Subtitles Codec',
- 'DXT1': 'Microsoft DirectX Compressed Texture (DXT1)',
- 'DXT2': 'Microsoft DirectX Compressed Texture (DXT2)',
- 'DXT3': 'Microsoft DirectX Compressed Texture (DXT3)',
- 'DXT4': 'Microsoft DirectX Compressed Texture (DXT4)',
- 'DXT5': 'Microsoft DirectX Compressed Texture (DXT5)',
- 'DXTC': 'Microsoft DirectX Compressed Texture (DXTC)',
- 'DXTN': 'Microsoft DirectX Compressed Texture (DXTn)',
- 'EKQ0': 'Elsa EKQ0',
- 'ELK0': 'Elsa ELK0',
- 'EM2V': 'Etymonix MPEG-2 I-frame',
- 'EQK0': 'Elsa graphics card quick codec',
- 'ESCP': 'Eidos Escape',
- 'ETV1': 'eTreppid Video ETV1',
- 'ETV2': 'eTreppid Video ETV2',
- 'ETVC': 'eTreppid Video ETVC',
- 'FFDS': 'FFDShow supported',
- 'FFV1': 'FFDShow supported',
- 'FFVH': 'FFVH codec',
- 'FLIC': 'Autodesk FLI/FLC Animation',
- 'FLJP': 'D-Vision Field Encoded Motion JPEG',
- 'FLV1': 'FLV1 codec',
- 'FMJP': 'D-Vision fieldbased ISO MJPEG',
- 'FRLE': 'SoftLab-NSK Y16 + Alpha RLE',
- 'FRWA': 'SoftLab-Nsk Forward Motion JPEG w/ alpha channel',
- 'FRWD': 'SoftLab-Nsk Forward Motion JPEG',
- 'FRWT': 'SoftLab-NSK Vision Forward Motion JPEG with Alpha-channel',
- 'FRWU': 'SoftLab-NSK Vision Forward Uncompressed',
- 'FVF1': 'Iterated Systems Fractal Video Frame',
- 'FVFW': 'ff MPEG-4 based on XviD codec',
- 'GEPJ': 'White Pine (ex Paradigm Matrix) Motion JPEG Codec',
- 'GJPG': 'Grand Tech GT891x Codec',
- 'GLCC': 'GigaLink AV Capture codec',
- 'GLZW': 'Motion LZW',
- 'GPEG': 'Motion JPEG',
- 'GPJM': 'Pinnacle ReelTime MJPEG Codec',
- 'GREY': 'Apparently a duplicate of Y800',
- 'GWLT': 'Microsoft Greyscale WLT DIB',
- 'H260': 'H.260',
- 'H261': 'H.261',
- 'H262': 'H.262',
- 'H263': 'H.263',
- 'H264': 'H.264 AVC',
- 'H265': 'H.265',
- 'H266': 'H.266',
- 'H267': 'H.267',
- 'H268': 'H.268',
- 'H269': 'H.269',
- 'HD10': 'BlueFish444 (lossless RGBA, YUV 10-bit)',
- 'HDX4': 'Jomigo HDX4',
- 'HFYU': 'Huffman Lossless Codec',
- 'HMCR': 'Rendition Motion Compensation Format (HMCR)',
- 'HMRR': 'Rendition Motion Compensation Format (HMRR)',
- 'I263': 'Intel ITU H.263 Videoconferencing (i263)',
- 'I420': 'Intel Indeo 4',
- 'IAN ': 'Intel RDX',
- 'ICLB': 'InSoft CellB Videoconferencing',
- 'IDM0': 'IDM Motion Wavelets 2.0',
- 'IF09': 'Microsoft H.261',
- 'IGOR': 'Power DVD',
- 'IJPG': 'Intergraph JPEG',
- 'ILVC': 'Intel Layered Video',
- 'ILVR': 'ITU-T H.263+',
- 'IMC1': 'IMC1',
- 'IMC2': 'IMC2',
- 'IMC3': 'IMC3',
- 'IMC4': 'IMC4',
- 'IMJG': 'Accom SphereOUS MJPEG with Alpha-channel',
- 'IPDV': 'I-O Data Device Giga AVI DV Codec',
- 'IPJ2': 'Image Power JPEG2000',
- 'IR21': 'Intel Indeo 2.1',
- 'IRAW': 'Intel YUV Uncompressed',
- 'IUYV': 'Interlaced version of UYVY (line order 0,2,4 then 1,3,5 etc)',
- 'IV30': 'Ligos Indeo 3.0',
- 'IV31': 'Ligos Indeo 3.1',
- 'IV32': 'Ligos Indeo 3.2',
- 'IV33': 'Ligos Indeo 3.3',
- 'IV34': 'Ligos Indeo 3.4',
- 'IV35': 'Ligos Indeo 3.5',
- 'IV36': 'Ligos Indeo 3.6',
- 'IV37': 'Ligos Indeo 3.7',
- 'IV38': 'Ligos Indeo 3.8',
- 'IV39': 'Ligos Indeo 3.9',
- 'IV40': 'Ligos Indeo Interactive 4.0',
- 'IV41': 'Ligos Indeo Interactive 4.1',
- 'IV42': 'Ligos Indeo Interactive 4.2',
- 'IV43': 'Ligos Indeo Interactive 4.3',
- 'IV44': 'Ligos Indeo Interactive 4.4',
- 'IV45': 'Ligos Indeo Interactive 4.5',
- 'IV46': 'Ligos Indeo Interactive 4.6',
- 'IV47': 'Ligos Indeo Interactive 4.7',
- 'IV48': 'Ligos Indeo Interactive 4.8',
- 'IV49': 'Ligos Indeo Interactive 4.9',
- 'IV50': 'Ligos Indeo Interactive 5.0',
- 'IY41': 'Interlaced version of Y41P (line order 0,2,4,...,1,3,5...)',
- 'IYU1': '12 bit format used in mode 2 of the IEEE 1394 Digital Camera 1.04 spec',
- 'IYU2': '24 bit format used in mode 2 of the IEEE 1394 Digital Camera 1.04 spec',
- 'IYUV': 'Intel Indeo iYUV 4:2:0',
- 'JBYR': 'Kensington JBYR',
- 'JFIF': 'Motion JPEG (FFmpeg)',
- 'JPEG': 'Still Image JPEG DIB',
- 'JPG ': 'JPEG compressed',
- 'JPGL': 'Webcam JPEG Light',
- 'KMVC': 'Karl Morton\'s Video Codec',
- 'KPCD': 'Kodak Photo CD',
- 'L261': 'Lead Technologies H.261',
- 'L263': 'Lead Technologies H.263',
- 'LAGS': 'Lagarith LossLess',
- 'LBYR': 'Creative WebCam codec',
- 'LCMW': 'Lead Technologies Motion CMW Codec',
- 'LCW2': 'LEADTools MCMW 9Motion Wavelet)',
- 'LEAD': 'LEAD Video Codec',
- 'LGRY': 'Lead Technologies Grayscale Image',
- 'LJ2K': 'LEADTools JPEG2000',
- 'LJPG': 'LEAD MJPEG Codec',
- 'LMP2': 'LEADTools MPEG2',
- 'LOCO': 'LOCO Lossless Codec',
- 'LSCR': 'LEAD Screen Capture',
- 'LSVM': 'Vianet Lighting Strike Vmail (Streaming)',
- 'LZO1': 'LZO compressed (lossless codec)',
- 'M261': 'Microsoft H.261',
- 'M263': 'Microsoft H.263',
- 'M4CC': 'ESS MPEG4 Divio codec',
- 'M4S2': 'Microsoft MPEG-4 (M4S2)',
- 'MC12': 'ATI Motion Compensation Format (MC12)',
- 'MC24': 'MainConcept Motion JPEG Codec',
- 'MCAM': 'ATI Motion Compensation Format (MCAM)',
- 'MCZM': 'Theory MicroCosm Lossless 64bit RGB with Alpha-channel',
- 'MDVD': 'Alex MicroDVD Video (hacked MS MPEG-4)',
- 'MDVF': 'Pinnacle DV/DV50/DVHD100',
- 'MHFY': 'A.M.Paredes mhuffyYUV (LossLess)',
- 'MJ2C': 'Morgan Multimedia Motion JPEG2000',
- 'MJPA': 'Pinnacle ReelTime MJPG hardware codec',
- 'MJPB': 'Motion JPEG codec',
- 'MJPG': 'Motion JPEG DIB',
- 'MJPX': 'Pegasus PICVideo Motion JPEG',
- 'MMES': 'Matrox MPEG-2 I-frame',
- 'MNVD': 'MindBend MindVid LossLess',
- 'MP2A': 'MPEG-2 Audio',
- 'MP2T': 'MPEG-2 Transport Stream',
- 'MP2V': 'MPEG-2 Video',
- 'MP41': 'Microsoft MPEG-4 V1 (enhansed H263)',
- 'MP42': 'Microsoft MPEG-4 (low-motion)',
- 'MP43': 'Microsoft MPEG-4 (fast-motion)',
- 'MP4A': 'MPEG-4 Audio',
- 'MP4S': 'Microsoft MPEG-4 (MP4S)',
- 'MP4T': 'MPEG-4 Transport Stream',
- 'MP4V': 'Apple QuickTime MPEG-4 native',
- 'MPEG': 'MPEG-1',
- 'MPG1': 'FFmpeg-1',
- 'MPG2': 'FFmpeg-1',
- 'MPG3': 'Same as Low motion DivX MPEG-4',
- 'MPG4': 'Microsoft MPEG-4 Video High Speed Compressor',
- 'MPGI': 'Sigma Designs MPEG',
- 'MPNG': 'Motion PNG codec',
- 'MRCA': 'Martin Regen Codec',
- 'MRLE': 'Run Length Encoding',
- 'MSS1': 'Windows Screen Video',
- 'MSS2': 'Windows Media 9',
- 'MSUC': 'MSU LossLess',
- 'MSVC': 'Microsoft Video 1',
- 'MSZH': 'Lossless codec (ZIP compression)',
- 'MTGA': 'Motion TGA images (24, 32 bpp)',
- 'MTX1': 'Matrox MTX1',
- 'MTX2': 'Matrox MTX2',
- 'MTX3': 'Matrox MTX3',
- 'MTX4': 'Matrox MTX4',
- 'MTX5': 'Matrox MTX5',
- 'MTX6': 'Matrox MTX6',
- 'MTX7': 'Matrox MTX7',
- 'MTX8': 'Matrox MTX8',
- 'MTX9': 'Matrox MTX9',
- 'MV12': 'MV12',
- 'MVI1': 'Motion Pixels MVI',
- 'MVI2': 'Motion Pixels MVI',
- 'MWV1': 'Aware Motion Wavelets',
- 'MYUV': 'Media-100 844/X Uncompressed',
- 'NAVI': 'nAVI',
- 'NDIG': 'Ahead Nero Digital MPEG-4 Codec',
- 'NHVU': 'NVidia Texture Format (GEForce 3)',
- 'NO16': 'Theory None16 64bit uncompressed RAW',
- 'NT00': 'NewTek LigtWave HDTV YUV with Alpha-channel',
- 'NTN1': 'Nogatech Video Compression 1',
- 'NTN2': 'Nogatech Video Compression 2 (GrabBee hardware coder)',
- 'NUV1': 'NuppelVideo',
- 'NV12': '8-bit Y plane followed by an interleaved U/V plane with 2x2 subsampling',
- 'NV21': 'As NV12 with U and V reversed in the interleaved plane',
- 'NVDS': 'nVidia Texture Format',
- 'NVHS': 'NVidia Texture Format (GEForce 3)',
- 'NVS0': 'nVidia GeForce Texture',
- 'NVS1': 'nVidia GeForce Texture',
- 'NVS2': 'nVidia GeForce Texture',
- 'NVS3': 'nVidia GeForce Texture',
- 'NVS4': 'nVidia GeForce Texture',
- 'NVS5': 'nVidia GeForce Texture',
- 'NVT0': 'nVidia GeForce Texture',
- 'NVT1': 'nVidia GeForce Texture',
- 'NVT2': 'nVidia GeForce Texture',
- 'NVT3': 'nVidia GeForce Texture',
- 'NVT4': 'nVidia GeForce Texture',
- 'NVT5': 'nVidia GeForce Texture',
- 'PDVC': 'I-O Data Device Digital Video Capture DV codec',
- 'PGVV': 'Radius Video Vision',
- 'PHMO': 'IBM Photomotion',
- 'PIM1': 'Pegasus Imaging',
- 'PIM2': 'Pegasus Imaging',
- 'PIMJ': 'Pegasus Imaging Lossless JPEG',
- 'PIXL': 'MiroVideo XL (Motion JPEG)',
- 'PNG ': 'Apple PNG',
- 'PNG1': 'Corecodec.org CorePNG Codec',
- 'PVEZ': 'Horizons Technology PowerEZ',
- 'PVMM': 'PacketVideo Corporation MPEG-4',
- 'PVW2': 'Pegasus Imaging Wavelet Compression',
- 'PVWV': 'Pegasus Imaging Wavelet 2000',
- 'PXLT': 'Apple Pixlet (Wavelet)',
- 'Q1.0': 'Q-Team QPEG 1.0 (www.q-team.de)',
- 'Q1.1': 'Q-Team QPEG 1.1 (www.q-team.de)',
- 'QDGX': 'Apple QuickDraw GX',
- 'QPEG': 'Q-Team QPEG 1.0',
- 'QPEQ': 'Q-Team QPEG 1.1',
- 'R210': 'BlackMagic YUV (Quick Time)',
- 'R411': 'Radius DV NTSC YUV',
- 'R420': 'Radius DV PAL YUV',
- 'RAVI': 'GroupTRON ReferenceAVI codec (dummy for MPEG compressor)',
- 'RAV_': 'GroupTRON ReferenceAVI codec (dummy for MPEG compressor)',
- 'RAW ': 'Full Frames (Uncompressed)',
- 'RGB ': 'Full Frames (Uncompressed)',
- 'RGB(15)': 'Uncompressed RGB15 5:5:5',
- 'RGB(16)': 'Uncompressed RGB16 5:6:5',
- 'RGB(24)': 'Uncompressed RGB24 8:8:8',
- 'RGB1': 'Uncompressed RGB332 3:3:2',
- 'RGBA': 'Raw RGB with alpha',
- 'RGBO': 'Uncompressed RGB555 5:5:5',
- 'RGBP': 'Uncompressed RGB565 5:6:5',
- 'RGBQ': 'Uncompressed RGB555X 5:5:5 BE',
- 'RGBR': 'Uncompressed RGB565X 5:6:5 BE',
- 'RGBT': 'Computer Concepts 32-bit support',
- 'RL4 ': 'RLE 4bpp RGB',
- 'RL8 ': 'RLE 8bpp RGB',
- 'RLE ': 'Microsoft Run Length Encoder',
- 'RLE4': 'Run Length Encoded 4',
- 'RLE8': 'Run Length Encoded 8',
- 'RMP4': 'REALmagic MPEG-4 Video Codec',
- 'ROQV': 'Id RoQ File Video Decoder',
- 'RPZA': 'Apple Video 16 bit "road pizza"',
- 'RT21': 'Intel Real Time Video 2.1',
- 'RTV0': 'NewTek VideoToaster',
- 'RUD0': 'Rududu video codec',
- 'RV10': 'RealVideo codec',
- 'RV13': 'RealVideo codec',
- 'RV20': 'RealVideo G2',
- 'RV30': 'RealVideo 8',
- 'RV40': 'RealVideo 9',
- 'RVX ': 'Intel RDX (RVX )',
- 'S263': 'Sorenson Vision H.263',
- 'S422': 'Tekram VideoCap C210 YUV 4:2:2',
- 'SAMR': 'Adaptive Multi-Rate (AMR) audio codec',
- 'SAN3': 'MPEG-4 codec (direct copy of DivX 3.11a)',
- 'SDCC': 'Sun Communication Digital Camera Codec',
- 'SEDG': 'Samsung MPEG-4 codec',
- 'SFMC': 'CrystalNet Surface Fitting Method',
- 'SHR0': 'BitJazz SheerVideo',
- 'SHR1': 'BitJazz SheerVideo',
- 'SHR2': 'BitJazz SheerVideo',
- 'SHR3': 'BitJazz SheerVideo',
- 'SHR4': 'BitJazz SheerVideo',
- 'SHR5': 'BitJazz SheerVideo',
- 'SHR6': 'BitJazz SheerVideo',
- 'SHR7': 'BitJazz SheerVideo',
- 'SJPG': 'CUseeMe Networks Codec',
- 'SL25': 'SoftLab-NSK DVCPRO',
- 'SL50': 'SoftLab-NSK DVCPRO50',
- 'SLDV': 'SoftLab-NSK Forward DV Draw codec',
- 'SLIF': 'SoftLab-NSK MPEG2 I-frames',
- 'SLMJ': 'SoftLab-NSK Forward MJPEG',
- 'SMC ': 'Apple Graphics (SMC) codec (256 color)',
- 'SMSC': 'Radius SMSC',
- 'SMSD': 'Radius SMSD',
- 'SMSV': 'WorldConnect Wavelet Video',
- 'SNOW': 'SNOW codec',
- 'SP40': 'SunPlus YUV',
- 'SP44': 'SunPlus Aiptek MegaCam Codec',
- 'SP53': 'SunPlus Aiptek MegaCam Codec',
- 'SP54': 'SunPlus Aiptek MegaCam Codec',
- 'SP55': 'SunPlus Aiptek MegaCam Codec',
- 'SP56': 'SunPlus Aiptek MegaCam Codec',
- 'SP57': 'SunPlus Aiptek MegaCam Codec',
- 'SP58': 'SunPlus Aiptek MegaCam Codec',
- 'SPIG': 'Radius Spigot',
- 'SPLC': 'Splash Studios ACM Audio Codec',
- 'SPRK': 'Sorenson Spark',
- 'SQZ2': 'Microsoft VXTreme Video Codec V2',
- 'STVA': 'ST CMOS Imager Data (Bayer)',
- 'STVB': 'ST CMOS Imager Data (Nudged Bayer)',
- 'STVC': 'ST CMOS Imager Data (Bunched)',
- 'STVX': 'ST CMOS Imager Data (Extended CODEC Data Format)',
- 'STVY': 'ST CMOS Imager Data (Extended CODEC Data Format with Correction Data)',
- 'SV10': 'Sorenson Video R1',
- 'SVQ1': 'Sorenson Video R3',
- 'SVQ3': 'Sorenson Video 3 (Apple Quicktime 5)',
- 'SWC1': 'MainConcept Motion JPEG Codec',
- 'T420': 'Toshiba YUV 4:2:0',
- 'TGA ': 'Apple TGA (with Alpha-channel)',
- 'THEO': 'FFVFW Supported Codec',
- 'TIFF': 'Apple TIFF (with Alpha-channel)',
- 'TIM2': 'Pinnacle RAL DVI',
- 'TLMS': 'TeraLogic Motion Intraframe Codec (TLMS)',
- 'TLST': 'TeraLogic Motion Intraframe Codec (TLST)',
- 'TM20': 'Duck TrueMotion 2.0',
- 'TM2A': 'Duck TrueMotion Archiver 2.0',
- 'TM2X': 'Duck TrueMotion 2X',
- 'TMIC': 'TeraLogic Motion Intraframe Codec (TMIC)',
- 'TMOT': 'Horizons Technology TrueMotion S',
- 'TR20': 'Duck TrueMotion RealTime 2.0',
- 'TRLE': 'Akula Alpha Pro Custom AVI (LossLess)',
- 'TSCC': 'TechSmith Screen Capture Codec',
- 'TV10': 'Tecomac Low-Bit Rate Codec',
- 'TVJP': 'TrueVision Field Encoded Motion JPEG',
- 'TVMJ': 'Truevision TARGA MJPEG Hardware Codec',
- 'TY0N': 'Trident TY0N',
- 'TY2C': 'Trident TY2C',
- 'TY2N': 'Trident TY2N',
- 'U263': 'UB Video StreamForce H.263',
- 'U<Y ': 'Discreet UC YUV 4:2:2:4 10 bit',
- 'U<YA': 'Discreet UC YUV 4:2:2:4 10 bit (with Alpha-channel)',
- 'UCOD': 'eMajix.com ClearVideo',
- 'ULTI': 'IBM Ultimotion',
- 'UMP4': 'UB Video MPEG 4',
- 'UYNV': 'UYVY',
- 'UYVP': 'YCbCr 4:2:2',
- 'UYVU': 'SoftLab-NSK Forward YUV codec',
- 'UYVY': 'UYVY 4:2:2 byte ordering',
- 'V210': 'Optibase VideoPump 10-bit 4:2:2 Component YCbCr',
- 'V261': 'Lucent VX2000S',
- 'V422': '24 bit YUV 4:2:2 Format',
- 'V655': '16 bit YUV 4:2:2 Format',
- 'VBLE': 'MarcFD VBLE Lossless Codec',
- 'VCR1': 'ATI VCR 1.0',
- 'VCR2': 'ATI VCR 2.0',
- 'VCR3': 'ATI VCR 3.0',
- 'VCR4': 'ATI VCR 4.0',
- 'VCR5': 'ATI VCR 5.0',
- 'VCR6': 'ATI VCR 6.0',
- 'VCR7': 'ATI VCR 7.0',
- 'VCR8': 'ATI VCR 8.0',
- 'VCR9': 'ATI VCR 9.0',
- 'VDCT': 'Video Maker Pro DIB',
- 'VDOM': 'VDOnet VDOWave',
- 'VDOW': 'VDOnet VDOLive (H.263)',
- 'VDST': 'VirtualDub remote frameclient ICM driver',
- 'VDTZ': 'Darim Vison VideoTizer YUV',
- 'VGPX': 'VGPixel Codec',
- 'VIDM': 'DivX 5.0 Pro Supported Codec',
- 'VIDS': 'YUV 4:2:2 CCIR 601 for V422',
- 'VIFP': 'VIFP',
- 'VIV1': 'Vivo H.263',
- 'VIV2': 'Vivo H.263',
- 'VIVO': 'Vivo H.263 v2.00',
- 'VIXL': 'Miro Video XL',
- 'VLV1': 'Videologic VLCAP.DRV',
- 'VP30': 'On2 VP3.0',
- 'VP31': 'On2 VP3.1',
- 'VP40': 'On2 TrueCast VP4',
- 'VP50': 'On2 TrueCast VP5',
- 'VP60': 'On2 TrueCast VP6',
- 'VP61': 'On2 TrueCast VP6.1',
- 'VP62': 'On2 TrueCast VP6.2',
- 'VP70': 'On2 TrueMotion VP7',
- 'VQC1': 'Vector-quantised codec 1',
- 'VQC2': 'Vector-quantised codec 2',
- 'VR21': 'BlackMagic YUV (Quick Time)',
- 'VSSH': 'Vanguard VSS H.264',
- 'VSSV': 'Vanguard Software Solutions Video Codec',
- 'VSSW': 'Vanguard VSS H.264',
- 'VTLP': 'Alaris VideoGramPixel Codec',
- 'VX1K': 'VX1000S Video Codec',
- 'VX2K': 'VX2000S Video Codec',
- 'VXSP': 'VX1000SP Video Codec',
- 'VYU9': 'ATI Technologies YUV',
- 'VYUY': 'ATI Packed YUV Data',
- 'WBVC': 'Winbond W9960',
- 'WHAM': 'Microsoft Video 1 (WHAM)',
- 'WINX': 'Winnov Software Compression',
- 'WJPG': 'AverMedia Winbond JPEG',
- 'WMV1': 'Windows Media Video V7',
- 'WMV2': 'Windows Media Video V8',
- 'WMV3': 'Windows Media Video V9',
- 'WMVA': 'WMVA codec',
- 'WMVP': 'Windows Media Video V9',
- 'WNIX': 'WniWni Codec',
- 'WNV1': 'Winnov Hardware Compression',
- 'WNVA': 'Winnov hw compress',
- 'WRLE': 'Apple QuickTime BMP Codec',
- 'WRPR': 'VideoTools VideoServer Client Codec',
- 'WV1F': 'WV1F codec',
- 'WVLT': 'IllusionHope Wavelet 9/7',
- 'WVP2': 'WVP2 codec',
- 'X263': 'Xirlink H.263',
- 'X264': 'XiWave GNU GPL x264 MPEG-4 Codec',
- 'XLV0': 'NetXL Video Decoder',
- 'XMPG': 'Xing MPEG (I-Frame only)',
- 'XVID': 'XviD MPEG-4',
- 'XVIX': 'Based on XviD MPEG-4 codec',
- 'XWV0': 'XiWave Video Codec',
- 'XWV1': 'XiWave Video Codec',
- 'XWV2': 'XiWave Video Codec',
- 'XWV3': 'XiWave Video Codec (Xi-3 Video)',
- 'XWV4': 'XiWave Video Codec',
- 'XWV5': 'XiWave Video Codec',
- 'XWV6': 'XiWave Video Codec',
- 'XWV7': 'XiWave Video Codec',
- 'XWV8': 'XiWave Video Codec',
- 'XWV9': 'XiWave Video Codec',
- 'XXAN': 'XXAN',
- 'XYZP': 'Extended PAL format XYZ palette',
- 'Y211': 'YUV 2:1:1 Packed',
- 'Y216': 'Pinnacle TARGA CineWave YUV (Quick Time)',
- 'Y411': 'YUV 4:1:1 Packed',
- 'Y41B': 'YUV 4:1:1 Planar',
- 'Y41P': 'PC1 4:1:1',
- 'Y41T': 'PC1 4:1:1 with transparency',
- 'Y422': 'Y422',
- 'Y42B': 'YUV 4:2:2 Planar',
- 'Y42T': 'PCI 4:2:2 with transparency',
- 'Y444': 'IYU2',
- 'Y8 ': 'Grayscale video',
- 'Y800': 'Simple grayscale video',
- 'YC12': 'Intel YUV12 Codec',
- 'YMPG': 'YMPEG Alpha',
- 'YU12': 'ATI YV12 4:2:0 Planar',
- 'YU92': 'Intel - YUV',
- 'YUNV': 'YUNV',
- 'YUV2': 'Apple Component Video (YUV 4:2:2)',
- 'YUV8': 'Winnov Caviar YUV8',
- 'YUV9': 'Intel YUV9',
- 'YUVP': 'YCbCr 4:2:2',
- 'YUY2': 'Uncompressed YUV 4:2:2',
- 'YUYV': 'Canopus YUV',
- 'YV12': 'YVU12 Planar',
- 'YV16': 'Elecard YUV 4:2:2 Planar',
- 'YV92': 'Intel Smart Video Recorder YVU9',
- 'YVU9': 'Intel YVU9 Planar',
- 'YVYU': 'YVYU 4:2:2 byte ordering',
- 'ZLIB': 'ZLIB',
- 'ZPEG': 'Metheus Video Zipper',
- 'ZYGO': 'ZyGo Video Codec'
-}
-
-# make it fool prove
-for code, value in FOURCC.items():
- if not code.upper() in FOURCC:
- FOURCC[code.upper()] = value
- if code.endswith(' '):
- FOURCC[code.strip().upper()] = value
diff --git a/lib/enzyme/infos.py b/lib/enzyme/infos.py
deleted file mode 100644
index a6f0bcc5913e44975aeeca9e1a80568a007ca055..0000000000000000000000000000000000000000
--- a/lib/enzyme/infos.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-__version__ = '0.2'
diff --git a/lib/enzyme/language.py b/lib/enzyme/language.py
deleted file mode 100644
index 3957f9d9f62a351357bf92fc2ee5f983e94ea14e..0000000000000000000000000000000000000000
--- a/lib/enzyme/language.py
+++ /dev/null
@@ -1,535 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-import re
-
-__all__ = ['resolve']
-
-
-def resolve(code):
- """
- Transform the given (2- or 3-letter) language code to a human readable
- language name. The return value is a 2-tuple containing the given
- language code and the language name. If the language code cannot be
- resolved, name will be 'Unknown (<code>)'.
- """
- if not code:
- return None, None
- if not isinstance(code, basestring):
- raise ValueError('Invalid language code specified by parser')
-
- # Take up to 3 letters from the code.
- code = re.split(r'[^a-z]', code.lower())[0][:3]
-
- for spec in codes:
- if code in spec[:-1]:
- return code, spec[-1]
-
- return code, u'Unknown (%r)' % code
-
-
-# Parsed from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
-codes = (
- ('aar', 'aa', u'Afar'),
- ('abk', 'ab', u'Abkhazian'),
- ('ace', u'Achinese'),
- ('ach', u'Acoli'),
- ('ada', u'Adangme'),
- ('ady', u'Adyghe'),
- ('afa', u'Afro-Asiatic '),
- ('afh', u'Afrihili'),
- ('afr', 'af', u'Afrikaans'),
- ('ain', u'Ainu'),
- ('aka', 'ak', u'Akan'),
- ('akk', u'Akkadian'),
- ('alb', 'sq', u'Albanian'),
- ('ale', u'Aleut'),
- ('alg', u'Algonquian languages'),
- ('alt', u'Southern Altai'),
- ('amh', 'am', u'Amharic'),
- ('ang', u'English, Old '),
- ('anp', u'Angika'),
- ('apa', u'Apache languages'),
- ('ara', 'ar', u'Arabic'),
- ('arc', u'Official Aramaic '),
- ('arg', 'an', u'Aragonese'),
- ('arm', 'hy', u'Armenian'),
- ('arn', u'Mapudungun'),
- ('arp', u'Arapaho'),
- ('art', u'Artificial '),
- ('arw', u'Arawak'),
- ('asm', 'as', u'Assamese'),
- ('ast', u'Asturian'),
- ('ath', u'Athapascan languages'),
- ('aus', u'Australian languages'),
- ('ava', 'av', u'Avaric'),
- ('ave', 'ae', u'Avestan'),
- ('awa', u'Awadhi'),
- ('aym', 'ay', u'Aymara'),
- ('aze', 'az', u'Azerbaijani'),
- ('bad', u'Banda languages'),
- ('bai', u'Bamileke languages'),
- ('bak', 'ba', u'Bashkir'),
- ('bal', u'Baluchi'),
- ('bam', 'bm', u'Bambara'),
- ('ban', u'Balinese'),
- ('baq', 'eu', u'Basque'),
- ('bas', u'Basa'),
- ('bat', u'Baltic '),
- ('bej', u'Beja'),
- ('bel', 'be', u'Belarusian'),
- ('bem', u'Bemba'),
- ('ben', 'bn', u'Bengali'),
- ('ber', u'Berber '),
- ('bho', u'Bhojpuri'),
- ('bih', 'bh', u'Bihari'),
- ('bik', u'Bikol'),
- ('bin', u'Bini'),
- ('bis', 'bi', u'Bislama'),
- ('bla', u'Siksika'),
- ('bnt', u'Bantu '),
- ('bos', 'bs', u'Bosnian'),
- ('bra', u'Braj'),
- ('bre', 'br', u'Breton'),
- ('btk', u'Batak languages'),
- ('bua', u'Buriat'),
- ('bug', u'Buginese'),
- ('bul', 'bg', u'Bulgarian'),
- ('bur', 'my', u'Burmese'),
- ('byn', u'Blin'),
- ('cad', u'Caddo'),
- ('cai', u'Central American Indian '),
- ('car', u'Galibi Carib'),
- ('cat', 'ca', u'Catalan'),
- ('cau', u'Caucasian '),
- ('ceb', u'Cebuano'),
- ('cel', u'Celtic '),
- ('cha', 'ch', u'Chamorro'),
- ('chb', u'Chibcha'),
- ('che', 'ce', u'Chechen'),
- ('chg', u'Chagatai'),
- ('chi', 'zh', u'Chinese'),
- ('chk', u'Chuukese'),
- ('chm', u'Mari'),
- ('chn', u'Chinook jargon'),
- ('cho', u'Choctaw'),
- ('chp', u'Chipewyan'),
- ('chr', u'Cherokee'),
- ('chu', 'cu', u'Church Slavic'),
- ('chv', 'cv', u'Chuvash'),
- ('chy', u'Cheyenne'),
- ('cmc', u'Chamic languages'),
- ('cop', u'Coptic'),
- ('cor', 'kw', u'Cornish'),
- ('cos', 'co', u'Corsican'),
- ('cpe', u'Creoles and pidgins, English based '),
- ('cpf', u'Creoles and pidgins, French-based '),
- ('cpp', u'Creoles and pidgins, Portuguese-based '),
- ('cre', 'cr', u'Cree'),
- ('crh', u'Crimean Tatar'),
- ('crp', u'Creoles and pidgins '),
- ('csb', u'Kashubian'),
- ('cus', u'Cushitic '),
- ('cze', 'cs', u'Czech'),
- ('dak', u'Dakota'),
- ('dan', 'da', u'Danish'),
- ('dar', u'Dargwa'),
- ('day', u'Land Dayak languages'),
- ('del', u'Delaware'),
- ('den', u'Slave '),
- ('dgr', u'Dogrib'),
- ('din', u'Dinka'),
- ('div', 'dv', u'Divehi'),
- ('doi', u'Dogri'),
- ('dra', u'Dravidian '),
- ('dsb', u'Lower Sorbian'),
- ('dua', u'Duala'),
- ('dum', u'Dutch, Middle '),
- ('dut', 'nl', u'Dutch'),
- ('dyu', u'Dyula'),
- ('dzo', 'dz', u'Dzongkha'),
- ('efi', u'Efik'),
- ('egy', u'Egyptian '),
- ('eka', u'Ekajuk'),
- ('elx', u'Elamite'),
- ('eng', 'en', u'English'),
- ('enm', u'English, Middle '),
- ('epo', 'eo', u'Esperanto'),
- ('est', 'et', u'Estonian'),
- ('ewe', 'ee', u'Ewe'),
- ('ewo', u'Ewondo'),
- ('fan', u'Fang'),
- ('fao', 'fo', u'Faroese'),
- ('fat', u'Fanti'),
- ('fij', 'fj', u'Fijian'),
- ('fil', u'Filipino'),
- ('fin', 'fi', u'Finnish'),
- ('fiu', u'Finno-Ugrian '),
- ('fon', u'Fon'),
- ('fre', 'fr', u'French'),
- ('frm', u'French, Middle '),
- ('fro', u'French, Old '),
- ('frr', u'Northern Frisian'),
- ('frs', u'Eastern Frisian'),
- ('fry', 'fy', u'Western Frisian'),
- ('ful', 'ff', u'Fulah'),
- ('fur', u'Friulian'),
- ('gaa', u'Ga'),
- ('gay', u'Gayo'),
- ('gba', u'Gbaya'),
- ('gem', u'Germanic '),
- ('geo', 'ka', u'Georgian'),
- ('ger', 'de', u'German'),
- ('gez', u'Geez'),
- ('gil', u'Gilbertese'),
- ('gla', 'gd', u'Gaelic'),
- ('gle', 'ga', u'Irish'),
- ('glg', 'gl', u'Galician'),
- ('glv', 'gv', u'Manx'),
- ('gmh', u'German, Middle High '),
- ('goh', u'German, Old High '),
- ('gon', u'Gondi'),
- ('gor', u'Gorontalo'),
- ('got', u'Gothic'),
- ('grb', u'Grebo'),
- ('grc', u'Greek, Ancient '),
- ('gre', 'el', u'Greek, Modern '),
- ('grn', 'gn', u'Guarani'),
- ('gsw', u'Swiss German'),
- ('guj', 'gu', u'Gujarati'),
- ('gwi', u"Gwich'in"),
- ('hai', u'Haida'),
- ('hat', 'ht', u'Haitian'),
- ('hau', 'ha', u'Hausa'),
- ('haw', u'Hawaiian'),
- ('heb', 'he', u'Hebrew'),
- ('her', 'hz', u'Herero'),
- ('hil', u'Hiligaynon'),
- ('him', u'Himachali'),
- ('hin', 'hi', u'Hindi'),
- ('hit', u'Hittite'),
- ('hmn', u'Hmong'),
- ('hmo', 'ho', u'Hiri Motu'),
- ('hsb', u'Upper Sorbian'),
- ('hun', 'hu', u'Hungarian'),
- ('hup', u'Hupa'),
- ('iba', u'Iban'),
- ('ibo', 'ig', u'Igbo'),
- ('ice', 'is', u'Icelandic'),
- ('ido', 'io', u'Ido'),
- ('iii', 'ii', u'Sichuan Yi'),
- ('ijo', u'Ijo languages'),
- ('iku', 'iu', u'Inuktitut'),
- ('ile', 'ie', u'Interlingue'),
- ('ilo', u'Iloko'),
- ('ina', 'ia', u'Interlingua '),
- ('inc', u'Indic '),
- ('ind', 'id', u'Indonesian'),
- ('ine', u'Indo-European '),
- ('inh', u'Ingush'),
- ('ipk', 'ik', u'Inupiaq'),
- ('ira', u'Iranian '),
- ('iro', u'Iroquoian languages'),
- ('ita', 'it', u'Italian'),
- ('jav', 'jv', u'Javanese'),
- ('jbo', u'Lojban'),
- ('jpn', 'ja', u'Japanese'),
- ('jpr', u'Judeo-Persian'),
- ('jrb', u'Judeo-Arabic'),
- ('kaa', u'Kara-Kalpak'),
- ('kab', u'Kabyle'),
- ('kac', u'Kachin'),
- ('kal', 'kl', u'Kalaallisut'),
- ('kam', u'Kamba'),
- ('kan', 'kn', u'Kannada'),
- ('kar', u'Karen languages'),
- ('kas', 'ks', u'Kashmiri'),
- ('kau', 'kr', u'Kanuri'),
- ('kaw', u'Kawi'),
- ('kaz', 'kk', u'Kazakh'),
- ('kbd', u'Kabardian'),
- ('kha', u'Khasi'),
- ('khi', u'Khoisan '),
- ('khm', 'km', u'Central Khmer'),
- ('kho', u'Khotanese'),
- ('kik', 'ki', u'Kikuyu'),
- ('kin', 'rw', u'Kinyarwanda'),
- ('kir', 'ky', u'Kirghiz'),
- ('kmb', u'Kimbundu'),
- ('kok', u'Konkani'),
- ('kom', 'kv', u'Komi'),
- ('kon', 'kg', u'Kongo'),
- ('kor', 'ko', u'Korean'),
- ('kos', u'Kosraean'),
- ('kpe', u'Kpelle'),
- ('krc', u'Karachay-Balkar'),
- ('krl', u'Karelian'),
- ('kro', u'Kru languages'),
- ('kru', u'Kurukh'),
- ('kua', 'kj', u'Kuanyama'),
- ('kum', u'Kumyk'),
- ('kur', 'ku', u'Kurdish'),
- ('kut', u'Kutenai'),
- ('lad', u'Ladino'),
- ('lah', u'Lahnda'),
- ('lam', u'Lamba'),
- ('lao', 'lo', u'Lao'),
- ('lat', 'la', u'Latin'),
- ('lav', 'lv', u'Latvian'),
- ('lez', u'Lezghian'),
- ('lim', 'li', u'Limburgan'),
- ('lin', 'ln', u'Lingala'),
- ('lit', 'lt', u'Lithuanian'),
- ('lol', u'Mongo'),
- ('loz', u'Lozi'),
- ('ltz', 'lb', u'Luxembourgish'),
- ('lua', u'Luba-Lulua'),
- ('lub', 'lu', u'Luba-Katanga'),
- ('lug', 'lg', u'Ganda'),
- ('lui', u'Luiseno'),
- ('lun', u'Lunda'),
- ('luo', u'Luo '),
- ('lus', u'Lushai'),
- ('mac', 'mk', u'Macedonian'),
- ('mad', u'Madurese'),
- ('mag', u'Magahi'),
- ('mah', 'mh', u'Marshallese'),
- ('mai', u'Maithili'),
- ('mak', u'Makasar'),
- ('mal', 'ml', u'Malayalam'),
- ('man', u'Mandingo'),
- ('mao', 'mi', u'Maori'),
- ('map', u'Austronesian '),
- ('mar', 'mr', u'Marathi'),
- ('mas', u'Masai'),
- ('may', 'ms', u'Malay'),
- ('mdf', u'Moksha'),
- ('mdr', u'Mandar'),
- ('men', u'Mende'),
- ('mga', u'Irish, Middle '),
- ('mic', u"Mi'kmaq"),
- ('min', u'Minangkabau'),
- ('mis', u'Uncoded languages'),
- ('mkh', u'Mon-Khmer '),
- ('mlg', 'mg', u'Malagasy'),
- ('mlt', 'mt', u'Maltese'),
- ('mnc', u'Manchu'),
- ('mni', u'Manipuri'),
- ('mno', u'Manobo languages'),
- ('moh', u'Mohawk'),
- ('mol', 'mo', u'Moldavian'),
- ('mon', 'mn', u'Mongolian'),
- ('mos', u'Mossi'),
- ('mul', u'Multiple languages'),
- ('mun', u'Munda languages'),
- ('mus', u'Creek'),
- ('mwl', u'Mirandese'),
- ('mwr', u'Marwari'),
- ('myn', u'Mayan languages'),
- ('myv', u'Erzya'),
- ('nah', u'Nahuatl languages'),
- ('nai', u'North American Indian'),
- ('nap', u'Neapolitan'),
- ('nau', 'na', u'Nauru'),
- ('nav', 'nv', u'Navajo'),
- ('nbl', 'nr', u'Ndebele, South'),
- ('nde', 'nd', u'Ndebele, North'),
- ('ndo', 'ng', u'Ndonga'),
- ('nds', u'Low German'),
- ('nep', 'ne', u'Nepali'),
- ('new', u'Nepal Bhasa'),
- ('nia', u'Nias'),
- ('nic', u'Niger-Kordofanian '),
- ('niu', u'Niuean'),
- ('nno', 'nn', u'Norwegian Nynorsk'),
- ('nob', 'nb', u'Bokm\xe5l, Norwegian'),
- ('nog', u'Nogai'),
- ('non', u'Norse, Old'),
- ('nor', 'no', u'Norwegian'),
- ('nqo', u"N'Ko"),
- ('nso', u'Pedi'),
- ('nub', u'Nubian languages'),
- ('nwc', u'Classical Newari'),
- ('nya', 'ny', u'Chichewa'),
- ('nym', u'Nyamwezi'),
- ('nyn', u'Nyankole'),
- ('nyo', u'Nyoro'),
- ('nzi', u'Nzima'),
- ('oci', 'oc', u'Occitan '),
- ('oji', 'oj', u'Ojibwa'),
- ('ori', 'or', u'Oriya'),
- ('orm', 'om', u'Oromo'),
- ('osa', u'Osage'),
- ('oss', 'os', u'Ossetian'),
- ('ota', u'Turkish, Ottoman '),
- ('oto', u'Otomian languages'),
- ('paa', u'Papuan '),
- ('pag', u'Pangasinan'),
- ('pal', u'Pahlavi'),
- ('pam', u'Pampanga'),
- ('pan', 'pa', u'Panjabi'),
- ('pap', u'Papiamento'),
- ('pau', u'Palauan'),
- ('peo', u'Persian, Old '),
- ('per', 'fa', u'Persian'),
- ('phi', u'Philippine '),
- ('phn', u'Phoenician'),
- ('pli', 'pi', u'Pali'),
- ('pol', 'pl', u'Polish'),
- ('pon', u'Pohnpeian'),
- ('por', 'pt', u'Portuguese'),
- ('pra', u'Prakrit languages'),
- ('pro', u'Proven\xe7al, Old '),
- ('pus', 'ps', u'Pushto'),
- ('qaa-qtz', u'Reserved for local use'),
- ('que', 'qu', u'Quechua'),
- ('raj', u'Rajasthani'),
- ('rap', u'Rapanui'),
- ('rar', u'Rarotongan'),
- ('roa', u'Romance '),
- ('roh', 'rm', u'Romansh'),
- ('rom', u'Romany'),
- ('rum', 'ro', u'Romanian'),
- ('run', 'rn', u'Rundi'),
- ('rup', u'Aromanian'),
- ('rus', 'ru', u'Russian'),
- ('sad', u'Sandawe'),
- ('sag', 'sg', u'Sango'),
- ('sah', u'Yakut'),
- ('sai', u'South American Indian '),
- ('sal', u'Salishan languages'),
- ('sam', u'Samaritan Aramaic'),
- ('san', 'sa', u'Sanskrit'),
- ('sas', u'Sasak'),
- ('sat', u'Santali'),
- ('scc', 'sr', u'Serbian'),
- ('scn', u'Sicilian'),
- ('sco', u'Scots'),
- ('scr', 'hr', u'Croatian'),
- ('sel', u'Selkup'),
- ('sem', u'Semitic '),
- ('sga', u'Irish, Old '),
- ('sgn', u'Sign Languages'),
- ('shn', u'Shan'),
- ('sid', u'Sidamo'),
- ('sin', 'si', u'Sinhala'),
- ('sio', u'Siouan languages'),
- ('sit', u'Sino-Tibetan '),
- ('sla', u'Slavic '),
- ('slo', 'sk', u'Slovak'),
- ('slv', 'sl', u'Slovenian'),
- ('sma', u'Southern Sami'),
- ('sme', 'se', u'Northern Sami'),
- ('smi', u'Sami languages '),
- ('smj', u'Lule Sami'),
- ('smn', u'Inari Sami'),
- ('smo', 'sm', u'Samoan'),
- ('sms', u'Skolt Sami'),
- ('sna', 'sn', u'Shona'),
- ('snd', 'sd', u'Sindhi'),
- ('snk', u'Soninke'),
- ('sog', u'Sogdian'),
- ('som', 'so', u'Somali'),
- ('son', u'Songhai languages'),
- ('sot', 'st', u'Sotho, Southern'),
- ('spa', 'es', u'Spanish'),
- ('srd', 'sc', u'Sardinian'),
- ('srn', u'Sranan Tongo'),
- ('srr', u'Serer'),
- ('ssa', u'Nilo-Saharan '),
- ('ssw', 'ss', u'Swati'),
- ('suk', u'Sukuma'),
- ('sun', 'su', u'Sundanese'),
- ('sus', u'Susu'),
- ('sux', u'Sumerian'),
- ('swa', 'sw', u'Swahili'),
- ('swe', 'sv', u'Swedish'),
- ('syc', u'Classical Syriac'),
- ('syr', u'Syriac'),
- ('tah', 'ty', u'Tahitian'),
- ('tai', u'Tai '),
- ('tam', 'ta', u'Tamil'),
- ('tat', 'tt', u'Tatar'),
- ('tel', 'te', u'Telugu'),
- ('tem', u'Timne'),
- ('ter', u'Tereno'),
- ('tet', u'Tetum'),
- ('tgk', 'tg', u'Tajik'),
- ('tgl', 'tl', u'Tagalog'),
- ('tha', 'th', u'Thai'),
- ('tib', 'bo', u'Tibetan'),
- ('tig', u'Tigre'),
- ('tir', 'ti', u'Tigrinya'),
- ('tiv', u'Tiv'),
- ('tkl', u'Tokelau'),
- ('tlh', u'Klingon'),
- ('tli', u'Tlingit'),
- ('tmh', u'Tamashek'),
- ('tog', u'Tonga '),
- ('ton', 'to', u'Tonga '),
- ('tpi', u'Tok Pisin'),
- ('tsi', u'Tsimshian'),
- ('tsn', 'tn', u'Tswana'),
- ('tso', 'ts', u'Tsonga'),
- ('tuk', 'tk', u'Turkmen'),
- ('tum', u'Tumbuka'),
- ('tup', u'Tupi languages'),
- ('tur', 'tr', u'Turkish'),
- ('tut', u'Altaic '),
- ('tvl', u'Tuvalu'),
- ('twi', 'tw', u'Twi'),
- ('tyv', u'Tuvinian'),
- ('udm', u'Udmurt'),
- ('uga', u'Ugaritic'),
- ('uig', 'ug', u'Uighur'),
- ('ukr', 'uk', u'Ukrainian'),
- ('umb', u'Umbundu'),
- ('und', u'Undetermined'),
- ('urd', 'ur', u'Urdu'),
- ('uzb', 'uz', u'Uzbek'),
- ('vai', u'Vai'),
- ('ven', 've', u'Venda'),
- ('vie', 'vi', u'Vietnamese'),
- ('vol', 'vo', u'Volap\xfck'),
- ('vot', u'Votic'),
- ('wak', u'Wakashan languages'),
- ('wal', u'Walamo'),
- ('war', u'Waray'),
- ('was', u'Washo'),
- ('wel', 'cy', u'Welsh'),
- ('wen', u'Sorbian languages'),
- ('wln', 'wa', u'Walloon'),
- ('wol', 'wo', u'Wolof'),
- ('xal', u'Kalmyk'),
- ('xho', 'xh', u'Xhosa'),
- ('yao', u'Yao'),
- ('yap', u'Yapese'),
- ('yid', 'yi', u'Yiddish'),
- ('yor', 'yo', u'Yoruba'),
- ('ypk', u'Yupik languages'),
- ('zap', u'Zapotec'),
- ('zbl', u'Blissymbols'),
- ('zen', u'Zenaga'),
- ('zha', 'za', u'Zhuang'),
- ('znd', u'Zande languages'),
- ('zul', 'zu', u'Zulu'),
- ('zun', u'Zuni'),
- ('zxx', u'No linguistic content'),
- ('zza', u'Zaza'),
-)
diff --git a/lib/enzyme/mkv.py b/lib/enzyme/mkv.py
index aba5325e29501e2637ef7168236ff8d9b0193eab..9dbce9bce9e1f6ff633b82400f866d396e62024c 100644
--- a/lib/enzyme/mkv.py
+++ b/lib/enzyme/mkv.py
@@ -1,840 +1,351 @@
# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
-# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
-# Copyright 2003-2006 Jason Tackaberry <tack@urandom.ca>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-from datetime import datetime
-from exceptions import ParseError
-from struct import unpack
-import core
+from .exceptions import ParserError, MalformedMKVError
+from .parsers import ebml
+from datetime import timedelta
import logging
-import re
-
-__all__ = ['Parser']
-
-
-# get logging object
-log = logging.getLogger(__name__)
-
-# Main IDs for the Matroska streams
-MATROSKA_VIDEO_TRACK = 0x01
-MATROSKA_AUDIO_TRACK = 0x02
-MATROSKA_SUBTITLES_TRACK = 0x11
-
-MATROSKA_HEADER_ID = 0x1A45DFA3
-MATROSKA_TRACKS_ID = 0x1654AE6B
-MATROSKA_CUES_ID = 0x1C53BB6B
-MATROSKA_SEGMENT_ID = 0x18538067
-MATROSKA_SEGMENT_INFO_ID = 0x1549A966
-MATROSKA_CLUSTER_ID = 0x1F43B675
-MATROSKA_VOID_ID = 0xEC
-MATROSKA_CRC_ID = 0xBF
-MATROSKA_TIMECODESCALE_ID = 0x2AD7B1
-MATROSKA_DURATION_ID = 0x4489
-MATROSKA_CRC32_ID = 0xBF
-MATROSKA_TIMECODESCALE_ID = 0x2AD7B1
-MATROSKA_MUXING_APP_ID = 0x4D80
-MATROSKA_WRITING_APP_ID = 0x5741
-MATROSKA_CODEC_ID = 0x86
-MATROSKA_CODEC_PRIVATE_ID = 0x63A2
-MATROSKA_FRAME_DURATION_ID = 0x23E383
-MATROSKA_VIDEO_SETTINGS_ID = 0xE0
-MATROSKA_VIDEO_WIDTH_ID = 0xB0
-MATROSKA_VIDEO_HEIGHT_ID = 0xBA
-MATROSKA_VIDEO_INTERLACED_ID = 0x9A
-MATROSKA_VIDEO_DISPLAY_WIDTH_ID = 0x54B0
-MATROSKA_VIDEO_DISPLAY_HEIGHT_ID = 0x54BA
-MATROSKA_AUDIO_SETTINGS_ID = 0xE1
-MATROSKA_AUDIO_SAMPLERATE_ID = 0xB5
-MATROSKA_AUDIO_CHANNELS_ID = 0x9F
-MATROSKA_TRACK_UID_ID = 0x73C5
-MATROSKA_TRACK_NUMBER_ID = 0xD7
-MATROSKA_TRACK_TYPE_ID = 0x83
-MATROSKA_TRACK_LANGUAGE_ID = 0x22B59C
-MATROSKA_TRACK_OFFSET = 0x537F
-MATROSKA_TRACK_FLAG_DEFAULT_ID = 0x88
-MATROSKA_TRACK_FLAG_ENABLED_ID = 0xB9
-MATROSKA_TITLE_ID = 0x7BA9
-MATROSKA_DATE_UTC_ID = 0x4461
-MATROSKA_NAME_ID = 0x536E
-
-MATROSKA_CHAPTERS_ID = 0x1043A770
-MATROSKA_CHAPTER_UID_ID = 0x73C4
-MATROSKA_EDITION_ENTRY_ID = 0x45B9
-MATROSKA_CHAPTER_ATOM_ID = 0xB6
-MATROSKA_CHAPTER_TIME_START_ID = 0x91
-MATROSKA_CHAPTER_TIME_END_ID = 0x92
-MATROSKA_CHAPTER_FLAG_ENABLED_ID = 0x4598
-MATROSKA_CHAPTER_DISPLAY_ID = 0x80
-MATROSKA_CHAPTER_LANGUAGE_ID = 0x437C
-MATROSKA_CHAPTER_STRING_ID = 0x85
-
-MATROSKA_ATTACHMENTS_ID = 0x1941A469
-MATROSKA_ATTACHED_FILE_ID = 0x61A7
-MATROSKA_FILE_DESC_ID = 0x467E
-MATROSKA_FILE_NAME_ID = 0x466E
-MATROSKA_FILE_MIME_TYPE_ID = 0x4660
-MATROSKA_FILE_DATA_ID = 0x465C
-
-MATROSKA_SEEKHEAD_ID = 0x114D9B74
-MATROSKA_SEEK_ID = 0x4DBB
-MATROSKA_SEEKID_ID = 0x53AB
-MATROSKA_SEEK_POSITION_ID = 0x53AC
-
-MATROSKA_TAGS_ID = 0x1254C367
-MATROSKA_TAG_ID = 0x7373
-MATROSKA_TARGETS_ID = 0x63C0
-MATROSKA_TARGET_TYPE_VALUE_ID = 0x68CA
-MATROSKA_TARGET_TYPE_ID = 0x63CA
-MATRSOKA_TAGS_TRACK_UID_ID = 0x63C5
-MATRSOKA_TAGS_EDITION_UID_ID = 0x63C9
-MATRSOKA_TAGS_CHAPTER_UID_ID = 0x63C4
-MATRSOKA_TAGS_ATTACHMENT_UID_ID = 0x63C6
-MATROSKA_SIMPLE_TAG_ID = 0x67C8
-MATROSKA_TAG_NAME_ID = 0x45A3
-MATROSKA_TAG_LANGUAGE_ID = 0x447A
-MATROSKA_TAG_STRING_ID = 0x4487
-MATROSKA_TAG_BINARY_ID = 0x4485
-
-
-# See mkv spec for details:
-# http://www.matroska.org/technical/specs/index.html
-
-# Map to convert to well known codes
-# http://haali.cs.msu.ru/mkv/codecs.pdf
-FOURCCMap = {
- 'V_THEORA': 'THEO',
- 'V_SNOW': 'SNOW',
- 'V_MPEG4/ISO/ASP': 'MP4V',
- 'V_MPEG4/ISO/AVC': 'AVC1',
- 'A_AC3': 0x2000,
- 'A_MPEG/L3': 0x0055,
- 'A_MPEG/L2': 0x0050,
- 'A_MPEG/L1': 0x0050,
- 'A_DTS': 0x2001,
- 'A_PCM/INT/LIT': 0x0001,
- 'A_PCM/FLOAT/IEEE': 0x003,
- 'A_TTA1': 0x77a1,
- 'A_WAVPACK4': 0x5756,
- 'A_VORBIS': 0x6750,
- 'A_FLAC': 0xF1AC,
- 'A_AAC': 0x00ff,
- 'A_AAC/': 0x00ff
-}
-
-
-def matroska_date_to_datetime(date):
- """
- Converts a date in Matroska's date format to a python datetime object.
- Returns the given date string if it could not be converted.
- """
- # From the specs:
- # The fields with dates should have the following format: YYYY-MM-DD
- # HH:MM:SS.MSS [...] To store less accuracy, you remove items starting
- # from the right. To store only the year, you would use, "2004". To store
- # a specific day such as May 1st, 2003, you would use "2003-05-01".
- format = re.split(r'([-:. ])', '%Y-%m-%d %H:%M:%S.%f')
- while format:
- try:
- return datetime.strptime(date, ''.join(format))
- except ValueError:
- format = format[:-2]
- return date
-def matroska_bps_to_bitrate(bps):
- """
- Tries to convert a free-form bps string into a bitrate (bits per second).
- """
- m = re.search('([\d.]+)\s*(\D.*)', bps)
- if m:
- bps, suffix = m.groups()
- if 'kbit' in suffix:
- return float(bps) * 1024
- elif 'kbyte' in suffix:
- return float(bps) * 1024 * 8
- elif 'byte' in suffix:
- return float(bps) * 8
- elif 'bps' in suffix or 'bit' in suffix:
- return float(bps)
- if bps.replace('.', '').isdigit():
- if float(bps) < 30000:
- # Assume kilobits and convert to bps
- return float(bps) * 1024
- return float(bps)
-
-
-# Used to convert the official matroska tag names (only lower-cased) to core
-# attributes. tag name -> attr, filter
-TAGS_MAP = {
- # From Media core
- u'title': ('title', None),
- u'subtitle': ('caption', None),
- u'comment': ('comment', None),
- u'url': ('url', None),
- u'artist': ('artist', None),
- u'keywords': ('keywords', lambda s: [word.strip() for word in s.split(',')]),
- u'composer_nationality': ('country', None),
- u'date_released': ('datetime', None),
- u'date_recorded': ('datetime', None),
- u'date_written': ('datetime', None),
-
- # From Video core
- u'encoder': ('encoder', None),
- u'bps': ('bitrate', matroska_bps_to_bitrate),
- u'part_number': ('trackno', int),
- u'total_parts': ('trackof', int),
- u'copyright': ('copyright', None),
- u'genre': ('genre', None),
- u'actor': ('actors', None),
- u'written_by': ('writer', None),
- u'producer': ('producer', None),
- u'production_studio': ('studio', None),
- u'law_rating': ('rating', None),
- u'summary': ('summary', None),
- u'synopsis': ('synopsis', None),
-}
-
-
-class EbmlEntity:
- """
- This is class that is responsible to handle one Ebml entity as described in
- the Matroska/Ebml spec
- """
- def __init__(self, inbuf):
- # Compute the EBML id
- # Set the CRC len to zero
- self.crc_len = 0
- # Now loop until we find an entity without CRC
- try:
- self.build_entity(inbuf)
- except IndexError:
- raise ParseError()
- while self.get_id() == MATROSKA_CRC32_ID:
- self.crc_len += self.get_total_len()
- inbuf = inbuf[self.get_total_len():]
- self.build_entity(inbuf)
-
- def build_entity(self, inbuf):
- self.compute_id(inbuf)
-
- if self.id_len == 0:
- log.error(u'EBML entity not found, bad file format')
- raise ParseError()
-
- self.entity_len, self.len_size = self.compute_len(inbuf[self.id_len:])
- self.entity_data = inbuf[self.get_header_len() : self.get_total_len()]
- self.ebml_length = self.entity_len
- self.entity_len = min(len(self.entity_data), self.entity_len)
-
- # if the data size is 8 or less, it could be a numeric value
- self.value = 0
- if self.entity_len <= 8:
- for pos, shift in zip(range(self.entity_len), range((self.entity_len - 1) * 8, -1, -8)):
- self.value |= ord(self.entity_data[pos]) << shift
-
+__all__ = ['VIDEO_TRACK', 'AUDIO_TRACK', 'SUBTITLE_TRACK', 'MKV', 'Info', 'Track', 'VideoTrack',
+ 'AudioTrack', 'SubtitleTrack', 'Tag', 'SimpleTag', 'Chapter']
+logger = logging.getLogger(__name__)
- def add_data(self, data):
- maxlen = self.ebml_length - len(self.entity_data)
- if maxlen <= 0:
- return
- self.entity_data += data[:maxlen]
- self.entity_len = len(self.entity_data)
+# Track types
+VIDEO_TRACK, AUDIO_TRACK, SUBTITLE_TRACK = 0x01, 0x02, 0x11
- def compute_id(self, inbuf):
- self.id_len = 0
- if len(inbuf) < 1:
- return 0
- first = ord(inbuf[0])
- if first & 0x80:
- self.id_len = 1
- self.entity_id = first
- elif first & 0x40:
- if len(inbuf) < 2:
- return 0
- self.id_len = 2
- self.entity_id = ord(inbuf[0]) << 8 | ord(inbuf[1])
- elif first & 0x20:
- if len(inbuf) < 3:
- return 0
- self.id_len = 3
- self.entity_id = (ord(inbuf[0]) << 16) | (ord(inbuf[1]) << 8) | \
- (ord(inbuf[2]))
- elif first & 0x10:
- if len(inbuf) < 4:
- return 0
- self.id_len = 4
- self.entity_id = (ord(inbuf[0]) << 24) | (ord(inbuf[1]) << 16) | \
- (ord(inbuf[2]) << 8) | (ord(inbuf[3]))
- self.entity_str = inbuf[0:self.id_len]
+class MKV(object):
+ """Matroska Video file
- def compute_len(self, inbuf):
- if not inbuf:
- return 0, 0
- i = num_ffs = 0
- len_mask = 0x80
- len = ord(inbuf[0])
- while not len & len_mask:
- i += 1
- len_mask >>= 1
- if i >= 8:
- return 0, 0
+ :param stream: seekable file-like object
- len &= len_mask - 1
- if len == len_mask - 1:
- num_ffs += 1
- for p in range(i):
- len = (len << 8) | ord(inbuf[p + 1])
- if len & 0xff == 0xff:
- num_ffs += 1
- if num_ffs == i + 1:
- len = 0
- return len, i + 1
+ """
+ def __init__(self, stream, recurse_seek_head=False):
+ # default attributes
+ self.info = None
+ self.video_tracks = []
+ self.audio_tracks = []
+ self.subtitle_tracks = []
+ self.chapters = []
+ self.tags = []
+
+ # keep track of the elements parsed
+ self.recurse_seek_head = recurse_seek_head
+ self._parsed_positions = set()
+ try:
+ # get the Segment element
+ logger.info('Reading Segment element')
+ specs = ebml.get_matroska_specs()
+ segments = ebml.parse(stream, specs, ignore_element_names=['EBML'], max_level=0)
+ if not segments:
+ raise MalformedMKVError('No Segment found')
+ if len(segments) > 1:
+ logger.warning('%d segments found, using the first one', len(segments))
+ segment = segments[0]
+
+ # get and recursively parse the SeekHead element
+ logger.info('Reading SeekHead element')
+ stream.seek(segment.position)
+ seek_head = ebml.parse_element(stream, specs)
+ if seek_head.name != 'SeekHead':
+ raise MalformedMKVError('No SeekHead found')
+ seek_head.load(stream, specs, ignore_element_names=['Void', 'CRC-32'])
+ self._parse_seekhead(seek_head, segment, stream, specs)
+ except ParserError as e:
+ raise MalformedMKVError('Parsing error: %s' % e)
+
+ def _parse_seekhead(self, seek_head, segment, stream, specs):
+ for seek in seek_head:
+ element_id = ebml.read_element_id(seek['SeekID'].data)
+ element_name = specs[element_id][1]
+ element_position = seek['SeekPosition'].data + segment.position
+ if element_position in self._parsed_positions:
+ logger.warning('Skipping already parsed %s element at position %d', element_name, element_position)
+ continue
+ if element_name == 'Info':
+ logger.info('Processing element %s from SeekHead at position %d', element_name, element_position)
+ stream.seek(element_position)
+ self.info = Info.fromelement(ebml.parse_element(stream, specs, True, ignore_element_names=['Void', 'CRC-32']))
+ elif element_name == 'Tracks':
+ logger.info('Processing element %s from SeekHead at position %d', element_name, element_position)
+ stream.seek(element_position)
+ tracks = ebml.parse_element(stream, specs, True, ignore_element_names=['Void', 'CRC-32'])
+ self.video_tracks.extend([VideoTrack.fromelement(t) for t in tracks if t['TrackType'].data == VIDEO_TRACK])
+ self.audio_tracks.extend([AudioTrack.fromelement(t) for t in tracks if t['TrackType'].data == AUDIO_TRACK])
+ self.subtitle_tracks.extend([SubtitleTrack.fromelement(t) for t in tracks if t['TrackType'].data == SUBTITLE_TRACK])
+ elif element_name == 'Chapters':
+ logger.info('Processing element %s from SeekHead at position %d', element_name, element_position)
+ stream.seek(element_position)
+ self.chapters.extend([Chapter.fromelement(c) for c in ebml.parse_element(stream, specs, True, ignore_element_names=['Void', 'CRC-32'])[0] if c.name == 'ChapterAtom'])
+ elif element_name == 'Tags':
+ logger.info('Processing element %s from SeekHead at position %d', element_name, element_position)
+ stream.seek(element_position)
+ self.tags.extend([Tag.fromelement(t) for t in ebml.parse_element(stream, specs, True, ignore_element_names=['Void', 'CRC-32'])])
+ elif element_name == 'SeekHead' and self.recurse_seek_head:
+ logger.info('Processing element %s from SeekHead at position %d', element_name, element_position)
+ stream.seek(element_position)
+ self._parse_seekhead(ebml.parse_element(stream, specs, True, ignore_element_names=['Void', 'CRC-32']), segment, stream, specs)
+ else:
+ logger.debug('Element %s ignored', element_name)
+ self._parsed_positions.add(element_position)
- def get_crc_len(self):
- return self.crc_len
+ def to_dict(self):
+ return {'info': self.info.__dict__, 'video_tracks': [t.__dict__ for t in self.video_tracks],
+ 'audio_tracks': [t.__dict__ for t in self.audio_tracks], 'subtitle_tracks': [t.__dict__ for t in self.subtitle_tracks],
+ 'chapters': [c.__dict__ for c in self.chapters], 'tags': [t.__dict__ for t in self.tags]}
+ def __repr__(self):
+ return '<%s [%r, %r, %r, %r]>' % (self.__class__.__name__, self.info, self.video_tracks, self.audio_tracks, self.subtitle_tracks)
- def get_value(self):
- return self.value
+class Info(object):
+ """Object for the Info EBML element"""
+ def __init__(self, title=None, duration=None, date_utc=None, timecode_scale=None, muxing_app=None, writing_app=None):
+ self.title = title
+ self.duration = timedelta(microseconds=duration * (timecode_scale or 1000000) // 1000) if duration else None
+ self.date_utc = date_utc
+ self.muxing_app = muxing_app
+ self.writing_app = writing_app
- def get_float_value(self):
- if len(self.entity_data) == 4:
- return unpack('!f', self.entity_data)[0]
- elif len(self.entity_data) == 8:
- return unpack('!d', self.entity_data)[0]
- return 0.0
+ @classmethod
+ def fromelement(cls, element):
+ """Load the :class:`Info` from an :class:`~enzyme.parsers.ebml.Element`
+ :param element: the Info element
+ :type element: :class:`~enzyme.parsers.ebml.Element`
- def get_data(self):
- return self.entity_data
+ """
+ title = element.get('Title')
+ duration = element.get('Duration')
+ date_utc = element.get('DateUTC')
+ timecode_scale = element.get('TimecodeScale')
+ muxing_app = element.get('MuxingApp')
+ writing_app = element.get('WritingApp')
+ return cls(title, duration, date_utc, timecode_scale, muxing_app, writing_app)
+
+ def __repr__(self):
+ return '<%s [title=%r, duration=%s, date=%s]>' % (self.__class__.__name__, self.title, self.duration, self.date_utc)
+
+ def __str__(self):
+ return repr(self.__dict__)
+
+
+class Track(object):
+ """Base object for the Tracks EBML element"""
+ def __init__(self, type=None, number=None, name=None, language=None, enabled=None, default=None, forced=None, lacing=None, # @ReservedAssignment
+ codec_id=None, codec_name=None):
+ self.type = type
+ self.number = number
+ self.name = name
+ self.language = language
+ self.enabled = enabled
+ self.default = default
+ self.forced = forced
+ self.lacing = lacing
+ self.codec_id = codec_id
+ self.codec_name = codec_name
+
+ @classmethod
+ def fromelement(cls, element):
+ """Load the :class:`Track` from an :class:`~enzyme.parsers.ebml.Element`
+
+ :param element: the Track element
+ :type element: :class:`~enzyme.parsers.ebml.Element`
+ """
+ type = element.get('TrackType') # @ReservedAssignment
+ number = element.get('TrackNumber', 0)
+ name = element.get('Name')
+ language = element.get('Language')
+ enabled = bool(element.get('FlagEnabled', 1))
+ default = bool(element.get('FlagDefault', 1))
+ forced = bool(element.get('FlagForced', 0))
+ lacing = bool(element.get('FlagLacing', 1))
+ codec_id = element.get('CodecID')
+ codec_name = element.get('CodecName')
+ return cls(type=type, number=number, name=name, language=language, enabled=enabled, default=default,
+ forced=forced, lacing=lacing, codec_id=codec_id, codec_name=codec_name)
+
+ def __repr__(self):
+ return '<%s [%d, name=%r, language=%s]>' % (self.__class__.__name__, self.number, self.name, self.language)
+
+ def __str__(self):
+ return str(self.__dict__)
+
+
+class VideoTrack(Track):
+ """Object for the Tracks EBML element with :data:`VIDEO_TRACK` TrackType"""
+ def __init__(self, width=0, height=0, interlaced=False, stereo_mode=None, crop=None,
+ display_width=None, display_height=None, display_unit=None, aspect_ratio_type=None, **kwargs):
+ super(VideoTrack, self).__init__(**kwargs)
+ self.width = width
+ self.height = height
+ self.interlaced = interlaced
+ self.stereo_mode = stereo_mode
+ self.crop = crop
+ self.display_width = display_width
+ self.display_height = display_height
+ self.display_unit = display_unit
+ self.aspect_ratio_type = aspect_ratio_type
+
+ @classmethod
+ def fromelement(cls, element):
+ """Load the :class:`VideoTrack` from an :class:`~enzyme.parsers.ebml.Element`
+
+ :param element: the Track element with :data:`VIDEO_TRACK` TrackType
+ :type element: :class:`~enzyme.parsers.ebml.Element`
- def get_utf8(self):
- return unicode(self.entity_data, 'utf-8', 'replace')
+ """
+ videotrack = super(VideoTrack, cls).fromelement(element)
+ videotrack.width = element['Video'].get('PixelWidth', 0)
+ videotrack.height = element['Video'].get('PixelHeight', 0)
+ videotrack.interlaced = bool(element['Video'].get('FlagInterlaced', False))
+ videotrack.stereo_mode = element['Video'].get('StereoMode')
+ videotrack.crop = {}
+ if 'PixelCropTop' in element['Video']:
+ videotrack.crop['top'] = element['Video']['PixelCropTop']
+ if 'PixelCropBottom' in element['Video']:
+ videotrack.crop['bottom'] = element['Video']['PixelCropBottom']
+ if 'PixelCropLeft' in element['Video']:
+ videotrack.crop['left'] = element['Video']['PixelCropLeft']
+ if 'PixelCropRight' in element['Video']:
+ videotrack.crop['right'] = element['Video']['PixelCropRight']
+ videotrack.display_width = element['Video'].get('DisplayWidth')
+ videotrack.display_height = element['Video'].get('DisplayHeight')
+ videotrack.display_unit = element['Video'].get('DisplayUnit')
+ videotrack.aspect_ratio_type = element['Video'].get('AspectRatioType')
+ return videotrack
+
+ def __repr__(self):
+ return '<%s [%d, %dx%d, %s, name=%r, language=%s]>' % (self.__class__.__name__, self.number, self.width, self.height,
+ self.codec_id, self.name, self.language)
+
+ def __str__(self):
+ return str(self.__dict__)
+
+
+class AudioTrack(Track):
+ """Object for the Tracks EBML element with :data:`AUDIO_TRACK` TrackType"""
+ def __init__(self, sampling_frequency=None, channels=None, output_sampling_frequency=None, bit_depth=None, **kwargs):
+ super(AudioTrack, self).__init__(**kwargs)
+ self.sampling_frequency = sampling_frequency
+ self.channels = channels
+ self.output_sampling_frequency = output_sampling_frequency
+ self.bit_depth = bit_depth
+
+ @classmethod
+ def fromelement(cls, element):
+ """Load the :class:`AudioTrack` from an :class:`~enzyme.parsers.ebml.Element`
+
+ :param element: the Track element with :data:`AUDIO_TRACK` TrackType
+ :type element: :class:`~enzyme.parsers.ebml.Element`
+ """
+ audiotrack = super(AudioTrack, cls).fromelement(element)
+ audiotrack.sampling_frequency = element['Audio'].get('SamplingFrequency', 8000.0)
+ audiotrack.channels = element['Audio'].get('Channels', 1)
+ audiotrack.output_sampling_frequency = element['Audio'].get('OutputSamplingFrequency')
+ audiotrack.bit_depth = element['Audio'].get('BitDepth')
+ return audiotrack
- def get_str(self):
- return unicode(self.entity_data, 'ascii', 'replace')
+ def __repr__(self):
+ return '<%s [%d, %d channel(s), %.0fHz, %s, name=%r, language=%s]>' % (self.__class__.__name__, self.number, self.channels,
+ self.sampling_frequency, self.codec_id, self.name, self.language)
- def get_id(self):
- return self.entity_id
+class SubtitleTrack(Track):
+ """Object for the Tracks EBML element with :data:`SUBTITLE_TRACK` TrackType"""
+ pass
- def get_str_id(self):
- return self.entity_str
+class Tag(object):
+ """Object for the Tag EBML element"""
+ def __init__(self, targets=None, simpletags=None):
+ self.targets = targets if targets is not None else []
+ self.simpletags = simpletags if simpletags is not None else []
+ @classmethod
+ def fromelement(cls, element):
+ """Load the :class:`Tag` from an :class:`~enzyme.parsers.ebml.Element`
- def get_len(self):
- return self.entity_len
+ :param element: the Tag element
+ :type element: :class:`~enzyme.parsers.ebml.Element`
+ """
+ targets = element['Targets'] if 'Targets' in element else []
+ simpletags = [SimpleTag.fromelement(s) for s in element if s.name == 'SimpleTag']
+ return cls(targets, simpletags)
- def get_total_len(self):
- return self.entity_len + self.id_len + self.len_size
+ def __repr__(self):
+ return '<%s [targets=%r, simpletags=%r]>' % (self.__class__.__name__, self.targets, self.simpletags)
- def get_header_len(self):
- return self.id_len + self.len_size
+class SimpleTag(object):
+ """Object for the SimpleTag EBML element"""
+ def __init__(self, name, language='und', default=True, string=None, binary=None):
+ self.name = name
+ self.language = language
+ self.default = default
+ self.string = string
+ self.binary = binary
+ @classmethod
+ def fromelement(cls, element):
+ """Load the :class:`SimpleTag` from an :class:`~enzyme.parsers.ebml.Element`
+ :param element: the SimpleTag element
+ :type element: :class:`~enzyme.parsers.ebml.Element`
-class Matroska(core.AVContainer):
- """
- Matroska video and audio parser. If at least one video stream is
- detected it will set the type to MEDIA_AV.
- """
- def __init__(self, file):
- core.AVContainer.__init__(self)
- self.samplerate = 1
-
- self.file = file
- # Read enough that we're likely to get the full seekhead (FIXME: kludge)
- buffer = file.read(2000)
- if len(buffer) == 0:
- # Regular File end
- raise ParseError()
-
- # Check the Matroska header
- header = EbmlEntity(buffer)
- if header.get_id() != MATROSKA_HEADER_ID:
- raise ParseError()
-
- log.debug(u'HEADER ID found %08X' % header.get_id())
- self.mime = 'video/x-matroska'
- self.type = 'Matroska'
- self.has_idx = False
- self.objects_by_uid = {}
-
- # Now get the segment
- self.segment = segment = EbmlEntity(buffer[header.get_total_len():])
- # Record file offset of segment data for seekheads
- self.segment.offset = header.get_total_len() + segment.get_header_len()
- if segment.get_id() != MATROSKA_SEGMENT_ID:
- log.debug(u'SEGMENT ID not found %08X' % segment.get_id())
- return
-
- log.debug(u'SEGMENT ID found %08X' % segment.get_id())
- try:
- for elem in self.process_one_level(segment):
- if elem.get_id() == MATROSKA_SEEKHEAD_ID:
- self.process_elem(elem)
- except ParseError:
- pass
-
- if not self.has_idx:
- log.warning(u'File has no index')
- self._set('corrupt', True)
-
- def process_elem(self, elem):
- elem_id = elem.get_id()
- log.debug(u'BEGIN: process element %r' % hex(elem_id))
- if elem_id == MATROSKA_SEGMENT_INFO_ID:
- duration = 0
- scalecode = 1000000.0
-
- for ielem in self.process_one_level(elem):
- ielem_id = ielem.get_id()
- if ielem_id == MATROSKA_TIMECODESCALE_ID:
- scalecode = ielem.get_value()
- elif ielem_id == MATROSKA_DURATION_ID:
- duration = ielem.get_float_value()
- elif ielem_id == MATROSKA_TITLE_ID:
- self.title = ielem.get_utf8()
- elif ielem_id == MATROSKA_DATE_UTC_ID:
- timestamp = unpack('!q', ielem.get_data())[0] / 10.0 ** 9
- # Date is offset 2001-01-01 00:00:00 (timestamp 978307200.0)
- self.timestamp = int(timestamp + 978307200)
-
- self.length = duration * scalecode / 1000000000.0
-
- elif elem_id == MATROSKA_TRACKS_ID:
- self.process_tracks(elem)
-
- elif elem_id == MATROSKA_CHAPTERS_ID:
- self.process_chapters(elem)
-
- elif elem_id == MATROSKA_ATTACHMENTS_ID:
- self.process_attachments(elem)
-
- elif elem_id == MATROSKA_SEEKHEAD_ID:
- self.process_seekhead(elem)
-
- elif elem_id == MATROSKA_TAGS_ID:
- self.process_tags(elem)
-
- elif elem_id == MATROSKA_CUES_ID:
- self.has_idx = True
-
- log.debug(u'END: process element %r' % hex(elem_id))
- return True
-
-
- def process_seekhead(self, elem):
- for seek_elem in self.process_one_level(elem):
- if seek_elem.get_id() != MATROSKA_SEEK_ID:
- continue
- for sub_elem in self.process_one_level(seek_elem):
- if sub_elem.get_id() == MATROSKA_SEEKID_ID:
- if sub_elem.get_value() == MATROSKA_CLUSTER_ID:
- # Not interested in these.
- return
-
- elif sub_elem.get_id() == MATROSKA_SEEK_POSITION_ID:
- self.file.seek(self.segment.offset + sub_elem.get_value())
- buffer = self.file.read(100)
- try:
- elem = EbmlEntity(buffer)
- except ParseError:
- continue
-
- # Fetch all data necessary for this element.
- elem.add_data(self.file.read(elem.ebml_length))
- self.process_elem(elem)
-
-
- def process_tracks(self, tracks):
- tracksbuf = tracks.get_data()
- index = 0
- while index < tracks.get_len():
- trackelem = EbmlEntity(tracksbuf[index:])
- log.debug (u'ELEMENT %X found' % trackelem.get_id())
- self.process_track(trackelem)
- index += trackelem.get_total_len() + trackelem.get_crc_len()
-
-
- def process_one_level(self, item):
- buf = item.get_data()
- index = 0
- while index < item.get_len():
- if len(buf[index:]) == 0:
- break
- elem = EbmlEntity(buf[index:])
- yield elem
- index += elem.get_total_len() + elem.get_crc_len()
-
- def set_track_defaults(self, track):
- track.language = 'eng'
-
- def process_track(self, track):
- # Collapse generator into a list since we need to iterate over it
- # twice.
- elements = [x for x in self.process_one_level(track)]
- track_type = [x.get_value() for x in elements if x.get_id() == MATROSKA_TRACK_TYPE_ID]
- if not track_type:
- log.debug(u'Bad track: no type id found')
- return
-
- track_type = track_type[0]
- track = None
-
- if track_type == MATROSKA_VIDEO_TRACK:
- log.debug(u'Video track found')
- track = self.process_video_track(elements)
- elif track_type == MATROSKA_AUDIO_TRACK:
- log.debug(u'Audio track found')
- track = self.process_audio_track(elements)
- elif track_type == MATROSKA_SUBTITLES_TRACK:
- log.debug(u'Subtitle track found')
- track = core.Subtitle()
- self.set_track_defaults(track)
- track.id = len(self.subtitles)
- self.subtitles.append(track)
- for elem in elements:
- self.process_track_common(elem, track)
-
-
- def process_track_common(self, elem, track):
- elem_id = elem.get_id()
- if elem_id == MATROSKA_TRACK_LANGUAGE_ID:
- track.language = elem.get_str()
- log.debug(u'Track language found: %r' % track.language)
- elif elem_id == MATROSKA_NAME_ID:
- track.title = elem.get_utf8()
- elif elem_id == MATROSKA_TRACK_NUMBER_ID:
- track.trackno = elem.get_value()
- elif elem_id == MATROSKA_TRACK_FLAG_ENABLED_ID:
- track.enabled = bool(elem.get_value())
- elif elem_id == MATROSKA_TRACK_FLAG_DEFAULT_ID:
- track.default = bool(elem.get_value())
- elif elem_id == MATROSKA_CODEC_ID:
- track.codec = elem.get_str()
- elif elem_id == MATROSKA_CODEC_PRIVATE_ID:
- track.codec_private = elem.get_data()
- elif elem_id == MATROSKA_TRACK_UID_ID:
- self.objects_by_uid[elem.get_value()] = track
-
-
- def process_video_track(self, elements):
- track = core.VideoStream()
- # Defaults
- track.codec = u'Unknown'
- track.fps = 0
- self.set_track_defaults(track)
-
- for elem in elements:
- elem_id = elem.get_id()
- if elem_id == MATROSKA_CODEC_ID:
- track.codec = elem.get_str()
-
- elif elem_id == MATROSKA_FRAME_DURATION_ID:
- try:
- track.fps = 1 / (pow(10, -9) * (elem.get_value()))
- except ZeroDivisionError:
- pass
-
- elif elem_id == MATROSKA_VIDEO_SETTINGS_ID:
- d_width = d_height = None
- for settings_elem in self.process_one_level(elem):
- settings_elem_id = settings_elem.get_id()
- if settings_elem_id == MATROSKA_VIDEO_WIDTH_ID:
- track.width = settings_elem.get_value()
- elif settings_elem_id == MATROSKA_VIDEO_HEIGHT_ID:
- track.height = settings_elem.get_value()
- elif settings_elem_id == MATROSKA_VIDEO_DISPLAY_WIDTH_ID:
- d_width = settings_elem.get_value()
- elif settings_elem_id == MATROSKA_VIDEO_DISPLAY_HEIGHT_ID:
- d_height = settings_elem.get_value()
- elif settings_elem_id == MATROSKA_VIDEO_INTERLACED_ID:
- value = int(settings_elem.get_value())
- self._set('interlaced', value)
-
- if None not in [d_width, d_height]:
- track.aspect = float(d_width) / d_height
+ """
+ name = element.get('TagName')
+ language = element.get('TagLanguage', 'und')
+ default = element.get('TagDefault', True)
+ string = element.get('TagString')
+ binary = element.get('TagBinary')
+ return cls(name, language, default, string, binary)
- else:
- self.process_track_common(elem, track)
-
- # convert codec information
- # http://haali.cs.msu.ru/mkv/codecs.pdf
- if track.codec in FOURCCMap:
- track.codec = FOURCCMap[track.codec]
- elif '/' in track.codec and track.codec.split('/')[0] + '/' in FOURCCMap:
- track.codec = FOURCCMap[track.codec.split('/')[0] + '/']
- elif track.codec.endswith('FOURCC') and len(track.codec_private or '') == 40:
- track.codec = track.codec_private[16:20]
- elif track.codec.startswith('V_REAL/'):
- track.codec = track.codec[7:]
- elif track.codec.startswith('V_'):
- # FIXME: add more video codecs here
- track.codec = track.codec[2:]
-
- track.id = len(self.video)
- self.video.append(track)
- return track
-
-
- def process_audio_track(self, elements):
- track = core.AudioStream()
- track.codec = u'Unknown'
- self.set_track_defaults(track)
-
- for elem in elements:
- elem_id = elem.get_id()
- if elem_id == MATROSKA_CODEC_ID:
- track.codec = elem.get_str()
- elif elem_id == MATROSKA_AUDIO_SETTINGS_ID:
- for settings_elem in self.process_one_level(elem):
- settings_elem_id = settings_elem.get_id()
- if settings_elem_id == MATROSKA_AUDIO_SAMPLERATE_ID:
- track.samplerate = settings_elem.get_float_value()
- elif settings_elem_id == MATROSKA_AUDIO_CHANNELS_ID:
- track.channels = settings_elem.get_value()
- else:
- self.process_track_common(elem, track)
-
-
- if track.codec in FOURCCMap:
- track.codec = FOURCCMap[track.codec]
- elif '/' in track.codec and track.codec.split('/')[0] + '/' in FOURCCMap:
- track.codec = FOURCCMap[track.codec.split('/')[0] + '/']
- elif track.codec.startswith('A_'):
- track.codec = track.codec[2:]
-
- track.id = len(self.audio)
- self.audio.append(track)
- return track
-
-
- def process_chapters(self, chapters):
- elements = self.process_one_level(chapters)
- for elem in elements:
- if elem.get_id() == MATROSKA_EDITION_ENTRY_ID:
- buf = elem.get_data()
- index = 0
- while index < elem.get_len():
- sub_elem = EbmlEntity(buf[index:])
- if sub_elem.get_id() == MATROSKA_CHAPTER_ATOM_ID:
- self.process_chapter_atom(sub_elem)
- index += sub_elem.get_total_len() + sub_elem.get_crc_len()
-
-
- def process_chapter_atom(self, atom):
- elements = self.process_one_level(atom)
- chap = core.Chapter()
-
- for elem in elements:
- elem_id = elem.get_id()
- if elem_id == MATROSKA_CHAPTER_TIME_START_ID:
- # Scale timecode to seconds (float)
- chap.pos = elem.get_value() / 1000000 / 1000.0
- elif elem_id == MATROSKA_CHAPTER_FLAG_ENABLED_ID:
- chap.enabled = elem.get_value()
- elif elem_id == MATROSKA_CHAPTER_DISPLAY_ID:
- # Matroska supports multiple (chapter name, language) pairs for
- # each chapter, so chapter names can be internationalized. This
- # logic will only take the last one in the list.
- for display_elem in self.process_one_level(elem):
- if display_elem.get_id() == MATROSKA_CHAPTER_STRING_ID:
- chap.name = display_elem.get_utf8()
- elif elem_id == MATROSKA_CHAPTER_UID_ID:
- self.objects_by_uid[elem.get_value()] = chap
-
- log.debug(u'Chapter %r found', chap.name)
- chap.id = len(self.chapters)
- self.chapters.append(chap)
-
-
- def process_attachments(self, attachments):
- buf = attachments.get_data()
- index = 0
- while index < attachments.get_len():
- elem = EbmlEntity(buf[index:])
- if elem.get_id() == MATROSKA_ATTACHED_FILE_ID:
- self.process_attachment(elem)
- index += elem.get_total_len() + elem.get_crc_len()
-
-
- def process_attachment(self, attachment):
- elements = self.process_one_level(attachment)
- name = desc = mimetype = ""
- data = None
-
- for elem in elements:
- elem_id = elem.get_id()
- if elem_id == MATROSKA_FILE_NAME_ID:
- name = elem.get_utf8()
- elif elem_id == MATROSKA_FILE_DESC_ID:
- desc = elem.get_utf8()
- elif elem_id == MATROSKA_FILE_MIME_TYPE_ID:
- mimetype = elem.get_data()
- elif elem_id == MATROSKA_FILE_DATA_ID:
- data = elem.get_data()
-
- # Right now we only support attachments that could be cover images.
- # Make a guess to see if this attachment is a cover image.
- if mimetype.startswith("image/") and u"cover" in (name + desc).lower() and data:
- self.thumbnail = data
-
- log.debug(u'Attachment %r found' % name)
-
-
- def process_tags(self, tags):
- # Tags spec: http://www.matroska.org/technical/specs/tagging/index.html
- # Iterate over Tags children. Tags element children is a
- # Tag element (whose children are SimpleTags) and a Targets element
- # whose children specific what objects the tags apply to.
- for tag_elem in self.process_one_level(tags):
- # Start a new dict to hold all SimpleTag elements.
- tags_dict = core.Tags()
- # A list of target uids this tags dict applies too. If empty,
- # tags are global.
- targets = []
- for sub_elem in self.process_one_level(tag_elem):
- if sub_elem.get_id() == MATROSKA_SIMPLE_TAG_ID:
- self.process_simple_tag(sub_elem, tags_dict)
- elif sub_elem.get_id() == MATROSKA_TARGETS_ID:
- # Targets element: if there is no uid child (track uid,
- # chapter uid, etc.) then the tags dict applies to the
- # whole file (top-level Media object).
- for target_elem in self.process_one_level(sub_elem):
- target_elem_id = target_elem.get_id()
- if target_elem_id in (MATRSOKA_TAGS_TRACK_UID_ID, MATRSOKA_TAGS_EDITION_UID_ID,
- MATRSOKA_TAGS_CHAPTER_UID_ID, MATRSOKA_TAGS_ATTACHMENT_UID_ID):
- targets.append(target_elem.get_value())
- elif target_elem_id == MATROSKA_TARGET_TYPE_VALUE_ID:
- # Target types not supported for now. (Unclear how this
- # would fit with kaa.metadata.)
- pass
- if targets:
- # Assign tags to all listed uids
- for target in targets:
- try:
- self.objects_by_uid[target].tags.update(tags_dict)
- self.tags_to_attributes(self.objects_by_uid[target], tags_dict)
- except KeyError:
- log.warning(u'Tags assigned to unknown/unsupported target uid %d', target)
- else:
- self.tags.update(tags_dict)
- self.tags_to_attributes(self, tags_dict)
+ def __repr__(self):
+ return '<%s [%s, language=%s, default=%s, string=%s]>' % (self.__class__.__name__, self.name, self.language, self.default, self.string)
- def process_simple_tag(self, simple_tag_elem, tags_dict):
- """
- Returns a dict representing the Tag element.
- """
- name = lang = value = children = None
- binary = False
- for elem in self.process_one_level(simple_tag_elem):
- elem_id = elem.get_id()
- if elem_id == MATROSKA_TAG_NAME_ID:
- name = elem.get_utf8().lower()
- elif elem_id == MATROSKA_TAG_STRING_ID:
- value = elem.get_utf8()
- elif elem_id == MATROSKA_TAG_BINARY_ID:
- value = elem.get_data()
- binary = True
- elif elem_id == MATROSKA_TAG_LANGUAGE_ID:
- lang = elem.get_utf8()
- elif elem_id == MATROSKA_SIMPLE_TAG_ID:
- if children is None:
- children = core.Tags()
- self.process_simple_tag(elem, children)
-
- if children:
- # Convert ourselves to a Tags object.
- children.value = value
- children.langcode = lang
- value = children
- else:
- if name.startswith('date_'):
- # Try to convert date to a datetime object.
- value = matroska_date_to_datetime(value)
- value = core.Tag(value, lang, binary)
-
- if name in tags_dict:
- # Multiple items of this tag name.
- if not isinstance(tags_dict[name], list):
- # Convert to a list
- tags_dict[name] = [tags_dict[name]]
- # Append to list
- tags_dict[name].append(value)
- else:
- tags_dict[name] = value
-
-
- def tags_to_attributes(self, obj, tags):
- # Convert tags to core attributes.
- for name, tag in tags.items():
- if isinstance(tag, dict):
- # Nested tags dict, recurse.
- self.tags_to_attributes(obj, tag)
- continue
- elif name not in TAGS_MAP:
- continue
+class Chapter(object):
+ """Object for the ChapterAtom and ChapterDisplay EBML element
- attr, filter = TAGS_MAP[name]
- if attr not in obj._keys and attr not in self._keys:
- # Tag is not in any core attribute for this object or global,
- # so skip.
- continue
+ .. note::
+ For the sake of simplicity, it is assumed that the ChapterAtom element
+ has no more than 1 ChapterDisplay child element and informations it contains
+ are merged into the :class:`Chapter`
- # Pull value out of Tag object or list of Tag objects.
- value = [item.value for item in tag] if isinstance(tag, list) else tag.value
- if filter:
- try:
- value = [filter(item) for item in value] if isinstance(value, list) else filter(value)
- except Exception, e:
- log.warning(u'Failed to convert tag to core attribute: %r', e)
- # Special handling for tv series recordings. The 'title' tag
- # can be used for both the series and the episode name. The
- # same is true for trackno which may refer to the season
- # and the episode number. Therefore, if we find these
- # attributes already set we try some guessing.
- if attr == 'trackno' and getattr(self, attr) is not None:
- # delete trackno and save season and episode
- self.season = self.trackno
- self.episode = value
- self.trackno = None
- continue
- if attr == 'title' and getattr(self, attr) is not None:
- # store current value of title as series and use current
- # value of title as title
- self.series = self.title
- if attr in obj._keys:
- setattr(obj, attr, value)
- else:
- setattr(self, attr, value)
+ """
+ def __init__(self, start, hidden=False, enabled=False, end=None, string=None, language=None):
+ self.start = start
+ self.hidden = hidden
+ self.enabled = enabled
+ self.end = end
+ self.string = string
+ self.language = language
+ @classmethod
+ def fromelement(cls, element):
+ """Load the :class:`Chapter` from an :class:`~enzyme.parsers.ebml.Element`
-Parser = Matroska
+ :param element: the ChapterAtom element
+ :type element: :class:`~enzyme.parsers.ebml.Element`
+
+ """
+ start = timedelta(microseconds=element.get('ChapterTimeStart') // 1000)
+ hidden = element.get('ChapterFlagHidden', False)
+ enabled = element.get('ChapterFlagEnabled', True)
+ end = element.get('ChapterTimeEnd')
+ chapterdisplays = [c for c in element if c.name == 'ChapterDisplay']
+ if len(chapterdisplays) > 1:
+ logger.warning('More than 1 (%d) ChapterDisplay element in the ChapterAtom, using the first one', len(chapterdisplays))
+ if chapterdisplays:
+ string = chapterdisplays[0].get('ChapString')
+ language = chapterdisplays[0].get('ChapLanguage')
+ return cls(start, hidden, enabled, end, string, language)
+ return cls(start, hidden, enabled, end)
+
+ def __repr__(self):
+ return '<%s [%s, enabled=%s]>' % (self.__class__.__name__, self.start, self.enabled)
diff --git a/lib/enzyme/mp4.py b/lib/enzyme/mp4.py
deleted file mode 100644
index e8a2c329c98a823db22ef3dc538d5c1f1a8215ed..0000000000000000000000000000000000000000
--- a/lib/enzyme/mp4.py
+++ /dev/null
@@ -1,486 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2007 Thomas Schueppel <stain@acm.org>
-# Copyright 2003-2007 Dirk Meyer <dischi@freevo.org>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-__all__ = ['Parser']
-
-import zlib
-import logging
-import StringIO
-import struct
-from exceptions import ParseError
-import core
-
-# get logging object
-log = logging.getLogger(__name__)
-
-
-# http://developer.apple.com/documentation/QuickTime/QTFF/index.html
-# http://developer.apple.com/documentation/QuickTime/QTFF/QTFFChap4/\
-# chapter_5_section_2.html#//apple_ref/doc/uid/TP40000939-CH206-BBCBIICE
-# Note: May need to define custom log level to work like ATOM_DEBUG did here
-
-QTUDTA = {
- 'nam': 'title',
- 'aut': 'artist',
- 'cpy': 'copyright'
-}
-
-QTLANGUAGES = {
- 0: "en",
- 1: "fr",
- 2: "de",
- 3: "it",
- 4: "nl",
- 5: "sv",
- 6: "es",
- 7: "da",
- 8: "pt",
- 9: "no",
- 10: "he",
- 11: "ja",
- 12: "ar",
- 13: "fi",
- 14: "el",
- 15: "is",
- 16: "mt",
- 17: "tr",
- 18: "hr",
- 19: "Traditional Chinese",
- 20: "ur",
- 21: "hi",
- 22: "th",
- 23: "ko",
- 24: "lt",
- 25: "pl",
- 26: "hu",
- 27: "et",
- 28: "lv",
- 29: "Lappish",
- 30: "fo",
- 31: "Farsi",
- 32: "ru",
- 33: "Simplified Chinese",
- 34: "Flemish",
- 35: "ga",
- 36: "sq",
- 37: "ro",
- 38: "cs",
- 39: "sk",
- 40: "sl",
- 41: "yi",
- 42: "sr",
- 43: "mk",
- 44: "bg",
- 45: "uk",
- 46: "be",
- 47: "uz",
- 48: "kk",
- 49: "az",
- 50: "AzerbaijanAr",
- 51: "hy",
- 52: "ka",
- 53: "mo",
- 54: "ky",
- 55: "tg",
- 56: "tk",
- 57: "mn",
- 58: "MongolianCyr",
- 59: "ps",
- 60: "ku",
- 61: "ks",
- 62: "sd",
- 63: "bo",
- 64: "ne",
- 65: "sa",
- 66: "mr",
- 67: "bn",
- 68: "as",
- 69: "gu",
- 70: "pa",
- 71: "or",
- 72: "ml",
- 73: "kn",
- 74: "ta",
- 75: "te",
- 76: "si",
- 77: "my",
- 78: "Khmer",
- 79: "lo",
- 80: "vi",
- 81: "id",
- 82: "tl",
- 83: "MalayRoman",
- 84: "MalayArabic",
- 85: "am",
- 86: "ti",
- 87: "om",
- 88: "so",
- 89: "sw",
- 90: "Ruanda",
- 91: "Rundi",
- 92: "Chewa",
- 93: "mg",
- 94: "eo",
- 128: "cy",
- 129: "eu",
- 130: "ca",
- 131: "la",
- 132: "qu",
- 133: "gn",
- 134: "ay",
- 135: "tt",
- 136: "ug",
- 137: "Dzongkha",
- 138: "JavaneseRom",
-}
-
-class MPEG4(core.AVContainer):
- """
- Parser for the MP4 container format. This format is mostly
- identical to Apple Quicktime and 3GP files. It maps to mp4, mov,
- qt and some other extensions.
- """
- table_mapping = {'QTUDTA': QTUDTA}
-
- def __init__(self, file):
- core.AVContainer.__init__(self)
- self._references = []
-
- self.mime = 'video/quicktime'
- self.type = 'Quicktime Video'
- h = file.read(8)
- try:
- (size, type) = struct.unpack('>I4s', h)
- except struct.error:
- # EOF.
- raise ParseError()
-
- if type == 'ftyp':
- # file type information
- if size >= 12:
- # this should always happen
- if file.read(4) != 'qt ':
- # not a quicktime movie, it is a mpeg4 container
- self.mime = 'video/mp4'
- self.type = 'MPEG-4 Video'
- size -= 4
- file.seek(size - 8, 1)
- h = file.read(8)
- (size, type) = struct.unpack('>I4s', h)
-
- while type in ['mdat', 'skip']:
- # movie data at the beginning, skip
- file.seek(size - 8, 1)
- h = file.read(8)
- (size, type) = struct.unpack('>I4s', h)
-
- if not type in ['moov', 'wide', 'free']:
- log.debug(u'invalid header: %r' % type)
- raise ParseError()
-
- # Extended size
- if size == 1:
- size = struct.unpack('>Q', file.read(8))
-
- # Back over the atom header we just read, since _readatom expects the
- # file position to be at the start of an atom.
- file.seek(-8, 1)
- while self._readatom(file):
- pass
-
- if self._references:
- self._set('references', self._references)
-
-
- def _readatom(self, file):
- s = file.read(8)
- if len(s) < 8:
- return 0
-
- atomsize, atomtype = struct.unpack('>I4s', s)
- if not str(atomtype).decode('latin1').isalnum():
- # stop at nonsense data
- return 0
-
- log.debug(u'%r [%X]' % (atomtype, atomsize))
-
- if atomtype == 'udta':
- # Userdata (Metadata)
- pos = 0
- tabl = {}
- i18ntabl = {}
- atomdata = file.read(atomsize - 8)
- while pos < atomsize - 12:
- (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
- if ord(datatype[0]) == 169:
- # i18n Metadata...
- mypos = 8 + pos
- while mypos + 4 < datasize + pos:
- # first 4 Bytes are i18n header
- (tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos + 4])
- i18ntabl[lang] = i18ntabl.get(lang, {})
- l = atomdata[mypos + 4:mypos + tlen + 4]
- i18ntabl[lang][datatype[1:]] = l
- mypos += tlen + 4
- elif datatype == 'WLOC':
- # Drop Window Location
- pass
- else:
- if ord(atomdata[pos + 8:pos + datasize][0]) > 1:
- tabl[datatype] = atomdata[pos + 8:pos + datasize]
- pos += datasize
- if len(i18ntabl.keys()) > 0:
- for k in i18ntabl.keys():
- if QTLANGUAGES.has_key(k) and QTLANGUAGES[k] == 'en':
- self._appendtable('QTUDTA', i18ntabl[k])
- self._appendtable('QTUDTA', tabl)
- else:
- log.debug(u'NO i18')
- self._appendtable('QTUDTA', tabl)
-
- elif atomtype == 'trak':
- atomdata = file.read(atomsize - 8)
- pos = 0
- trackinfo = {}
- tracktype = None
- while pos < atomsize - 8:
- (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
-
- if datatype == 'tkhd':
- tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos + 8:pos + datasize])
- trackinfo['width'] = tkhd[10] >> 16
- trackinfo['height'] = tkhd[11] >> 16
- trackinfo['id'] = tkhd[3]
-
- try:
- # XXX Timestamp of Seconds is since January 1st 1904!
- # XXX 2082844800 is the difference between Unix and
- # XXX Apple time. FIXME to work on Apple, too
- self.timestamp = int(tkhd[1]) - 2082844800
- except Exception, e:
- log.exception(u'There was trouble extracting timestamp')
-
- elif datatype == 'mdia':
- pos += 8
- datasize -= 8
- log.debug(u'--> mdia information')
-
- while datasize:
- mdia = struct.unpack('>I4s', atomdata[pos:pos + 8])
- if mdia[1] == 'mdhd':
- # Parse based on version of mdhd header. See
- # http://wiki.multimedia.cx/index.php?title=QuickTime_container#mdhd
- ver = ord(atomdata[pos + 8])
- if ver == 0:
- mdhd = struct.unpack('>IIIIIhh', atomdata[pos + 8:pos + 8 + 24])
- elif ver == 1:
- mdhd = struct.unpack('>IQQIQhh', atomdata[pos + 8:pos + 8 + 36])
- else:
- mdhd = None
-
- if mdhd:
- # duration / time scale
- trackinfo['length'] = mdhd[4] / mdhd[3]
- if mdhd[5] in QTLANGUAGES:
- trackinfo['language'] = QTLANGUAGES[mdhd[5]]
- elif mdhd[5] == 0x7FF:
- trackinfo['language'] = 'und'
- elif mdhd[5] >= 0x400:
- # language code detected as explained in:
- # https://developer.apple.com/library/mac/documentation/QuickTime/qtff/QTFFChap4/qtff4.html#//apple_ref/doc/uid/TP40000939-CH206-35103
- language = bytearray([ ((mdhd[5] & 0x7C00) >> 10) + 0x60, ((mdhd[5] & 0x3E0) >> 5) + 0x60, (mdhd[5] & 0x1F) + 0x60])
- trackinfo['language'] = str(language)
- # mdhd[6] == quality
- self.length = max(self.length, mdhd[4] / mdhd[3])
- elif mdia[1] == 'minf':
- # minf has only atoms inside
- pos -= (mdia[0] - 8)
- datasize += (mdia[0] - 8)
- elif mdia[1] == 'stbl':
- # stbl has only atoms inside
- pos -= (mdia[0] - 8)
- datasize += (mdia[0] - 8)
- elif mdia[1] == 'hdlr':
- hdlr = struct.unpack('>I4s4s', atomdata[pos + 8:pos + 8 + 12])
- if hdlr[1] == 'mhlr' or hdlr[1] == '\0\0\0\0':
- if hdlr[2] == 'vide':
- tracktype = 'video'
- if hdlr[2] == 'soun':
- tracktype = 'audio'
- if hdlr[2] == 'subt' or hdlr[2] == 'sbtl' or hdlr[2] == 'subp' or hdlr[2] == 'text':
- tracktype = 'subtitle'
- elif mdia[1] == 'stsd':
- stsd = struct.unpack('>2I', atomdata[pos + 8:pos + 8 + 8])
- if stsd[1] > 0:
- codec = atomdata[pos + 16:pos + 16 + 8]
- codec = struct.unpack('>I4s', codec)
- trackinfo['codec'] = codec[1]
- if codec[1] == 'jpeg':
- tracktype = 'image'
- elif mdia[1] == 'dinf':
- dref = struct.unpack('>I4s', atomdata[pos + 8:pos + 8 + 8])
- log.debug(u' --> %r, %r (useless)' % mdia)
- if dref[1] == 'dref':
- num = struct.unpack('>I', atomdata[pos + 20:pos + 20 + 4])[0]
- rpos = pos + 20 + 4
- for ref in range(num):
- # FIXME: do somthing if this references
- ref = struct.unpack('>I3s', atomdata[rpos:rpos + 7])
- data = atomdata[rpos + 7:rpos + ref[0]]
- rpos += ref[0]
- else:
- if mdia[1].startswith('st'):
- log.debug(u' --> %r, %r (sample)' % mdia)
- elif mdia[1] == 'vmhd' and not tracktype:
- # indicates that this track is video
- tracktype = 'video'
- elif mdia[1] in ['vmhd', 'smhd'] and not tracktype:
- # indicates that this track is audio
- tracktype = 'audio'
- else:
- log.debug(u' --> %r, %r (unknown)' % mdia)
-
- pos += mdia[0]
- datasize -= mdia[0]
-
- elif datatype == 'udta':
- log.debug(u'udta: %r' % struct.unpack('>I4s', atomdata[:8]))
- else:
- if datatype == 'edts':
- log.debug(u'--> %r [%d] (edit list)' % \
- (datatype, datasize))
- else:
- log.debug(u'--> %r [%d] (unknown)' % \
- (datatype, datasize))
- pos += datasize
-
- info = None
- if tracktype == 'video':
- info = core.VideoStream()
- self.video.append(info)
- if tracktype == 'audio':
- info = core.AudioStream()
- self.audio.append(info)
- if tracktype == 'subtitle':
- info = core.Subtitle()
- self.subtitles.append(info)
- if info:
- for key, value in trackinfo.items():
- setattr(info, key, value)
-
- elif atomtype == 'mvhd':
- # movie header
- mvhd = struct.unpack('>6I2h', file.read(28))
- self.length = max(self.length, mvhd[4] / mvhd[3])
- self.volume = mvhd[6]
- file.seek(atomsize - 8 - 28, 1)
-
-
- elif atomtype == 'cmov':
- # compressed movie
- datasize, atomtype = struct.unpack('>I4s', file.read(8))
- if not atomtype == 'dcom':
- return atomsize
-
- method = struct.unpack('>4s', file.read(datasize - 8))[0]
-
- datasize, atomtype = struct.unpack('>I4s', file.read(8))
- if not atomtype == 'cmvd':
- return atomsize
-
- if method == 'zlib':
- data = file.read(datasize - 8)
- try:
- decompressed = zlib.decompress(data)
- except Exception, e:
- try:
- decompressed = zlib.decompress(data[4:])
- except Exception, e:
- log.exception(u'There was a proble decompressiong atom')
- return atomsize
-
- decompressedIO = StringIO.StringIO(decompressed)
- while self._readatom(decompressedIO):
- pass
-
- else:
- log.info(u'unknown compression %r' % method)
- # unknown compression method
- file.seek(datasize - 8, 1)
-
- elif atomtype == 'moov':
- # decompressed movie info
- while self._readatom(file):
- pass
-
- elif atomtype == 'mdat':
- pos = file.tell() + atomsize - 8
- # maybe there is data inside the mdat
- log.info(u'parsing mdat')
- while self._readatom(file):
- pass
- log.info(u'end of mdat')
- file.seek(pos, 0)
-
-
- elif atomtype == 'rmra':
- # reference list
- while self._readatom(file):
- pass
-
- elif atomtype == 'rmda':
- # reference
- atomdata = file.read(atomsize - 8)
- pos = 0
- url = ''
- quality = 0
- datarate = 0
- while pos < atomsize - 8:
- (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
- if datatype == 'rdrf':
- rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos + 8:pos + 20])
- if rtype == 'url ':
- url = atomdata[pos + 20:pos + 20 + rlen]
- if url.find('\0') > 0:
- url = url[:url.find('\0')]
- elif datatype == 'rmqu':
- quality = struct.unpack('>I', atomdata[pos + 8:pos + 12])[0]
-
- elif datatype == 'rmdr':
- datarate = struct.unpack('>I', atomdata[pos + 12:pos + 16])[0]
-
- pos += datasize
- if url:
- self._references.append((url, quality, datarate))
-
- else:
- if not atomtype in ['wide', 'free']:
- log.info(u'unhandled base atom %r' % atomtype)
-
- # Skip unknown atoms
- try:
- file.seek(atomsize - 8, 1)
- except IOError:
- return 0
-
- return atomsize
-
-
-Parser = MPEG4
diff --git a/lib/enzyme/mpeg.py b/lib/enzyme/mpeg.py
deleted file mode 100644
index 3d43ba4fd73ab855d87570168b7af3ee1b6b4d3e..0000000000000000000000000000000000000000
--- a/lib/enzyme/mpeg.py
+++ /dev/null
@@ -1,913 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
-# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-__all__ = ['Parser']
-
-import os
-import struct
-import logging
-import stat
-from exceptions import ParseError
-import core
-
-# get logging object
-log = logging.getLogger(__name__)
-
-##------------------------------------------------------------------------
-## START_CODE
-##
-## Start Codes, with 'slice' occupying 0x01..0xAF
-##------------------------------------------------------------------------
-START_CODE = {
- 0x00 : 'picture_start_code',
- 0xB0 : 'reserved',
- 0xB1 : 'reserved',
- 0xB2 : 'user_data_start_code',
- 0xB3 : 'sequence_header_code',
- 0xB4 : 'sequence_error_code',
- 0xB5 : 'extension_start_code',
- 0xB6 : 'reserved',
- 0xB7 : 'sequence end',
- 0xB8 : 'group of pictures',
-}
-for i in range(0x01, 0xAF):
- START_CODE[i] = 'slice_start_code'
-
-##------------------------------------------------------------------------
-## START CODES
-##------------------------------------------------------------------------
-PICTURE = 0x00
-USERDATA = 0xB2
-SEQ_HEAD = 0xB3
-SEQ_ERR = 0xB4
-EXT_START = 0xB5
-SEQ_END = 0xB7
-GOP = 0xB8
-
-SEQ_START_CODE = 0xB3
-PACK_PKT = 0xBA
-SYS_PKT = 0xBB
-PADDING_PKT = 0xBE
-AUDIO_PKT = 0xC0
-VIDEO_PKT = 0xE0
-PRIVATE_STREAM1 = 0xBD
-PRIVATE_STREAM2 = 0xBf
-
-TS_PACKET_LENGTH = 188
-TS_SYNC = 0x47
-
-##------------------------------------------------------------------------
-## FRAME_RATE
-##
-## A lookup table of all the standard frame rates. Some rates adhere to
-## a particular profile that ensures compatibility with VLSI capabilities
-## of the early to mid 1990s.
-##
-## CPB
-## Constrained Parameters Bitstreams, an MPEG-1 set of sampling and
-## bitstream parameters designed to normalize decoder computational
-## complexity, buffer size, and memory bandwidth while still addressing
-## the widest possible range of applications.
-##
-## Main Level
-## MPEG-2 Video Main Profile and Main Level is analogous to MPEG-1's
-## CPB, with sampling limits at CCIR 601 parameters (720x480x30 Hz or
-## 720x576x24 Hz).
-##
-##------------------------------------------------------------------------
-FRAME_RATE = [
- 0,
- 24000.0 / 1001, ## 3-2 pulldown NTSC (CPB/Main Level)
- 24, ## Film (CPB/Main Level)
- 25, ## PAL/SECAM or 625/60 video
- 30000.0 / 1001, ## NTSC (CPB/Main Level)
- 30, ## drop-frame NTSC or component 525/60 (CPB/Main Level)
- 50, ## double-rate PAL
- 60000.0 / 1001, ## double-rate NTSC
- 60, ## double-rate, drop-frame NTSC/component 525/60 video
- ]
-
-##------------------------------------------------------------------------
-## ASPECT_RATIO -- INCOMPLETE?
-##
-## This lookup table maps the header aspect ratio index to a float value.
-## These are just the defined ratios for CPB I believe. As I understand
-## it, a stream that doesn't adhere to one of these aspect ratios is
-## technically considered non-compliant.
-##------------------------------------------------------------------------
-ASPECT_RATIO = (None, # Forbidden
- 1.0, # 1/1 (VGA)
- 4.0 / 3, # 4/3 (TV)
- 16.0 / 9, # 16/9 (Widescreen)
- 2.21 # (Cinema)
- )
-
-
-class MPEG(core.AVContainer):
- """
- Parser for various MPEG files. This includes MPEG-1 and MPEG-2
- program streams, elementary streams and transport streams. The
- reported length differs from the length reported by most video
- players but the provides length here is correct. An MPEG file has
- no additional metadata like title, etc; only codecs, length and
- resolution is reported back.
- """
- def __init__(self, file):
- core.AVContainer.__init__(self)
- self.sequence_header_offset = 0
- self.mpeg_version = 2
-
- # detect TS (fast scan)
- if not self.isTS(file):
- # detect system mpeg (many infos)
- if not self.isMPEG(file):
- # detect PES
- if not self.isPES(file):
- # Maybe it's MPEG-ES
- if self.isES(file):
- # If isES() succeeds, we needn't do anything further.
- return
- if file.name.lower().endswith('mpeg') or \
- file.name.lower().endswith('mpg'):
- # This has to be an mpeg file. It could be a bad
- # recording from an ivtv based hardware encoder with
- # same bytes missing at the beginning.
- # Do some more digging...
- if not self.isMPEG(file, force=True) or \
- not self.video or not self.audio:
- # does not look like an mpeg at all
- raise ParseError()
- else:
- # no mpeg at all
- raise ParseError()
-
- self.mime = 'video/mpeg'
- if not self.video:
- self.video.append(core.VideoStream())
-
- if self.sequence_header_offset <= 0:
- return
-
- self.progressive(file)
-
- for vi in self.video:
- vi.width, vi.height = self.dxy(file)
- vi.fps, vi.aspect = self.framerate_aspect(file)
- vi.bitrate = self.bitrate(file)
- if self.length:
- vi.length = self.length
-
- if not self.type:
- self.type = 'MPEG Video'
-
- # set fourcc codec for video and audio
- vc, ac = 'MP2V', 'MP2A'
- if self.mpeg_version == 1:
- vc, ac = 'MPEG', 0x0050
- for v in self.video:
- v.codec = vc
- for a in self.audio:
- if not a.codec:
- a.codec = ac
-
-
- def dxy(self, file):
- """
- get width and height of the video
- """
- file.seek(self.sequence_header_offset + 4, 0)
- v = file.read(4)
- x = struct.unpack('>H', v[:2])[0] >> 4
- y = struct.unpack('>H', v[1:3])[0] & 0x0FFF
- return (x, y)
-
-
- def framerate_aspect(self, file):
- """
- read framerate and aspect ratio
- """
- file.seek(self.sequence_header_offset + 7, 0)
- v = struct.unpack('>B', file.read(1))[0]
- try:
- fps = FRAME_RATE[v & 0xf]
- except IndexError:
- fps = None
- if v >> 4 < len(ASPECT_RATIO):
- aspect = ASPECT_RATIO[v >> 4]
- else:
- aspect = None
- return (fps, aspect)
-
-
- def progressive(self, file):
- """
- Try to find out with brute force if the mpeg is interlaced or not.
- Search for the Sequence_Extension in the extension header (01B5)
- """
- file.seek(0)
- buffer = ''
- count = 0
- while 1:
- if len(buffer) < 1000:
- count += 1
- if count > 1000:
- break
- buffer += file.read(1024)
- if len(buffer) < 1000:
- break
- pos = buffer.find('\x00\x00\x01\xb5')
- if pos == -1 or len(buffer) - pos < 5:
- buffer = buffer[-10:]
- continue
- ext = (ord(buffer[pos + 4]) >> 4)
- if ext == 8:
- pass
- elif ext == 1:
- if (ord(buffer[pos + 5]) >> 3) & 1:
- self._set('progressive', True)
- else:
- self._set('interlaced', True)
- return True
- else:
- log.debug(u'ext: %r' % ext)
- buffer = buffer[pos + 4:]
- return False
-
-
- ##------------------------------------------------------------------------
- ## bitrate()
- ##
- ## From the MPEG-2.2 spec:
- ##
- ## bit_rate -- This is a 30-bit integer. The lower 18 bits of the
- ## integer are in bit_rate_value and the upper 12 bits are in
- ## bit_rate_extension. The 30-bit integer specifies the bitrate of the
- ## bitstream measured in units of 400 bits/second, rounded upwards.
- ## The value zero is forbidden.
- ##
- ## So ignoring all the variable bitrate stuff for now, this 30 bit integer
- ## multiplied times 400 bits/sec should give the rate in bits/sec.
- ##
- ## TODO: Variable bitrates? I need one that implements this.
- ##
- ## Continued from the MPEG-2.2 spec:
- ##
- ## If the bitstream is a constant bitrate stream, the bitrate specified
- ## is the actual rate of operation of the VBV specified in annex C. If
- ## the bitstream is a variable bitrate stream, the STD specifications in
- ## ISO/IEC 13818-1 supersede the VBV, and the bitrate specified here is
- ## used to dimension the transport stream STD (2.4.2 in ITU-T Rec. xxx |
- ## ISO/IEC 13818-1), or the program stream STD (2.4.5 in ITU-T Rec. xxx |
- ## ISO/IEC 13818-1).
- ##
- ## If the bitstream is not a constant rate bitstream the vbv_delay
- ## field shall have the value FFFF in hexadecimal.
- ##
- ## Given the value encoded in the bitrate field, the bitstream shall be
- ## generated so that the video encoding and the worst case multiplex
- ## jitter do not cause STD buffer overflow or underflow.
- ##
- ##
- ##------------------------------------------------------------------------
-
-
- ## Some parts in the code are based on mpgtx (mpgtx.sf.net)
-
- def bitrate(self, file):
- """
- read the bitrate (most of the time broken)
- """
- file.seek(self.sequence_header_offset + 8, 0)
- t, b = struct.unpack('>HB', file.read(3))
- vrate = t << 2 | b >> 6
- return vrate * 400
-
-
- def ReadSCRMpeg2(self, buffer):
- """
- read SCR (timestamp) for MPEG2 at the buffer beginning (6 Bytes)
- """
- if len(buffer) < 6:
- return None
-
- highbit = (ord(buffer[0]) & 0x20) >> 5
-
- low4Bytes = ((long(ord(buffer[0])) & 0x18) >> 3) << 30
- low4Bytes |= (ord(buffer[0]) & 0x03) << 28
- low4Bytes |= ord(buffer[1]) << 20
- low4Bytes |= (ord(buffer[2]) & 0xF8) << 12
- low4Bytes |= (ord(buffer[2]) & 0x03) << 13
- low4Bytes |= ord(buffer[3]) << 5
- low4Bytes |= (ord(buffer[4])) >> 3
-
- sys_clock_ref = (ord(buffer[4]) & 0x3) << 7
- sys_clock_ref |= (ord(buffer[5]) >> 1)
-
- return (long(highbit * (1 << 16) * (1 << 16)) + low4Bytes) / 90000
-
-
- def ReadSCRMpeg1(self, buffer):
- """
- read SCR (timestamp) for MPEG1 at the buffer beginning (5 Bytes)
- """
- if len(buffer) < 5:
- return None
-
- highbit = (ord(buffer[0]) >> 3) & 0x01
-
- low4Bytes = ((long(ord(buffer[0])) >> 1) & 0x03) << 30
- low4Bytes |= ord(buffer[1]) << 22;
- low4Bytes |= (ord(buffer[2]) >> 1) << 15;
- low4Bytes |= ord(buffer[3]) << 7;
- low4Bytes |= ord(buffer[4]) >> 1;
-
- return (long(highbit) * (1 << 16) * (1 << 16) + low4Bytes) / 90000;
-
-
- def ReadPTS(self, buffer):
- """
- read PTS (PES timestamp) at the buffer beginning (5 Bytes)
- """
- high = ((ord(buffer[0]) & 0xF) >> 1)
- med = (ord(buffer[1]) << 7) + (ord(buffer[2]) >> 1)
- low = (ord(buffer[3]) << 7) + (ord(buffer[4]) >> 1)
- return ((long(high) << 30) + (med << 15) + low) / 90000
-
-
- def ReadHeader(self, buffer, offset):
- """
- Handle MPEG header in buffer on position offset
- Return None on error, new offset or 0 if the new offset can't be scanned
- """
- if buffer[offset:offset + 3] != '\x00\x00\x01':
- return None
-
- id = ord(buffer[offset + 3])
-
- if id == PADDING_PKT:
- return offset + (ord(buffer[offset + 4]) << 8) + \
- ord(buffer[offset + 5]) + 6
-
- if id == PACK_PKT:
- if ord(buffer[offset + 4]) & 0xF0 == 0x20:
- self.type = 'MPEG-1 Video'
- self.get_time = self.ReadSCRMpeg1
- self.mpeg_version = 1
- return offset + 12
- elif (ord(buffer[offset + 4]) & 0xC0) == 0x40:
- self.type = 'MPEG-2 Video'
- self.get_time = self.ReadSCRMpeg2
- return offset + (ord(buffer[offset + 13]) & 0x07) + 14
- else:
- # I have no idea what just happened, but for some DVB
- # recordings done with mencoder this points to a
- # PACK_PKT describing something odd. Returning 0 here
- # (let's hope there are no extensions in the header)
- # fixes it.
- return 0
-
- if 0xC0 <= id <= 0xDF:
- # code for audio stream
- for a in self.audio:
- if a.id == id:
- break
- else:
- self.audio.append(core.AudioStream())
- self.audio[-1]._set('id', id)
- return 0
-
- if 0xE0 <= id <= 0xEF:
- # code for video stream
- for v in self.video:
- if v.id == id:
- break
- else:
- self.video.append(core.VideoStream())
- self.video[-1]._set('id', id)
- return 0
-
- if id == SEQ_HEAD:
- # sequence header, remember that position for later use
- self.sequence_header_offset = offset
- return 0
-
- if id in [PRIVATE_STREAM1, PRIVATE_STREAM2]:
- # private stream. we don't know, but maybe we can guess later
- add = ord(buffer[offset + 8])
- # if (ord(buffer[offset+6]) & 4) or 1:
- # id = ord(buffer[offset+10+add])
- if buffer[offset + 11 + add:offset + 15 + add].find('\x0b\x77') != -1:
- # AC3 stream
- for a in self.audio:
- if a.id == id:
- break
- else:
- self.audio.append(core.AudioStream())
- self.audio[-1]._set('id', id)
- self.audio[-1].codec = 0x2000 # AC3
- return 0
-
- if id == SYS_PKT:
- return 0
-
- if id == EXT_START:
- return 0
-
- return 0
-
-
- # Normal MPEG (VCD, SVCD) ========================================
-
- def isMPEG(self, file, force=False):
- """
- This MPEG starts with a sequence of 0x00 followed by a PACK Header
- http://dvd.sourceforge.net/dvdinfo/packhdr.html
- """
- file.seek(0, 0)
- buffer = file.read(10000)
- offset = 0
-
- # seek until the 0 byte stop
- while offset < len(buffer) - 100 and buffer[offset] == '\0':
- offset += 1
- offset -= 2
-
- # test for mpeg header 0x00 0x00 0x01
- header = '\x00\x00\x01%s' % chr(PACK_PKT)
- if offset < 0 or not buffer[offset:offset + 4] == header:
- if not force:
- return 0
- # brute force and try to find the pack header in the first
- # 10000 bytes somehow
- offset = buffer.find(header)
- if offset < 0:
- return 0
-
- # scan the 100000 bytes of data
- buffer += file.read(100000)
-
- # scan first header, to get basic info about
- # how to read a timestamp
- self.ReadHeader(buffer, offset)
-
- # store first timestamp
- self.start = self.get_time(buffer[offset + 4:])
- while len(buffer) > offset + 1000 and \
- buffer[offset:offset + 3] == '\x00\x00\x01':
- # read the mpeg header
- new_offset = self.ReadHeader(buffer, offset)
-
- # header scanning detected error, this is no mpeg
- if new_offset == None:
- return 0
-
- if new_offset:
- # we have a new offset
- offset = new_offset
-
- # skip padding 0 before a new header
- while len(buffer) > offset + 10 and \
- not ord(buffer[offset + 2]):
- offset += 1
-
- else:
- # seek to new header by brute force
- offset += buffer[offset + 4:].find('\x00\x00\x01') + 4
-
- # fill in values for support functions:
- self.__seek_size__ = 1000000
- self.__sample_size__ = 10000
- self.__search__ = self._find_timer_
- self.filename = file.name
-
- # get length of the file
- self.length = self.get_length()
- return 1
-
-
- def _find_timer_(self, buffer):
- """
- Return position of timer in buffer or None if not found.
- This function is valid for 'normal' mpeg files
- """
- pos = buffer.find('\x00\x00\x01%s' % chr(PACK_PKT))
- if pos == -1:
- return None
- return pos + 4
-
-
-
- # PES ============================================================
-
-
- def ReadPESHeader(self, offset, buffer, id=0):
- """
- Parse a PES header.
- Since it starts with 0x00 0x00 0x01 like 'normal' mpegs, this
- function will return (0, None) when it is no PES header or
- (packet length, timestamp position (maybe None))
-
- http://dvd.sourceforge.net/dvdinfo/pes-hdr.html
- """
- if not buffer[0:3] == '\x00\x00\x01':
- return 0, None
-
- packet_length = (ord(buffer[4]) << 8) + ord(buffer[5]) + 6
- align = ord(buffer[6]) & 4
- header_length = ord(buffer[8])
-
- # PES ID (starting with 001)
- if ord(buffer[3]) & 0xE0 == 0xC0:
- id = id or ord(buffer[3]) & 0x1F
- for a in self.audio:
- if a.id == id:
- break
- else:
- self.audio.append(core.AudioStream())
- self.audio[-1]._set('id', id)
-
- elif ord(buffer[3]) & 0xF0 == 0xE0:
- id = id or ord(buffer[3]) & 0xF
- for v in self.video:
- if v.id == id:
- break
- else:
- self.video.append(core.VideoStream())
- self.video[-1]._set('id', id)
-
- # new mpeg starting
- if buffer[header_length + 9:header_length + 13] == \
- '\x00\x00\x01\xB3' and not self.sequence_header_offset:
- # yes, remember offset for later use
- self.sequence_header_offset = offset + header_length + 9
- elif ord(buffer[3]) == 189 or ord(buffer[3]) == 191:
- # private stream. we don't know, but maybe we can guess later
- id = id or ord(buffer[3]) & 0xF
- if align and \
- buffer[header_length + 9:header_length + 11] == '\x0b\x77':
- # AC3 stream
- for a in self.audio:
- if a.id == id:
- break
- else:
- self.audio.append(core.AudioStream())
- self.audio[-1]._set('id', id)
- self.audio[-1].codec = 0x2000 # AC3
-
- else:
- # unknown content
- pass
-
- ptsdts = ord(buffer[7]) >> 6
-
- if ptsdts and ptsdts == ord(buffer[9]) >> 4:
- if ord(buffer[9]) >> 4 != ptsdts:
- log.warning(u'WARNING: bad PTS/DTS, please contact us')
- return packet_length, None
-
- # timestamp = self.ReadPTS(buffer[9:14])
- high = ((ord(buffer[9]) & 0xF) >> 1)
- med = (ord(buffer[10]) << 7) + (ord(buffer[11]) >> 1)
- low = (ord(buffer[12]) << 7) + (ord(buffer[13]) >> 1)
- return packet_length, 9
-
- return packet_length, None
-
-
-
- def isPES(self, file):
- log.info(u'trying mpeg-pes scan')
- file.seek(0, 0)
- buffer = file.read(3)
-
- # header (also valid for all mpegs)
- if not buffer == '\x00\x00\x01':
- return 0
-
- self.sequence_header_offset = 0
- buffer += file.read(10000)
-
- offset = 0
- while offset + 1000 < len(buffer):
- pos, timestamp = self.ReadPESHeader(offset, buffer[offset:])
- if not pos:
- return 0
- if timestamp != None and not hasattr(self, 'start'):
- self.get_time = self.ReadPTS
- bpos = buffer[offset + timestamp:offset + timestamp + 5]
- self.start = self.get_time(bpos)
- if self.sequence_header_offset and hasattr(self, 'start'):
- # we have all informations we need
- break
-
- offset += pos
- if offset + 1000 < len(buffer) and len(buffer) < 1000000 or 1:
- # looks like a pes, read more
- buffer += file.read(10000)
-
- if not self.video and not self.audio:
- # no video and no audio?
- return 0
-
- self.type = 'MPEG-PES'
-
- # fill in values for support functions:
- self.__seek_size__ = 10000000 # 10 MB
- self.__sample_size__ = 500000 # 500 k scanning
- self.__search__ = self._find_timer_PES_
- self.filename = file.name
-
- # get length of the file
- self.length = self.get_length()
- return 1
-
-
- def _find_timer_PES_(self, buffer):
- """
- Return position of timer in buffer or -1 if not found.
- This function is valid for PES files
- """
- pos = buffer.find('\x00\x00\x01')
- offset = 0
- if pos == -1 or offset + 1000 >= len(buffer):
- return None
-
- retpos = -1
- ackcount = 0
- while offset + 1000 < len(buffer):
- pos, timestamp = self.ReadPESHeader(offset, buffer[offset:])
- if timestamp != None and retpos == -1:
- retpos = offset + timestamp
- if pos == 0:
- # Oops, that was a mpeg header, no PES header
- offset += buffer[offset:].find('\x00\x00\x01')
- retpos = -1
- ackcount = 0
- else:
- offset += pos
- if retpos != -1:
- ackcount += 1
- if ackcount > 10:
- # looks ok to me
- return retpos
- return None
-
-
- # Elementary Stream ===============================================
-
- def isES(self, file):
- file.seek(0, 0)
- try:
- header = struct.unpack('>LL', file.read(8))
- except (struct.error, IOError):
- return False
-
- if header[0] != 0x1B3:
- return False
-
- # Is an mpeg video elementary stream
-
- self.mime = 'video/mpeg'
- video = core.VideoStream()
- video.width = header[1] >> 20
- video.height = (header[1] >> 8) & 0xfff
- if header[1] & 0xf < len(FRAME_RATE):
- video.fps = FRAME_RATE[header[1] & 0xf]
- if (header[1] >> 4) & 0xf < len(ASPECT_RATIO):
- # FIXME: Empirically the aspect looks like PAR rather than DAR
- video.aspect = ASPECT_RATIO[(header[1] >> 4) & 0xf]
- self.video.append(video)
- return True
-
-
- # Transport Stream ===============================================
-
- def isTS(self, file):
- file.seek(0, 0)
-
- buffer = file.read(TS_PACKET_LENGTH * 2)
- c = 0
-
- while c + TS_PACKET_LENGTH < len(buffer):
- if ord(buffer[c]) == ord(buffer[c + TS_PACKET_LENGTH]) == TS_SYNC:
- break
- c += 1
- else:
- return 0
-
- buffer += file.read(10000)
- self.type = 'MPEG-TS'
-
- while c + TS_PACKET_LENGTH < len(buffer):
- start = ord(buffer[c + 1]) & 0x40
- # maybe load more into the buffer
- if c + 2 * TS_PACKET_LENGTH > len(buffer) and c < 500000:
- buffer += file.read(10000)
-
- # wait until the ts payload contains a payload header
- if not start:
- c += TS_PACKET_LENGTH
- continue
-
- tsid = ((ord(buffer[c + 1]) & 0x3F) << 8) + ord(buffer[c + 2])
- adapt = (ord(buffer[c + 3]) & 0x30) >> 4
-
- offset = 4
- if adapt & 0x02:
- # meta info present, skip it for now
- adapt_len = ord(buffer[c + offset])
- offset += adapt_len + 1
-
- if not ord(buffer[c + 1]) & 0x40:
- # no new pes or psi in stream payload starting
- pass
- elif adapt & 0x01:
- # PES
- timestamp = self.ReadPESHeader(c + offset, buffer[c + offset:],
- tsid)[1]
- if timestamp != None:
- if not hasattr(self, 'start'):
- self.get_time = self.ReadPTS
- timestamp = c + offset + timestamp
- self.start = self.get_time(buffer[timestamp:timestamp + 5])
- elif not hasattr(self, 'audio_ok'):
- timestamp = c + offset + timestamp
- start = self.get_time(buffer[timestamp:timestamp + 5])
- if start is not None and self.start is not None and \
- abs(start - self.start) < 10:
- # looks ok
- self.audio_ok = True
- else:
- # timestamp broken
- del self.start
- log.warning(u'Timestamp error, correcting')
-
- if hasattr(self, 'start') and self.start and \
- self.sequence_header_offset and self.video and self.audio:
- break
-
- c += TS_PACKET_LENGTH
-
-
- if not self.sequence_header_offset:
- return 0
-
- # fill in values for support functions:
- self.__seek_size__ = 10000000 # 10 MB
- self.__sample_size__ = 100000 # 100 k scanning
- self.__search__ = self._find_timer_TS_
- self.filename = file.name
-
- # get length of the file
- self.length = self.get_length()
- return 1
-
-
- def _find_timer_TS_(self, buffer):
- c = 0
-
- while c + TS_PACKET_LENGTH < len(buffer):
- if ord(buffer[c]) == ord(buffer[c + TS_PACKET_LENGTH]) == TS_SYNC:
- break
- c += 1
- else:
- return None
-
- while c + TS_PACKET_LENGTH < len(buffer):
- start = ord(buffer[c + 1]) & 0x40
- if not start:
- c += TS_PACKET_LENGTH
- continue
-
- tsid = ((ord(buffer[c + 1]) & 0x3F) << 8) + ord(buffer[c + 2])
- adapt = (ord(buffer[c + 3]) & 0x30) >> 4
-
- offset = 4
- if adapt & 0x02:
- # meta info present, skip it for now
- offset += ord(buffer[c + offset]) + 1
-
- if adapt & 0x01:
- timestamp = self.ReadPESHeader(c + offset, buffer[c + offset:], tsid)[1]
- if timestamp is None:
- # this should not happen
- log.error(u'bad TS')
- return None
- return c + offset + timestamp
- c += TS_PACKET_LENGTH
- return None
-
-
-
- # Support functions ==============================================
-
- def get_endpos(self):
- """
- get the last timestamp of the mpeg, return -1 if this is not possible
- """
- if not hasattr(self, 'filename') or not hasattr(self, 'start'):
- return None
-
- length = os.stat(self.filename)[stat.ST_SIZE]
- if length < self.__sample_size__:
- return
-
- file = open(self.filename)
- file.seek(length - self.__sample_size__)
- buffer = file.read(self.__sample_size__)
-
- end = None
- while 1:
- pos = self.__search__(buffer)
- if pos == None:
- break
- end = self.get_time(buffer[pos:]) or end
- buffer = buffer[pos + 100:]
-
- file.close()
- return end
-
-
- def get_length(self):
- """
- get the length in seconds, return -1 if this is not possible
- """
- end = self.get_endpos()
- if end == None or self.start == None:
- return None
- if self.start > end:
- return int(((long(1) << 33) - 1) / 90000) - self.start + end
- return end - self.start
-
-
- def seek(self, end_time):
- """
- Return the byte position in the file where the time position
- is 'pos' seconds. Return 0 if this is not possible
- """
- if not hasattr(self, 'filename') or not hasattr(self, 'start'):
- return 0
-
- file = open(self.filename)
- seek_to = 0
-
- while 1:
- file.seek(self.__seek_size__, 1)
- buffer = file.read(self.__sample_size__)
- if len(buffer) < 10000:
- break
- pos = self.__search__(buffer)
- if pos != None:
- # found something
- nt = self.get_time(buffer[pos:])
- if nt is not None and nt >= end_time:
- # too much, break
- break
- # that wasn't enough
- seek_to = file.tell()
-
- file.close()
- return seek_to
-
-
- def __scan__(self):
- """
- scan file for timestamps (may take a long time)
- """
- if not hasattr(self, 'filename') or not hasattr(self, 'start'):
- return 0
-
- file = open(self.filename)
- log.debug(u'scanning file...')
- while 1:
- file.seek(self.__seek_size__ * 10, 1)
- buffer = file.read(self.__sample_size__)
- if len(buffer) < 10000:
- break
- pos = self.__search__(buffer)
- if pos == None:
- continue
- log.debug(u'buffer position: %r' % self.get_time(buffer[pos:]))
-
- file.close()
- log.debug(u'done scanning file')
-
-
-Parser = MPEG
diff --git a/lib/enzyme/ogm.py b/lib/enzyme/ogm.py
deleted file mode 100644
index 4198be24af34b412886d03eaa2dd0fb5dfbd44c4..0000000000000000000000000000000000000000
--- a/lib/enzyme/ogm.py
+++ /dev/null
@@ -1,299 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
-# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-__all__ = ['Parser']
-
-import struct
-import re
-import stat
-import os
-import logging
-from exceptions import ParseError
-import core
-
-# get logging object
-log = logging.getLogger(__name__)
-
-PACKET_TYPE_HEADER = 0x01
-PACKED_TYPE_METADATA = 0x03
-PACKED_TYPE_SETUP = 0x05
-PACKET_TYPE_BITS = 0x07
-PACKET_IS_SYNCPOINT = 0x08
-
-#VORBIS_VIDEO_PACKET_INFO = 'video'
-
-STREAM_HEADER_VIDEO = '<4sIQQIIHII'
-STREAM_HEADER_AUDIO = '<4sIQQIIHHHI'
-
-VORBISCOMMENT = { 'TITLE': 'title',
- 'ALBUM': 'album',
- 'ARTIST': 'artist',
- 'COMMENT': 'comment',
- 'ENCODER': 'encoder',
- 'TRACKNUMBER': 'trackno',
- 'LANGUAGE': 'language',
- 'GENRE': 'genre',
- }
-
-# FIXME: check VORBISCOMMENT date and convert to timestamp
-# Deactived tag: 'DATE': 'date',
-
-MAXITERATIONS = 30
-
-class Ogm(core.AVContainer):
-
- table_mapping = { 'VORBISCOMMENT' : VORBISCOMMENT }
-
- def __init__(self, file):
- core.AVContainer.__init__(self)
- self.samplerate = 1
- self.all_streams = [] # used to add meta data to streams
- self.all_header = []
-
- for i in range(MAXITERATIONS):
- granule, nextlen = self._parseOGGS(file)
- if granule == None:
- if i == 0:
- # oops, bad file
- raise ParseError()
- break
- elif granule > 0:
- # ok, file started
- break
-
- # seek to the end of the stream, to avoid scanning the whole file
- if (os.stat(file.name)[stat.ST_SIZE] > 50000):
- file.seek(os.stat(file.name)[stat.ST_SIZE] - 49000)
-
- # read the rest of the file into a buffer
- h = file.read()
-
- # find last OggS to get length info
- if len(h) > 200:
- idx = h.find('OggS')
- pos = -49000 + idx
- if idx:
- file.seek(os.stat(file.name)[stat.ST_SIZE] + pos)
- while 1:
- granule, nextlen = self._parseOGGS(file)
- if not nextlen:
- break
-
- # Copy metadata to the streams
- if len(self.all_header) == len(self.all_streams):
- for i in range(len(self.all_header)):
-
- # get meta info
- for key in self.all_streams[i].keys():
- if self.all_header[i].has_key(key):
- self.all_streams[i][key] = self.all_header[i][key]
- del self.all_header[i][key]
- if self.all_header[i].has_key(key.upper()):
- asi = self.all_header[i][key.upper()]
- self.all_streams[i][key] = asi
- del self.all_header[i][key.upper()]
-
- # Chapter parser
- if self.all_header[i].has_key('CHAPTER01') and \
- not self.chapters:
- while 1:
- s = 'CHAPTER%02d' % (len(self.chapters) + 1)
- if self.all_header[i].has_key(s) and \
- self.all_header[i].has_key(s + 'NAME'):
- pos = self.all_header[i][s]
- try:
- pos = int(pos)
- except ValueError:
- new_pos = 0
- for v in pos.split(':'):
- new_pos = new_pos * 60 + float(v)
- pos = int(new_pos)
-
- c = self.all_header[i][s + 'NAME']
- c = core.Chapter(c, pos)
- del self.all_header[i][s + 'NAME']
- del self.all_header[i][s]
- self.chapters.append(c)
- else:
- break
-
- # If there are no video streams in this ogg container, it
- # must be an audio file. Raise an exception to cause the
- # factory to fall back to audio.ogg.
- if len(self.video) == 0:
- raise ParseError
-
- # Copy Metadata from tables into the main set of attributes
- for header in self.all_header:
- self._appendtable('VORBISCOMMENT', header)
-
-
- def _parseOGGS(self, file):
- h = file.read(27)
- if len(h) == 0:
- # Regular File end
- return None, None
- elif len(h) < 27:
- log.debug(u'%d Bytes of Garbage found after End.' % len(h))
- return None, None
- if h[:4] != "OggS":
- log.debug(u'Invalid Ogg')
- raise ParseError()
-
- version = ord(h[4])
- if version != 0:
- log.debug(u'Unsupported OGG/OGM Version %d' % version)
- return None, None
-
- head = struct.unpack('<BQIIIB', h[5:])
- headertype, granulepos, serial, pageseqno, checksum, \
- pageSegCount = head
-
- self.mime = 'application/ogm'
- self.type = 'OGG Media'
- tab = file.read(pageSegCount)
- nextlen = 0
- for i in range(len(tab)):
- nextlen += ord(tab[i])
- else:
- h = file.read(1)
- packettype = ord(h[0]) & PACKET_TYPE_BITS
- if packettype == PACKET_TYPE_HEADER:
- h += file.read(nextlen - 1)
- self._parseHeader(h, granulepos)
- elif packettype == PACKED_TYPE_METADATA:
- h += file.read(nextlen - 1)
- self._parseMeta(h)
- else:
- file.seek(nextlen - 1, 1)
- if len(self.all_streams) > serial:
- stream = self.all_streams[serial]
- if hasattr(stream, 'samplerate') and \
- stream.samplerate:
- stream.length = granulepos / stream.samplerate
- elif hasattr(stream, 'bitrate') and \
- stream.bitrate:
- stream.length = granulepos / stream.bitrate
-
- return granulepos, nextlen + 27 + pageSegCount
-
-
- def _parseMeta(self, h):
- flags = ord(h[0])
- headerlen = len(h)
- if headerlen >= 7 and h[1:7] == 'vorbis':
- header = {}
- nextlen, self.encoder = self._extractHeaderString(h[7:])
- numItems = struct.unpack('<I', h[7 + nextlen:7 + nextlen + 4])[0]
- start = 7 + 4 + nextlen
- for _ in range(numItems):
- (nextlen, s) = self._extractHeaderString(h[start:])
- start += nextlen
- if s:
- a = re.split('=', s)
- header[(a[0]).upper()] = a[1]
- # Put Header fields into info fields
- self.type = 'OGG Vorbis'
- self.subtype = ''
- self.all_header.append(header)
-
-
- def _parseHeader(self, header, granule):
- headerlen = len(header)
- flags = ord(header[0])
-
- if headerlen >= 30 and header[1:7] == 'vorbis':
- ai = core.AudioStream()
- ai.version, ai.channels, ai.samplerate, bitrate_max, ai.bitrate, \
- bitrate_min, blocksize, framing = \
- struct.unpack('<IBIiiiBB', header[7:7 + 23])
- ai.codec = 'Vorbis'
- #ai.granule = granule
- #ai.length = granule / ai.samplerate
- self.audio.append(ai)
- self.all_streams.append(ai)
-
- elif headerlen >= 7 and header[1:7] == 'theora':
- # Theora Header
- # XXX Finish Me
- vi = core.VideoStream()
- vi.codec = 'theora'
- self.video.append(vi)
- self.all_streams.append(vi)
-
- elif headerlen >= 142 and \
- header[1:36] == 'Direct Show Samples embedded in Ogg':
- # Old Directshow format
- # XXX Finish Me
- vi = core.VideoStream()
- vi.codec = 'dshow'
- self.video.append(vi)
- self.all_streams.append(vi)
-
- elif flags & PACKET_TYPE_BITS == PACKET_TYPE_HEADER and \
- headerlen >= struct.calcsize(STREAM_HEADER_VIDEO) + 1:
- # New Directshow Format
- htype = header[1:9]
-
- if htype[:5] == 'video':
- sh = header[9:struct.calcsize(STREAM_HEADER_VIDEO) + 9]
- streamheader = struct.unpack(STREAM_HEADER_VIDEO, sh)
- vi = core.VideoStream()
- (type, ssize, timeunit, samplerate, vi.length, buffersize, \
- vi.bitrate, vi.width, vi.height) = streamheader
-
- vi.width /= 65536
- vi.height /= 65536
- # XXX length, bitrate are very wrong
- vi.codec = type
- vi.fps = 10000000 / timeunit
- self.video.append(vi)
- self.all_streams.append(vi)
-
- elif htype[:5] == 'audio':
- sha = header[9:struct.calcsize(STREAM_HEADER_AUDIO) + 9]
- streamheader = struct.unpack(STREAM_HEADER_AUDIO, sha)
- ai = core.AudioStream()
- (type, ssize, timeunit, ai.samplerate, ai.length, buffersize, \
- ai.bitrate, ai.channels, bloc, ai.bitrate) = streamheader
- self.samplerate = ai.samplerate
- log.debug(u'Samplerate %d' % self.samplerate)
- self.audio.append(ai)
- self.all_streams.append(ai)
-
- elif htype[:4] == 'text':
- subtitle = core.Subtitle()
- # FIXME: add more info
- self.subtitles.append(subtitle)
- self.all_streams.append(subtitle)
-
- else:
- log.debug(u'Unknown Header')
-
-
- def _extractHeaderString(self, header):
- len = struct.unpack('<I', header[:4])[0]
- try:
- return (len + 4, unicode(header[4:4 + len], 'utf-8'))
- except (KeyError, IndexError, UnicodeDecodeError):
- return (len + 4, None)
-
-
-Parser = Ogm
diff --git a/lib/enzyme/parsers/__init__.py b/lib/enzyme/parsers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..40a96afc6ff09d58a702b76e3f7dd412fe975e26
--- /dev/null
+++ b/lib/enzyme/parsers/__init__.py
@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-
diff --git a/lib/enzyme/parsers/ebml/__init__.py b/lib/enzyme/parsers/ebml/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..04219f8c32e03ff4a12e17ad5e0bd0e5a0c52383
--- /dev/null
+++ b/lib/enzyme/parsers/ebml/__init__.py
@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+from .core import *
+from .readers import *
diff --git a/lib/enzyme/parsers/ebml/core.py b/lib/enzyme/parsers/ebml/core.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae025ac739c58a74bf7674afec47a6e94726dd41
--- /dev/null
+++ b/lib/enzyme/parsers/ebml/core.py
@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+from ...exceptions import ReadError
+from .readers import *
+from pkg_resources import resource_stream # @UnresolvedImport
+from xml.dom import minidom
+import logging
+
+
+__all__ = ['INTEGER', 'UINTEGER', 'FLOAT', 'STRING', 'UNICODE', 'DATE', 'MASTER', 'BINARY',
+ 'SPEC_TYPES', 'READERS', 'Element', 'MasterElement', 'parse', 'parse_element',
+ 'get_matroska_specs']
+logger = logging.getLogger(__name__)
+
+
+# EBML types
+INTEGER, UINTEGER, FLOAT, STRING, UNICODE, DATE, MASTER, BINARY = range(8)
+
+# Spec types to EBML types mapping
+SPEC_TYPES = {
+ 'integer': INTEGER,
+ 'uinteger': UINTEGER,
+ 'float': FLOAT,
+ 'string': STRING,
+ 'utf-8': UNICODE,
+ 'date': DATE,
+ 'master': MASTER,
+ 'binary': BINARY
+}
+
+# Readers to use per EBML type
+READERS = {
+ INTEGER: read_element_integer,
+ UINTEGER: read_element_uinteger,
+ FLOAT: read_element_float,
+ STRING: read_element_string,
+ UNICODE: read_element_unicode,
+ DATE: read_element_date,
+ BINARY: read_element_binary
+}
+
+
+class Element(object):
+ """Base object of EBML
+
+ :param int id: id of the element, best represented as hexadecimal (0x18538067 for Matroska Segment element)
+ :param type: type of the element
+ :type type: :data:`INTEGER`, :data:`UINTEGER`, :data:`FLOAT`, :data:`STRING`, :data:`UNICODE`, :data:`DATE`, :data:`MASTER` or :data:`BINARY`
+ :param string name: name of the element
+ :param int level: level of the element
+ :param int position: position of element's data
+ :param int size: size of element's data
+ :param data: data as read by the corresponding :data:`READERS`
+
+ """
+ def __init__(self, id=None, type=None, name=None, level=None, position=None, size=None, data=None): # @ReservedAssignment
+ self.id = id
+ self.type = type
+ self.name = name
+ self.level = level
+ self.position = position
+ self.size = size
+ self.data = data
+
+ def __repr__(self):
+ return '<%s [%s, %r]>' % (self.__class__.__name__, self.name, self.data)
+
+
+class MasterElement(Element):
+ """Element of type :data:`MASTER` that has a list of :class:`Element` as its data
+
+ :param int id: id of the element, best represented as hexadecimal (0x18538067 for Matroska Segment element)
+ :param string name: name of the element
+ :param int level: level of the element
+ :param int position: position of element's data
+ :param int size: size of element's data
+ :param data: child elements
+ :type data: list of :class:`Element`
+
+ :class:`MasterElement` implements some magic methods to ease manipulation. Thus, a MasterElement supports
+ the `in` keyword to test for the presence of a child element by its name and gives access to it
+ with a container getter::
+
+ >>> ebml_element = parse(open('test1.mkv', 'rb'), get_matroska_specs())[0]
+ >>> 'EBMLVersion' in ebml_element
+ False
+ >>> 'DocType' in ebml_element
+ True
+ >>> ebml_element['DocType']
+ Element(DocType, u'matroska')
+
+ """
+ def __init__(self, id=None, name=None, level=None, position=None, size=None, data=None): # @ReservedAssignment
+ super(MasterElement, self).__init__(id, MASTER, name, level, position, size, data)
+
+ def load(self, stream, specs, ignore_element_types=None, ignore_element_names=None, max_level=None):
+ """Load children :class:`Elements <Element>` with level lower or equal to the `max_level`
+ from the `stream` according to the `specs`
+
+ :param stream: file-like object from which to read
+ :param dict specs: see :ref:`specs`
+ :param int max_level: maximum level for children elements
+ :param list ignore_element_types: list of element types to ignore
+ :param list ignore_element_names: list of element names to ignore
+ :param int max_level: maximum level of elements
+
+ """
+ self.data = parse(stream, specs, self.size, ignore_element_types, ignore_element_names, max_level)
+
+ def get(self, name, default=None):
+ """Convenience method for ``master_element[name].data if name in master_element else default``
+
+ :param string name: the name of the child to get
+ :param default: default value if `name` is not in the :class:`MasterElement`
+ :return: the data of the child :class:`Element` or `default`
+
+ """
+ if name not in self:
+ return default
+ element = self[name]
+ if element.type == MASTER:
+ raise ValueError('%s is a MasterElement' % name)
+ return element.data
+
+ def __getitem__(self, key):
+ if isinstance(key, int):
+ return self.data[key]
+ children = [e for e in self.data if e.name == key]
+ if not children:
+ raise KeyError(key)
+ if len(children) > 1:
+ raise KeyError('More than 1 child with key %s (%d)' % (key, len(children)))
+ return children[0]
+
+ def __contains__(self, item):
+ return len([e for e in self.data if e.name == item]) > 0
+
+ def __iter__(self):
+ return iter(self.data)
+
+
+def parse(stream, specs, size=None, ignore_element_types=None, ignore_element_names=None, max_level=None):
+ """Parse a stream for `size` bytes according to the `specs`
+
+ :param stream: file-like object from which to read
+ :param size: maximum number of bytes to read, None to read all the stream
+ :type size: int or None
+ :param dict specs: see :ref:`specs`
+ :param list ignore_element_types: list of element types to ignore
+ :param list ignore_element_names: list of element names to ignore
+ :param int max_level: maximum level of elements
+ :return: parsed data as a tree of :class:`~enzyme.parsers.ebml.core.Element`
+ :rtype: list
+
+ .. note::
+ If `size` is reached in a middle of an element, reading will continue
+ until the element is fully parsed.
+
+ """
+ ignore_element_types = ignore_element_types if ignore_element_types is not None else []
+ ignore_element_names = ignore_element_names if ignore_element_names is not None else []
+ start = stream.tell()
+ elements = []
+ while size is None or stream.tell() - start < size:
+ try:
+ element = parse_element(stream, specs)
+ if element is None:
+ continue
+ logger.debug('%s %s parsed', element.__class__.__name__, element.name)
+ if element.type in ignore_element_types or element.name in ignore_element_names:
+ logger.info('%s %s ignored', element.__class__.__name__, element.name)
+ if element.type == MASTER:
+ stream.seek(element.size, 1)
+ continue
+ if element.type == MASTER:
+ if max_level is not None and element.level >= max_level:
+ logger.info('Maximum level %d reached for children of %s %s', max_level, element.__class__.__name__, element.name)
+ stream.seek(element.size, 1)
+ else:
+ logger.debug('Loading child elements for %s %s with size %d', element.__class__.__name__, element.name, element.size)
+ element.data = parse(stream, specs, element.size, ignore_element_types, ignore_element_names, max_level)
+ elements.append(element)
+ except ReadError:
+ if size is not None:
+ raise
+ break
+ return elements
+
+
+def parse_element(stream, specs, load_children=False, ignore_element_types=None, ignore_element_names=None, max_level=None):
+ """Extract a single :class:`Element` from the `stream` according to the `specs`
+
+ :param stream: file-like object from which to read
+ :param dict specs: see :ref:`specs`
+ :param bool load_children: load children elements if the parsed element is a :class:`MasterElement`
+ :param list ignore_element_types: list of element types to ignore
+ :param list ignore_element_names: list of element names to ignore
+ :param int max_level: maximum level for children elements
+ :return: the parsed element
+ :rtype: :class:`Element`
+
+ """
+ ignore_element_types = ignore_element_types if ignore_element_types is not None else []
+ ignore_element_names = ignore_element_names if ignore_element_names is not None else []
+ element_id = read_element_id(stream)
+ if element_id is None:
+ raise ReadError('Cannot read element id')
+ element_size = read_element_size(stream)
+ if element_size is None:
+ raise ReadError('Cannot read element size')
+ if element_id not in specs:
+ logger.error('Element with id 0x%x is not in the specs' % element_id)
+ stream.seek(element_size, 1)
+ return None
+ element_type, element_name, element_level = specs[element_id]
+ if element_type == MASTER:
+ element = MasterElement(element_id, element_name, element_level, stream.tell(), element_size)
+ if load_children:
+ element.data = parse(stream, specs, element.size, ignore_element_types, ignore_element_names, max_level)
+ else:
+ element = Element(element_id, element_type, element_name, element_level, stream.tell(), element_size)
+ element.data = READERS[element_type](stream, element_size)
+ return element
+
+
+def get_matroska_specs(webm_only=False):
+ """Get the Matroska specs
+
+ :param bool webm_only: load *only* WebM specs
+ :return: the specs in the appropriate format. See :ref:`specs`
+ :rtype: dict
+
+ """
+ specs = {}
+ with resource_stream(__name__, 'specs/matroska.xml') as resource:
+ xmldoc = minidom.parse(resource)
+ for element in xmldoc.getElementsByTagName('element'):
+ if not webm_only or element.hasAttribute('webm') and element.getAttribute('webm') == '1':
+ specs[int(element.getAttribute('id'), 16)] = (SPEC_TYPES[element.getAttribute('type')], element.getAttribute('name'), int(element.getAttribute('level')))
+ return specs
diff --git a/lib/enzyme/parsers/ebml/readers.py b/lib/enzyme/parsers/ebml/readers.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c9709b77ddd86acd6e99f7a7e804edc5a60a8ce
--- /dev/null
+++ b/lib/enzyme/parsers/ebml/readers.py
@@ -0,0 +1,235 @@
+# -*- coding: utf-8 -*-
+from ...compat import bytes
+from ...exceptions import ReadError, SizeError
+from datetime import datetime, timedelta
+from io import BytesIO
+from struct import unpack
+
+
+__all__ = ['read_element_id', 'read_element_size', 'read_element_integer', 'read_element_uinteger',
+ 'read_element_float', 'read_element_string', 'read_element_unicode', 'read_element_date',
+ 'read_element_binary']
+
+
+def _read(stream, size):
+ """Read the `stream` for *exactly* `size` bytes and raise an exception if
+ less than `size` bytes are actually read
+
+ :param stream: file-like object from which to read
+ :param int size: number of bytes to read
+ :raise ReadError: when less than `size` bytes are actually read
+ :return: read data from the `stream`
+ :rtype: bytes
+
+ """
+ data = stream.read(size)
+ if len(data) < size:
+ raise ReadError('Less than %d bytes read (%d)' % (size, len(data)))
+ return data
+
+
+def read_element_id(stream):
+ """Read the Element ID
+
+ :param stream: file-like object from which to read
+ :raise ReadError: when not all the required bytes could be read
+ :return: the id of the element
+ :rtype: int
+
+ """
+ char = _read(stream, 1)
+ byte = ord(char)
+ if byte & 0x80:
+ return byte
+ elif byte & 0x40:
+ return unpack('>H', char + _read(stream, 1))[0]
+ elif byte & 0x20:
+ b, h = unpack('>BH', char + _read(stream, 2))
+ return b * 2 ** 16 + h
+ elif byte & 0x10:
+ return unpack('>L', char + _read(stream, 3))[0]
+ else:
+ ValueError('Not an Element ID')
+
+
+def read_element_size(stream):
+ """Read the Element Size
+
+ :param stream: file-like object from which to read
+ :raise ReadError: when not all the required bytes could be read
+ :return: the size of element's data
+ :rtype: int
+
+ """
+ char = _read(stream, 1)
+ byte = ord(char)
+ if byte & 0x80:
+ return unpack('>B', bytes((byte ^ 0x80,)))[0]
+ elif byte & 0x40:
+ return unpack('>H', bytes((byte ^ 0x40,)) + _read(stream, 1))[0]
+ elif byte & 0x20:
+ b, h = unpack('>BH', bytes((byte ^ 0x20,)) + _read(stream, 2))
+ return b * 2 ** 16 + h
+ elif byte & 0x10:
+ return unpack('>L', bytes((byte ^ 0x10,)) + _read(stream, 3))[0]
+ elif byte & 0x08:
+ b, l = unpack('>BL', bytes((byte ^ 0x08,)) + _read(stream, 4))
+ return b * 2 ** 32 + l
+ elif byte & 0x04:
+ h, l = unpack('>HL', bytes((byte ^ 0x04,)) + _read(stream, 5))
+ return h * 2 ** 32 + l
+ elif byte & 0x02:
+ b, h, l = unpack('>BHL', bytes((byte ^ 0x02,)) + _read(stream, 6))
+ return b * 2 ** 48 + h * 2 ** 32 + l
+ elif byte & 0x01:
+ return unpack('>Q', bytes((byte ^ 0x01,)) + _read(stream, 7))[0]
+ else:
+ ValueError('Not an Element Size')
+
+
+def read_element_integer(stream, size):
+ """Read the Element Data of type :data:`INTEGER`
+
+ :param stream: file-like object from which to read
+ :param int size: size of element's data
+ :raise ReadError: when not all the required bytes could be read
+ :raise SizeError: if size is incorrect
+ :return: the read integer
+ :rtype: int
+
+ """
+ if size == 1:
+ return unpack('>b', _read(stream, 1))[0]
+ elif size == 2:
+ return unpack('>h', _read(stream, 2))[0]
+ elif size == 3:
+ b, h = unpack('>bH', _read(stream, 3))
+ return b * 2 ** 16 + h
+ elif size == 4:
+ return unpack('>l', _read(stream, 4))[0]
+ elif size == 5:
+ b, l = unpack('>bL', _read(stream, 5))
+ return b * 2 ** 32 + l
+ elif size == 6:
+ h, l = unpack('>hL', _read(stream, 6))
+ return h * 2 ** 32 + l
+ elif size == 7:
+ b, h, l = unpack('>bHL', _read(stream, 7))
+ return b * 2 ** 48 + h * 2 ** 32 + l
+ elif size == 8:
+ return unpack('>q', _read(stream, 8))[0]
+ else:
+ raise SizeError(size)
+
+
+def read_element_uinteger(stream, size):
+ """Read the Element Data of type :data:`UINTEGER`
+
+ :param stream: file-like object from which to read
+ :param int size: size of element's data
+ :raise ReadError: when not all the required bytes could be read
+ :raise SizeError: if size is incorrect
+ :return: the read unsigned integer
+ :rtype: int
+
+ """
+ if size == 1:
+ return unpack('>B', _read(stream, 1))[0]
+ elif size == 2:
+ return unpack('>H', _read(stream, 2))[0]
+ elif size == 3:
+ b, h = unpack('>BH', _read(stream, 3))
+ return b * 2 ** 16 + h
+ elif size == 4:
+ return unpack('>L', _read(stream, 4))[0]
+ elif size == 5:
+ b, l = unpack('>BL', _read(stream, 5))
+ return b * 2 ** 32 + l
+ elif size == 6:
+ h, l = unpack('>HL', _read(stream, 6))
+ return h * 2 ** 32 + l
+ elif size == 7:
+ b, h, l = unpack('>BHL', _read(stream, 7))
+ return b * 2 ** 48 + h * 2 ** 32 + l
+ elif size == 8:
+ return unpack('>Q', _read(stream, 8))[0]
+ else:
+ raise SizeError(size)
+
+
+def read_element_float(stream, size):
+ """Read the Element Data of type :data:`FLOAT`
+
+ :param stream: file-like object from which to read
+ :param int size: size of element's data
+ :raise ReadError: when not all the required bytes could be read
+ :raise SizeError: if size is incorrect
+ :return: the read float
+ :rtype: float
+
+ """
+ if size == 4:
+ return unpack('>f', _read(stream, 4))[0]
+ elif size == 8:
+ return unpack('>d', _read(stream, 8))[0]
+ else:
+ raise SizeError(size)
+
+
+def read_element_string(stream, size):
+ """Read the Element Data of type :data:`STRING`
+
+ :param stream: file-like object from which to read
+ :param int size: size of element's data
+ :raise ReadError: when not all the required bytes could be read
+ :raise SizeError: if size is incorrect
+ :return: the read ascii-decoded string
+ :rtype: unicode
+
+ """
+ return _read(stream, size).decode('ascii')
+
+
+def read_element_unicode(stream, size):
+ """Read the Element Data of type :data:`UNICODE`
+
+ :param stream: file-like object from which to read
+ :param int size: size of element's data
+ :raise ReadError: when not all the required bytes could be read
+ :raise SizeError: if size is incorrect
+ :return: the read utf-8-decoded string
+ :rtype: unicode
+
+ """
+ return _read(stream, size).decode('utf-8')
+
+
+def read_element_date(stream, size):
+ """Read the Element Data of type :data:`DATE`
+
+ :param stream: file-like object from which to read
+ :param int size: size of element's data
+ :raise ReadError: when not all the required bytes could be read
+ :raise SizeError: if size is incorrect
+ :return: the read date
+ :rtype: datetime
+
+ """
+ if size != 8:
+ raise SizeError(size)
+ nanoseconds = unpack('>q', _read(stream, 8))[0]
+ return datetime(2001, 1, 1, 0, 0, 0, 0, None) + timedelta(microseconds=nanoseconds // 1000)
+
+
+def read_element_binary(stream, size):
+ """Read the Element Data of type :data:`BINARY`
+
+ :param stream: file-like object from which to read
+ :param int size: size of element's data
+ :raise ReadError: when not all the required bytes could be read
+ :raise SizeError: if size is incorrect
+ :return: raw binary data
+ :rtype: bytes
+
+ """
+ return BytesIO(stream.read(size))
diff --git a/lib/enzyme/parsers/ebml/specs/matroska.xml b/lib/enzyme/parsers/ebml/specs/matroska.xml
new file mode 100644
index 0000000000000000000000000000000000000000..4f3f79fcb80d8d9c8af75dbba48b834e3795f11e
--- /dev/null
+++ b/lib/enzyme/parsers/ebml/specs/matroska.xml
@@ -0,0 +1,224 @@
+<?xml version="1.0" encoding="utf-8"?>
+<table>
+ <element name="EBML" level="0" id="0x1A45DFA3" type="master" mandatory="1" multiple="1" minver="1">Set the EBML characteristics of the data to follow. Each EBML document has to start with this.</element>
+ <element name="EBMLVersion" level="1" id="0x4286" type="uinteger" mandatory="1" default="1" minver="1">The version of EBML parser used to create the file.</element>
+ <element name="EBMLReadVersion" level="1" id="0x42F7" type="uinteger" mandatory="1" default="1" minver="1">The minimum EBML version a parser has to support to read this file.</element>
+ <element name="EBMLMaxIDLength" level="1" id="0x42F2" type="uinteger" mandatory="1" default="4" minver="1">The maximum length of the IDs you'll find in this file (4 or less in Matroska).</element>
+ <element name="EBMLMaxSizeLength" level="1" id="0x42F3" type="uinteger" mandatory="1" default="8" minver="1">The maximum length of the sizes you'll find in this file (8 or less in Matroska). This does not override the element size indicated at the beginning of an element. Elements that have an indicated size which is larger than what is allowed by EBMLMaxSizeLength shall be considered invalid.</element>
+ <element name="DocType" level="1" id="0x4282" type="string" mandatory="1" default="matroska" minver="1">A string that describes the type of document that follows this EBML header. 'matroska' in our case or 'webm' for webm files.</element>
+ <element name="DocTypeVersion" level="1" id="0x4287" type="uinteger" mandatory="1" default="1" minver="1">The version of DocType interpreter used to create the file.</element>
+ <element name="DocTypeReadVersion" level="1" id="0x4285" type="uinteger" mandatory="1" default="1" minver="1">The minimum DocType version an interpreter has to support to read this file.</element>
+ <element name="Void" level="-1" id="0xEC" type="binary" minver="1">Used to void damaged data, to avoid unexpected behaviors when using damaged data. The content is discarded. Also used to reserve space in a sub-element for later use.</element>
+ <element name="CRC-32" level="-1" id="0xBF" type="binary" minver="1" webm="0">The CRC is computed on all the data of the Master element it's in. The CRC element should be the first in it's parent master for easier reading. All level 1 elements should include a CRC-32. The CRC in use is the IEEE CRC32 Little Endian</element>
+ <element name="SignatureSlot" level="-1" id="0x1B538667" type="master" multiple="1" webm="0">Contain signature of some (coming) elements in the stream.</element>
+ <element name="SignatureAlgo" level="1" id="0x7E8A" type="uinteger" webm="0">Signature algorithm used (1=RSA, 2=elliptic).</element>
+ <element name="SignatureHash" level="1" id="0x7E9A" type="uinteger" webm="0">Hash algorithm used (1=SHA1-160, 2=MD5).</element>
+ <element name="SignaturePublicKey" level="1" id="0x7EA5" type="binary" webm="0">The public key to use with the algorithm (in the case of a PKI-based signature).</element>
+ <element name="Signature" level="1" id="0x7EB5" type="binary" webm="0">The signature of the data (until a new.</element>
+ <element name="SignatureElements" level="1" id="0x7E5B" type="master" webm="0">Contains elements that will be used to compute the signature.</element>
+ <element name="SignatureElementList" level="2" id="0x7E7B" type="master" multiple="1" webm="0">A list consists of a number of consecutive elements that represent one case where data is used in signature. Ex: <i>Cluster|Block|BlockAdditional</i> means that the BlockAdditional of all Blocks in all Clusters is used for encryption.</element>
+ <element name="SignedElement" level="3" id="0x6532" type="binary" multiple="1" webm="0">An element ID whose data will be used to compute the signature.</element>
+ <element name="Segment" level="0" id="0x18538067" type="master" mandatory="1" multiple="1" minver="1">This element contains all other top-level (level 1) elements. Typically a Matroska file is composed of 1 segment.</element>
+ <element name="SeekHead" cppname="SeekHeader" level="1" id="0x114D9B74" type="master" multiple="1" minver="1">Contains the <a href="http://www.matroska.org/technical/specs/notes.html#Position_References">position</a> of other level 1 elements.</element>
+ <element name="Seek" cppname="SeekPoint" level="2" id="0x4DBB" type="master" mandatory="1" multiple="1" minver="1">Contains a single seek entry to an EBML element.</element>
+ <element name="SeekID" level="3" id="0x53AB" type="binary" mandatory="1" minver="1">The binary ID corresponding to the element name.</element>
+ <element name="SeekPosition" level="3" id="0x53AC" type="uinteger" mandatory="1" minver="1">The <a href="http://www.matroska.org/technical/specs/notes.html#Position_References">position</a> of the element in the segment in octets (0 = first level 1 element).</element>
+ <element name="Info" level="1" id="0x1549A966" type="master" mandatory="1" multiple="1" minver="1">Contains miscellaneous general information and statistics on the file.</element>
+ <element name="SegmentUID" level="2" id="0x73A4" type="binary" minver="1" webm="0" range="not 0" bytesize="16">A randomly generated unique ID to identify the current segment between many others (128 bits).</element>
+ <element name="SegmentFilename" level="2" id="0x7384" type="utf-8" minver="1" webm="0">A filename corresponding to this segment.</element>
+ <element name="PrevUID" level="2" id="0x3CB923" type="binary" minver="1" webm="0" bytesize="16">A unique ID to identify the previous chained segment (128 bits).</element>
+ <element name="PrevFilename" level="2" id="0x3C83AB" type="utf-8" minver="1" webm="0">An escaped filename corresponding to the previous segment.</element>
+ <element name="NextUID" level="2" id="0x3EB923" type="binary" minver="1" webm="0" bytesize="16">A unique ID to identify the next chained segment (128 bits).</element>
+ <element name="NextFilename" level="2" id="0x3E83BB" type="utf-8" minver="1" webm="0">An escaped filename corresponding to the next segment.</element>
+ <element name="SegmentFamily" level="2" id="0x4444" type="binary" multiple="1" minver="1" webm="0" bytesize="16">A randomly generated unique ID that all segments related to each other must use (128 bits).</element>
+ <element name="ChapterTranslate" level="2" id="0x6924" type="master" multiple="1" minver="1" webm="0">A tuple of corresponding ID used by chapter codecs to represent this segment.</element>
+ <element name="ChapterTranslateEditionUID" level="3" id="0x69FC" type="uinteger" multiple="1" minver="1" webm="0">Specify an edition UID on which this correspondance applies. When not specified, it means for all editions found in the segment.</element>
+ <element name="ChapterTranslateCodec" level="3" id="0x69BF" type="uinteger" mandatory="1" minver="1" webm="0">The <a href="http://www.matroska.org/technical/specs/index.html#ChapProcessCodecID">chapter codec</a> using this ID (0: Matroska Script, 1: DVD-menu).</element>
+ <element name="ChapterTranslateID" level="3" id="0x69A5" type="binary" mandatory="1" minver="1" webm="0">The binary value used to represent this segment in the chapter codec data. The format depends on the <a href="http://www.matroska.org/technical/specs/chapters/index.html#ChapProcessCodecID">ChapProcessCodecID</a> used.</element>
+ <element name="TimecodeScale" level="2" id="0x2AD7B1" type="uinteger" mandatory="1" minver="1" default="1000000">Timecode scale in nanoseconds (1.000.000 means all timecodes in the segment are expressed in milliseconds).</element>
+ <!-- <element name="TimecodeScaleDenominator" level="2" id="0x2AD7B2" type="uinteger" mandatory="1" minver="4" default="1000000000">Timecode scale numerator, see <a href="http://www.matroska.org/technical/specs/index.html#TimecodeScale">TimecodeScale</a>.</element>
+ TimecodeScale When combined with <a href="http://www.matroska.org/technical/specs/index.html#TimecodeScaleDenominator">TimecodeScaleDenominator</a> the Timecode scale is given by the fraction TimecodeScale/TimecodeScaleDenominator in seconds.-->
+ <element name="Duration" level="2" id="0x4489" type="float" minver="1" range="> 0">Duration of the segment (based on TimecodeScale).</element>
+ <element name="DateUTC" level="2" id="0x4461" type="date" minver="1">Date of the origin of timecode (value 0), i.e. production date.</element>
+ <element name="Title" level="2" id="0x7BA9" type="utf-8" minver="1" webm="0">General name of the segment.</element>
+ <element name="MuxingApp" level="2" id="0x4D80" type="utf-8" mandatory="1" minver="1">Muxing application or library ("libmatroska-0.4.3").</element>
+ <element name="WritingApp" level="2" id="0x5741" type="utf-8" mandatory="1" minver="1">Writing application ("mkvmerge-0.3.3").</element>
+ <element name="Cluster" level="1" id="0x1F43B675" type="master" multiple="1" minver="1">The lower level element containing the (monolithic) Block structure.</element>
+ <element name="Timecode" cppname="ClusterTimecode" level="2" id="0xE7" type="uinteger" mandatory="1" minver="1">Absolute timecode of the cluster (based on TimecodeScale).</element>
+ <element name="SilentTracks" cppname="ClusterSilentTracks" level="2" id="0x5854" type="master" minver="1" webm="0">The list of tracks that are not used in that part of the stream. It is useful when using overlay tracks on seeking. Then you should decide what track to use.</element>
+ <element name="SilentTrackNumber" cppname="ClusterSilentTrackNumber" level="3" id="0x58D7" type="uinteger" multiple="1" minver="1" webm="0">One of the track number that are not used from now on in the stream. It could change later if not specified as silent in a further Cluster.</element>
+ <element name="Position" cppname="ClusterPosition" level="2" id="0xA7" type="uinteger" minver="1" webm="0">The <a href="http://www.matroska.org/technical/specs/notes.html#Position_References">Position</a> of the Cluster in the segment (0 in live broadcast streams). It might help to resynchronise offset on damaged streams.</element>
+ <element name="PrevSize" cppname="ClusterPrevSize" level="2" id="0xAB" type="uinteger" minver="1">Size of the previous Cluster, in octets. Can be useful for backward playing.</element>
+ <element name="SimpleBlock" level="2" id="0xA3" type="binary" multiple="1" minver="2" webm="1" divx="1">Similar to <a href="http://www.matroska.org/technical/specs/index.html#Block">Block</a> but without all the extra information, mostly used to reduced overhead when no extra feature is needed. (see <a href="http://www.matroska.org/technical/specs/index.html#simpleblock_structure">SimpleBlock Structure</a>)</element>
+ <element name="BlockGroup" level="2" id="0xA0" type="master" multiple="1" minver="1">Basic container of information containing a single Block or BlockVirtual, and information specific to that Block/VirtualBlock.</element>
+ <element name="Block" level="3" id="0xA1" type="binary" mandatory="1" minver="1">Block containing the actual data to be rendered and a timecode relative to the Cluster Timecode. (see <a href="http://www.matroska.org/technical/specs/index.html#block_structure">Block Structure</a>)</element>
+ <element name="BlockVirtual" level="3" id="0xA2" type="binary" webm="0">A Block with no data. It must be stored in the stream at the place the real Block should be in display order. (see <a href="http://www.matroska.org/technical/specs/index.html#block_virtual">Block Virtual</a>)</element>
+ <element name="BlockAdditions" level="3" id="0x75A1" type="master" minver="1" webm="0">Contain additional blocks to complete the main one. An EBML parser that has no knowledge of the Block structure could still see and use/skip these data.</element>
+ <element name="BlockMore" level="4" id="0xA6" type="master" mandatory="1" multiple="1" minver="1" webm="0">Contain the BlockAdditional and some parameters.</element>
+ <element name="BlockAddID" level="5" id="0xEE" type="uinteger" mandatory="1" minver="1" webm="0" default="1" range="not 0">An ID to identify the BlockAdditional level.</element>
+ <element name="BlockAdditional" level="5" id="0xA5" type="binary" mandatory="1" minver="1" webm="0">Interpreted by the codec as it wishes (using the BlockAddID).</element>
+ <element name="BlockDuration" level="3" id="0x9B" type="uinteger" minver="1" default="TrackDuration">The duration of the Block (based on TimecodeScale). This element is mandatory when DefaultDuration is set for the track (but can be omitted as other default values). When not written and with no DefaultDuration, the value is assumed to be the difference between the timecode of this Block and the timecode of the next Block in "display" order (not coding order). This element can be useful at the end of a Track (as there is not other Block available), or when there is a break in a track like for subtitle tracks. When set to 0 that means the frame is not a keyframe.</element>
+ <element name="ReferencePriority" cppname="FlagReferenced" level="3" id="0xFA" type="uinteger" mandatory="1" minver="1" webm="0" default="0">This frame is referenced and has the specified cache priority. In cache only a frame of the same or higher priority can replace this frame. A value of 0 means the frame is not referenced.</element>
+ <element name="ReferenceBlock" level="3" id="0xFB" type="integer" multiple="1" minver="1">Timecode of another frame used as a reference (ie: B or P frame). The timecode is relative to the block it's attached to.</element>
+ <element name="ReferenceVirtual" level="3" id="0xFD" type="integer" webm="0">Relative <a href="http://www.matroska.org/technical/specs/notes.html#Position_References">position</a> of the data that should be in position of the virtual block.</element>
+ <element name="CodecState" level="3" id="0xA4" type="binary" minver="2" webm="0">The new codec state to use. Data interpretation is private to the codec. This information should always be referenced by a seek entry.</element>
+ <element name="Slices" level="3" id="0x8E" type="master" minver="1" divx="0">Contains slices description.</element>
+ <element name="TimeSlice" level="4" id="0xE8" type="master" multiple="1" minver="1" divx="0">Contains extra time information about the data contained in the Block. While there are a few files in the wild with this element, it is no longer in use and has been deprecated. Being able to interpret this element is not required for playback.</element>
+ <element name="LaceNumber" cppname="SliceLaceNumber" level="5" id="0xCC" type="uinteger" minver="1" default="0" divx="0">The reverse number of the frame in the lace (0 is the last frame, 1 is the next to last, etc). While there are a few files in the wild with this element, it is no longer in use and has been deprecated. Being able to interpret this element is not required for playback.</element>
+ <element name="FrameNumber" cppname="SliceFrameNumber" level="5" id="0xCD" type="uinteger" default="0">The number of the frame to generate from this lace with this delay (allow you to generate many frames from the same Block/Frame).</element>
+ <element name="BlockAdditionID" cppname="SliceBlockAddID" level="5" id="0xCB" type="uinteger" default="0">The ID of the BlockAdditional element (0 is the main Block).</element>
+ <element name="Delay" cppname="SliceDelay" level="5" id="0xCE" type="uinteger" default="0">The (scaled) delay to apply to the element.</element>
+ <element name="SliceDuration" level="5" id="0xCF" type="uinteger" default="0">The (scaled) duration to apply to the element.</element>
+ <element name="ReferenceFrame" level="3" id="0xC8" type="master" multiple="0" minver="0" webm="0" divx="1"><a href="http://developer.divx.com/docs/divx_plus_hd/format_features/Smooth_FF_RW">DivX trick track extenstions</a></element>
+ <element name="ReferenceOffset" level="4" id="0xC9" type="uinteger" multiple="0" mandatory="1" minver="0" webm="0" divx="1"><a href="http://developer.divx.com/docs/divx_plus_hd/format_features/Smooth_FF_RW">DivX trick track extenstions</a></element>
+ <element name="ReferenceTimeCode" level="4" id="0xCA" type="uinteger" multiple="0" mandatory="1" minver="0" webm="0" divx="1"><a href="http://developer.divx.com/docs/divx_plus_hd/format_features/Smooth_FF_RW">DivX trick track extenstions</a></element>
+ <element name="EncryptedBlock" level="2" id="0xAF" type="binary" multiple="1" webm="0">Similar to <a href="http://www.matroska.org/technical/specs/index.html#SimpleBlock">SimpleBlock</a> but the data inside the Block are Transformed (encrypt and/or signed). (see <a href="http://www.matroska.org/technical/specs/index.html#encryptedblock_structure">EncryptedBlock Structure</a>)</element>
+ <element name="Tracks" level="1" id="0x1654AE6B" type="master" multiple="1" minver="1">A top-level block of information with many tracks described.</element>
+ <element name="TrackEntry" level="2" id="0xAE" type="master" mandatory="1" multiple="1" minver="1">Describes a track with all elements.</element>
+ <element name="TrackNumber" level="3" id="0xD7" type="uinteger" mandatory="1" minver="1" range="not 0">The track number as used in the Block Header (using more than 127 tracks is not encouraged, though the design allows an unlimited number).</element>
+ <element name="TrackUID" level="3" id="0x73C5" type="uinteger" mandatory="1" minver="1" range="not 0">A unique ID to identify the Track. This should be kept the same when making a direct stream copy of the Track to another file.</element>
+ <element name="TrackType" level="3" id="0x83" type="uinteger" mandatory="1" minver="1" range="1-254">A set of track types coded on 8 bits (1: video, 2: audio, 3: complex, 0x10: logo, 0x11: subtitle, 0x12: buttons, 0x20: control).</element>
+ <element name="FlagEnabled" cppname="TrackFlagEnabled" level="3" id="0xB9" type="uinteger" mandatory="1" minver="2" webm="1" default="1" range="0-1">Set if the track is usable. (1 bit)</element>
+ <element name="FlagDefault" cppname="TrackFlagDefault" level="3" id="0x88" type="uinteger" mandatory="1" minver="1" default="1" range="0-1">Set if that track (audio, video or subs) SHOULD be active if no language found matches the user preference. (1 bit)</element>
+ <element name="FlagForced" cppname="TrackFlagForced" level="3" id="0x55AA" type="uinteger" mandatory="1" minver="1" default="0" range="0-1">Set if that track MUST be active during playback. There can be many forced track for a kind (audio, video or subs), the player should select the one which language matches the user preference or the default + forced track. Overlay MAY happen between a forced and non-forced track of the same kind. (1 bit)</element>
+ <element name="FlagLacing" cppname="TrackFlagLacing" level="3" id="0x9C" type="uinteger" mandatory="1" minver="1" default="1" range="0-1">Set if the track may contain blocks using lacing. (1 bit)</element>
+ <element name="MinCache" cppname="TrackMinCache" level="3" id="0x6DE7" type="uinteger" mandatory="1" minver="1" webm="0" default="0">The minimum number of frames a player should be able to cache during playback. If set to 0, the reference pseudo-cache system is not used.</element>
+ <element name="MaxCache" cppname="TrackMaxCache" level="3" id="0x6DF8" type="uinteger" minver="1" webm="0">The maximum cache size required to store referenced frames in and the current frame. 0 means no cache is needed.</element>
+ <element name="DefaultDuration" cppname="TrackDefaultDuration" level="3" id="0x23E383" type="uinteger" minver="1" range="not 0">Number of nanoseconds (not scaled via TimecodeScale) per frame ('frame' in the Matroska sense -- one element put into a (Simple)Block).</element>
+ <element name="TrackTimecodeScale" level="3" id="0x23314F" type="float" mandatory="1" minver="1" maxver="3" webm="0" default="1.0" range="> 0">DEPRECATED, DO NOT USE. The scale to apply on this track to work at normal speed in relation with other tracks (mostly used to adjust video speed when the audio length differs).</element>
+ <element name="TrackOffset" level="3" id="0x537F" type="integer" webm="0" default="0">A value to add to the Block's Timecode. This can be used to adjust the playback offset of a track.</element>
+ <element name="MaxBlockAdditionID" level="3" id="0x55EE" type="uinteger" mandatory="1" minver="1" webm="0" default="0">The maximum value of <a href="http://www.matroska.org/technical/specs/index.html#BlockAddID">BlockAddID</a>. A value 0 means there is no <a href="http://www.matroska.org/technical/specs/index.html#BlockAdditions">BlockAdditions</a> for this track.</element>
+ <element name="Name" cppname="TrackName" level="3" id="0x536E" type="utf-8" minver="1">A human-readable track name.</element>
+ <element name="Language" cppname="TrackLanguage" level="3" id="0x22B59C" type="string" minver="1" default="eng">Specifies the language of the track in the <a href="http://www.matroska.org/technical/specs/index.html#languages">Matroska languages form</a>.</element>
+ <element name="CodecID" level="3" id="0x86" type="string" mandatory="1" minver="1">An ID corresponding to the codec, see the <a href="http://www.matroska.org/technical/specs/codecid/index.html">codec page</a> for more info.</element>
+ <element name="CodecPrivate" level="3" id="0x63A2" type="binary" minver="1">Private data only known to the codec.</element>
+ <element name="CodecName" level="3" id="0x258688" type="utf-8" minver="1">A human-readable string specifying the codec.</element>
+ <element name="AttachmentLink" cppname="TrackAttachmentLink" level="3" id="0x7446" type="uinteger" minver="1" webm="0" range="not 0">The UID of an attachment that is used by this codec.</element>
+ <element name="CodecSettings" level="3" id="0x3A9697" type="utf-8" webm="0">A string describing the encoding setting used.</element>
+ <element name="CodecInfoURL" level="3" id="0x3B4040" type="string" multiple="1" webm="0">A URL to find information about the codec used.</element>
+ <element name="CodecDownloadURL" level="3" id="0x26B240" type="string" multiple="1" webm="0">A URL to download about the codec used.</element>
+ <element name="CodecDecodeAll" level="3" id="0xAA" type="uinteger" mandatory="1" minver="2" webm="0" default="1" range="0-1">The codec can decode potentially damaged data (1 bit).</element>
+ <element name="TrackOverlay" level="3" id="0x6FAB" type="uinteger" multiple="1" minver="1" webm="0">Specify that this track is an overlay track for the Track specified (in the u-integer). That means when this track has a gap (see <a href="http://www.matroska.org/technical/specs/index.html#SilentTracks">SilentTracks</a>) the overlay track should be used instead. The order of multiple TrackOverlay matters, the first one is the one that should be used. If not found it should be the second, etc.</element>
+ <element name="TrackTranslate" level="3" id="0x6624" type="master" multiple="1" minver="1" webm="0">The track identification for the given Chapter Codec.</element>
+ <element name="TrackTranslateEditionUID" level="4" id="0x66FC" type="uinteger" multiple="1" minver="1" webm="0">Specify an edition UID on which this translation applies. When not specified, it means for all editions found in the segment.</element>
+ <element name="TrackTranslateCodec" level="4" id="0x66BF" type="uinteger" mandatory="1" minver="1" webm="0">The <a href="http://www.matroska.org/technical/specs/index.html#ChapProcessCodecID">chapter codec</a> using this ID (0: Matroska Script, 1: DVD-menu).</element>
+ <element name="TrackTranslateTrackID" level="4" id="0x66A5" type="binary" mandatory="1" minver="1" webm="0">The binary value used to represent this track in the chapter codec data. The format depends on the <a href="http://www.matroska.org/technical/specs/index.html#ChapProcessCodecID">ChapProcessCodecID</a> used.</element>
+ <element name="Video" cppname="TrackVideo" level="3" id="0xE0" type="master" minver="1">Video settings.</element>
+ <element name="FlagInterlaced" cppname="VideoFlagInterlaced" level="4" id="0x9A" type="uinteger" mandatory="1" minver="2" webm="1" default="0" range="0-1">Set if the video is interlaced. (1 bit)</element>
+ <element name="StereoMode" cppname="VideoStereoMode" level="4" id="0x53B8" type="uinteger" minver="3" webm="1" default="0">Stereo-3D video mode (0: mono, 1: side by side (left eye is first), 2: top-bottom (right eye is first), 3: top-bottom (left eye is first), 4: checkboard (right is first), 5: checkboard (left is first), 6: row interleaved (right is first), 7: row interleaved (left is first), 8: column interleaved (right is first), 9: column interleaved (left is first), 10: anaglyph (cyan/red), 11: side by side (right eye is first), 12: anaglyph (green/magenta), 13 both eyes laced in one Block (left eye is first), 14 both eyes laced in one Block (right eye is first)) . There are some more details on <a href="http://www.matroska.org/technical/specs/notes.html#3D">3D support in the Specification Notes</a>.</element>
+ <element name="OldStereoMode" level="4" id="0x53B9" type="uinteger" maxver="0" webm="0" divx="0">DEPRECATED, DO NOT USE. Bogus StereoMode value used in old versions of libmatroska. (0: mono, 1: right eye, 2: left eye, 3: both eyes).</element>
+ <element name="PixelWidth" cppname="VideoPixelWidth" level="4" id="0xB0" type="uinteger" mandatory="1" minver="1" range="not 0">Width of the encoded video frames in pixels.</element>
+ <element name="PixelHeight" cppname="VideoPixelHeight" level="4" id="0xBA" type="uinteger" mandatory="1" minver="1" range="not 0">Height of the encoded video frames in pixels.</element>
+ <element name="PixelCropBottom" cppname="VideoPixelCropBottom" level="4" id="0x54AA" type="uinteger" minver="1" default="0">The number of video pixels to remove at the bottom of the image (for HDTV content).</element>
+ <element name="PixelCropTop" cppname="VideoPixelCropTop" level="4" id="0x54BB" type="uinteger" minver="1" default="0">The number of video pixels to remove at the top of the image.</element>
+ <element name="PixelCropLeft" cppname="VideoPixelCropLeft" level="4" id="0x54CC" type="uinteger" minver="1" default="0">The number of video pixels to remove on the left of the image.</element>
+ <element name="PixelCropRight" cppname="VideoPixelCropRight" level="4" id="0x54DD" type="uinteger" minver="1" default="0">The number of video pixels to remove on the right of the image.</element>
+ <element name="DisplayWidth" cppname="VideoDisplayWidth" level="4" id="0x54B0" type="uinteger" minver="1" default="PixelWidth" range="not 0">Width of the video frames to display. The default value is only valid when <a href="http://www.matroska.org/technical/specs/index.html#DisplayUnit">DisplayUnit</a> is 0.</element>
+ <element name="DisplayHeight" cppname="VideoDisplayHeight" level="4" id="0x54BA" type="uinteger" minver="1" default="PixelHeight" range="not 0">Height of the video frames to display. The default value is only valid when <a href="http://www.matroska.org/technical/specs/index.html#DisplayUnit">DisplayUnit</a> is 0.</element>
+ <element name="DisplayUnit" cppname="VideoDisplayUnit" level="4" id="0x54B2" type="uinteger" minver="1" default="0">How DisplayWidth & DisplayHeight should be interpreted (0: pixels, 1: centimeters, 2: inches, 3: Display Aspect Ratio).</element>
+ <element name="AspectRatioType" cppname="VideoAspectRatio" level="4" id="0x54B3" type="uinteger" minver="1" default="0">Specify the possible modifications to the aspect ratio (0: free resizing, 1: keep aspect ratio, 2: fixed).</element>
+ <element name="ColourSpace" cppname="VideoColourSpace" level="4" id="0x2EB524" type="binary" minver="1" webm="0" bytesize="4">Same value as in AVI (32 bits).</element>
+ <element name="GammaValue" cppname="VideoGamma" level="4" id="0x2FB523" type="float" webm="0" range="> 0">Gamma Value.</element>
+ <element name="FrameRate" cppname="VideoFrameRate" level="4" id="0x2383E3" type="float" range="> 0">Number of frames per second. <strong>Informational</strong> only.</element>
+ <element name="Audio" cppname="TrackAudio" level="3" id="0xE1" type="master" minver="1">Audio settings.</element>
+ <element name="SamplingFrequency" cppname="AudioSamplingFreq" level="4" id="0xB5" type="float" mandatory="1" minver="1" default="8000.0" range="> 0">Sampling frequency in Hz.</element>
+ <element name="OutputSamplingFrequency" cppname="AudioOutputSamplingFreq" level="4" id="0x78B5" type="float" minver="1" default="Sampling Frequency" range="> 0">Real output sampling frequency in Hz (used for SBR techniques).</element>
+ <element name="Channels" cppname="AudioChannels" level="4" id="0x9F" type="uinteger" mandatory="1" minver="1" default="1" range="not 0">Numbers of channels in the track.</element>
+ <element name="ChannelPositions" cppname="AudioPosition" level="4" id="0x7D7B" type="binary" webm="0">Table of horizontal angles for each successive channel, see <a href="http://www.matroska.org/technical/specs/index.html#channelposition">appendix</a>.</element>
+ <element name="BitDepth" cppname="AudioBitDepth" level="4" id="0x6264" type="uinteger" minver="1" range="not 0">Bits per sample, mostly used for PCM.</element>
+ <element name="TrackOperation" level="3" id="0xE2" type="master" minver="3" webm="0">Operation that needs to be applied on tracks to create this virtual track. For more details <a href="http://www.matroska.org/technical/specs/notes.html#TrackOperation">look at the Specification Notes</a> on the subject.</element>
+ <element name="TrackCombinePlanes" level="4" id="0xE3" type="master" minver="3" webm="0">Contains the list of all video plane tracks that need to be combined to create this 3D track</element>
+ <element name="TrackPlane" level="5" id="0xE4" type="master" mandatory="1" multiple="1" minver="3" webm="0">Contains a video plane track that need to be combined to create this 3D track</element>
+ <element name="TrackPlaneUID" level="6" id="0xE5" type="uinteger" mandatory="1" minver="3" webm="0" range="not 0">The trackUID number of the track representing the plane.</element>
+ <element name="TrackPlaneType" level="6" id="0xE6" type="uinteger" mandatory="1" minver="3" webm="0">The kind of plane this track corresponds to (0: left eye, 1: right eye, 2: background).</element>
+ <element name="TrackJoinBlocks" level="4" id="0xE9" type="master" minver="3" webm="0">Contains the list of all tracks whose Blocks need to be combined to create this virtual track</element>
+ <element name="TrackJoinUID" level="5" id="0xED" type="uinteger" mandatory="1" multiple="1" minver="3" webm="0" range="not 0">The trackUID number of a track whose blocks are used to create this virtual track.</element>
+ <element name="TrickTrackUID" level="3" id="0xC0" type="uinteger" divx="1"><a href="http://developer.divx.com/docs/divx_plus_hd/format_features/Smooth_FF_RW">DivX trick track extenstions</a></element>
+ <element name="TrickTrackSegmentUID" level="3" id="0xC1" type="binary" divx="1" bytesize="16"><a href="http://developer.divx.com/docs/divx_plus_hd/format_features/Smooth_FF_RW">DivX trick track extenstions</a></element>
+ <element name="TrickTrackFlag" level="3" id="0xC6" type="uinteger" divx="1" default="0"><a href="http://developer.divx.com/docs/divx_plus_hd/format_features/Smooth_FF_RW">DivX trick track extenstions</a></element>
+ <element name="TrickMasterTrackUID" level="3" id="0xC7" type="uinteger" divx="1"><a href="http://developer.divx.com/docs/divx_plus_hd/format_features/Smooth_FF_RW">DivX trick track extenstions</a></element>
+ <element name="TrickMasterTrackSegmentUID" level="3" id="0xC4" type="binary" divx="1" bytesize="16"><a href="http://developer.divx.com/docs/divx_plus_hd/format_features/Smooth_FF_RW">DivX trick track extenstions</a></element>
+ <element name="ContentEncodings" level="3" id="0x6D80" type="master" minver="1" webm="0">Settings for several content encoding mechanisms like compression or encryption.</element>
+ <element name="ContentEncoding" level="4" id="0x6240" type="master" mandatory="1" multiple="1" minver="1" webm="0">Settings for one content encoding like compression or encryption.</element>
+ <element name="ContentEncodingOrder" level="5" id="0x5031" type="uinteger" mandatory="1" minver="1" webm="0" default="0">Tells when this modification was used during encoding/muxing starting with 0 and counting upwards. The decoder/demuxer has to start with the highest order number it finds and work its way down. This value has to be unique over all ContentEncodingOrder elements in the segment.</element>
+ <element name="ContentEncodingScope" level="5" id="0x5032" type="uinteger" mandatory="1" minver="1" webm="0" default="1" range="not 0">A bit field that describes which elements have been modified in this way. Values (big endian) can be OR'ed. Possible values:<br/> 1 - all frame contents,<br/> 2 - the track's private data,<br/> 4 - the next ContentEncoding (next ContentEncodingOrder. Either the data inside ContentCompression and/or ContentEncryption)</element>
+ <element name="ContentEncodingType" level="5" id="0x5033" type="uinteger" mandatory="1" minver="1" webm="0" default="0">A value describing what kind of transformation has been done. Possible values:<br/> 0 - compression,<br/> 1 - encryption</element>
+ <element name="ContentCompression" level="5" id="0x5034" type="master" minver="1" webm="0">Settings describing the compression used. Must be present if the value of ContentEncodingType is 0 and absent otherwise. Each block must be decompressable even if no previous block is available in order not to prevent seeking.</element>
+ <element name="ContentCompAlgo" level="6" id="0x4254" type="uinteger" mandatory="1" minver="1" webm="0" default="0">The compression algorithm used. Algorithms that have been specified so far are:<br/> 0 - zlib,<br/> <del>1 - bzlib,</del><br/> <del>2 - lzo1x</del><br/> 3 - Header Stripping</element>
+ <element name="ContentCompSettings" level="6" id="0x4255" type="binary" minver="1" webm="0">Settings that might be needed by the decompressor. For Header Stripping (ContentCompAlgo=3), the bytes that were removed from the beggining of each frames of the track.</element>
+ <element name="ContentEncryption" level="5" id="0x5035" type="master" minver="1" webm="0">Settings describing the encryption used. Must be present if the value of ContentEncodingType is 1 and absent otherwise.</element>
+ <element name="ContentEncAlgo" level="6" id="0x47E1" type="uinteger" minver="1" webm="0" default="0">The encryption algorithm used. The value '0' means that the contents have not been encrypted but only signed. Predefined values:<br/> 1 - DES, 2 - 3DES, 3 - Twofish, 4 - Blowfish, 5 - AES</element>
+ <element name="ContentEncKeyID" level="6" id="0x47E2" type="binary" minver="1" webm="0">For public key algorithms this is the ID of the public key the the data was encrypted with.</element>
+ <element name="ContentSignature" level="6" id="0x47E3" type="binary" minver="1" webm="0">A cryptographic signature of the contents.</element>
+ <element name="ContentSigKeyID" level="6" id="0x47E4" type="binary" minver="1" webm="0">This is the ID of the private key the data was signed with.</element>
+ <element name="ContentSigAlgo" level="6" id="0x47E5" type="uinteger" minver="1" webm="0" default="0">The algorithm used for the signature. A value of '0' means that the contents have not been signed but only encrypted. Predefined values:<br/> 1 - RSA</element>
+ <element name="ContentSigHashAlgo" level="6" id="0x47E6" type="uinteger" minver="1" webm="0" default="0">The hash algorithm used for the signature. A value of '0' means that the contents have not been signed but only encrypted. Predefined values:<br/> 1 - SHA1-160<br/> 2 - MD5</element>
+ <element name="Cues" level="1" id="0x1C53BB6B" type="master" minver="1">A top-level element to speed seeking access. All entries are local to the segment. Should be mandatory for non <a href="http://www.matroska.org/technical/streaming/index.hmtl">"live" streams</a>.</element>
+ <element name="CuePoint" level="2" id="0xBB" type="master" mandatory="1" multiple="1" minver="1">Contains all information relative to a seek point in the segment.</element>
+ <element name="CueTime" level="3" id="0xB3" type="uinteger" mandatory="1" minver="1">Absolute timecode according to the segment time base.</element>
+ <element name="CueTrackPositions" level="3" id="0xB7" type="master" mandatory="1" multiple="1" minver="1">Contain positions for different tracks corresponding to the timecode.</element>
+ <element name="CueTrack" level="4" id="0xF7" type="uinteger" mandatory="1" minver="1" range="not 0">The track for which a position is given.</element>
+ <element name="CueClusterPosition" level="4" id="0xF1" type="uinteger" mandatory="1" minver="1">The <a href="http://www.matroska.org/technical/specs/notes.html#Position_References">position</a> of the Cluster containing the required Block.</element>
+ <element name="CueRelativePosition" level="4" id="0xF0" type="uinteger" mandatory="0" minver="4" webm="0">The relative position of the referenced block inside the cluster with 0 being the first possible position for an element inside that cluster.</element>
+ <element name="CueDuration" level="4" id="0xB2" type="uinteger" mandatory="0" minver="4" webm="0">The duration of the block according to the segment time base. If missing the track's DefaultDuration does not apply and no duration information is available in terms of the cues.</element>
+ <element name="CueBlockNumber" level="4" id="0x5378" type="uinteger" minver="1" default="1" range="not 0">Number of the Block in the specified Cluster.</element>
+ <element name="CueCodecState" level="4" id="0xEA" type="uinteger" minver="2" webm="0" default="0">The <a href="http://www.matroska.org/technical/specs/notes.html#Position_References">position</a> of the Codec State corresponding to this Cue element. 0 means that the data is taken from the initial Track Entry.</element>
+ <element name="CueReference" level="4" id="0xDB" type="master" multiple="1" minver="2" webm="0">The Clusters containing the required referenced Blocks.</element>
+ <element name="CueRefTime" level="5" id="0x96" type="uinteger" mandatory="1" minver="2" webm="0">Timecode of the referenced Block.</element>
+ <element name="CueRefCluster" level="5" id="0x97" type="uinteger" mandatory="1" webm="0">The <a href="http://www.matroska.org/technical/specs/notes.html#Position_References">Position</a> of the Cluster containing the referenced Block.</element>
+ <element name="CueRefNumber" level="5" id="0x535F" type="uinteger" webm="0" default="1" range="not 0">Number of the referenced Block of Track X in the specified Cluster.</element>
+ <element name="CueRefCodecState" level="5" id="0xEB" type="uinteger" webm="0" default="0">The <a href="http://www.matroska.org/technical/specs/notes.html#Position_References">position</a> of the Codec State corresponding to this referenced element. 0 means that the data is taken from the initial Track Entry.</element>
+ <element name="Attachments" level="1" id="0x1941A469" type="master" minver="1" webm="0">Contain attached files.</element>
+ <element name="AttachedFile" level="2" id="0x61A7" type="master" mandatory="1" multiple="1" minver="1" webm="0">An attached file.</element>
+ <element name="FileDescription" level="3" id="0x467E" type="utf-8" minver="1" webm="0">A human-friendly name for the attached file.</element>
+ <element name="FileName" level="3" id="0x466E" type="utf-8" mandatory="1" minver="1" webm="0">Filename of the attached file.</element>
+ <element name="FileMimeType" level="3" id="0x4660" type="string" mandatory="1" minver="1" webm="0">MIME type of the file.</element>
+ <element name="FileData" level="3" id="0x465C" type="binary" mandatory="1" minver="1" webm="0">The data of the file.</element>
+ <element name="FileUID" level="3" id="0x46AE" type="uinteger" mandatory="1" minver="1" webm="0" range="not 0">Unique ID representing the file, as random as possible.</element>
+ <element name="FileReferral" level="3" id="0x4675" type="binary" webm="0">A binary value that a track/codec can refer to when the attachment is needed.</element>
+ <element name="FileUsedStartTime" level="3" id="0x4661" type="uinteger" divx="1"><a href="http://developer.divx.com/docs/divx_plus_hd/format_features/World_Fonts">DivX font extension</a></element>
+ <element name="FileUsedEndTime" level="3" id="0x4662" type="uinteger" divx="1"><a href="http://developer.divx.com/docs/divx_plus_hd/format_features/World_Fonts">DivX font extension</a></element>
+ <element name="Chapters" level="1" id="0x1043A770" type="master" minver="1" webm="1">A system to define basic menus and partition data. For more detailed information, look at the <a href="http://www.matroska.org/technical/specs/chapters/index.html">Chapters Explanation</a>.</element>
+ <element name="EditionEntry" level="2" id="0x45B9" type="master" mandatory="1" multiple="1" minver="1" webm="1">Contains all information about a segment edition.</element>
+ <element name="EditionUID" level="3" id="0x45BC" type="uinteger" minver="1" webm="0" range="not 0">A unique ID to identify the edition. It's useful for tagging an edition.</element>
+ <element name="EditionFlagHidden" level="3" id="0x45BD" type="uinteger" mandatory="1" minver="1" webm="0" default="0" range="0-1">If an edition is hidden (1), it should not be available to the user interface (but still to Control Tracks). (1 bit)</element>
+ <element name="EditionFlagDefault" level="3" id="0x45DB" type="uinteger" mandatory="1" minver="1" webm="0" default="0" range="0-1">If a flag is set (1) the edition should be used as the default one. (1 bit)</element>
+ <element name="EditionFlagOrdered" level="3" id="0x45DD" type="uinteger" minver="1" webm="0" default="0" range="0-1">Specify if the chapters can be defined multiple times and the order to play them is enforced. (1 bit)</element>
+ <element name="ChapterAtom" level="3" recursive="1" id="0xB6" type="master" mandatory="1" multiple="1" minver="1" webm="1">Contains the atom information to use as the chapter atom (apply to all tracks).</element>
+ <element name="ChapterUID" level="4" id="0x73C4" type="uinteger" mandatory="1" minver="1" webm="1" range="not 0">A unique ID to identify the Chapter.</element>
+ <element name="ChapterStringUID" level="4" id="0x5654" type="utf-8" mandatory="0" minver="3" webm="1">A unique string ID to identify the Chapter. Use for <a href="http://dev.w3.org/html5/webvtt/#webvtt-cue-identifier">WebVTT cue identifier storage</a>.</element>
+ <element name="ChapterTimeStart" level="4" id="0x91" type="uinteger" mandatory="1" minver="1" webm="1">Timecode of the start of Chapter (not scaled).</element>
+ <element name="ChapterTimeEnd" level="4" id="0x92" type="uinteger" minver="1" webm="0">Timecode of the end of Chapter (timecode excluded, not scaled).</element>
+ <element name="ChapterFlagHidden" level="4" id="0x98" type="uinteger" mandatory="1" minver="1" webm="0" default="0" range="0-1">If a chapter is hidden (1), it should not be available to the user interface (but still to Control Tracks). (1 bit)</element>
+ <element name="ChapterFlagEnabled" level="4" id="0x4598" type="uinteger" mandatory="1" minver="1" webm="0" default="1" range="0-1">Specify wether the chapter is enabled. It can be enabled/disabled by a Control Track. When disabled, the movie should skip all the content between the TimeStart and TimeEnd of this chapter. (1 bit)</element>
+ <element name="ChapterSegmentUID" level="4" id="0x6E67" type="binary" minver="1" webm="0" range=">0" bytesize="16">A segment to play in place of this chapter. Edition ChapterSegmentEditionUID should be used for this segment, otherwise no edition is used.</element>
+ <element name="ChapterSegmentEditionUID" level="4" id="0x6EBC" type="uinteger" minver="1" webm="0" range="not 0">The EditionUID to play from the segment linked in ChapterSegmentUID.</element>
+ <element name="ChapterPhysicalEquiv" level="4" id="0x63C3" type="uinteger" minver="1" webm="0">Specify the physical equivalent of this ChapterAtom like "DVD" (60) or "SIDE" (50), see <a href="http://www.matroska.org/technical/specs/index.html#physical">complete list of values</a>.</element>
+ <element name="ChapterTrack" level="4" id="0x8F" type="master" minver="1" webm="0">List of tracks on which the chapter applies. If this element is not present, all tracks apply</element>
+ <element name="ChapterTrackNumber" level="5" id="0x89" type="uinteger" mandatory="1" multiple="1" minver="1" webm="0" range="not 0">UID of the Track to apply this chapter too. In the absense of a control track, choosing this chapter will select the listed Tracks and deselect unlisted tracks. Absense of this element indicates that the Chapter should be applied to any currently used Tracks.</element>
+ <element name="ChapterDisplay" level="4" id="0x80" type="master" multiple="1" minver="1" webm="1">Contains all possible strings to use for the chapter display.</element>
+ <element name="ChapString" cppname="ChapterString" level="5" id="0x85" type="utf-8" mandatory="1" minver="1" webm="1">Contains the string to use as the chapter atom.</element>
+ <element name="ChapLanguage" cppname="ChapterLanguage" level="5" id="0x437C" type="string" mandatory="1" multiple="1" minver="1" webm="1" default="eng">The languages corresponding to the string, in the <a href="http://lcweb.loc.gov/standards/iso639-2/englangn.html#two">bibliographic ISO-639-2 form</a>.</element>
+ <element name="ChapCountry" cppname="ChapterCountry" level="5" id="0x437E" type="string" multiple="1" minver="1" webm="0">The countries corresponding to the string, same 2 octets as in <a href="http://www.iana.org/cctld/cctld-whois.htm">Internet domains</a>.</element>
+ <element name="ChapProcess" cppname="ChapterProcess" level="4" id="0x6944" type="master" multiple="1" minver="1" webm="0">Contains all the commands associated to the Atom.</element>
+ <element name="ChapProcessCodecID" cppname="ChapterProcessCodecID" level="5" id="0x6955" type="uinteger" mandatory="1" minver="1" webm="0" default="0">Contains the type of the codec used for the processing. A value of 0 means native Matroska processing (to be defined), a value of 1 means the <a href="http://www.matroska.org/technical/specs/chapters/index.html#dvd">DVD</a> command set is used. More codec IDs can be added later.</element>
+ <element name="ChapProcessPrivate" cppname="ChapterProcessPrivate" level="5" id="0x450D" type="binary" minver="1" webm="0">Some optional data attached to the ChapProcessCodecID information. <a href="http://www.matroska.org/technical/specs/chapters/index.html#dvd">For ChapProcessCodecID = 1</a>, it is the "DVD level" equivalent.</element>
+ <element name="ChapProcessCommand" cppname="ChapterProcessCommand" level="5" id="0x6911" type="master" multiple="1" minver="1" webm="0">Contains all the commands associated to the Atom.</element>
+ <element name="ChapProcessTime" cppname="ChapterProcessTime" level="6" id="0x6922" type="uinteger" mandatory="1" minver="1" webm="0">Defines when the process command should be handled (0: during the whole chapter, 1: before starting playback, 2: after playback of the chapter).</element>
+ <element name="ChapProcessData" cppname="ChapterProcessData" level="6" id="0x6933" type="binary" mandatory="1" minver="1" webm="0">Contains the command information. The data should be interpreted depending on the ChapProcessCodecID value. <a href="http://www.matroska.org/technical/specs/chapters/index.html#dvd">For ChapProcessCodecID = 1</a>, the data correspond to the binary DVD cell pre/post commands.</element>
+ <element name="Tags" level="1" id="0x1254C367" type="master" multiple="1" minver="1" webm="0">Element containing elements specific to Tracks/Chapters. A list of valid tags can be found <a href="http://www.matroska.org/technical/specs/tagging/index.html">here.</a></element>
+ <element name="Tag" level="2" id="0x7373" type="master" mandatory="1" multiple="1" minver="1" webm="0">Element containing elements specific to Tracks/Chapters.</element>
+ <element name="Targets" cppname="TagTargets" level="3" id="0x63C0" type="master" mandatory="1" minver="1" webm="0">Contain all UIDs where the specified meta data apply. It is empty to describe everything in the segment.</element>
+ <element name="TargetTypeValue" cppname="TagTargetTypeValue" level="4" id="0x68CA" type="uinteger" minver="1" webm="0" default="50">A number to indicate the logical level of the target (see <a href="http://www.matroska.org/technical/specs/tagging/index.html#targettypes">TargetType</a>).</element>
+ <element name="TargetType" cppname="TagTargetType" level="4" id="0x63CA" type="string" minver="1" webm="0">An <strong>informational</strong> string that can be used to display the logical level of the target like "ALBUM", "TRACK", "MOVIE", "CHAPTER", etc (see <a href="http://www.matroska.org/technical/specs/tagging/index.html#targettypes">TargetType</a>).</element>
+ <element name="TagTrackUID" level="4" id="0x63C5" type="uinteger" multiple="1" minver="1" webm="0" default="0">A unique ID to identify the Track(s) the tags belong to. If the value is 0 at this level, the tags apply to all tracks in the Segment.</element>
+ <element name="TagEditionUID" level="4" id="0x63C9" type="uinteger" multiple="1" minver="1" webm="0" default="0">A unique ID to identify the EditionEntry(s) the tags belong to. If the value is 0 at this level, the tags apply to all editions in the Segment.</element>
+ <element name="TagChapterUID" level="4" id="0x63C4" type="uinteger" multiple="1" minver="1" webm="0" default="0">A unique ID to identify the Chapter(s) the tags belong to. If the value is 0 at this level, the tags apply to all chapters in the Segment.</element>
+ <element name="TagAttachmentUID" level="4" id="0x63C6" type="uinteger" multiple="1" minver="1" webm="0" default="0">A unique ID to identify the Attachment(s) the tags belong to. If the value is 0 at this level, the tags apply to all the attachments in the Segment.</element>
+ <element name="SimpleTag" cppname="TagSimple" level="3" recursive="1" id="0x67C8" type="master" mandatory="1" multiple="1" minver="1" webm="0">Contains general information about the target.</element>
+ <element name="TagName" level="4" id="0x45A3" type="utf-8" mandatory="1" minver="1" webm="0">The name of the Tag that is going to be stored.</element>
+ <element name="TagLanguage" level="4" id="0x447A" type="string" mandatory="1" minver="1" webm="0" default="und">Specifies the language of the tag specified, in the <a href="http://www.matroska.org/technical/specs/index.html#languages">Matroska languages form</a>.</element>
+ <element name="TagDefault" level="4" id="0x4484" type="uinteger" mandatory="1" minver="1" webm="0" default="1" range="0-1">Indication to know if this is the default/original language to use for the given tag. (1 bit)</element>
+ <element name="TagString" level="4" id="0x4487" type="utf-8" minver="1" webm="0">The value of the Tag.</element>
+ <element name="TagBinary" level="4" id="0x4485" type="binary" minver="1" webm="0">The values of the Tag if it is binary. Note that this cannot be used in the same SimpleTag as TagString.</element>
+</table>
diff --git a/lib/enzyme/real.py b/lib/enzyme/real.py
deleted file mode 100644
index e7c69e930f324dfb45142d47cdfa255ad588c200..0000000000000000000000000000000000000000
--- a/lib/enzyme/real.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
-# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-__all__ = ['Parser']
-
-import struct
-import logging
-from exceptions import ParseError
-import core
-
-# http://www.pcisys.net/~melanson/codecs/rmff.htm
-# http://www.pcisys.net/~melanson/codecs/
-
-# get logging object
-log = logging.getLogger(__name__)
-
-class RealVideo(core.AVContainer):
- def __init__(self, file):
- core.AVContainer.__init__(self)
- self.mime = 'video/real'
- self.type = 'Real Video'
- h = file.read(10)
- try:
- (object_id, object_size, object_version) = struct.unpack('>4sIH', h)
- except struct.error:
- # EOF.
- raise ParseError()
-
- if not object_id == '.RMF':
- raise ParseError()
-
- file_version, num_headers = struct.unpack('>II', file.read(8))
- log.debug(u'size: %d, ver: %d, headers: %d' % \
- (object_size, file_version, num_headers))
- for _ in range(0, num_headers):
- try:
- oi = struct.unpack('>4sIH', file.read(10))
- except (struct.error, IOError):
- # Header data we expected wasn't there. File may be
- # only partially complete.
- break
-
- if object_id == 'DATA' and oi[0] != 'INDX':
- log.debug(u'INDX chunk expected after DATA but not found -- file corrupt')
- break
-
- (object_id, object_size, object_version) = oi
- if object_id == 'DATA':
- # Seek over the data chunk rather than reading it in.
- file.seek(object_size - 10, 1)
- else:
- self._read_header(object_id, file.read(object_size - 10))
- log.debug(u'%r [%d]' % (object_id, object_size - 10))
- # Read all the following headers
-
-
- def _read_header(self, object_id, s):
- if object_id == 'PROP':
- prop = struct.unpack('>9IHH', s)
- log.debug(u'PROP: %r' % prop)
- if object_id == 'MDPR':
- mdpr = struct.unpack('>H7I', s[:30])
- log.debug(u'MDPR: %r' % mdpr)
- self.length = mdpr[7] / 1000.0
- (stream_name_size,) = struct.unpack('>B', s[30:31])
- stream_name = s[31:31 + stream_name_size]
- pos = 31 + stream_name_size
- (mime_type_size,) = struct.unpack('>B', s[pos:pos + 1])
- mime = s[pos + 1:pos + 1 + mime_type_size]
- pos += mime_type_size + 1
- (type_specific_len,) = struct.unpack('>I', s[pos:pos + 4])
- type_specific = s[pos + 4:pos + 4 + type_specific_len]
- pos += 4 + type_specific_len
- if mime[:5] == 'audio':
- ai = core.AudioStream()
- ai.id = mdpr[0]
- ai.bitrate = mdpr[2]
- self.audio.append(ai)
- elif mime[:5] == 'video':
- vi = core.VideoStream()
- vi.id = mdpr[0]
- vi.bitrate = mdpr[2]
- self.video.append(vi)
- else:
- log.debug(u'Unknown: %r' % mime)
- if object_id == 'CONT':
- pos = 0
- (title_len,) = struct.unpack('>H', s[pos:pos + 2])
- self.title = s[2:title_len + 2]
- pos += title_len + 2
- (author_len,) = struct.unpack('>H', s[pos:pos + 2])
- self.artist = s[pos + 2:pos + author_len + 2]
- pos += author_len + 2
- (copyright_len,) = struct.unpack('>H', s[pos:pos + 2])
- self.copyright = s[pos + 2:pos + copyright_len + 2]
- pos += copyright_len + 2
- (comment_len,) = struct.unpack('>H', s[pos:pos + 2])
- self.comment = s[pos + 2:pos + comment_len + 2]
-
-
-Parser = RealVideo
diff --git a/lib/enzyme/riff.py b/lib/enzyme/riff.py
deleted file mode 100644
index 516c727b10f951ff45feff64089024341ecf3899..0000000000000000000000000000000000000000
--- a/lib/enzyme/riff.py
+++ /dev/null
@@ -1,566 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2003-2006 Thomas Schueppel <stain@acm.org>
-# Copyright 2003-2006 Dirk Meyer <dischi@freevo.org>
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-__all__ = ['Parser']
-
-import os
-import struct
-import string
-import logging
-import time
-from exceptions import ParseError
-import core
-
-# get logging object
-log = logging.getLogger(__name__)
-
-# List of tags
-# http://kibus1.narod.ru/frames_eng.htm?sof/abcavi/infotags.htm
-# http://www.divx-digest.com/software/avitags_dll.html
-# File Format: google for odmlff2.pdf
-
-AVIINFO = {
- 'INAM': 'title',
- 'IART': 'artist',
- 'IPRD': 'product',
- 'ISFT': 'software',
- 'ICMT': 'comment',
- 'ILNG': 'language',
- 'IKEY': 'keywords',
- 'IPRT': 'trackno',
- 'IFRM': 'trackof',
- 'IPRO': 'producer',
- 'IWRI': 'writer',
- 'IGNR': 'genre',
- 'ICOP': 'copyright'
-}
-
-# Taken from libavcodec/mpeg4data.h (pixel_aspect struct)
-PIXEL_ASPECT = {
- 1: (1, 1),
- 2: (12, 11),
- 3: (10, 11),
- 4: (16, 11),
- 5: (40, 33)
-}
-
-
-class Riff(core.AVContainer):
- """
- AVI parser also parsing metadata like title, languages, etc.
- """
- table_mapping = { 'AVIINFO' : AVIINFO }
-
- def __init__(self, file):
- core.AVContainer.__init__(self)
- # read the header
- h = file.read(12)
- if h[:4] != "RIFF" and h[:4] != 'SDSS':
- raise ParseError()
-
- self.has_idx = False
- self.header = {}
- self.junkStart = None
- self.infoStart = None
- self.type = h[8:12]
- if self.type == 'AVI ':
- self.mime = 'video/avi'
- elif self.type == 'WAVE':
- self.mime = 'audio/wav'
- try:
- while self._parseRIFFChunk(file):
- pass
- except IOError:
- log.exception(u'error in file, stop parsing')
-
- self._find_subtitles(file.name)
-
- if not self.has_idx and isinstance(self, core.AVContainer):
- log.debug(u'WARNING: avi has no index')
- self._set('corrupt', True)
-
-
- def _find_subtitles(self, filename):
- """
- Search for subtitle files. Right now only VobSub is supported
- """
- base = os.path.splitext(filename)[0]
- if os.path.isfile(base + '.idx') and \
- (os.path.isfile(base + '.sub') or os.path.isfile(base + '.rar')):
- file = open(base + '.idx')
- if file.readline().find('VobSub index file') > 0:
- for line in file.readlines():
- if line.find('id') == 0:
- sub = core.Subtitle()
- sub.language = line[4:6]
- sub.trackno = base + '.idx' # Maybe not?
- self.subtitles.append(sub)
- file.close()
-
-
- def _parseAVIH(self, t):
- retval = {}
- v = struct.unpack('<IIIIIIIIIIIIII', t[0:56])
- (retval['dwMicroSecPerFrame'],
- retval['dwMaxBytesPerSec'],
- retval['dwPaddingGranularity'],
- retval['dwFlags'],
- retval['dwTotalFrames'],
- retval['dwInitialFrames'],
- retval['dwStreams'],
- retval['dwSuggestedBufferSize'],
- retval['dwWidth'],
- retval['dwHeight'],
- retval['dwScale'],
- retval['dwRate'],
- retval['dwStart'],
- retval['dwLength']) = v
- if retval['dwMicroSecPerFrame'] == 0:
- log.warning(u'ERROR: Corrupt AVI')
- raise ParseError()
-
- return retval
-
-
- def _parseSTRH(self, t):
- retval = {}
- retval['fccType'] = t[0:4]
- log.debug(u'_parseSTRH(%r) : %d bytes' % (retval['fccType'], len(t)))
- if retval['fccType'] != 'auds':
- retval['fccHandler'] = t[4:8]
- v = struct.unpack('<IHHIIIIIIIII', t[8:52])
- (retval['dwFlags'],
- retval['wPriority'],
- retval['wLanguage'],
- retval['dwInitialFrames'],
- retval['dwScale'],
- retval['dwRate'],
- retval['dwStart'],
- retval['dwLength'],
- retval['dwSuggestedBufferSize'],
- retval['dwQuality'],
- retval['dwSampleSize'],
- retval['rcFrame']) = v
- else:
- try:
- v = struct.unpack('<IHHIIIIIIIII', t[8:52])
- (retval['dwFlags'],
- retval['wPriority'],
- retval['wLanguage'],
- retval['dwInitialFrames'],
- retval['dwScale'],
- retval['dwRate'],
- retval['dwStart'],
- retval['dwLength'],
- retval['dwSuggestedBufferSize'],
- retval['dwQuality'],
- retval['dwSampleSize'],
- retval['rcFrame']) = v
- self.delay = float(retval['dwStart']) / \
- (float(retval['dwRate']) / retval['dwScale'])
- except (KeyError, IndexError, ValueError, ZeroDivisionError):
- pass
-
- return retval
-
-
- def _parseSTRF(self, t, strh):
- fccType = strh['fccType']
- retval = {}
- if fccType == 'auds':
- v = struct.unpack('<HHHHHH', t[0:12])
- (retval['wFormatTag'],
- retval['nChannels'],
- retval['nSamplesPerSec'],
- retval['nAvgBytesPerSec'],
- retval['nBlockAlign'],
- retval['nBitsPerSample'],
- ) = v
- ai = core.AudioStream()
- ai.samplerate = retval['nSamplesPerSec']
- ai.channels = retval['nChannels']
- # FIXME: Bitrate calculation is completely wrong.
- #ai.samplebits = retval['nBitsPerSample']
- #ai.bitrate = retval['nAvgBytesPerSec'] * 8
-
- # TODO: set code if possible
- # http://www.stats.uwa.edu.au/Internal/Specs/DXALL/FileSpec/\
- # Languages
- # ai.language = strh['wLanguage']
- ai.codec = retval['wFormatTag']
- self.audio.append(ai)
- elif fccType == 'vids':
- v = struct.unpack('<IIIHH', t[0:16])
- (retval['biSize'],
- retval['biWidth'],
- retval['biHeight'],
- retval['biPlanes'],
- retval['biBitCount']) = v
- v = struct.unpack('IIIII', t[20:40])
- (retval['biSizeImage'],
- retval['biXPelsPerMeter'],
- retval['biYPelsPerMeter'],
- retval['biClrUsed'],
- retval['biClrImportant']) = v
- vi = core.VideoStream()
- vi.codec = t[16:20]
- vi.width = retval['biWidth']
- vi.height = retval['biHeight']
- # FIXME: Bitrate calculation is completely wrong.
- #vi.bitrate = strh['dwRate']
- vi.fps = float(strh['dwRate']) / strh['dwScale']
- vi.length = strh['dwLength'] / vi.fps
- self.video.append(vi)
- return retval
-
-
- def _parseSTRL(self, t):
- retval = {}
- size = len(t)
- i = 0
-
- while i < len(t) - 8:
- key = t[i:i + 4]
- sz = struct.unpack('<I', t[i + 4:i + 8])[0]
- i += 8
- value = t[i:]
-
- if key == 'strh':
- retval[key] = self._parseSTRH(value)
- elif key == 'strf':
- retval[key] = self._parseSTRF(value, retval['strh'])
- else:
- log.debug(u'_parseSTRL: unsupported stream tag %r', key)
-
- i += sz
-
- return retval, i
-
-
- def _parseODML(self, t):
- retval = {}
- size = len(t)
- i = 0
- key = t[i:i + 4]
- sz = struct.unpack('<I', t[i + 4:i + 8])[0]
- i += 8
- value = t[i:]
- if key != 'dmlh':
- log.debug(u'_parseODML: Error')
-
- i += sz - 8
- return (retval, i)
-
-
- def _parseVPRP(self, t):
- retval = {}
- v = struct.unpack('<IIIIIIIIII', t[:4 * 10])
-
- (retval['VideoFormat'],
- retval['VideoStandard'],
- retval['RefreshRate'],
- retval['HTotalIn'],
- retval['VTotalIn'],
- retval['FrameAspectRatio'],
- retval['wPixel'],
- retval['hPixel']) = v[1:-1]
-
- # I need an avi with more informations
- # enum {FORMAT_UNKNOWN, FORMAT_PAL_SQUARE, FORMAT_PAL_CCIR_601,
- # FORMAT_NTSC_SQUARE, FORMAT_NTSC_CCIR_601,...} VIDEO_FORMAT;
- # enum {STANDARD_UNKNOWN, STANDARD_PAL, STANDARD_NTSC, STANDARD_SECAM}
- # VIDEO_STANDARD;
- #
- r = retval['FrameAspectRatio']
- r = float(r >> 16) / (r & 0xFFFF)
- retval['FrameAspectRatio'] = r
- if self.video:
- map(lambda v: setattr(v, 'aspect', r), self.video)
- return (retval, v[0])
-
-
- def _parseLISTmovi(self, size, file):
- """
- Digs into movi list, looking for a Video Object Layer header in an
- mpeg4 stream in order to determine aspect ratio.
- """
- i = 0
- n_dc = 0
- done = False
- # If the VOL header doesn't appear within 5MB or 5 video chunks,
- # give up. The 5MB limit is not likely to apply except in
- # pathological cases.
- while i < min(1024 * 1024 * 5, size - 8) and n_dc < 5:
- data = file.read(8)
- if ord(data[0]) == 0:
- # Eat leading nulls.
- data = data[1:] + file.read(1)
- i += 1
-
- key, sz = struct.unpack('<4sI', data)
- if key[2:] != 'dc' or sz > 1024 * 500:
- # This chunk is not video or is unusually big (> 500KB);
- # skip it.
- file.seek(sz, 1)
- i += 8 + sz
- continue
-
- n_dc += 1
- # Read video chunk into memory
- data = file.read(sz)
-
- #for p in range(0,min(80, sz)):
- # print "%02x " % ord(data[p]),
- #print "\n\n"
-
- # Look through the picture header for VOL startcode. The basic
- # logic for this is taken from libavcodec, h263.c
- pos = 0
- startcode = 0xff
- def bits(v, o, n):
- # Returns n bits in v, offset o bits.
- return (v & 2 ** n - 1 << (64 - n - o)) >> 64 - n - o
-
- while pos < sz:
- startcode = ((startcode << 8) | ord(data[pos])) & 0xffffffff
- pos += 1
- if startcode & 0xFFFFFF00 != 0x100:
- # No startcode found yet
- continue
-
- if startcode >= 0x120 and startcode <= 0x12F:
- # We have the VOL startcode. Pull 64 bits of it and treat
- # as a bitstream
- v = struct.unpack(">Q", data[pos : pos + 8])[0]
- offset = 10
- if bits(v, 9, 1):
- # is_ol_id, skip over vo_ver_id and vo_priority
- offset += 7
- ar_info = bits(v, offset, 4)
- if ar_info == 15:
- # Extended aspect
- num = bits(v, offset + 4, 8)
- den = bits(v, offset + 12, 8)
- else:
- # A standard pixel aspect
- num, den = PIXEL_ASPECT.get(ar_info, (0, 0))
-
- # num/den indicates pixel aspect; convert to video aspect,
- # so we need frame width and height.
- if 0 not in [num, den]:
- width, height = self.video[-1].width, self.video[-1].height
- self.video[-1].aspect = num / float(den) * width / height
-
- done = True
- break
-
- startcode = 0xff
-
- i += 8 + len(data)
-
- if done:
- # We have the aspect, no need to continue parsing the movi
- # list, so break out of the loop.
- break
-
-
- if i < size:
- # Seek past whatever might be remaining of the movi list.
- file.seek(size - i, 1)
-
-
-
- def _parseLIST(self, t):
- retval = {}
- i = 0
- size = len(t)
-
- while i < size - 8:
- # skip zero
- if ord(t[i]) == 0: i += 1
- key = t[i:i + 4]
- sz = 0
-
- if key == 'LIST':
- sz = struct.unpack('<I', t[i + 4:i + 8])[0]
- i += 8
- key = "LIST:" + t[i:i + 4]
- value = self._parseLIST(t[i:i + sz])
- if key == 'strl':
- for k in value.keys():
- retval[k] = value[k]
- else:
- retval[key] = value
- i += sz
- elif key == 'avih':
- sz = struct.unpack('<I', t[i + 4:i + 8])[0]
- i += 8
- value = self._parseAVIH(t[i:i + sz])
- i += sz
- retval[key] = value
- elif key == 'strl':
- i += 4
- (value, sz) = self._parseSTRL(t[i:])
- key = value['strh']['fccType']
- i += sz
- retval[key] = value
- elif key == 'odml':
- i += 4
- (value, sz) = self._parseODML(t[i:])
- i += sz
- elif key == 'vprp':
- i += 4
- (value, sz) = self._parseVPRP(t[i:])
- retval[key] = value
- i += sz
- elif key == 'JUNK':
- sz = struct.unpack('<I', t[i + 4:i + 8])[0]
- i += sz + 8
- else:
- sz = struct.unpack('<I', t[i + 4:i + 8])[0]
- i += 8
- # in most cases this is some info stuff
- if not key in AVIINFO.keys() and key != 'IDIT':
- log.debug(u'Unknown Key: %r, len: %d' % (key, sz))
- value = t[i:i + sz]
- if key == 'ISFT':
- # product information
- if value.find('\0') > 0:
- # works for Casio S500 camera videos
- value = value[:value.find('\0')]
- value = value.replace('\0', '').lstrip().rstrip()
- value = value.replace('\0', '').lstrip().rstrip()
- if value:
- retval[key] = value
- if key in ['IDIT', 'ICRD']:
- # Timestamp the video was created. Spec says it
- # should be a format like "Wed Jan 02 02:03:55 1990"
- # Casio S500 uses "2005/12/24/ 14:11", but I've
- # also seen "December 24, 2005"
- specs = ('%a %b %d %H:%M:%S %Y', '%Y/%m/%d/ %H:%M', '%B %d, %Y')
- for tmspec in specs:
- try:
- tm = time.strptime(value, tmspec)
- # save timestamp as int
- self.timestamp = int(time.mktime(tm))
- break
- except ValueError:
- pass
- else:
- log.debug(u'no support for time format %r', value)
- i += sz
- return retval
-
-
- def _parseRIFFChunk(self, file):
- h = file.read(8)
- if len(h) < 8:
- return False
- name = h[:4]
- size = struct.unpack('<I', h[4:8])[0]
-
- if name == 'LIST':
- pos = file.tell() - 8
- key = file.read(4)
- if key == 'movi' and self.video and not self.video[-1].aspect and \
- self.video[-1].width and self.video[-1].height and \
- self.video[-1].format in ['DIVX', 'XVID', 'FMP4']: # any others?
- # If we don't have the aspect (i.e. it isn't in odml vprp
- # header), but we do know the video's dimensions, and
- # we're dealing with an mpeg4 stream, try to get the aspect
- # from the VOL header in the mpeg4 stream.
- self._parseLISTmovi(size - 4, file)
- return True
- elif size > 80000:
- log.debug(u'RIFF LIST %r too long to parse: %r bytes' % (key, size))
- t = file.seek(size - 4, 1)
- return True
- elif size < 5:
- log.debug(u'RIFF LIST %r too short: %r bytes' % (key, size))
- return True
-
- t = file.read(size - 4)
- log.debug(u'parse RIFF LIST %r: %d bytes' % (key, size))
- value = self._parseLIST(t)
- self.header[key] = value
- if key == 'INFO':
- self.infoStart = pos
- self._appendtable('AVIINFO', value)
- elif key == 'MID ':
- self._appendtable('AVIMID', value)
- elif key == 'hdrl':
- # no need to add this info to a table
- pass
- else:
- log.debug(u'Skipping table info %r' % key)
-
- elif name == 'JUNK':
- self.junkStart = file.tell() - 8
- self.junkSize = size
- file.seek(size, 1)
- elif name == 'idx1':
- self.has_idx = True
- log.debug(u'idx1: %r bytes' % size)
- # no need to parse this
- t = file.seek(size, 1)
- elif name == 'RIFF':
- log.debug(u'New RIFF chunk, extended avi [%i]' % size)
- type = file.read(4)
- if type != 'AVIX':
- log.debug(u'Second RIFF chunk is %r, not AVIX, skipping', type)
- file.seek(size - 4, 1)
- # that's it, no new informations should be in AVIX
- return False
- elif name == 'fmt ' and size <= 50:
- # This is a wav file.
- data = file.read(size)
- fmt = struct.unpack("<HHLLHH", data[:16])
- self._set('codec', hex(fmt[0]))
- self._set('samplerate', fmt[2])
- # fmt[3] is average bytes per second, so we must divide it
- # by 125 to get kbits per second
- self._set('bitrate', fmt[3] / 125)
- # ugly hack: remember original rate in bytes per second
- # so that the length can be calculated in next elif block
- self._set('byterate', fmt[3])
- # Set a dummy fourcc so codec will be resolved in finalize.
- self._set('fourcc', 'dummy')
- elif name == 'data':
- # XXX: this is naive and may not be right. For example if the
- # stream is something that supports VBR like mp3, the value
- # will be off. The only way to properly deal with this issue
- # is to decode part of the stream based on its codec, but
- # kaa.metadata doesn't have this capability (yet?)
- # ugly hack: use original rate in bytes per second
- self._set('length', size / float(self.byterate))
- file.seek(size, 1)
- elif not name.strip(string.printable + string.whitespace):
- # check if name is something usefull at all, maybe it is no
- # avi or broken
- t = file.seek(size, 1)
- log.debug(u'Skipping %r [%i]' % (name, size))
- else:
- # bad avi
- log.debug(u'Bad or broken avi')
- return False
- return True
-
-
-Parser = Riff
diff --git a/lib/enzyme/strutils.py b/lib/enzyme/strutils.py
deleted file mode 100644
index 8578aefa85abd123818bfb6ed79ccbe1d4b0529e..0000000000000000000000000000000000000000
--- a/lib/enzyme/strutils.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# -*- coding: utf-8 -*-
-# enzyme - Video metadata parser
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-# Copyright 2006-2009 Dirk Meyer <dischi@freevo.org>
-# Copyright 2006-2009 Jason Tackaberry
-#
-# This file is part of enzyme.
-#
-# enzyme is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# enzyme is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with enzyme. If not, see <http://www.gnu.org/licenses/>.
-__all__ = ['ENCODING', 'str_to_unicode', 'unicode_to_str']
-
-import locale
-
-# find the correct encoding
-try:
- ENCODING = locale.getdefaultlocale()[1]
- ''.encode(ENCODING)
-except (UnicodeError, TypeError):
- ENCODING = 'latin-1'
-
-
-def str_to_unicode(s, encoding=None):
- """
- Attempts to convert a string of unknown character set to a unicode
- string. First it tries to decode the string based on the locale's
- preferred encoding, and if that fails, fall back to UTF-8 and then
- latin-1. If all fails, it will force encoding to the preferred
- charset, replacing unknown characters. If the given object is no
- string, this function will return the given object.
- """
- if not type(s) == str:
- return s
-
- if not encoding:
- encoding = ENCODING
-
- for c in [encoding, "utf-8", "latin-1"]:
- try:
- return s.decode(c)
- except UnicodeDecodeError:
- pass
-
- return s.decode(encoding, "replace")
-
-
-def unicode_to_str(s, encoding=None):
- """
- Attempts to convert a unicode string of unknown character set to a
- string. First it tries to encode the string based on the locale's
- preferred encoding, and if that fails, fall back to UTF-8 and then
- latin-1. If all fails, it will force encoding to the preferred
- charset, replacing unknown characters. If the given object is no
- unicode string, this function will return the given object.
- """
- if not type(s) == unicode:
- return s
-
- if not encoding:
- encoding = ENCODING
-
- for c in [encoding, "utf-8", "latin-1"]:
- try:
- return s.encode(c)
- except UnicodeDecodeError:
- pass
-
- return s.encode(encoding, "replace")
diff --git a/lib/enzyme/tests/__init__.py b/lib/enzyme/tests/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..426d3598ffef7f2c1c6d2172569b78d378df35b0
--- /dev/null
+++ b/lib/enzyme/tests/__init__.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+from . import test_mkv, test_parsers
+import unittest
+
+
+suite = unittest.TestSuite([test_mkv.suite(), test_parsers.suite()])
+
+
+if __name__ == '__main__':
+ unittest.TextTestRunner().run(suite)
diff --git a/lib/enzyme/tests/parsers/ebml/test1.mkv.yml b/lib/enzyme/tests/parsers/ebml/test1.mkv.yml
new file mode 100644
index 0000000000000000000000000000000000000000..92642ec5d3cc7efcbd39a110b5b0e145be81b3d5
--- /dev/null
+++ b/lib/enzyme/tests/parsers/ebml/test1.mkv.yml
@@ -0,0 +1,2974 @@
+- - 440786851
+ - 6
+ - EBML
+ - 0
+ - 5
+ - 19
+ - - [17026, 3, DocType, 1, 8, 8, matroska]
+ - [17031, 1, DocTypeVersion, 1, 19, 1, 2]
+ - [17029, 1, DocTypeReadVersion, 1, 23, 1, 2]
+- - 408125543
+ - 6
+ - Segment
+ - 0
+ - 32
+ - 23339305
+ - - - 290298740
+ - 6
+ - SeekHead
+ - 1
+ - 37
+ - 59
+ - - - 19899
+ - 6
+ - Seek
+ - 2
+ - 40
+ - 11
+ - - [21419, 7, SeekID, 3, 43, 4, null]
+ - [21420, 1, SeekPosition, 3, 50, 1, 64]
+ - - 19899
+ - 6
+ - Seek
+ - 2
+ - 54
+ - 12
+ - - [21419, 7, SeekID, 3, 57, 4, null]
+ - [21420, 1, SeekPosition, 3, 64, 2, 275]
+ - - 19899
+ - 6
+ - Seek
+ - 2
+ - 69
+ - 12
+ - - [21419, 7, SeekID, 3, 72, 4, null]
+ - [21420, 1, SeekPosition, 3, 79, 2, 440]
+ - - 19899
+ - 6
+ - Seek
+ - 2
+ - 84
+ - 12
+ - - [21419, 7, SeekID, 3, 87, 4, null]
+ - [21420, 1, SeekPosition, 3, 94, 2, 602]
+ - - 357149030
+ - 6
+ - Info
+ - 1
+ - 102
+ - 205
+ - - [17545, 2, Duration, 2, 105, 4, 87336.0]
+ - [19840, 4, MuxingApp, 2, 112, 39, libebml2 v0.10.0 + libmatroska2 v0.10.1]
+ - [22337, 4, WritingApp, 2, 154, 123, 'mkclean 0.5.5 ru from libebml v1.0.0
+ + libmatroska v1.0.0 + mkvmerge v4.1.1 (''Bouncin'' Back'') built on Jul 3
+ 2010 22:54:08']
+ - [17505, 5, DateUTC, 2, 280, 8, !!timestamp '2010-08-21 07:23:03']
+ - [29604, 7, SegmentUID, 2, 291, 16, null]
+ - - 374648427
+ - 6
+ - Tracks
+ - 1
+ - 313
+ - 159
+ - - - 174
+ - 6
+ - TrackEntry
+ - 2
+ - 315
+ - 105
+ - - [215, 1, TrackNumber, 3, 317, 1, 1]
+ - [131, 1, TrackType, 3, 320, 1, 1]
+ - [134, 3, CodecID, 3, 323, 15, V_MS/VFW/FOURCC]
+ - [29637, 1, TrackUID, 3, 341, 4, 2422994868]
+ - [156, 1, FlagLacing, 3, 347, 1, 0]
+ - [28135, 1, MinCache, 3, 351, 1, 1]
+ - [25506, 7, CodecPrivate, 3, 355, 40, null]
+ - [2352003, 1, DefaultDuration, 3, 399, 4, 41666666]
+ - [2274716, 3, Language, 3, 407, 3, und]
+ - - 224
+ - 6
+ - Video
+ - 3
+ - 412
+ - 8
+ - - [176, 1, PixelWidth, 4, 414, 2, 854]
+ - [186, 1, PixelHeight, 4, 418, 2, 480]
+ - - 174
+ - 6
+ - TrackEntry
+ - 2
+ - 422
+ - 50
+ - - [215, 1, TrackNumber, 3, 424, 1, 2]
+ - [131, 1, TrackType, 3, 427, 1, 2]
+ - [134, 3, CodecID, 3, 430, 9, A_MPEG/L3]
+ - [29637, 1, TrackUID, 3, 442, 4, 3653291187]
+ - [2352003, 1, DefaultDuration, 3, 450, 4, 24000000]
+ - [2274716, 3, Language, 3, 458, 3, und]
+ - - 225
+ - 6
+ - Audio
+ - 3
+ - 463
+ - 9
+ - - [181, 2, SamplingFrequency, 4, 465, 4, 48000.0]
+ - [159, 1, Channels, 4, 471, 1, 2]
+ - - 307544935
+ - 6
+ - Tags
+ - 1
+ - 478
+ - 156
+ - - - 29555
+ - 6
+ - Tag
+ - 2
+ - 482
+ - 152
+ - - - 25536
+ - 6
+ - Targets
+ - 3
+ - 485
+ - 0
+ - []
+ - - 26568
+ - 6
+ - SimpleTag
+ - 3
+ - 488
+ - 34
+ - - [17827, 4, TagName, 4, 491, 5, TITLE]
+ - [17543, 4, TagString, 4, 499, 23, Big Buck Bunny - test 1]
+ - - 26568
+ - 6
+ - SimpleTag
+ - 3
+ - 525
+ - 23
+ - - [17827, 4, TagName, 4, 528, 13, DATE_RELEASED]
+ - [17543, 4, TagString, 4, 544, 4, '2010']
+ - - 26568
+ - 6
+ - SimpleTag
+ - 3
+ - 551
+ - 83
+ - - [17827, 4, TagName, 4, 554, 7, COMMENT]
+ - [17543, 4, TagString, 4, 564, 70, 'Matroska Validation File1, basic
+ MPEG4.2 and MP3 with only SimpleBlock']
+ - - 475249515
+ - 6
+ - Cues
+ - 1
+ - 640
+ - 163
+ - - - 187
+ - 6
+ - CuePoint
+ - 2
+ - 642
+ - 12
+ - - [179, 1, CueTime, 3, 644, 1, 0]
+ - - 183
+ - 6
+ - CueTrackPositions
+ - 3
+ - 647
+ - 7
+ - - [247, 1, CueTrack, 4, 649, 1, 1]
+ - [241, 1, CueClusterPosition, 4, 652, 2, 771]
+ - - 187
+ - 6
+ - CuePoint
+ - 2
+ - 656
+ - 14
+ - - [179, 1, CueTime, 3, 658, 2, 1042]
+ - - 183
+ - 6
+ - CueTrackPositions
+ - 3
+ - 662
+ - 8
+ - - [247, 1, CueTrack, 4, 664, 1, 1]
+ - [241, 1, CueClusterPosition, 4, 667, 3, 145582]
+ - - 187
+ - 6
+ - CuePoint
+ - 2
+ - 672
+ - 14
+ - - [179, 1, CueTime, 3, 674, 2, 11667]
+ - - 183
+ - 6
+ - CueTrackPositions
+ - 3
+ - 678
+ - 8
+ - - [247, 1, CueTrack, 4, 680, 1, 1]
+ - [241, 1, CueClusterPosition, 4, 683, 3, 3131552]
+ - - 187
+ - 6
+ - CuePoint
+ - 2
+ - 688
+ - 14
+ - - [179, 1, CueTime, 3, 690, 2, 22083]
+ - - 183
+ - 6
+ - CueTrackPositions
+ - 3
+ - 694
+ - 8
+ - - [247, 1, CueTrack, 4, 696, 1, 1]
+ - [241, 1, CueClusterPosition, 4, 699, 3, 5654336]
+ - - 187
+ - 6
+ - CuePoint
+ - 2
+ - 704
+ - 14
+ - - [179, 1, CueTime, 3, 706, 2, 32500]
+ - - 183
+ - 6
+ - CueTrackPositions
+ - 3
+ - 710
+ - 8
+ - - [247, 1, CueTrack, 4, 712, 1, 1]
+ - [241, 1, CueClusterPosition, 4, 715, 3, 9696374]
+ - - 187
+ - 6
+ - CuePoint
+ - 2
+ - 720
+ - 14
+ - - [179, 1, CueTime, 3, 722, 2, 42917]
+ - - 183
+ - 6
+ - CueTrackPositions
+ - 3
+ - 726
+ - 8
+ - - [247, 1, CueTrack, 4, 728, 1, 1]
+ - [241, 1, CueClusterPosition, 4, 731, 3, 13440514]
+ - - 187
+ - 6
+ - CuePoint
+ - 2
+ - 736
+ - 14
+ - - [179, 1, CueTime, 3, 738, 2, 53333]
+ - - 183
+ - 6
+ - CueTrackPositions
+ - 3
+ - 742
+ - 8
+ - - [247, 1, CueTrack, 4, 744, 1, 1]
+ - [241, 1, CueClusterPosition, 4, 747, 3, 16690071]
+ - - 187
+ - 6
+ - CuePoint
+ - 2
+ - 752
+ - 15
+ - - [179, 1, CueTime, 3, 754, 2, 56083]
+ - - 183
+ - 6
+ - CueTrackPositions
+ - 3
+ - 758
+ - 9
+ - - [247, 1, CueTrack, 4, 760, 1, 1]
+ - [241, 1, CueClusterPosition, 4, 763, 4, 17468879]
+ - - 187
+ - 6
+ - CuePoint
+ - 2
+ - 769
+ - 16
+ - - [179, 1, CueTime, 3, 771, 3, 66500]
+ - - 183
+ - 6
+ - CueTrackPositions
+ - 3
+ - 776
+ - 9
+ - - [247, 1, CueTrack, 4, 778, 1, 1]
+ - [241, 1, CueClusterPosition, 4, 781, 4, 18628759]
+ - - 187
+ - 6
+ - CuePoint
+ - 2
+ - 787
+ - 16
+ - - [179, 1, CueTime, 3, 789, 3, 76917]
+ - - 183
+ - 6
+ - CueTrackPositions
+ - 3
+ - 794
+ - 9
+ - - [247, 1, CueTrack, 4, 796, 1, 1]
+ - [241, 1, CueClusterPosition, 4, 799, 4, 20732433]
+ - - 524531317
+ - 6
+ - Cluster
+ - 1
+ - 810
+ - 144804
+ - - [231, 1, Timecode, 2, 812, 1, 0]
+ - [163, 7, SimpleBlock, 2, 816, 5008, null]
+ - [163, 7, SimpleBlock, 2, 5827, 4464, null]
+ - [163, 7, SimpleBlock, 2, 10294, 303, null]
+ - [163, 7, SimpleBlock, 2, 10600, 303, null]
+ - [163, 7, SimpleBlock, 2, 10906, 208, null]
+ - [163, 7, SimpleBlock, 2, 11117, 676, null]
+ - [163, 7, SimpleBlock, 2, 11796, 2465, null]
+ - [163, 7, SimpleBlock, 2, 14264, 2794, null]
+ - [163, 7, SimpleBlock, 2, 17061, 4486, null]
+ - [163, 7, SimpleBlock, 2, 21550, 4966, null]
+ - [163, 7, SimpleBlock, 2, 26519, 580, null]
+ - [163, 7, SimpleBlock, 2, 27102, 4476, null]
+ - [163, 7, SimpleBlock, 2, 31581, 3077, null]
+ - [163, 7, SimpleBlock, 2, 34661, 4485, null]
+ - [163, 7, SimpleBlock, 2, 39149, 5117, null]
+ - [163, 7, SimpleBlock, 2, 44269, 1639, null]
+ - [163, 7, SimpleBlock, 2, 45911, 4521, null]
+ - [163, 7, SimpleBlock, 2, 50435, 772, null]
+ - [163, 7, SimpleBlock, 2, 51210, 4543, null]
+ - [163, 7, SimpleBlock, 2, 55756, 3371, null]
+ - [163, 7, SimpleBlock, 2, 59130, 4602, null]
+ - [163, 7, SimpleBlock, 2, 63735, 5427, null]
+ - [163, 7, SimpleBlock, 2, 69165, 1735, null]
+ - [163, 7, SimpleBlock, 2, 70903, 4790, null]
+ - [163, 7, SimpleBlock, 2, 75696, 772, null]
+ - [163, 7, SimpleBlock, 2, 76471, 4905, null]
+ - [163, 7, SimpleBlock, 2, 81379, 1639, null]
+ - [163, 7, SimpleBlock, 2, 83021, 5052, null]
+ - [163, 7, SimpleBlock, 2, 88076, 2697, null]
+ - [163, 7, SimpleBlock, 2, 90776, 5215, null]
+ - [163, 7, SimpleBlock, 2, 95994, 3371, null]
+ - [163, 7, SimpleBlock, 2, 99368, 5630, null]
+ - [163, 7, SimpleBlock, 2, 105001, 5582, null]
+ - [163, 7, SimpleBlock, 2, 110586, 5696, null]
+ - [163, 7, SimpleBlock, 2, 116285, 2505, null]
+ - [163, 7, SimpleBlock, 2, 118793, 6002, null]
+ - [163, 7, SimpleBlock, 2, 124798, 5794, null]
+ - [163, 7, SimpleBlock, 2, 130595, 2601, null]
+ - [163, 7, SimpleBlock, 2, 133199, 6520, null]
+ - [163, 7, SimpleBlock, 2, 139722, 5892, null]
+ - - 524531317
+ - 6
+ - Cluster
+ - 1
+ - 145621
+ - 41405
+ - - [231, 1, Timecode, 2, 145623, 2, 1042]
+ - [163, 7, SimpleBlock, 2, 145628, 964, null]
+ - [163, 7, SimpleBlock, 2, 146595, 2504, null]
+ - [163, 7, SimpleBlock, 2, 149102, 7082, null]
+ - [163, 7, SimpleBlock, 2, 156187, 6024, null]
+ - [163, 7, SimpleBlock, 2, 162214, 4237, null]
+ - [163, 7, SimpleBlock, 2, 166454, 7739, null]
+ - [163, 7, SimpleBlock, 2, 174196, 6210, null]
+ - [163, 7, SimpleBlock, 2, 180409, 6617, null]
+ - - 524531317
+ - 6
+ - Cluster
+ - 1
+ - 187034
+ - 2944550
+ - - [231, 1, Timecode, 2, 187036, 2, 1250]
+ - [163, 7, SimpleBlock, 2, 187041, 772, null]
+ - [163, 7, SimpleBlock, 2, 187816, 6736, null]
+ - [163, 7, SimpleBlock, 2, 194555, 8731, null]
+ - [163, 7, SimpleBlock, 2, 203289, 6522, null]
+ - [163, 7, SimpleBlock, 2, 209814, 7087, null]
+ - [163, 7, SimpleBlock, 2, 216904, 7323, null]
+ - [163, 7, SimpleBlock, 2, 224230, 7629, null]
+ - [163, 7, SimpleBlock, 2, 231862, 6546, null]
+ - [163, 7, SimpleBlock, 2, 238411, 7860, null]
+ - [163, 7, SimpleBlock, 2, 246274, 7989, null]
+ - [163, 7, SimpleBlock, 2, 254266, 8281, null]
+ - [163, 7, SimpleBlock, 2, 262550, 8399, null]
+ - [163, 7, SimpleBlock, 2, 270952, 5967, null]
+ - [163, 7, SimpleBlock, 2, 276922, 8557, null]
+ - [163, 7, SimpleBlock, 2, 285482, 8820, null]
+ - [163, 7, SimpleBlock, 2, 294305, 8886, null]
+ - [163, 7, SimpleBlock, 2, 303194, 8997, null]
+ - [163, 7, SimpleBlock, 2, 312194, 9160, null]
+ - [163, 7, SimpleBlock, 2, 321357, 6643, null]
+ - [163, 7, SimpleBlock, 2, 328003, 9359, null]
+ - [163, 7, SimpleBlock, 2, 337365, 9630, null]
+ - [163, 7, SimpleBlock, 2, 346998, 10035, null]
+ - [163, 7, SimpleBlock, 2, 357036, 10450, null]
+ - [163, 7, SimpleBlock, 2, 367489, 6641, null]
+ - [163, 7, SimpleBlock, 2, 374133, 11054, null]
+ - [163, 7, SimpleBlock, 2, 385190, 11571, null]
+ - [163, 7, SimpleBlock, 2, 396764, 11910, null]
+ - [163, 7, SimpleBlock, 2, 408677, 4492, null]
+ - [163, 7, SimpleBlock, 2, 413172, 4513, null]
+ - [163, 7, SimpleBlock, 2, 417688, 6931, null]
+ - [163, 7, SimpleBlock, 2, 424622, 5450, null]
+ - [163, 7, SimpleBlock, 2, 430075, 5226, null]
+ - [163, 7, SimpleBlock, 2, 435304, 5387, null]
+ - [163, 7, SimpleBlock, 2, 440694, 5433, null]
+ - [163, 7, SimpleBlock, 2, 446130, 5557, null]
+ - [163, 7, SimpleBlock, 2, 451690, 6163, null]
+ - [163, 7, SimpleBlock, 2, 457856, 5576, null]
+ - [163, 7, SimpleBlock, 2, 463435, 5832, null]
+ - [163, 7, SimpleBlock, 2, 469270, 5718, null]
+ - [163, 7, SimpleBlock, 2, 474991, 5658, null]
+ - [163, 7, SimpleBlock, 2, 480652, 6161, null]
+ - [163, 7, SimpleBlock, 2, 486816, 5455, null]
+ - [163, 7, SimpleBlock, 2, 492274, 5361, null]
+ - [163, 7, SimpleBlock, 2, 497638, 5391, null]
+ - [163, 7, SimpleBlock, 2, 503032, 5249, null]
+ - [163, 7, SimpleBlock, 2, 508284, 5241, null]
+ - [163, 7, SimpleBlock, 2, 513528, 6161, null]
+ - [163, 7, SimpleBlock, 2, 519692, 5189, null]
+ - [163, 7, SimpleBlock, 2, 524884, 5186, null]
+ - [163, 7, SimpleBlock, 2, 530073, 5185, null]
+ - [163, 7, SimpleBlock, 2, 535261, 5443, null]
+ - [163, 7, SimpleBlock, 2, 540707, 5587, null]
+ - [163, 7, SimpleBlock, 2, 546297, 5559, null]
+ - [163, 7, SimpleBlock, 2, 551859, 5899, null]
+ - [163, 7, SimpleBlock, 2, 557761, 6247, null]
+ - [163, 7, SimpleBlock, 2, 564011, 6210, null]
+ - [163, 7, SimpleBlock, 2, 570224, 6362, null]
+ - [163, 7, SimpleBlock, 2, 576589, 5776, null]
+ - [163, 7, SimpleBlock, 2, 582368, 6608, null]
+ - [163, 7, SimpleBlock, 2, 588979, 6560, null]
+ - [163, 7, SimpleBlock, 2, 595542, 6658, null]
+ - [163, 7, SimpleBlock, 2, 602203, 7020, null]
+ - [163, 7, SimpleBlock, 2, 609226, 7107, null]
+ - [163, 7, SimpleBlock, 2, 616336, 6063, null]
+ - [163, 7, SimpleBlock, 2, 622402, 7022, null]
+ - [163, 7, SimpleBlock, 2, 629427, 7149, null]
+ - [163, 7, SimpleBlock, 2, 636579, 7180, null]
+ - [163, 7, SimpleBlock, 2, 643762, 7213, null]
+ - [163, 7, SimpleBlock, 2, 650978, 5967, null]
+ - [163, 7, SimpleBlock, 2, 656948, 7189, null]
+ - [163, 7, SimpleBlock, 2, 664140, 7478, null]
+ - [163, 7, SimpleBlock, 2, 671621, 7488, null]
+ - [163, 7, SimpleBlock, 2, 679112, 7491, null]
+ - [163, 7, SimpleBlock, 2, 686606, 7515, null]
+ - [163, 7, SimpleBlock, 2, 694124, 5873, null]
+ - [163, 7, SimpleBlock, 2, 700000, 7718, null]
+ - [163, 7, SimpleBlock, 2, 707721, 7485, null]
+ - [163, 7, SimpleBlock, 2, 715209, 7448, null]
+ - [163, 7, SimpleBlock, 2, 722660, 7483, null]
+ - [163, 7, SimpleBlock, 2, 730146, 7497, null]
+ - [163, 7, SimpleBlock, 2, 737646, 5682, null]
+ - [163, 7, SimpleBlock, 2, 743331, 7583, null]
+ - [163, 7, SimpleBlock, 2, 750917, 7666, null]
+ - [163, 7, SimpleBlock, 2, 758586, 7792, null]
+ - [163, 7, SimpleBlock, 2, 766381, 7810, null]
+ - [163, 7, SimpleBlock, 2, 774194, 5778, null]
+ - [163, 7, SimpleBlock, 2, 779975, 7823, null]
+ - [163, 7, SimpleBlock, 2, 787801, 7962, null]
+ - [163, 7, SimpleBlock, 2, 795766, 8032, null]
+ - [163, 7, SimpleBlock, 2, 803801, 8119, null]
+ - [163, 7, SimpleBlock, 2, 811923, 8142, null]
+ - [163, 7, SimpleBlock, 2, 820068, 5874, null]
+ - [163, 7, SimpleBlock, 2, 825945, 8045, null]
+ - [163, 7, SimpleBlock, 2, 833993, 8247, null]
+ - [163, 7, SimpleBlock, 2, 842243, 8393, null]
+ - [163, 7, SimpleBlock, 2, 850639, 8264, null]
+ - [163, 7, SimpleBlock, 2, 858906, 6062, null]
+ - [163, 7, SimpleBlock, 2, 864971, 8456, null]
+ - [163, 7, SimpleBlock, 2, 873430, 8595, null]
+ - [163, 7, SimpleBlock, 2, 882028, 8604, null]
+ - [163, 7, SimpleBlock, 2, 890635, 8690, null]
+ - [163, 7, SimpleBlock, 2, 899328, 8682, null]
+ - [163, 7, SimpleBlock, 2, 908013, 5874, null]
+ - [163, 7, SimpleBlock, 2, 913890, 8927, null]
+ - [163, 7, SimpleBlock, 2, 922820, 8768, null]
+ - [163, 7, SimpleBlock, 2, 931591, 9073, null]
+ - [163, 7, SimpleBlock, 2, 940667, 9001, null]
+ - [163, 7, SimpleBlock, 2, 949671, 8907, null]
+ - [163, 7, SimpleBlock, 2, 958581, 5873, null]
+ - [163, 7, SimpleBlock, 2, 964457, 8930, null]
+ - [163, 7, SimpleBlock, 2, 973390, 8900, null]
+ - [163, 7, SimpleBlock, 2, 982293, 9019, null]
+ - [163, 7, SimpleBlock, 2, 991315, 9005, null]
+ - [163, 7, SimpleBlock, 2, 1000323, 5873, null]
+ - [163, 7, SimpleBlock, 2, 1006199, 9000, null]
+ - [163, 7, SimpleBlock, 2, 1015202, 9075, null]
+ - [163, 7, SimpleBlock, 2, 1024280, 9002, null]
+ - [163, 7, SimpleBlock, 2, 1033285, 9161, null]
+ - [163, 7, SimpleBlock, 2, 1042449, 9136, null]
+ - [163, 7, SimpleBlock, 2, 1051588, 5682, null]
+ - [163, 7, SimpleBlock, 2, 1057273, 9178, null]
+ - [163, 7, SimpleBlock, 2, 1066454, 9207, null]
+ - [163, 7, SimpleBlock, 2, 1075664, 9305, null]
+ - [163, 7, SimpleBlock, 2, 1084972, 9626, null]
+ - [163, 7, SimpleBlock, 2, 1094601, 5873, null]
+ - [163, 7, SimpleBlock, 2, 1100477, 9755, null]
+ - [163, 7, SimpleBlock, 2, 1110235, 9724, null]
+ - [163, 7, SimpleBlock, 2, 1119962, 9933, null]
+ - [163, 7, SimpleBlock, 2, 1129898, 9880, null]
+ - [163, 7, SimpleBlock, 2, 1139781, 10249, null]
+ - [163, 7, SimpleBlock, 2, 1150033, 6350, null]
+ - [163, 7, SimpleBlock, 2, 1156386, 10265, null]
+ - [163, 7, SimpleBlock, 2, 1166654, 10385, null]
+ - [163, 7, SimpleBlock, 2, 1177042, 10350, null]
+ - [163, 7, SimpleBlock, 2, 1187395, 10340, null]
+ - [163, 7, SimpleBlock, 2, 1197738, 10483, null]
+ - [163, 7, SimpleBlock, 2, 1208224, 6739, null]
+ - [163, 7, SimpleBlock, 2, 1214966, 10579, null]
+ - [163, 7, SimpleBlock, 2, 1225548, 10512, null]
+ - [163, 7, SimpleBlock, 2, 1236063, 10449, null]
+ - [163, 7, SimpleBlock, 2, 1246515, 10633, null]
+ - [163, 7, SimpleBlock, 2, 1257151, 6642, null]
+ - [163, 7, SimpleBlock, 2, 1263796, 10454, null]
+ - [163, 7, SimpleBlock, 2, 1274253, 10695, null]
+ - [163, 7, SimpleBlock, 2, 1284951, 10452, null]
+ - [163, 7, SimpleBlock, 2, 1295406, 10663, null]
+ - [163, 7, SimpleBlock, 2, 1306072, 10309, null]
+ - [163, 7, SimpleBlock, 2, 1316384, 6547, null]
+ - [163, 7, SimpleBlock, 2, 1322934, 10359, null]
+ - [163, 7, SimpleBlock, 2, 1333296, 10337, null]
+ - [163, 7, SimpleBlock, 2, 1343636, 10027, null]
+ - [163, 7, SimpleBlock, 2, 1353666, 9883, null]
+ - [163, 7, SimpleBlock, 2, 1363552, 6451, null]
+ - [163, 7, SimpleBlock, 2, 1370006, 9643, null]
+ - [163, 7, SimpleBlock, 2, 1379652, 9148, null]
+ - [163, 7, SimpleBlock, 2, 1388803, 8794, null]
+ - [163, 7, SimpleBlock, 2, 1397600, 8468, null]
+ - [163, 7, SimpleBlock, 2, 1406071, 8372, null]
+ - [163, 7, SimpleBlock, 2, 1414446, 6835, null]
+ - [163, 7, SimpleBlock, 2, 1421284, 8121, null]
+ - [163, 7, SimpleBlock, 2, 1429408, 8022, null]
+ - [163, 7, SimpleBlock, 2, 1437433, 8096, null]
+ - [163, 7, SimpleBlock, 2, 1445532, 7920, null]
+ - [163, 7, SimpleBlock, 2, 1453455, 7699, null]
+ - [163, 7, SimpleBlock, 2, 1461157, 6545, null]
+ - [163, 7, SimpleBlock, 2, 1467705, 7707, null]
+ - [163, 7, SimpleBlock, 2, 1475415, 7821, null]
+ - [163, 7, SimpleBlock, 2, 1483239, 7978, null]
+ - [163, 7, SimpleBlock, 2, 1491220, 8241, null]
+ - [163, 7, SimpleBlock, 2, 1499464, 5778, null]
+ - [163, 7, SimpleBlock, 2, 1505245, 8282, null]
+ - [163, 7, SimpleBlock, 2, 1513530, 8598, null]
+ - [163, 7, SimpleBlock, 2, 1522131, 9098, null]
+ - [163, 7, SimpleBlock, 2, 1531232, 9644, null]
+ - [163, 7, SimpleBlock, 2, 1540879, 10086, null]
+ - [163, 7, SimpleBlock, 2, 1550968, 5779, null]
+ - [163, 7, SimpleBlock, 2, 1556750, 10191, null]
+ - [163, 7, SimpleBlock, 2, 1566944, 10458, null]
+ - [163, 7, SimpleBlock, 2, 1577405, 10570, null]
+ - [163, 7, SimpleBlock, 2, 1587978, 11074, null]
+ - [163, 7, SimpleBlock, 2, 1599055, 6158, null]
+ - [163, 7, SimpleBlock, 2, 1605216, 11120, null]
+ - [163, 7, SimpleBlock, 2, 1616339, 11421, null]
+ - [163, 7, SimpleBlock, 2, 1627763, 11589, null]
+ - [163, 7, SimpleBlock, 2, 1639355, 11727, null]
+ - [163, 7, SimpleBlock, 2, 1651085, 11990, null]
+ - [163, 7, SimpleBlock, 2, 1663078, 6352, null]
+ - [163, 7, SimpleBlock, 2, 1669433, 12178, null]
+ - [163, 7, SimpleBlock, 2, 1681614, 12242, null]
+ - [163, 7, SimpleBlock, 2, 1693859, 12403, null]
+ - [163, 7, SimpleBlock, 2, 1706265, 12268, null]
+ - [163, 7, SimpleBlock, 2, 1718536, 12507, null]
+ - [163, 7, SimpleBlock, 2, 1731046, 6450, null]
+ - [163, 7, SimpleBlock, 2, 1737499, 12548, null]
+ - [163, 7, SimpleBlock, 2, 1750050, 12540, null]
+ - [163, 7, SimpleBlock, 2, 1762593, 12616, null]
+ - [163, 7, SimpleBlock, 2, 1775212, 12497, null]
+ - [163, 7, SimpleBlock, 2, 1787712, 5586, null]
+ - [163, 7, SimpleBlock, 2, 1793301, 12619, null]
+ - [163, 7, SimpleBlock, 2, 1805923, 12645, null]
+ - [163, 7, SimpleBlock, 2, 1818571, 12819, null]
+ - [163, 7, SimpleBlock, 2, 1831393, 12553, null]
+ - [163, 7, SimpleBlock, 2, 1843949, 12186, null]
+ - [163, 7, SimpleBlock, 2, 1856138, 6349, null]
+ - [163, 7, SimpleBlock, 2, 1862490, 12232, null]
+ - [163, 7, SimpleBlock, 2, 1874725, 11787, null]
+ - [163, 7, SimpleBlock, 2, 1886515, 12022, null]
+ - [163, 7, SimpleBlock, 2, 1898540, 11715, null]
+ - [163, 7, SimpleBlock, 2, 1910258, 11778, null]
+ - [163, 7, SimpleBlock, 2, 1922039, 6258, null]
+ - [163, 7, SimpleBlock, 2, 1928300, 11504, null]
+ - [163, 7, SimpleBlock, 2, 1939807, 11427, null]
+ - [163, 7, SimpleBlock, 2, 1951237, 11323, null]
+ - [163, 7, SimpleBlock, 2, 1962563, 10800, null]
+ - [163, 7, SimpleBlock, 2, 1973366, 6258, null]
+ - [163, 7, SimpleBlock, 2, 1979627, 10602, null]
+ - [163, 7, SimpleBlock, 2, 1990232, 10219, null]
+ - [163, 7, SimpleBlock, 2, 2000454, 9952, null]
+ - [163, 7, SimpleBlock, 2, 2010409, 10054, null]
+ - [163, 7, SimpleBlock, 2, 2020466, 10129, null]
+ - [163, 7, SimpleBlock, 2, 2030598, 6065, null]
+ - [163, 7, SimpleBlock, 2, 2036666, 10124, null]
+ - [163, 7, SimpleBlock, 2, 2046793, 10209, null]
+ - [163, 7, SimpleBlock, 2, 2057005, 10584, null]
+ - [163, 7, SimpleBlock, 2, 2067592, 10618, null]
+ - [163, 7, SimpleBlock, 2, 2078213, 5970, null]
+ - [163, 7, SimpleBlock, 2, 2084186, 11182, null]
+ - [163, 7, SimpleBlock, 2, 2095371, 11631, null]
+ - [163, 7, SimpleBlock, 2, 2107005, 12268, null]
+ - [163, 7, SimpleBlock, 2, 2119276, 13038, null]
+ - [163, 7, SimpleBlock, 2, 2132317, 13455, null]
+ - [163, 7, SimpleBlock, 2, 2145775, 5970, null]
+ - [163, 7, SimpleBlock, 2, 2151748, 13833, null]
+ - [163, 7, SimpleBlock, 2, 2165584, 13984, null]
+ - [163, 7, SimpleBlock, 2, 2179571, 13708, null]
+ - [163, 7, SimpleBlock, 2, 2193282, 13782, null]
+ - [163, 7, SimpleBlock, 2, 2207067, 14245, null]
+ - [163, 7, SimpleBlock, 2, 2221315, 5680, null]
+ - [163, 7, SimpleBlock, 2, 2226998, 14394, null]
+ - [163, 7, SimpleBlock, 2, 2241395, 14877, null]
+ - [163, 7, SimpleBlock, 2, 2256275, 15072, null]
+ - [163, 7, SimpleBlock, 2, 2271350, 15391, null]
+ - [163, 7, SimpleBlock, 2, 2286744, 5680, null]
+ - [163, 7, SimpleBlock, 2, 2292427, 15642, null]
+ - [163, 7, SimpleBlock, 2, 2308072, 15860, null]
+ - [163, 7, SimpleBlock, 2, 2323935, 16213, null]
+ - [163, 7, SimpleBlock, 2, 2340152, 16528, null]
+ - [163, 7, SimpleBlock, 2, 2356684, 16926, null]
+ - [163, 7, SimpleBlock, 2, 2373613, 5585, null]
+ - [163, 7, SimpleBlock, 2, 2379202, 16873, null]
+ - [163, 7, SimpleBlock, 2, 2396079, 17018, null]
+ - [163, 7, SimpleBlock, 2, 2413101, 16919, null]
+ - [163, 7, SimpleBlock, 2, 2430024, 17045, null]
+ - [163, 7, SimpleBlock, 2, 2447072, 5392, null]
+ - [163, 7, SimpleBlock, 2, 2452468, 16885, null]
+ - [163, 7, SimpleBlock, 2, 2469357, 16916, null]
+ - [163, 7, SimpleBlock, 2, 2486277, 16981, null]
+ - [163, 7, SimpleBlock, 2, 2503262, 16714, null]
+ - [163, 7, SimpleBlock, 2, 2519980, 16876, null]
+ - [163, 7, SimpleBlock, 2, 2536859, 5583, null]
+ - [163, 7, SimpleBlock, 2, 2542446, 16975, null]
+ - [163, 7, SimpleBlock, 2, 2559425, 17112, null]
+ - [163, 7, SimpleBlock, 2, 2576541, 17040, null]
+ - [163, 7, SimpleBlock, 2, 2593585, 17198, null]
+ - [163, 7, SimpleBlock, 2, 2610787, 17325, null]
+ - [163, 7, SimpleBlock, 2, 2628115, 5967, null]
+ - [163, 7, SimpleBlock, 2, 2634086, 17301, null]
+ - [163, 7, SimpleBlock, 2, 2651391, 17363, null]
+ - [163, 7, SimpleBlock, 2, 2668758, 17444, null]
+ - [163, 7, SimpleBlock, 2, 2686206, 17214, null]
+ - [163, 7, SimpleBlock, 2, 2703423, 5968, null]
+ - [163, 7, SimpleBlock, 2, 2709395, 16998, null]
+ - [163, 7, SimpleBlock, 2, 2726397, 16808, null]
+ - [163, 7, SimpleBlock, 2, 2743208, 16300, null]
+ - [163, 7, SimpleBlock, 2, 2759511, 16046, null]
+ - [163, 7, SimpleBlock, 2, 2775560, 15219, null]
+ - [163, 7, SimpleBlock, 2, 2790782, 2313, null]
+ - [163, 7, SimpleBlock, 2, 2793098, 15047, null]
+ - [163, 7, SimpleBlock, 2, 2808148, 14767, null]
+ - [163, 7, SimpleBlock, 2, 2822918, 6352, null]
+ - [163, 7, SimpleBlock, 2, 2829273, 14386, null]
+ - [163, 7, SimpleBlock, 2, 2843662, 14226, null]
+ - [163, 7, SimpleBlock, 2, 2857891, 14208, null]
+ - [163, 7, SimpleBlock, 2, 2872102, 14241, null]
+ - [163, 7, SimpleBlock, 2, 2886346, 5970, null]
+ - [163, 7, SimpleBlock, 2, 2892319, 13992, null]
+ - [163, 7, SimpleBlock, 2, 2906314, 14075, null]
+ - [163, 7, SimpleBlock, 2, 2920392, 13939, null]
+ - [163, 7, SimpleBlock, 2, 2934334, 13791, null]
+ - [163, 7, SimpleBlock, 2, 2948128, 13671, null]
+ - [163, 7, SimpleBlock, 2, 2961802, 5874, null]
+ - [163, 7, SimpleBlock, 2, 2967679, 13547, null]
+ - [163, 7, SimpleBlock, 2, 2981229, 13453, null]
+ - [163, 7, SimpleBlock, 2, 2994685, 13272, null]
+ - [163, 7, SimpleBlock, 2, 3007960, 12962, null]
+ - [163, 7, SimpleBlock, 2, 3020925, 5777, null]
+ - [163, 7, SimpleBlock, 2, 3026705, 12709, null]
+ - [163, 7, SimpleBlock, 2, 3039417, 12244, null]
+ - [163, 7, SimpleBlock, 2, 3051664, 12266, null]
+ - [163, 7, SimpleBlock, 2, 3063933, 12052, null]
+ - [163, 7, SimpleBlock, 2, 3075988, 11674, null]
+ - [163, 7, SimpleBlock, 2, 3087665, 4334, null]
+ - [163, 7, SimpleBlock, 2, 3092002, 10707, null]
+ - [163, 7, SimpleBlock, 2, 3102712, 10379, null]
+ - [163, 7, SimpleBlock, 2, 3113094, 9656, null]
+ - [163, 7, SimpleBlock, 2, 3122753, 8831, null]
+ - - 524531317
+ - 6
+ - Cluster
+ - 1
+ - 3131592
+ - 2522776
+ - - [231, 1, Timecode, 2, 3131594, 2, 11667]
+ - [163, 7, SimpleBlock, 2, 3131599, 676, null]
+ - [163, 7, SimpleBlock, 2, 3132278, 6066, null]
+ - [163, 7, SimpleBlock, 2, 3138348, 76018, null]
+ - [163, 7, SimpleBlock, 2, 3214369, 1660, null]
+ - [163, 7, SimpleBlock, 2, 3216032, 2664, null]
+ - [163, 7, SimpleBlock, 2, 3218699, 2864, null]
+ - [163, 7, SimpleBlock, 2, 3221566, 2369, null]
+ - [163, 7, SimpleBlock, 2, 3223938, 6547, null]
+ - [163, 7, SimpleBlock, 2, 3230489, 91368, null]
+ - [163, 7, SimpleBlock, 2, 3321860, 8748, null]
+ - [163, 7, SimpleBlock, 2, 3330611, 13105, null]
+ - [163, 7, SimpleBlock, 2, 3343719, 13051, null]
+ - [163, 7, SimpleBlock, 2, 3356773, 6641, null]
+ - [163, 7, SimpleBlock, 2, 3363417, 13474, null]
+ - [163, 7, SimpleBlock, 2, 3376894, 14246, null]
+ - [163, 7, SimpleBlock, 2, 3391143, 14613, null]
+ - [163, 7, SimpleBlock, 2, 3405759, 15195, null]
+ - [163, 7, SimpleBlock, 2, 3420957, 15310, null]
+ - [163, 7, SimpleBlock, 2, 3436270, 6546, null]
+ - [163, 7, SimpleBlock, 2, 3442819, 15441, null]
+ - [163, 7, SimpleBlock, 2, 3458263, 15653, null]
+ - [163, 7, SimpleBlock, 2, 3473919, 15680, null]
+ - [163, 7, SimpleBlock, 2, 3489602, 15627, null]
+ - [163, 7, SimpleBlock, 2, 3505232, 6547, null]
+ - [163, 7, SimpleBlock, 2, 3511782, 15376, null]
+ - [163, 7, SimpleBlock, 2, 3527161, 15431, null]
+ - [163, 7, SimpleBlock, 2, 3542595, 15411, null]
+ - [163, 7, SimpleBlock, 2, 3558009, 15211, null]
+ - [163, 7, SimpleBlock, 2, 3573223, 15589, null]
+ - [163, 7, SimpleBlock, 2, 3588815, 6353, null]
+ - [163, 7, SimpleBlock, 2, 3595171, 15450, null]
+ - [163, 7, SimpleBlock, 2, 3610624, 15443, null]
+ - [163, 7, SimpleBlock, 2, 3626070, 15422, null]
+ - [163, 7, SimpleBlock, 2, 3641495, 15484, null]
+ - [163, 7, SimpleBlock, 2, 3656982, 15369, null]
+ - [163, 7, SimpleBlock, 2, 3672354, 6543, null]
+ - [163, 7, SimpleBlock, 2, 3678900, 15472, null]
+ - [163, 7, SimpleBlock, 2, 3694375, 15538, null]
+ - [163, 7, SimpleBlock, 2, 3709916, 15403, null]
+ - [163, 7, SimpleBlock, 2, 3725322, 15527, null]
+ - [163, 7, SimpleBlock, 2, 3740852, 6353, null]
+ - [163, 7, SimpleBlock, 2, 3747208, 15560, null]
+ - [163, 7, SimpleBlock, 2, 3762771, 15725, null]
+ - [163, 7, SimpleBlock, 2, 3778499, 15805, null]
+ - [163, 7, SimpleBlock, 2, 3794307, 16012, null]
+ - [163, 7, SimpleBlock, 2, 3810322, 15586, null]
+ - [163, 7, SimpleBlock, 2, 3825911, 6355, null]
+ - [163, 7, SimpleBlock, 2, 3832269, 15751, null]
+ - [163, 7, SimpleBlock, 2, 3848023, 15878, null]
+ - [163, 7, SimpleBlock, 2, 3863904, 16069, null]
+ - [163, 7, SimpleBlock, 2, 3879976, 16014, null]
+ - [163, 7, SimpleBlock, 2, 3895993, 6641, null]
+ - [163, 7, SimpleBlock, 2, 3902637, 15962, null]
+ - [163, 7, SimpleBlock, 2, 3918602, 16056, null]
+ - [163, 7, SimpleBlock, 2, 3934661, 16113, null]
+ - [163, 7, SimpleBlock, 2, 3950777, 15808, null]
+ - [163, 7, SimpleBlock, 2, 3966588, 15957, null]
+ - [163, 7, SimpleBlock, 2, 3982548, 5872, null]
+ - [163, 7, SimpleBlock, 2, 3988423, 16047, null]
+ - [163, 7, SimpleBlock, 2, 4004473, 15885, null]
+ - [163, 7, SimpleBlock, 2, 4020361, 15939, null]
+ - [163, 7, SimpleBlock, 2, 4036303, 16219, null]
+ - [163, 7, SimpleBlock, 2, 4052525, 16099, null]
+ - [163, 7, SimpleBlock, 2, 4068627, 5969, null]
+ - [163, 7, SimpleBlock, 2, 4074599, 16044, null]
+ - [163, 7, SimpleBlock, 2, 4090646, 15843, null]
+ - [163, 7, SimpleBlock, 2, 4106492, 15565, null]
+ - [163, 7, SimpleBlock, 2, 4122060, 15513, null]
+ - [163, 7, SimpleBlock, 2, 4137576, 5969, null]
+ - [163, 7, SimpleBlock, 2, 4143548, 15671, null]
+ - [163, 7, SimpleBlock, 2, 4159222, 15472, null]
+ - [163, 7, SimpleBlock, 2, 4174697, 15694, null]
+ - [163, 7, SimpleBlock, 2, 4190394, 15367, null]
+ - [163, 7, SimpleBlock, 2, 4205764, 15550, null]
+ - [163, 7, SimpleBlock, 2, 4221317, 5874, null]
+ - [163, 7, SimpleBlock, 2, 4227194, 15799, null]
+ - [163, 7, SimpleBlock, 2, 4242996, 15468, null]
+ - [163, 7, SimpleBlock, 2, 4258467, 15683, null]
+ - [163, 7, SimpleBlock, 2, 4274153, 15831, null]
+ - [163, 7, SimpleBlock, 2, 4289987, 15649, null]
+ - [163, 7, SimpleBlock, 2, 4305639, 6161, null]
+ - [163, 7, SimpleBlock, 2, 4311803, 15674, null]
+ - [163, 7, SimpleBlock, 2, 4327480, 15947, null]
+ - [163, 7, SimpleBlock, 2, 4343430, 15950, null]
+ - [163, 7, SimpleBlock, 2, 4359383, 16024, null]
+ - [163, 7, SimpleBlock, 2, 4375410, 6546, null]
+ - [163, 7, SimpleBlock, 2, 4381959, 15905, null]
+ - [163, 7, SimpleBlock, 2, 4397867, 15804, null]
+ - [163, 7, SimpleBlock, 2, 4413674, 15923, null]
+ - [163, 7, SimpleBlock, 2, 4429600, 16016, null]
+ - [163, 7, SimpleBlock, 2, 4445619, 15976, null]
+ - [163, 7, SimpleBlock, 2, 4461598, 6161, null]
+ - [163, 7, SimpleBlock, 2, 4467762, 15653, null]
+ - [163, 7, SimpleBlock, 2, 4483418, 15624, null]
+ - [163, 7, SimpleBlock, 2, 4499045, 15816, null]
+ - [163, 7, SimpleBlock, 2, 4514864, 15789, null]
+ - [163, 7, SimpleBlock, 2, 4530656, 6065, null]
+ - [163, 7, SimpleBlock, 2, 4536724, 15807, null]
+ - [163, 7, SimpleBlock, 2, 4552534, 15778, null]
+ - [163, 7, SimpleBlock, 2, 4568315, 16016, null]
+ - [163, 7, SimpleBlock, 2, 4584335, 16391, null]
+ - [163, 7, SimpleBlock, 2, 4600729, 16213, null]
+ - [163, 7, SimpleBlock, 2, 4616945, 5968, null]
+ - [163, 7, SimpleBlock, 2, 4622917, 16515, null]
+ - [163, 7, SimpleBlock, 2, 4639436, 16489, null]
+ - [163, 7, SimpleBlock, 2, 4655928, 16261, null]
+ - [163, 7, SimpleBlock, 2, 4672193, 16569, null]
+ - [163, 7, SimpleBlock, 2, 4688766, 16611, null]
+ - [163, 7, SimpleBlock, 2, 4705380, 6162, null]
+ - [163, 7, SimpleBlock, 2, 4711545, 16272, null]
+ - [163, 7, SimpleBlock, 2, 4727821, 16456, null]
+ - [163, 7, SimpleBlock, 2, 4744281, 16625, null]
+ - [163, 7, SimpleBlock, 2, 4760909, 16309, null]
+ - [163, 7, SimpleBlock, 2, 4777221, 6257, null]
+ - [163, 7, SimpleBlock, 2, 4783481, 16124, null]
+ - [163, 7, SimpleBlock, 2, 4799608, 16054, null]
+ - [163, 7, SimpleBlock, 2, 4815665, 16133, null]
+ - [163, 7, SimpleBlock, 2, 4831801, 16104, null]
+ - [163, 7, SimpleBlock, 2, 4847908, 16074, null]
+ - [163, 7, SimpleBlock, 2, 4863985, 6257, null]
+ - [163, 7, SimpleBlock, 2, 4870245, 15985, null]
+ - [163, 7, SimpleBlock, 2, 4886234, 30557, null]
+ - [163, 7, SimpleBlock, 2, 4916794, 1070, null]
+ - [163, 7, SimpleBlock, 2, 4917867, 1018, null]
+ - [163, 7, SimpleBlock, 2, 4918888, 6547, null]
+ - [163, 7, SimpleBlock, 2, 4925438, 999, null]
+ - [163, 7, SimpleBlock, 2, 4926440, 978, null]
+ - [163, 7, SimpleBlock, 2, 4927421, 1346, null]
+ - [163, 7, SimpleBlock, 2, 4928770, 961, null]
+ - [163, 7, SimpleBlock, 2, 4929734, 2286, null]
+ - [163, 7, SimpleBlock, 2, 4932023, 6739, null]
+ - [163, 7, SimpleBlock, 2, 4938765, 4122, null]
+ - [163, 7, SimpleBlock, 2, 4942890, 4871, null]
+ - [163, 7, SimpleBlock, 2, 4947764, 4809, null]
+ - [163, 7, SimpleBlock, 2, 4952576, 3777, null]
+ - [163, 7, SimpleBlock, 2, 4956356, 4788, null]
+ - [163, 7, SimpleBlock, 2, 4961147, 6451, null]
+ - [163, 7, SimpleBlock, 2, 4967601, 5463, null]
+ - [163, 7, SimpleBlock, 2, 4973067, 6989, null]
+ - [163, 7, SimpleBlock, 2, 4980059, 8594, null]
+ - [163, 7, SimpleBlock, 2, 4988656, 8170, null]
+ - [163, 7, SimpleBlock, 2, 4996829, 6545, null]
+ - [163, 7, SimpleBlock, 2, 5003377, 3838, null]
+ - [163, 7, SimpleBlock, 2, 5007218, 3437, null]
+ - [163, 7, SimpleBlock, 2, 5010658, 2846, null]
+ - [163, 7, SimpleBlock, 2, 5013507, 2664, null]
+ - [163, 7, SimpleBlock, 2, 5016174, 2312, null]
+ - [163, 7, SimpleBlock, 2, 5018489, 6449, null]
+ - [163, 7, SimpleBlock, 2, 5024941, 2172, null]
+ - [163, 7, SimpleBlock, 2, 5027116, 2268, null]
+ - [163, 7, SimpleBlock, 2, 5029387, 2394, null]
+ - [163, 7, SimpleBlock, 2, 5031784, 2501, null]
+ - [163, 7, SimpleBlock, 2, 5034288, 6450, null]
+ - [163, 7, SimpleBlock, 2, 5040741, 2616, null]
+ - [163, 7, SimpleBlock, 2, 5043360, 2571, null]
+ - [163, 7, SimpleBlock, 2, 5045934, 2547, null]
+ - [163, 7, SimpleBlock, 2, 5048484, 2487, null]
+ - [163, 7, SimpleBlock, 2, 5050974, 2602, null]
+ - [163, 7, SimpleBlock, 2, 5053579, 6354, null]
+ - [163, 7, SimpleBlock, 2, 5059936, 2173, null]
+ - [163, 7, SimpleBlock, 2, 5062112, 2151, null]
+ - [163, 7, SimpleBlock, 2, 5064266, 2176, null]
+ - [163, 7, SimpleBlock, 2, 5066445, 2030, null]
+ - [163, 7, SimpleBlock, 2, 5068478, 1997, null]
+ - [163, 7, SimpleBlock, 2, 5070478, 6257, null]
+ - [163, 7, SimpleBlock, 2, 5076738, 1716, null]
+ - [163, 7, SimpleBlock, 2, 5078457, 3963, null]
+ - [163, 7, SimpleBlock, 2, 5082423, 6863, null]
+ - [163, 7, SimpleBlock, 2, 5089289, 5119, null]
+ - [163, 7, SimpleBlock, 2, 5094411, 5199, null]
+ - [163, 7, SimpleBlock, 2, 5099613, 3255, null]
+ - [163, 7, SimpleBlock, 2, 5102871, 4286, null]
+ - [163, 7, SimpleBlock, 2, 5107160, 5759, null]
+ - [163, 7, SimpleBlock, 2, 5112922, 6331, null]
+ - [163, 7, SimpleBlock, 2, 5119256, 6585, null]
+ - [163, 7, SimpleBlock, 2, 5125844, 5201, null]
+ - [163, 7, SimpleBlock, 2, 5131048, 5612, null]
+ - [163, 7, SimpleBlock, 2, 5136663, 4421, null]
+ - [163, 7, SimpleBlock, 2, 5141087, 4525, null]
+ - [163, 7, SimpleBlock, 2, 5145615, 4141, null]
+ - [163, 7, SimpleBlock, 2, 5149759, 5490, null]
+ - [163, 7, SimpleBlock, 2, 5155252, 3473, null]
+ - [163, 7, SimpleBlock, 2, 5158728, 2837, null]
+ - [163, 7, SimpleBlock, 2, 5161568, 3132, null]
+ - [163, 7, SimpleBlock, 2, 5164703, 3646, null]
+ - [163, 7, SimpleBlock, 2, 5168352, 5469, null]
+ - [163, 7, SimpleBlock, 2, 5173824, 5873, null]
+ - [163, 7, SimpleBlock, 2, 5179700, 8756, null]
+ - [163, 7, SimpleBlock, 2, 5188459, 9327, null]
+ - [163, 7, SimpleBlock, 2, 5197789, 8557, null]
+ - [163, 7, SimpleBlock, 2, 5206349, 6774, null]
+ - [163, 7, SimpleBlock, 2, 5213126, 2800, null]
+ - [163, 7, SimpleBlock, 2, 5215929, 6159, null]
+ - [163, 7, SimpleBlock, 2, 5222091, 2426, null]
+ - [163, 7, SimpleBlock, 2, 5224520, 2308, null]
+ - [163, 7, SimpleBlock, 2, 5226831, 2065, null]
+ - [163, 7, SimpleBlock, 2, 5228899, 1848, null]
+ - [163, 7, SimpleBlock, 2, 5230750, 5969, null]
+ - [163, 7, SimpleBlock, 2, 5236722, 1791, null]
+ - [163, 7, SimpleBlock, 2, 5238516, 1759, null]
+ - [163, 7, SimpleBlock, 2, 5240278, 2394, null]
+ - [163, 7, SimpleBlock, 2, 5242675, 2589, null]
+ - [163, 7, SimpleBlock, 2, 5245267, 2474, null]
+ - [163, 7, SimpleBlock, 2, 5247744, 6062, null]
+ - [163, 7, SimpleBlock, 2, 5253809, 2594, null]
+ - [163, 7, SimpleBlock, 2, 5256406, 2693, null]
+ - [163, 7, SimpleBlock, 2, 5259102, 2275, null]
+ - [163, 7, SimpleBlock, 2, 5261380, 1749, null]
+ - [163, 7, SimpleBlock, 2, 5263132, 5968, null]
+ - [163, 7, SimpleBlock, 2, 5269103, 1866, null]
+ - [163, 7, SimpleBlock, 2, 5270972, 1849, null]
+ - [163, 7, SimpleBlock, 2, 5272824, 1718, null]
+ - [163, 7, SimpleBlock, 2, 5274545, 2034, null]
+ - [163, 7, SimpleBlock, 2, 5276582, 1945, null]
+ - [163, 7, SimpleBlock, 2, 5278530, 5969, null]
+ - [163, 7, SimpleBlock, 2, 5284502, 1836, null]
+ - [163, 7, SimpleBlock, 2, 5286341, 2041, null]
+ - [163, 7, SimpleBlock, 2, 5288385, 2254, null]
+ - [163, 7, SimpleBlock, 2, 5290642, 1765, null]
+ - [163, 7, SimpleBlock, 2, 5292410, 1135, null]
+ - [163, 7, SimpleBlock, 2, 5293548, 5872, null]
+ - [163, 7, SimpleBlock, 2, 5299423, 1202, null]
+ - [163, 7, SimpleBlock, 2, 5300628, 1294, null]
+ - [163, 7, SimpleBlock, 2, 5301925, 1459, null]
+ - [163, 7, SimpleBlock, 2, 5303387, 1521, null]
+ - [163, 7, SimpleBlock, 2, 5304911, 6066, null]
+ - [163, 7, SimpleBlock, 2, 5310980, 1531, null]
+ - [163, 7, SimpleBlock, 2, 5312514, 1475, null]
+ - [163, 7, SimpleBlock, 2, 5313992, 1411, null]
+ - [163, 7, SimpleBlock, 2, 5315406, 1211, null]
+ - [163, 7, SimpleBlock, 2, 5316620, 2324, null]
+ - [163, 7, SimpleBlock, 2, 5318947, 6257, null]
+ - [163, 7, SimpleBlock, 2, 5325207, 2000, null]
+ - [163, 7, SimpleBlock, 2, 5327210, 1445, null]
+ - [163, 7, SimpleBlock, 2, 5328658, 1469, null]
+ - [163, 7, SimpleBlock, 2, 5330130, 1727, null]
+ - [163, 7, SimpleBlock, 2, 5331860, 1755, null]
+ - [163, 7, SimpleBlock, 2, 5333618, 6162, null]
+ - [163, 7, SimpleBlock, 2, 5339783, 1839, null]
+ - [163, 7, SimpleBlock, 2, 5341625, 1878, null]
+ - [163, 7, SimpleBlock, 2, 5343506, 4785, null]
+ - [163, 7, SimpleBlock, 2, 5348294, 7508, null]
+ - [163, 7, SimpleBlock, 2, 5355805, 5489, null]
+ - [163, 7, SimpleBlock, 2, 5361297, 9645, null]
+ - [163, 7, SimpleBlock, 2, 5370945, 7838, null]
+ - [163, 7, SimpleBlock, 2, 5378786, 5736, null]
+ - [163, 7, SimpleBlock, 2, 5384525, 5252, null]
+ - [163, 7, SimpleBlock, 2, 5389780, 4668, null]
+ - [163, 7, SimpleBlock, 2, 5394451, 676, null]
+ - [163, 7, SimpleBlock, 2, 5395130, 6160, null]
+ - [163, 7, SimpleBlock, 2, 5401293, 5740, null]
+ - [163, 7, SimpleBlock, 2, 5407036, 5130, null]
+ - [163, 7, SimpleBlock, 2, 5412169, 4879, null]
+ - [163, 7, SimpleBlock, 2, 5417051, 4866, null]
+ - [163, 7, SimpleBlock, 2, 5421920, 6009, null]
+ - [163, 7, SimpleBlock, 2, 5427932, 5490, null]
+ - [163, 7, SimpleBlock, 2, 5433425, 6863, null]
+ - [163, 7, SimpleBlock, 2, 5440291, 7796, null]
+ - [163, 7, SimpleBlock, 2, 5448090, 11253, null]
+ - [163, 7, SimpleBlock, 2, 5459346, 15567, null]
+ - [163, 7, SimpleBlock, 2, 5474916, 12076, null]
+ - [163, 7, SimpleBlock, 2, 5486995, 4531, null]
+ - [163, 7, SimpleBlock, 2, 5491529, 13816, null]
+ - [163, 7, SimpleBlock, 2, 5505348, 11914, null]
+ - [163, 7, SimpleBlock, 2, 5517265, 10621, null]
+ - [163, 7, SimpleBlock, 2, 5527889, 9203, null]
+ - [163, 7, SimpleBlock, 2, 5537095, 4432, null]
+ - [163, 7, SimpleBlock, 2, 5541530, 11010, null]
+ - [163, 7, SimpleBlock, 2, 5552543, 10400, null]
+ - [163, 7, SimpleBlock, 2, 5562946, 10182, null]
+ - [163, 7, SimpleBlock, 2, 5573131, 10107, null]
+ - [163, 7, SimpleBlock, 2, 5583241, 7515, null]
+ - [163, 7, SimpleBlock, 2, 5590759, 4613, null]
+ - [163, 7, SimpleBlock, 2, 5595375, 2891, null]
+ - [163, 7, SimpleBlock, 2, 5598269, 2262, null]
+ - [163, 7, SimpleBlock, 2, 5600534, 2210, null]
+ - [163, 7, SimpleBlock, 2, 5602747, 1779, null]
+ - [163, 7, SimpleBlock, 2, 5604529, 5009, null]
+ - [163, 7, SimpleBlock, 2, 5609541, 1401, null]
+ - [163, 7, SimpleBlock, 2, 5610945, 1046, null]
+ - [163, 7, SimpleBlock, 2, 5611994, 882, null]
+ - [163, 7, SimpleBlock, 2, 5612879, 877, null]
+ - [163, 7, SimpleBlock, 2, 5613759, 984, null]
+ - [163, 7, SimpleBlock, 2, 5614746, 5104, null]
+ - [163, 7, SimpleBlock, 2, 5619853, 1173, null]
+ - [163, 7, SimpleBlock, 2, 5621029, 1175, null]
+ - [163, 7, SimpleBlock, 2, 5622207, 1082, null]
+ - [163, 7, SimpleBlock, 2, 5623292, 1103, null]
+ - [163, 7, SimpleBlock, 2, 5624398, 864, null]
+ - [163, 7, SimpleBlock, 2, 5625265, 5586, null]
+ - [163, 7, SimpleBlock, 2, 5630854, 766, null]
+ - [163, 7, SimpleBlock, 2, 5631623, 867, null]
+ - [163, 7, SimpleBlock, 2, 5632493, 866, null]
+ - [163, 7, SimpleBlock, 2, 5633362, 826, null]
+ - [163, 7, SimpleBlock, 2, 5634191, 5104, null]
+ - [163, 7, SimpleBlock, 2, 5639298, 929, null]
+ - [163, 7, SimpleBlock, 2, 5640230, 984, null]
+ - [163, 7, SimpleBlock, 2, 5641217, 893, null]
+ - [163, 7, SimpleBlock, 2, 5642113, 849, null]
+ - [163, 7, SimpleBlock, 2, 5642965, 751, null]
+ - [163, 7, SimpleBlock, 2, 5643719, 5874, null]
+ - [163, 7, SimpleBlock, 2, 5649596, 664, null]
+ - [163, 7, SimpleBlock, 2, 5650263, 704, null]
+ - [163, 7, SimpleBlock, 2, 5650970, 866, null]
+ - [163, 7, SimpleBlock, 2, 5651839, 932, null]
+ - [163, 7, SimpleBlock, 2, 5652774, 580, null]
+ - [163, 7, SimpleBlock, 2, 5653357, 1011, null]
+ - - 524531317
+ - 6
+ - Cluster
+ - 1
+ - 5654376
+ - 4042030
+ - - [231, 1, Timecode, 2, 5654378, 2, 22083]
+ - [163, 7, SimpleBlock, 2, 5654383, 5487, null]
+ - [163, 7, SimpleBlock, 2, 5659874, 25991, null]
+ - [163, 7, SimpleBlock, 2, 5685868, 1412, null]
+ - [163, 7, SimpleBlock, 2, 5687283, 875, null]
+ - [163, 7, SimpleBlock, 2, 5688161, 752, null]
+ - [163, 7, SimpleBlock, 2, 5688916, 652, null]
+ - [163, 7, SimpleBlock, 2, 5689571, 4816, null]
+ - [163, 7, SimpleBlock, 2, 5694390, 631, null]
+ - [163, 7, SimpleBlock, 2, 5695024, 780, null]
+ - [163, 7, SimpleBlock, 2, 5695807, 795, null]
+ - [163, 7, SimpleBlock, 2, 5696605, 832, null]
+ - [163, 7, SimpleBlock, 2, 5697440, 5491, null]
+ - [163, 7, SimpleBlock, 2, 5702934, 816, null]
+ - [163, 7, SimpleBlock, 2, 5703753, 840, null]
+ - [163, 7, SimpleBlock, 2, 5704596, 781, null]
+ - [163, 7, SimpleBlock, 2, 5705380, 678, null]
+ - [163, 7, SimpleBlock, 2, 5706061, 624, null]
+ - [163, 7, SimpleBlock, 2, 5706688, 5585, null]
+ - [163, 7, SimpleBlock, 2, 5712276, 496, null]
+ - [163, 7, SimpleBlock, 2, 5712775, 531, null]
+ - [163, 7, SimpleBlock, 2, 5713309, 559, null]
+ - [163, 7, SimpleBlock, 2, 5713871, 566, null]
+ - [163, 7, SimpleBlock, 2, 5714440, 6450, null]
+ - [163, 7, SimpleBlock, 2, 5720893, 513, null]
+ - [163, 7, SimpleBlock, 2, 5721409, 429, null]
+ - [163, 7, SimpleBlock, 2, 5721841, 485, null]
+ - [163, 7, SimpleBlock, 2, 5722329, 554, null]
+ - [163, 7, SimpleBlock, 2, 5722886, 512, null]
+ - [163, 7, SimpleBlock, 2, 5723401, 6450, null]
+ - [163, 7, SimpleBlock, 2, 5729855, 85844, null]
+ - [163, 7, SimpleBlock, 2, 5815702, 9241, null]
+ - [163, 7, SimpleBlock, 2, 5824946, 13021, null]
+ - [163, 7, SimpleBlock, 2, 5837970, 13020, null]
+ - [163, 7, SimpleBlock, 2, 5850993, 14475, null]
+ - [163, 7, SimpleBlock, 2, 5865471, 6835, null]
+ - [163, 7, SimpleBlock, 2, 5872309, 14579, null]
+ - [163, 7, SimpleBlock, 2, 5886891, 15342, null]
+ - [163, 7, SimpleBlock, 2, 5902236, 15053, null]
+ - [163, 7, SimpleBlock, 2, 5917292, 15560, null]
+ - [163, 7, SimpleBlock, 2, 5932855, 6642, null]
+ - [163, 7, SimpleBlock, 2, 5939500, 15399, null]
+ - [163, 7, SimpleBlock, 2, 5954902, 15560, null]
+ - [163, 7, SimpleBlock, 2, 5970465, 15577, null]
+ - [163, 7, SimpleBlock, 2, 5986045, 15647, null]
+ - [163, 7, SimpleBlock, 2, 6001695, 15358, null]
+ - [163, 7, SimpleBlock, 2, 6017056, 6835, null]
+ - [163, 7, SimpleBlock, 2, 6023894, 15537, null]
+ - [163, 7, SimpleBlock, 2, 6039434, 15598, null]
+ - [163, 7, SimpleBlock, 2, 6055035, 15730, null]
+ - [163, 7, SimpleBlock, 2, 6070768, 15582, null]
+ - [163, 7, SimpleBlock, 2, 6086353, 6351, null]
+ - [163, 7, SimpleBlock, 2, 6092707, 15441, null]
+ - [163, 7, SimpleBlock, 2, 6108151, 15429, null]
+ - [163, 7, SimpleBlock, 2, 6123583, 15534, null]
+ - [163, 7, SimpleBlock, 2, 6139120, 15550, null]
+ - [163, 7, SimpleBlock, 2, 6154673, 15537, null]
+ - [163, 7, SimpleBlock, 2, 6170213, 6546, null]
+ - [163, 7, SimpleBlock, 2, 6176762, 15619, null]
+ - [163, 7, SimpleBlock, 2, 6192384, 15707, null]
+ - [163, 7, SimpleBlock, 2, 6208094, 15679, null]
+ - [163, 7, SimpleBlock, 2, 6223776, 15407, null]
+ - [163, 7, SimpleBlock, 2, 6239186, 15554, null]
+ - [163, 7, SimpleBlock, 2, 6254743, 6448, null]
+ - [163, 7, SimpleBlock, 2, 6261194, 15613, null]
+ - [163, 7, SimpleBlock, 2, 6276810, 15697, null]
+ - [163, 7, SimpleBlock, 2, 6292510, 15583, null]
+ - [163, 7, SimpleBlock, 2, 6308096, 15663, null]
+ - [163, 7, SimpleBlock, 2, 6323762, 5971, null]
+ - [163, 7, SimpleBlock, 2, 6329736, 15636, null]
+ - [163, 7, SimpleBlock, 2, 6345375, 15711, null]
+ - [163, 7, SimpleBlock, 2, 6361089, 15877, null]
+ - [163, 7, SimpleBlock, 2, 6376969, 15632, null]
+ - [163, 7, SimpleBlock, 2, 6392604, 15880, null]
+ - [163, 7, SimpleBlock, 2, 6408487, 5299, null]
+ - [163, 7, SimpleBlock, 2, 6413789, 15875, null]
+ - [163, 7, SimpleBlock, 2, 6429667, 15671, null]
+ - [163, 7, SimpleBlock, 2, 6445341, 15803, null]
+ - [163, 7, SimpleBlock, 2, 6461147, 15793, null]
+ - [163, 7, SimpleBlock, 2, 6476943, 5872, null]
+ - [163, 7, SimpleBlock, 2, 6482818, 16039, null]
+ - [163, 7, SimpleBlock, 2, 6498860, 16305, null]
+ - [163, 7, SimpleBlock, 2, 6515169, 16465, null]
+ - [163, 7, SimpleBlock, 2, 6531638, 16499, null]
+ - [163, 7, SimpleBlock, 2, 6548141, 16741, null]
+ - [163, 7, SimpleBlock, 2, 6564885, 5584, null]
+ - [163, 7, SimpleBlock, 2, 6570473, 17095, null]
+ - [163, 7, SimpleBlock, 2, 6587572, 17131, null]
+ - [163, 7, SimpleBlock, 2, 6604707, 17304, null]
+ - [163, 7, SimpleBlock, 2, 6622015, 17294, null]
+ - [163, 7, SimpleBlock, 2, 6639313, 17485, null]
+ - [163, 7, SimpleBlock, 2, 6656801, 5490, null]
+ - [163, 7, SimpleBlock, 2, 6662295, 17982, null]
+ - [163, 7, SimpleBlock, 2, 6680281, 18072, null]
+ - [163, 7, SimpleBlock, 2, 6698357, 17845, null]
+ - [163, 7, SimpleBlock, 2, 6716206, 18220, null]
+ - [163, 7, SimpleBlock, 2, 6734429, 4914, null]
+ - [163, 7, SimpleBlock, 2, 6739347, 18198, null]
+ - [163, 7, SimpleBlock, 2, 6757549, 18229, null]
+ - [163, 7, SimpleBlock, 2, 6775782, 18246, null]
+ - [163, 7, SimpleBlock, 2, 6794032, 18232, null]
+ - [163, 7, SimpleBlock, 2, 6812268, 18081, null]
+ - [163, 7, SimpleBlock, 2, 6830352, 5586, null]
+ - [163, 7, SimpleBlock, 2, 6835942, 17839, null]
+ - [163, 7, SimpleBlock, 2, 6853785, 18150, null]
+ - [163, 7, SimpleBlock, 2, 6871939, 17811, null]
+ - [163, 7, SimpleBlock, 2, 6889754, 17733, null]
+ - [163, 7, SimpleBlock, 2, 6907491, 17342, null]
+ - [163, 7, SimpleBlock, 2, 6924836, 5393, null]
+ - [163, 7, SimpleBlock, 2, 6930233, 17401, null]
+ - [163, 7, SimpleBlock, 2, 6947638, 17334, null]
+ - [163, 7, SimpleBlock, 2, 6964976, 17208, null]
+ - [163, 7, SimpleBlock, 2, 6982188, 16806, null]
+ - [163, 7, SimpleBlock, 2, 6998997, 5199, null]
+ - [163, 7, SimpleBlock, 2, 7004200, 16683, null]
+ - [163, 7, SimpleBlock, 2, 7020887, 16765, null]
+ - [163, 7, SimpleBlock, 2, 7037656, 16489, null]
+ - [163, 7, SimpleBlock, 2, 7054149, 16415, null]
+ - [163, 7, SimpleBlock, 2, 7070567, 16272, null]
+ - [163, 7, SimpleBlock, 2, 7086842, 4910, null]
+ - [163, 7, SimpleBlock, 2, 7091755, 16065, null]
+ - [163, 7, SimpleBlock, 2, 7107823, 15683, null]
+ - [163, 7, SimpleBlock, 2, 7123509, 15669, null]
+ - [163, 7, SimpleBlock, 2, 7139181, 15353, null]
+ - [163, 7, SimpleBlock, 2, 7154537, 5395, null]
+ - [163, 7, SimpleBlock, 2, 7159935, 15367, null]
+ - [163, 7, SimpleBlock, 2, 7175305, 14998, null]
+ - [163, 7, SimpleBlock, 2, 7190306, 14862, null]
+ - [163, 7, SimpleBlock, 2, 7205171, 15044, null]
+ - [163, 7, SimpleBlock, 2, 7220218, 5872, null]
+ - [163, 7, SimpleBlock, 2, 7226093, 15078, null]
+ - [163, 7, SimpleBlock, 2, 7241174, 14735, null]
+ - [163, 7, SimpleBlock, 2, 7255912, 14895, null]
+ - [163, 7, SimpleBlock, 2, 7270810, 15001, null]
+ - [163, 7, SimpleBlock, 2, 7285814, 14921, null]
+ - [163, 7, SimpleBlock, 2, 7300738, 5778, null]
+ - [163, 7, SimpleBlock, 2, 7306519, 14923, null]
+ - [163, 7, SimpleBlock, 2, 7321445, 14971, null]
+ - [163, 7, SimpleBlock, 2, 7336419, 14927, null]
+ - [163, 7, SimpleBlock, 2, 7351349, 14900, null]
+ - [163, 7, SimpleBlock, 2, 7366252, 15092, null]
+ - [163, 7, SimpleBlock, 2, 7381347, 5485, null]
+ - [163, 7, SimpleBlock, 2, 7386835, 14913, null]
+ - [163, 7, SimpleBlock, 2, 7401751, 14865, null]
+ - [163, 7, SimpleBlock, 2, 7416619, 15019, null]
+ - [163, 7, SimpleBlock, 2, 7431641, 14883, null]
+ - [163, 7, SimpleBlock, 2, 7446527, 5682, null]
+ - [163, 7, SimpleBlock, 2, 7452212, 15002, null]
+ - [163, 7, SimpleBlock, 2, 7467217, 14870, null]
+ - [163, 7, SimpleBlock, 2, 7482090, 14810, null]
+ - [163, 7, SimpleBlock, 2, 7496903, 14940, null]
+ - [163, 7, SimpleBlock, 2, 7511846, 15141, null]
+ - [163, 7, SimpleBlock, 2, 7526990, 5874, null]
+ - [163, 7, SimpleBlock, 2, 7532867, 15044, null]
+ - [163, 7, SimpleBlock, 2, 7547914, 14799, null]
+ - [163, 7, SimpleBlock, 2, 7562716, 14863, null]
+ - [163, 7, SimpleBlock, 2, 7577582, 14982, null]
+ - [163, 7, SimpleBlock, 2, 7592567, 5873, null]
+ - [163, 7, SimpleBlock, 2, 7598443, 14843, null]
+ - [163, 7, SimpleBlock, 2, 7613289, 14979, null]
+ - [163, 7, SimpleBlock, 2, 7628271, 14680, null]
+ - [163, 7, SimpleBlock, 2, 7642954, 14874, null]
+ - [163, 7, SimpleBlock, 2, 7657831, 14871, null]
+ - [163, 7, SimpleBlock, 2, 7672705, 5491, null]
+ - [163, 7, SimpleBlock, 2, 7678199, 14981, null]
+ - [163, 7, SimpleBlock, 2, 7693183, 14699, null]
+ - [163, 7, SimpleBlock, 2, 7707885, 15065, null]
+ - [163, 7, SimpleBlock, 2, 7722953, 14820, null]
+ - [163, 7, SimpleBlock, 2, 7737776, 14760, null]
+ - [163, 7, SimpleBlock, 2, 7752539, 5584, null]
+ - [163, 7, SimpleBlock, 2, 7758126, 14847, null]
+ - [163, 7, SimpleBlock, 2, 7772976, 14937, null]
+ - [163, 7, SimpleBlock, 2, 7787916, 14800, null]
+ - [163, 7, SimpleBlock, 2, 7802719, 15108, null]
+ - [163, 7, SimpleBlock, 2, 7817830, 5107, null]
+ - [163, 7, SimpleBlock, 2, 7822940, 14980, null]
+ - [163, 7, SimpleBlock, 2, 7837923, 15035, null]
+ - [163, 7, SimpleBlock, 2, 7852961, 14959, null]
+ - [163, 7, SimpleBlock, 2, 7867923, 14964, null]
+ - [163, 7, SimpleBlock, 2, 7882890, 14914, null]
+ - [163, 7, SimpleBlock, 2, 7897807, 5490, null]
+ - [163, 7, SimpleBlock, 2, 7903300, 15071, null]
+ - [163, 7, SimpleBlock, 2, 7918374, 14910, null]
+ - [163, 7, SimpleBlock, 2, 7933287, 15206, null]
+ - [163, 7, SimpleBlock, 2, 7948496, 14820, null]
+ - [163, 7, SimpleBlock, 2, 7963319, 5583, null]
+ - [163, 7, SimpleBlock, 2, 7968905, 15074, null]
+ - [163, 7, SimpleBlock, 2, 7983982, 14970, null]
+ - [163, 7, SimpleBlock, 2, 7998955, 15396, null]
+ - [163, 7, SimpleBlock, 2, 8014354, 15402, null]
+ - [163, 7, SimpleBlock, 2, 8029759, 15417, null]
+ - [163, 7, SimpleBlock, 2, 8045179, 5873, null]
+ - [163, 7, SimpleBlock, 2, 8051055, 15643, null]
+ - [163, 7, SimpleBlock, 2, 8066701, 15741, null]
+ - [163, 7, SimpleBlock, 2, 8082445, 15823, null]
+ - [163, 7, SimpleBlock, 2, 8098271, 15968, null]
+ - [163, 7, SimpleBlock, 2, 8114242, 16024, null]
+ - [163, 7, SimpleBlock, 2, 8130269, 5681, null]
+ - [163, 7, SimpleBlock, 2, 8135953, 16190, null]
+ - [163, 7, SimpleBlock, 2, 8152146, 16229, null]
+ - [163, 7, SimpleBlock, 2, 8168378, 16320, null]
+ - [163, 7, SimpleBlock, 2, 8184702, 16427, null]
+ - [163, 7, SimpleBlock, 2, 8201132, 5487, null]
+ - [163, 7, SimpleBlock, 2, 8206623, 16674, null]
+ - [163, 7, SimpleBlock, 2, 8223301, 16862, null]
+ - [163, 7, SimpleBlock, 2, 8240167, 16715, null]
+ - [163, 7, SimpleBlock, 2, 8256886, 17261, null]
+ - [163, 7, SimpleBlock, 2, 8274151, 17477, null]
+ - [163, 7, SimpleBlock, 2, 8291631, 5778, null]
+ - [163, 7, SimpleBlock, 2, 8297413, 17112, null]
+ - [163, 7, SimpleBlock, 2, 8314529, 17366, null]
+ - [163, 7, SimpleBlock, 2, 8331899, 17553, null]
+ - [163, 7, SimpleBlock, 2, 8349456, 17762, null]
+ - [163, 7, SimpleBlock, 2, 8367222, 17629, null]
+ - [163, 7, SimpleBlock, 2, 8384854, 5778, null]
+ - [163, 7, SimpleBlock, 2, 8390636, 17749, null]
+ - [163, 7, SimpleBlock, 2, 8408389, 18009, null]
+ - [163, 7, SimpleBlock, 2, 8426402, 17943, null]
+ - [163, 7, SimpleBlock, 2, 8444349, 17886, null]
+ - [163, 7, SimpleBlock, 2, 8462238, 5777, null]
+ - [163, 7, SimpleBlock, 2, 8468019, 18051, null]
+ - [163, 7, SimpleBlock, 2, 8486074, 17991, null]
+ - [163, 7, SimpleBlock, 2, 8504069, 17883, null]
+ - [163, 7, SimpleBlock, 2, 8521956, 17835, null]
+ - [163, 7, SimpleBlock, 2, 8539795, 17949, null]
+ - [163, 7, SimpleBlock, 2, 8557747, 5775, null]
+ - [163, 7, SimpleBlock, 2, 8563526, 17955, null]
+ - [163, 7, SimpleBlock, 2, 8581485, 17740, null]
+ - [163, 7, SimpleBlock, 2, 8599229, 17608, null]
+ - [163, 7, SimpleBlock, 2, 8616841, 17646, null]
+ - [163, 7, SimpleBlock, 2, 8634490, 1541, null]
+ - [163, 7, SimpleBlock, 2, 8636035, 17615, null]
+ - [163, 7, SimpleBlock, 2, 8653653, 6065, null]
+ - [163, 7, SimpleBlock, 2, 8659722, 17519, null]
+ - [163, 7, SimpleBlock, 2, 8677245, 17265, null]
+ - [163, 7, SimpleBlock, 2, 8694514, 17299, null]
+ - [163, 7, SimpleBlock, 2, 8711817, 17000, null]
+ - [163, 7, SimpleBlock, 2, 8728821, 17199, null]
+ - [163, 7, SimpleBlock, 2, 8746023, 5776, null]
+ - [163, 7, SimpleBlock, 2, 8751803, 16948, null]
+ - [163, 7, SimpleBlock, 2, 8768755, 16961, null]
+ - [163, 7, SimpleBlock, 2, 8785720, 16941, null]
+ - [163, 7, SimpleBlock, 2, 8802665, 16778, null]
+ - [163, 7, SimpleBlock, 2, 8819447, 16807, null]
+ - [163, 7, SimpleBlock, 2, 8836257, 5488, null]
+ - [163, 7, SimpleBlock, 2, 8841749, 16640, null]
+ - [163, 7, SimpleBlock, 2, 8858393, 16528, null]
+ - [163, 7, SimpleBlock, 2, 8874925, 16728, null]
+ - [163, 7, SimpleBlock, 2, 8891656, 16326, null]
+ - [163, 7, SimpleBlock, 2, 8907985, 5489, null]
+ - [163, 7, SimpleBlock, 2, 8913478, 16478, null]
+ - [163, 7, SimpleBlock, 2, 8929959, 16373, null]
+ - [163, 7, SimpleBlock, 2, 8946336, 16728, null]
+ - [163, 7, SimpleBlock, 2, 8963068, 16548, null]
+ - [163, 7, SimpleBlock, 2, 8979620, 16730, null]
+ - [163, 7, SimpleBlock, 2, 8996353, 5779, null]
+ - [163, 7, SimpleBlock, 2, 9002136, 16702, null]
+ - [163, 7, SimpleBlock, 2, 9018842, 16695, null]
+ - [163, 7, SimpleBlock, 2, 9035541, 16575, null]
+ - [163, 7, SimpleBlock, 2, 9052120, 16558, null]
+ - [163, 7, SimpleBlock, 2, 9068682, 16576, null]
+ - [163, 7, SimpleBlock, 2, 9085261, 5585, null]
+ - [163, 7, SimpleBlock, 2, 9090850, 16410, null]
+ - [163, 7, SimpleBlock, 2, 9107264, 16615, null]
+ - [163, 7, SimpleBlock, 2, 9123883, 16629, null]
+ - [163, 7, SimpleBlock, 2, 9140516, 16572, null]
+ - [163, 7, SimpleBlock, 2, 9157091, 5487, null]
+ - [163, 7, SimpleBlock, 2, 9162582, 16740, null]
+ - [163, 7, SimpleBlock, 2, 9179326, 16688, null]
+ - [163, 7, SimpleBlock, 2, 9196018, 16625, null]
+ - [163, 7, SimpleBlock, 2, 9212647, 17417, null]
+ - [163, 7, SimpleBlock, 2, 9230068, 17527, null]
+ - [163, 7, SimpleBlock, 2, 9247598, 5487, null]
+ - [163, 7, SimpleBlock, 2, 9253089, 17348, null]
+ - [163, 7, SimpleBlock, 2, 9270441, 17060, null]
+ - [163, 7, SimpleBlock, 2, 9287505, 16552, null]
+ - [163, 7, SimpleBlock, 2, 9304060, 16344, null]
+ - [163, 7, SimpleBlock, 2, 9320407, 5296, null]
+ - [163, 7, SimpleBlock, 2, 9325706, 16256, null]
+ - [163, 7, SimpleBlock, 2, 9341965, 15758, null]
+ - [163, 7, SimpleBlock, 2, 9357726, 15896, null]
+ - [163, 7, SimpleBlock, 2, 9373625, 15296, null]
+ - [163, 7, SimpleBlock, 2, 9388924, 15339, null]
+ - [163, 7, SimpleBlock, 2, 9404266, 5489, null]
+ - [163, 7, SimpleBlock, 2, 9409758, 14934, null]
+ - [163, 7, SimpleBlock, 2, 9424695, 14798, null]
+ - [163, 7, SimpleBlock, 2, 9439496, 14636, null]
+ - [163, 7, SimpleBlock, 2, 9454135, 14532, null]
+ - [163, 7, SimpleBlock, 2, 9468670, 14366, null]
+ - [163, 7, SimpleBlock, 2, 9483039, 5583, null]
+ - [163, 7, SimpleBlock, 2, 9488625, 14350, null]
+ - [163, 7, SimpleBlock, 2, 9502978, 14273, null]
+ - [163, 7, SimpleBlock, 2, 9517254, 14005, null]
+ - [163, 7, SimpleBlock, 2, 9531262, 14068, null]
+ - [163, 7, SimpleBlock, 2, 9545333, 5583, null]
+ - [163, 7, SimpleBlock, 2, 9550919, 14134, null]
+ - [163, 7, SimpleBlock, 2, 9565056, 13834, null]
+ - [163, 7, SimpleBlock, 2, 9578893, 13920, null]
+ - [163, 7, SimpleBlock, 2, 9592816, 13837, null]
+ - [163, 7, SimpleBlock, 2, 9606656, 13788, null]
+ - [163, 7, SimpleBlock, 2, 9620447, 5487, null]
+ - [163, 7, SimpleBlock, 2, 9625937, 13746, null]
+ - [163, 7, SimpleBlock, 2, 9639686, 13819, null]
+ - [163, 7, SimpleBlock, 2, 9653508, 13907, null]
+ - [163, 7, SimpleBlock, 2, 9667418, 13995, null]
+ - [163, 7, SimpleBlock, 2, 9681416, 964, null]
+ - [163, 7, SimpleBlock, 2, 9682383, 14023, null]
+ - - 524531317
+ - 6
+ - Cluster
+ - 1
+ - 9696414
+ - 3744132
+ - - [231, 1, Timecode, 2, 9696416, 2, 32500]
+ - [163, 7, SimpleBlock, 2, 9696421, 5584, null]
+ - [163, 7, SimpleBlock, 2, 9702009, 58285, null]
+ - [163, 7, SimpleBlock, 2, 9760297, 8074, null]
+ - [163, 7, SimpleBlock, 2, 9768374, 11664, null]
+ - [163, 7, SimpleBlock, 2, 9780041, 12436, null]
+ - [163, 7, SimpleBlock, 2, 9792480, 13719, null]
+ - [163, 7, SimpleBlock, 2, 9806202, 5679, null]
+ - [163, 7, SimpleBlock, 2, 9811884, 14117, null]
+ - [163, 7, SimpleBlock, 2, 9826004, 14219, null]
+ - [163, 7, SimpleBlock, 2, 9840226, 14317, null]
+ - [163, 7, SimpleBlock, 2, 9854546, 14481, null]
+ - [163, 7, SimpleBlock, 2, 9869030, 5490, null]
+ - [163, 7, SimpleBlock, 2, 9874523, 14660, null]
+ - [163, 7, SimpleBlock, 2, 9889186, 14854, null]
+ - [163, 7, SimpleBlock, 2, 9904043, 14980, null]
+ - [163, 7, SimpleBlock, 2, 9919026, 15380, null]
+ - [163, 7, SimpleBlock, 2, 9934409, 15522, null]
+ - [163, 7, SimpleBlock, 2, 9949934, 5779, null]
+ - [163, 7, SimpleBlock, 2, 9955716, 15621, null]
+ - [163, 7, SimpleBlock, 2, 9971340, 15669, null]
+ - [163, 7, SimpleBlock, 2, 9987012, 15890, null]
+ - [163, 7, SimpleBlock, 2, 10002905, 16299, null]
+ - [163, 7, SimpleBlock, 2, 10019207, 5969, null]
+ - [163, 7, SimpleBlock, 2, 10025179, 16299, null]
+ - [163, 7, SimpleBlock, 2, 10041482, 16612, null]
+ - [163, 7, SimpleBlock, 2, 10058098, 17028, null]
+ - [163, 7, SimpleBlock, 2, 10075130, 17286, null]
+ - [163, 7, SimpleBlock, 2, 10092420, 17238, null]
+ - [163, 7, SimpleBlock, 2, 10109661, 5585, null]
+ - [163, 7, SimpleBlock, 2, 10115250, 17673, null]
+ - [163, 7, SimpleBlock, 2, 10132927, 17702, null]
+ - [163, 7, SimpleBlock, 2, 10150633, 18215, null]
+ - [163, 7, SimpleBlock, 2, 10168852, 18454, null]
+ - [163, 7, SimpleBlock, 2, 10187310, 18997, null]
+ - [163, 7, SimpleBlock, 2, 10206310, 5969, null]
+ - [163, 7, SimpleBlock, 2, 10212283, 19148, null]
+ - [163, 7, SimpleBlock, 2, 10231435, 19526, null]
+ - [163, 7, SimpleBlock, 2, 10250965, 16685, null]
+ - [163, 7, SimpleBlock, 2, 10267654, 16395, null]
+ - [163, 7, SimpleBlock, 2, 10284052, 5778, null]
+ - [163, 7, SimpleBlock, 2, 10289833, 16114, null]
+ - [163, 7, SimpleBlock, 2, 10305950, 16376, null]
+ - [163, 7, SimpleBlock, 2, 10322329, 16074, null]
+ - [163, 7, SimpleBlock, 2, 10338406, 16202, null]
+ - [163, 7, SimpleBlock, 2, 10354611, 16277, null]
+ - [163, 7, SimpleBlock, 2, 10370891, 5873, null]
+ - [163, 7, SimpleBlock, 2, 10376767, 16211, null]
+ - [163, 7, SimpleBlock, 2, 10392981, 16342, null]
+ - [163, 7, SimpleBlock, 2, 10409326, 16294, null]
+ - [163, 7, SimpleBlock, 2, 10425623, 16227, null]
+ - [163, 7, SimpleBlock, 2, 10441853, 5969, null]
+ - [163, 7, SimpleBlock, 2, 10447825, 16131, null]
+ - [163, 7, SimpleBlock, 2, 10463959, 16256, null]
+ - [163, 7, SimpleBlock, 2, 10480219, 16528, null]
+ - [163, 7, SimpleBlock, 2, 10496750, 16212, null]
+ - [163, 7, SimpleBlock, 2, 10512965, 16157, null]
+ - [163, 7, SimpleBlock, 2, 10529125, 5872, null]
+ - [163, 7, SimpleBlock, 2, 10535000, 16235, null]
+ - [163, 7, SimpleBlock, 2, 10551238, 15984, null]
+ - [163, 7, SimpleBlock, 2, 10567225, 16158, null]
+ - [163, 7, SimpleBlock, 2, 10583386, 16233, null]
+ - [163, 7, SimpleBlock, 2, 10599622, 16083, null]
+ - [163, 7, SimpleBlock, 2, 10615708, 6162, null]
+ - [163, 7, SimpleBlock, 2, 10621873, 16186, null]
+ - [163, 7, SimpleBlock, 2, 10638062, 16047, null]
+ - [163, 7, SimpleBlock, 2, 10654112, 15948, null]
+ - [163, 7, SimpleBlock, 2, 10670063, 16103, null]
+ - [163, 7, SimpleBlock, 2, 10686169, 6065, null]
+ - [163, 7, SimpleBlock, 2, 10692237, 16048, null]
+ - [163, 7, SimpleBlock, 2, 10708288, 16064, null]
+ - [163, 7, SimpleBlock, 2, 10724355, 15899, null]
+ - [163, 7, SimpleBlock, 2, 10740257, 15995, null]
+ - [163, 7, SimpleBlock, 2, 10756255, 16002, null]
+ - [163, 7, SimpleBlock, 2, 10772260, 6065, null]
+ - [163, 7, SimpleBlock, 2, 10778328, 16105, null]
+ - [163, 7, SimpleBlock, 2, 10794436, 15916, null]
+ - [163, 7, SimpleBlock, 2, 10810355, 16022, null]
+ - [163, 7, SimpleBlock, 2, 10826380, 15944, null]
+ - [163, 7, SimpleBlock, 2, 10842327, 6066, null]
+ - [163, 7, SimpleBlock, 2, 10848396, 15894, null]
+ - [163, 7, SimpleBlock, 2, 10864293, 15821, null]
+ - [163, 7, SimpleBlock, 2, 10880117, 15998, null]
+ - [163, 7, SimpleBlock, 2, 10896118, 15774, null]
+ - [163, 7, SimpleBlock, 2, 10911895, 15840, null]
+ - [163, 7, SimpleBlock, 2, 10927738, 6546, null]
+ - [163, 7, SimpleBlock, 2, 10934287, 15857, null]
+ - [163, 7, SimpleBlock, 2, 10950147, 15871, null]
+ - [163, 7, SimpleBlock, 2, 10966021, 15709, null]
+ - [163, 7, SimpleBlock, 2, 10981733, 15836, null]
+ - [163, 7, SimpleBlock, 2, 10997572, 15847, null]
+ - [163, 7, SimpleBlock, 2, 11013422, 6162, null]
+ - [163, 7, SimpleBlock, 2, 11019587, 15943, null]
+ - [163, 7, SimpleBlock, 2, 11035533, 15875, null]
+ - [163, 7, SimpleBlock, 2, 11051411, 15777, null]
+ - [163, 7, SimpleBlock, 2, 11067191, 15896, null]
+ - [163, 7, SimpleBlock, 2, 11083090, 6255, null]
+ - [163, 7, SimpleBlock, 2, 11089348, 15755, null]
+ - [163, 7, SimpleBlock, 2, 11105106, 15815, null]
+ - [163, 7, SimpleBlock, 2, 11120924, 15742, null]
+ - [163, 7, SimpleBlock, 2, 11136669, 15718, null]
+ - [163, 7, SimpleBlock, 2, 11152390, 15628, null]
+ - [163, 7, SimpleBlock, 2, 11168021, 964, null]
+ - [163, 7, SimpleBlock, 2, 11168988, 6161, null]
+ - [163, 7, SimpleBlock, 2, 11175152, 15743, null]
+ - [163, 7, SimpleBlock, 2, 11190898, 15651, null]
+ - [163, 7, SimpleBlock, 2, 11206552, 15646, null]
+ - [163, 7, SimpleBlock, 2, 11222201, 15666, null]
+ - [163, 7, SimpleBlock, 2, 11237870, 15600, null]
+ - [163, 7, SimpleBlock, 2, 11253473, 6354, null]
+ - [163, 7, SimpleBlock, 2, 11259830, 15472, null]
+ - [163, 7, SimpleBlock, 2, 11275305, 15276, null]
+ - [163, 7, SimpleBlock, 2, 11290584, 15429, null]
+ - [163, 7, SimpleBlock, 2, 11306016, 15363, null]
+ - [163, 7, SimpleBlock, 2, 11321382, 15264, null]
+ - [163, 7, SimpleBlock, 2, 11336649, 6256, null]
+ - [163, 7, SimpleBlock, 2, 11342908, 15189, null]
+ - [163, 7, SimpleBlock, 2, 11358100, 15429, null]
+ - [163, 7, SimpleBlock, 2, 11373532, 15182, null]
+ - [163, 7, SimpleBlock, 2, 11388717, 15176, null]
+ - [163, 7, SimpleBlock, 2, 11403896, 6258, null]
+ - [163, 7, SimpleBlock, 2, 11410157, 15160, null]
+ - [163, 7, SimpleBlock, 2, 11425320, 15084, null]
+ - [163, 7, SimpleBlock, 2, 11440407, 15027, null]
+ - [163, 7, SimpleBlock, 2, 11455437, 15087, null]
+ - [163, 7, SimpleBlock, 2, 11470527, 15308, null]
+ - [163, 7, SimpleBlock, 2, 11485838, 6643, null]
+ - [163, 7, SimpleBlock, 2, 11492484, 14960, null]
+ - [163, 7, SimpleBlock, 2, 11507447, 15005, null]
+ - [163, 7, SimpleBlock, 2, 11522455, 15128, null]
+ - [163, 7, SimpleBlock, 2, 11537586, 15124, null]
+ - [163, 7, SimpleBlock, 2, 11552713, 15103, null]
+ - [163, 7, SimpleBlock, 2, 11567819, 6354, null]
+ - [163, 7, SimpleBlock, 2, 11574176, 15080, null]
+ - [163, 7, SimpleBlock, 2, 11589259, 15014, null]
+ - [163, 7, SimpleBlock, 2, 11604276, 14951, null]
+ - [163, 7, SimpleBlock, 2, 11619230, 14915, null]
+ - [163, 7, SimpleBlock, 2, 11634148, 6256, null]
+ - [163, 7, SimpleBlock, 2, 11640407, 14819, null]
+ - [163, 7, SimpleBlock, 2, 11655229, 14729, null]
+ - [163, 7, SimpleBlock, 2, 11669961, 14715, null]
+ - [163, 7, SimpleBlock, 2, 11684679, 14801, null]
+ - [163, 7, SimpleBlock, 2, 11699483, 14828, null]
+ - [163, 7, SimpleBlock, 2, 11714314, 6258, null]
+ - [163, 7, SimpleBlock, 2, 11720575, 14604, null]
+ - [163, 7, SimpleBlock, 2, 11735182, 14649, null]
+ - [163, 7, SimpleBlock, 2, 11749834, 14712, null]
+ - [163, 7, SimpleBlock, 2, 11764549, 14456, null]
+ - [163, 7, SimpleBlock, 2, 11779008, 6162, null]
+ - [163, 7, SimpleBlock, 2, 11785173, 14612, null]
+ - [163, 7, SimpleBlock, 2, 11799788, 14464, null]
+ - [163, 7, SimpleBlock, 2, 11814255, 14548, null]
+ - [163, 7, SimpleBlock, 2, 11828806, 14477, null]
+ - [163, 7, SimpleBlock, 2, 11843286, 14547, null]
+ - [163, 7, SimpleBlock, 2, 11857836, 6162, null]
+ - [163, 7, SimpleBlock, 2, 11864001, 14432, null]
+ - [163, 7, SimpleBlock, 2, 11878436, 14322, null]
+ - [163, 7, SimpleBlock, 2, 11892761, 14270, null]
+ - [163, 7, SimpleBlock, 2, 11907034, 14174, null]
+ - [163, 7, SimpleBlock, 2, 11921211, 14244, null]
+ - [163, 7, SimpleBlock, 2, 11935458, 5968, null]
+ - [163, 7, SimpleBlock, 2, 11941429, 14147, null]
+ - [163, 7, SimpleBlock, 2, 11955579, 14105, null]
+ - [163, 7, SimpleBlock, 2, 11969687, 14050, null]
+ - [163, 7, SimpleBlock, 2, 11983740, 14133, null]
+ - [163, 7, SimpleBlock, 2, 11997876, 5966, null]
+ - [163, 7, SimpleBlock, 2, 12003845, 14114, null]
+ - [163, 7, SimpleBlock, 2, 12017962, 13853, null]
+ - [163, 7, SimpleBlock, 2, 12031818, 14074, null]
+ - [163, 7, SimpleBlock, 2, 12045895, 13788, null]
+ - [163, 7, SimpleBlock, 2, 12059686, 13645, null]
+ - [163, 7, SimpleBlock, 2, 12073334, 5872, null]
+ - [163, 7, SimpleBlock, 2, 12079209, 13645, null]
+ - [163, 7, SimpleBlock, 2, 12092857, 13742, null]
+ - [163, 7, SimpleBlock, 2, 12106602, 13511, null]
+ - [163, 7, SimpleBlock, 2, 12120116, 13642, null]
+ - [163, 7, SimpleBlock, 2, 12133761, 5871, null]
+ - [163, 7, SimpleBlock, 2, 12139635, 13525, null]
+ - [163, 7, SimpleBlock, 2, 12153163, 13417, null]
+ - [163, 7, SimpleBlock, 2, 12166583, 13399, null]
+ - [163, 7, SimpleBlock, 2, 12179985, 13388, null]
+ - [163, 7, SimpleBlock, 2, 12193376, 13531, null]
+ - [163, 7, SimpleBlock, 2, 12206910, 5776, null]
+ - [163, 7, SimpleBlock, 2, 12212689, 13370, null]
+ - [163, 7, SimpleBlock, 2, 12226062, 13288, null]
+ - [163, 7, SimpleBlock, 2, 12239353, 13187, null]
+ - [163, 7, SimpleBlock, 2, 12252543, 13400, null]
+ - [163, 7, SimpleBlock, 2, 12265946, 13324, null]
+ - [163, 7, SimpleBlock, 2, 12279273, 5776, null]
+ - [163, 7, SimpleBlock, 2, 12285052, 13383, null]
+ - [163, 7, SimpleBlock, 2, 12298438, 13326, null]
+ - [163, 7, SimpleBlock, 2, 12311767, 13233, null]
+ - [163, 7, SimpleBlock, 2, 12325003, 13224, null]
+ - [163, 7, SimpleBlock, 2, 12338230, 5393, null]
+ - [163, 7, SimpleBlock, 2, 12343626, 13395, null]
+ - [163, 7, SimpleBlock, 2, 12357024, 13150, null]
+ - [163, 7, SimpleBlock, 2, 12370177, 13085, null]
+ - [163, 7, SimpleBlock, 2, 12383265, 13142, null]
+ - [163, 7, SimpleBlock, 2, 12396410, 12979, null]
+ - [163, 7, SimpleBlock, 2, 12409392, 5391, null]
+ - [163, 7, SimpleBlock, 2, 12414786, 13151, null]
+ - [163, 7, SimpleBlock, 2, 12427940, 12982, null]
+ - [163, 7, SimpleBlock, 2, 12440925, 12778, null]
+ - [163, 7, SimpleBlock, 2, 12453706, 12569, null]
+ - [163, 7, SimpleBlock, 2, 12466278, 5681, null]
+ - [163, 7, SimpleBlock, 2, 12471962, 12672, null]
+ - [163, 7, SimpleBlock, 2, 12484637, 12435, null]
+ - [163, 7, SimpleBlock, 2, 12497075, 12408, null]
+ - [163, 7, SimpleBlock, 2, 12509486, 12258, null]
+ - [163, 7, SimpleBlock, 2, 12521747, 12327, null]
+ - [163, 7, SimpleBlock, 2, 12534077, 5970, null]
+ - [163, 7, SimpleBlock, 2, 12540050, 12242, null]
+ - [163, 7, SimpleBlock, 2, 12552295, 12099, null]
+ - [163, 7, SimpleBlock, 2, 12564397, 12248, null]
+ - [163, 7, SimpleBlock, 2, 12576648, 11962, null]
+ - [163, 7, SimpleBlock, 2, 12588613, 11927, null]
+ - [163, 7, SimpleBlock, 2, 12600543, 5874, null]
+ - [163, 7, SimpleBlock, 2, 12606420, 11981, null]
+ - [163, 7, SimpleBlock, 2, 12618404, 11902, null]
+ - [163, 7, SimpleBlock, 2, 12630309, 11921, null]
+ - [163, 7, SimpleBlock, 2, 12642233, 11689, null]
+ - [163, 7, SimpleBlock, 2, 12653925, 2794, null]
+ - [163, 7, SimpleBlock, 2, 12656722, 11864, null]
+ - [163, 7, SimpleBlock, 2, 12668589, 11531, null]
+ - [163, 7, SimpleBlock, 2, 12680123, 11632, null]
+ - [163, 7, SimpleBlock, 2, 12691758, 5777, null]
+ - [163, 7, SimpleBlock, 2, 12697538, 11429, null]
+ - [163, 7, SimpleBlock, 2, 12708970, 11526, null]
+ - [163, 7, SimpleBlock, 2, 12720499, 11214, null]
+ - [163, 7, SimpleBlock, 2, 12731716, 11362, null]
+ - [163, 7, SimpleBlock, 2, 12743081, 5585, null]
+ - [163, 7, SimpleBlock, 2, 12748669, 11338, null]
+ - [163, 7, SimpleBlock, 2, 12760010, 11249, null]
+ - [163, 7, SimpleBlock, 2, 12771262, 11295, null]
+ - [163, 7, SimpleBlock, 2, 12782560, 11137, null]
+ - [163, 7, SimpleBlock, 2, 12793700, 11203, null]
+ - [163, 7, SimpleBlock, 2, 12804906, 5395, null]
+ - [163, 7, SimpleBlock, 2, 12810304, 11042, null]
+ - [163, 7, SimpleBlock, 2, 12821349, 11145, null]
+ - [163, 7, SimpleBlock, 2, 12832497, 10864, null]
+ - [163, 7, SimpleBlock, 2, 12843364, 10885, null]
+ - [163, 7, SimpleBlock, 2, 12854252, 5680, null]
+ - [163, 7, SimpleBlock, 2, 12859935, 10829, null]
+ - [163, 7, SimpleBlock, 2, 12870767, 10656, null]
+ - [163, 7, SimpleBlock, 2, 12881426, 10698, null]
+ - [163, 7, SimpleBlock, 2, 12892127, 10718, null]
+ - [163, 7, SimpleBlock, 2, 12902848, 10663, null]
+ - [163, 7, SimpleBlock, 2, 12913514, 5393, null]
+ - [163, 7, SimpleBlock, 2, 12918910, 10615, null]
+ - [163, 7, SimpleBlock, 2, 12929528, 10614, null]
+ - [163, 7, SimpleBlock, 2, 12940145, 10528, null]
+ - [163, 7, SimpleBlock, 2, 12950676, 10479, null]
+ - [163, 7, SimpleBlock, 2, 12961158, 10358, null]
+ - [163, 7, SimpleBlock, 2, 12971519, 5487, null]
+ - [163, 7, SimpleBlock, 2, 12977009, 10405, null]
+ - [163, 7, SimpleBlock, 2, 12987417, 10152, null]
+ - [163, 7, SimpleBlock, 2, 12997572, 10226, null]
+ - [163, 7, SimpleBlock, 2, 13007801, 10300, null]
+ - [163, 7, SimpleBlock, 2, 13018104, 5104, null]
+ - [163, 7, SimpleBlock, 2, 13023211, 10268, null]
+ - [163, 7, SimpleBlock, 2, 13033482, 10148, null]
+ - [163, 7, SimpleBlock, 2, 13043633, 10309, null]
+ - [163, 7, SimpleBlock, 2, 13053945, 10178, null]
+ - [163, 7, SimpleBlock, 2, 13064126, 10096, null]
+ - [163, 7, SimpleBlock, 2, 13074225, 5201, null]
+ - [163, 7, SimpleBlock, 2, 13079429, 10085, null]
+ - [163, 7, SimpleBlock, 2, 13089517, 10239, null]
+ - [163, 7, SimpleBlock, 2, 13099759, 10113, null]
+ - [163, 7, SimpleBlock, 2, 13109875, 10129, null]
+ - [163, 7, SimpleBlock, 2, 13120007, 5008, null]
+ - [163, 7, SimpleBlock, 2, 13125018, 10090, null]
+ - [163, 7, SimpleBlock, 2, 13135111, 10152, null]
+ - [163, 7, SimpleBlock, 2, 13145266, 10211, null]
+ - [163, 7, SimpleBlock, 2, 13155480, 9935, null]
+ - [163, 7, SimpleBlock, 2, 13165418, 10088, null]
+ - [163, 7, SimpleBlock, 2, 13175509, 5202, null]
+ - [163, 7, SimpleBlock, 2, 13180714, 9887, null]
+ - [163, 7, SimpleBlock, 2, 13190604, 9798, null]
+ - [163, 7, SimpleBlock, 2, 13200405, 9855, null]
+ - [163, 7, SimpleBlock, 2, 13210263, 9677, null]
+ - [163, 7, SimpleBlock, 2, 13219943, 9497, null]
+ - [163, 7, SimpleBlock, 2, 13229443, 5585, null]
+ - [163, 7, SimpleBlock, 2, 13235031, 9425, null]
+ - [163, 7, SimpleBlock, 2, 13244459, 9598, null]
+ - [163, 7, SimpleBlock, 2, 13254060, 9206, null]
+ - [163, 7, SimpleBlock, 2, 13263269, 9294, null]
+ - [163, 7, SimpleBlock, 2, 13272566, 6354, null]
+ - [163, 7, SimpleBlock, 2, 13278923, 9149, null]
+ - [163, 7, SimpleBlock, 2, 13288075, 9011, null]
+ - [163, 7, SimpleBlock, 2, 13297089, 8892, null]
+ - [163, 7, SimpleBlock, 2, 13305984, 8677, null]
+ - [163, 7, SimpleBlock, 2, 13314664, 8752, null]
+ - [163, 7, SimpleBlock, 2, 13323419, 6547, null]
+ - [163, 7, SimpleBlock, 2, 13329969, 8803, null]
+ - [163, 7, SimpleBlock, 2, 13338775, 8670, null]
+ - [163, 7, SimpleBlock, 2, 13347448, 8642, null]
+ - [163, 7, SimpleBlock, 2, 13356093, 8631, null]
+ - [163, 7, SimpleBlock, 2, 13364727, 5682, null]
+ - [163, 7, SimpleBlock, 2, 13370412, 8570, null]
+ - [163, 7, SimpleBlock, 2, 13378985, 8420, null]
+ - [163, 7, SimpleBlock, 2, 13387408, 8489, null]
+ - [163, 7, SimpleBlock, 2, 13395900, 8492, null]
+ - [163, 7, SimpleBlock, 2, 13404395, 8290, null]
+ - [163, 7, SimpleBlock, 2, 13412688, 2793, null]
+ - [163, 7, SimpleBlock, 2, 13415484, 8296, null]
+ - [163, 7, SimpleBlock, 2, 13423783, 8405, null]
+ - [163, 7, SimpleBlock, 2, 13432191, 8355, null]
+ - - 524531317
+ - 6
+ - Cluster
+ - 1
+ - 13440554
+ - 3249549
+ - - [231, 1, Timecode, 2, 13440556, 2, 42917]
+ - [163, 7, SimpleBlock, 2, 13440561, 676, null]
+ - [163, 7, SimpleBlock, 2, 13441240, 5489, null]
+ - [163, 7, SimpleBlock, 2, 13446733, 51446, null]
+ - [163, 7, SimpleBlock, 2, 13498182, 4220, null]
+ - [163, 7, SimpleBlock, 2, 13502405, 6526, null]
+ - [163, 7, SimpleBlock, 2, 13508934, 6389, null]
+ - [163, 7, SimpleBlock, 2, 13515326, 7048, null]
+ - [163, 7, SimpleBlock, 2, 13522377, 5490, null]
+ - [163, 7, SimpleBlock, 2, 13527870, 6914, null]
+ - [163, 7, SimpleBlock, 2, 13534787, 7116, null]
+ - [163, 7, SimpleBlock, 2, 13541906, 7356, null]
+ - [163, 7, SimpleBlock, 2, 13549265, 7645, null]
+ - [163, 7, SimpleBlock, 2, 13556913, 5585, null]
+ - [163, 7, SimpleBlock, 2, 13562501, 7360, null]
+ - [163, 7, SimpleBlock, 2, 13569864, 7213, null]
+ - [163, 7, SimpleBlock, 2, 13577080, 7193, null]
+ - [163, 7, SimpleBlock, 2, 13584276, 7232, null]
+ - [163, 7, SimpleBlock, 2, 13591511, 7281, null]
+ - [163, 7, SimpleBlock, 2, 13598795, 5680, null]
+ - [163, 7, SimpleBlock, 2, 13604478, 7357, null]
+ - [163, 7, SimpleBlock, 2, 13611838, 7403, null]
+ - [163, 7, SimpleBlock, 2, 13619244, 7427, null]
+ - [163, 7, SimpleBlock, 2, 13626674, 7673, null]
+ - [163, 7, SimpleBlock, 2, 13634350, 5489, null]
+ - [163, 7, SimpleBlock, 2, 13639842, 7819, null]
+ - [163, 7, SimpleBlock, 2, 13647664, 8057, null]
+ - [163, 7, SimpleBlock, 2, 13655724, 8240, null]
+ - [163, 7, SimpleBlock, 2, 13663967, 8555, null]
+ - [163, 7, SimpleBlock, 2, 13672525, 8858, null]
+ - [163, 7, SimpleBlock, 2, 13681386, 5381, null]
+ - [163, 7, SimpleBlock, 2, 13686770, 9423, null]
+ - [163, 7, SimpleBlock, 2, 13696196, 9727, null]
+ - [163, 7, SimpleBlock, 2, 13705926, 10330, null]
+ - [163, 7, SimpleBlock, 2, 13716259, 10596, null]
+ - [163, 7, SimpleBlock, 2, 13726858, 10571, null]
+ - [163, 7, SimpleBlock, 2, 13737432, 5683, null]
+ - [163, 7, SimpleBlock, 2, 13743118, 10613, null]
+ - [163, 7, SimpleBlock, 2, 13753734, 10893, null]
+ - [163, 7, SimpleBlock, 2, 13764630, 11063, null]
+ - [163, 7, SimpleBlock, 2, 13775696, 10909, null]
+ - [163, 7, SimpleBlock, 2, 13786608, 5486, null]
+ - [163, 7, SimpleBlock, 2, 13792097, 10803, null]
+ - [163, 7, SimpleBlock, 2, 13802903, 10949, null]
+ - [163, 7, SimpleBlock, 2, 13813855, 10796, null]
+ - [163, 7, SimpleBlock, 2, 13824654, 10855, null]
+ - [163, 7, SimpleBlock, 2, 13835512, 10674, null]
+ - [163, 7, SimpleBlock, 2, 13846189, 5200, null]
+ - [163, 7, SimpleBlock, 2, 13851392, 10485, null]
+ - [163, 7, SimpleBlock, 2, 13861880, 10737, null]
+ - [163, 7, SimpleBlock, 2, 13872620, 10837, null]
+ - [163, 7, SimpleBlock, 2, 13883460, 11119, null]
+ - [163, 7, SimpleBlock, 2, 13894582, 6161, null]
+ - [163, 7, SimpleBlock, 2, 13900746, 11117, null]
+ - [163, 7, SimpleBlock, 2, 13911866, 11056, null]
+ - [163, 7, SimpleBlock, 2, 13922925, 11181, null]
+ - [163, 7, SimpleBlock, 2, 13934109, 11216, null]
+ - [163, 7, SimpleBlock, 2, 13945328, 11249, null]
+ - [163, 7, SimpleBlock, 2, 13956580, 6161, null]
+ - [163, 7, SimpleBlock, 2, 13962744, 11095, null]
+ - [163, 7, SimpleBlock, 2, 13973842, 11220, null]
+ - [163, 7, SimpleBlock, 2, 13985065, 11147, null]
+ - [163, 7, SimpleBlock, 2, 13996215, 11224, null]
+ - [163, 7, SimpleBlock, 2, 14007442, 11201, null]
+ - [163, 7, SimpleBlock, 2, 14018646, 5491, null]
+ - [163, 7, SimpleBlock, 2, 14024140, 11272, null]
+ - [163, 7, SimpleBlock, 2, 14035415, 11123, null]
+ - [163, 7, SimpleBlock, 2, 14046541, 11354, null]
+ - [163, 7, SimpleBlock, 2, 14057898, 11251, null]
+ - [163, 7, SimpleBlock, 2, 14069152, 5873, null]
+ - [163, 7, SimpleBlock, 2, 14075028, 11219, null]
+ - [163, 7, SimpleBlock, 2, 14086250, 11150, null]
+ - [163, 7, SimpleBlock, 2, 14097403, 11010, null]
+ - [163, 7, SimpleBlock, 2, 14108416, 11187, null]
+ - [163, 7, SimpleBlock, 2, 14119606, 11061, null]
+ - [163, 7, SimpleBlock, 2, 14130670, 5680, null]
+ - [163, 7, SimpleBlock, 2, 14136353, 10999, null]
+ - [163, 7, SimpleBlock, 2, 14147355, 10922, null]
+ - [163, 7, SimpleBlock, 2, 14158280, 10778, null]
+ - [163, 7, SimpleBlock, 2, 14169061, 10831, null]
+ - [163, 7, SimpleBlock, 2, 14179895, 5969, null]
+ - [163, 7, SimpleBlock, 2, 14185867, 10646, null]
+ - [163, 7, SimpleBlock, 2, 14196516, 10783, null]
+ - [163, 7, SimpleBlock, 2, 14207302, 10694, null]
+ - [163, 7, SimpleBlock, 2, 14217999, 10551, null]
+ - [163, 7, SimpleBlock, 2, 14228553, 10232, null]
+ - [163, 7, SimpleBlock, 2, 14238788, 5488, null]
+ - [163, 7, SimpleBlock, 2, 14244279, 10166, null]
+ - [163, 7, SimpleBlock, 2, 14254448, 10369, null]
+ - [163, 7, SimpleBlock, 2, 14264820, 10309, null]
+ - [163, 7, SimpleBlock, 2, 14275132, 10050, null]
+ - [163, 7, SimpleBlock, 2, 14285185, 9831, null]
+ - [163, 7, SimpleBlock, 2, 14295019, 5381, null]
+ - [163, 7, SimpleBlock, 2, 14300403, 9790, null]
+ - [163, 7, SimpleBlock, 2, 14310196, 9781, null]
+ - [163, 7, SimpleBlock, 2, 14319980, 9691, null]
+ - [163, 7, SimpleBlock, 2, 14329674, 9640, null]
+ - [163, 7, SimpleBlock, 2, 14339317, 5968, null]
+ - [163, 7, SimpleBlock, 2, 14345288, 9567, null]
+ - [163, 7, SimpleBlock, 2, 14354858, 9593, null]
+ - [163, 7, SimpleBlock, 2, 14364454, 9481, null]
+ - [163, 7, SimpleBlock, 2, 14373938, 9196, null]
+ - [163, 7, SimpleBlock, 2, 14383137, 9329, null]
+ - [163, 7, SimpleBlock, 2, 14392469, 5970, null]
+ - [163, 7, SimpleBlock, 2, 14398442, 9824, null]
+ - [163, 7, SimpleBlock, 2, 14408269, 10746, null]
+ - [163, 7, SimpleBlock, 2, 14419018, 11335, null]
+ - [163, 7, SimpleBlock, 2, 14430356, 10367, null]
+ - [163, 7, SimpleBlock, 2, 14440726, 12364, null]
+ - [163, 7, SimpleBlock, 2, 14453093, 5489, null]
+ - [163, 7, SimpleBlock, 2, 14458585, 12120, null]
+ - [163, 7, SimpleBlock, 2, 14470708, 12407, null]
+ - [163, 7, SimpleBlock, 2, 14483118, 11822, null]
+ - [163, 7, SimpleBlock, 2, 14494943, 9897, null]
+ - [163, 7, SimpleBlock, 2, 14504843, 5873, null]
+ - [163, 7, SimpleBlock, 2, 14510719, 9439, null]
+ - [163, 7, SimpleBlock, 2, 14520161, 8651, null]
+ - [163, 7, SimpleBlock, 2, 14528815, 7775, null]
+ - [163, 7, SimpleBlock, 2, 14536593, 6840, null]
+ - [163, 7, SimpleBlock, 2, 14543436, 6619, null]
+ - [163, 7, SimpleBlock, 2, 14550058, 5583, null]
+ - [163, 7, SimpleBlock, 2, 14555644, 6141, null]
+ - [163, 7, SimpleBlock, 2, 14561788, 6076, null]
+ - [163, 7, SimpleBlock, 2, 14567867, 6044, null]
+ - [163, 7, SimpleBlock, 2, 14573914, 5676, null]
+ - [163, 7, SimpleBlock, 2, 14579593, 5490, null]
+ - [163, 7, SimpleBlock, 2, 14585086, 5501, null]
+ - [163, 7, SimpleBlock, 2, 14590590, 5301, null]
+ - [163, 7, SimpleBlock, 2, 14595894, 5021, null]
+ - [163, 7, SimpleBlock, 2, 14600918, 4890, null]
+ - [163, 7, SimpleBlock, 2, 14605811, 4454, null]
+ - [163, 7, SimpleBlock, 2, 14610268, 5295, null]
+ - [163, 7, SimpleBlock, 2, 14615566, 4043, null]
+ - [163, 7, SimpleBlock, 2, 14619612, 3760, null]
+ - [163, 7, SimpleBlock, 2, 14623375, 3239, null]
+ - [163, 7, SimpleBlock, 2, 14626617, 3786, null]
+ - [163, 7, SimpleBlock, 2, 14630406, 6048, null]
+ - [163, 7, SimpleBlock, 2, 14636457, 5393, null]
+ - [163, 7, SimpleBlock, 2, 14641853, 7637, null]
+ - [163, 7, SimpleBlock, 2, 14649493, 9427, null]
+ - [163, 7, SimpleBlock, 2, 14658923, 10261, null]
+ - [163, 7, SimpleBlock, 2, 14669187, 10309, null]
+ - [163, 7, SimpleBlock, 2, 14679499, 5485, null]
+ - [163, 7, SimpleBlock, 2, 14684988, 81128, null]
+ - [163, 7, SimpleBlock, 2, 14766119, 2985, null]
+ - [163, 7, SimpleBlock, 2, 14769107, 4541, null]
+ - [163, 7, SimpleBlock, 2, 14773651, 5172, null]
+ - [163, 7, SimpleBlock, 2, 14778826, 7922, null]
+ - [163, 7, SimpleBlock, 2, 14786751, 5392, null]
+ - [163, 7, SimpleBlock, 2, 14792146, 9646, null]
+ - [163, 7, SimpleBlock, 2, 14801795, 12038, null]
+ - [163, 7, SimpleBlock, 2, 14813836, 13795, null]
+ - [163, 7, SimpleBlock, 2, 14827634, 14528, null]
+ - [163, 7, SimpleBlock, 2, 14842165, 5681, null]
+ - [163, 7, SimpleBlock, 2, 14847849, 15597, null]
+ - [163, 7, SimpleBlock, 2, 14863450, 16822, null]
+ - [163, 7, SimpleBlock, 2, 14880276, 18050, null]
+ - [163, 7, SimpleBlock, 2, 14898330, 18837, null]
+ - [163, 7, SimpleBlock, 2, 14917171, 19247, null]
+ - [163, 7, SimpleBlock, 2, 14936421, 5392, null]
+ - [163, 7, SimpleBlock, 2, 14941817, 19069, null]
+ - [163, 7, SimpleBlock, 2, 14960890, 19463, null]
+ - [163, 7, SimpleBlock, 2, 14980357, 19931, null]
+ - [163, 7, SimpleBlock, 2, 15000292, 20799, null]
+ - [163, 7, SimpleBlock, 2, 15021095, 21176, null]
+ - [163, 7, SimpleBlock, 2, 15042274, 5487, null]
+ - [163, 7, SimpleBlock, 2, 15047765, 21537, null]
+ - [163, 7, SimpleBlock, 2, 15069306, 22280, null]
+ - [163, 7, SimpleBlock, 2, 15091590, 22732, null]
+ - [163, 7, SimpleBlock, 2, 15114326, 23371, null]
+ - [163, 7, SimpleBlock, 2, 15137700, 5678, null]
+ - [163, 7, SimpleBlock, 2, 15143382, 23440, null]
+ - [163, 7, SimpleBlock, 2, 15166826, 22016, null]
+ - [163, 7, SimpleBlock, 2, 15188846, 21824, null]
+ - [163, 7, SimpleBlock, 2, 15210674, 21546, null]
+ - [163, 7, SimpleBlock, 2, 15232224, 21501, null]
+ - [163, 7, SimpleBlock, 2, 15253728, 5585, null]
+ - [163, 7, SimpleBlock, 2, 15259317, 22122, null]
+ - [163, 7, SimpleBlock, 2, 15281443, 21956, null]
+ - [163, 7, SimpleBlock, 2, 15303403, 22514, null]
+ - [163, 7, SimpleBlock, 2, 15325921, 22574, null]
+ - [163, 7, SimpleBlock, 2, 15348498, 5489, null]
+ - [163, 7, SimpleBlock, 2, 15353991, 22991, null]
+ - [163, 7, SimpleBlock, 2, 15376986, 23508, null]
+ - [163, 7, SimpleBlock, 2, 15400498, 23870, null]
+ - [163, 7, SimpleBlock, 2, 15424372, 24440, null]
+ - [163, 7, SimpleBlock, 2, 15448816, 25013, null]
+ - [163, 7, SimpleBlock, 2, 15473832, 5199, null]
+ - [163, 7, SimpleBlock, 2, 15479035, 25337, null]
+ - [163, 7, SimpleBlock, 2, 15504376, 24717, null]
+ - [163, 7, SimpleBlock, 2, 15529097, 24623, null]
+ - [163, 7, SimpleBlock, 2, 15553724, 24344, null]
+ - [163, 7, SimpleBlock, 2, 15578072, 23717, null]
+ - [163, 7, SimpleBlock, 2, 15601792, 5680, null]
+ - [163, 7, SimpleBlock, 2, 15607476, 23417, null]
+ - [163, 7, SimpleBlock, 2, 15630897, 23226, null]
+ - [163, 7, SimpleBlock, 2, 15654127, 22676, null]
+ - [163, 7, SimpleBlock, 2, 15676807, 21990, null]
+ - [163, 7, SimpleBlock, 2, 15698800, 5776, null]
+ - [163, 7, SimpleBlock, 2, 15704580, 21261, null]
+ - [163, 7, SimpleBlock, 2, 15725845, 20986, null]
+ - [163, 7, SimpleBlock, 2, 15746835, 20141, null]
+ - [163, 7, SimpleBlock, 2, 15766980, 19845, null]
+ - [163, 7, SimpleBlock, 2, 15786829, 19632, null]
+ - [163, 7, SimpleBlock, 2, 15806464, 5875, null]
+ - [163, 7, SimpleBlock, 2, 15812343, 19280, null]
+ - [163, 7, SimpleBlock, 2, 15831627, 19167, null]
+ - [163, 7, SimpleBlock, 2, 15850798, 19204, null]
+ - [163, 7, SimpleBlock, 2, 15870006, 18863, null]
+ - [163, 7, SimpleBlock, 2, 15888872, 5682, null]
+ - [163, 7, SimpleBlock, 2, 15894558, 18701, null]
+ - [163, 7, SimpleBlock, 2, 15913263, 18677, null]
+ - [163, 7, SimpleBlock, 2, 15931944, 18223, null]
+ - [163, 7, SimpleBlock, 2, 15950171, 18362, null]
+ - [163, 7, SimpleBlock, 2, 15968537, 17943, null]
+ - [163, 7, SimpleBlock, 2, 15986483, 5681, null]
+ - [163, 7, SimpleBlock, 2, 15992168, 17666, null]
+ - [163, 7, SimpleBlock, 2, 16009838, 17249, null]
+ - [163, 7, SimpleBlock, 2, 16027091, 16713, null]
+ - [163, 7, SimpleBlock, 2, 16043807, 16212, null]
+ - [163, 7, SimpleBlock, 2, 16060022, 15866, null]
+ - [163, 7, SimpleBlock, 2, 16075891, 5779, null]
+ - [163, 7, SimpleBlock, 2, 16081673, 15394, null]
+ - [163, 7, SimpleBlock, 2, 16097070, 15150, null]
+ - [163, 7, SimpleBlock, 2, 16112223, 14891, null]
+ - [163, 7, SimpleBlock, 2, 16127117, 14570, null]
+ - [163, 7, SimpleBlock, 2, 16141690, 5777, null]
+ - [163, 7, SimpleBlock, 2, 16147470, 14314, null]
+ - [163, 7, SimpleBlock, 2, 16161787, 13823, null]
+ - [163, 7, SimpleBlock, 2, 16175613, 13404, null]
+ - [163, 7, SimpleBlock, 2, 16189020, 12774, null]
+ - [163, 7, SimpleBlock, 2, 16201797, 12584, null]
+ - [163, 7, SimpleBlock, 2, 16214384, 5584, null]
+ - [163, 7, SimpleBlock, 2, 16219971, 12212, null]
+ - [163, 7, SimpleBlock, 2, 16232186, 11618, null]
+ - [163, 7, SimpleBlock, 2, 16243807, 11021, null]
+ - [163, 7, SimpleBlock, 2, 16254831, 10348, null]
+ - [163, 7, SimpleBlock, 2, 16265182, 2602, null]
+ - [163, 7, SimpleBlock, 2, 16267787, 9776, null]
+ - [163, 7, SimpleBlock, 2, 16277566, 9134, null]
+ - [163, 7, SimpleBlock, 2, 16286703, 8473, null]
+ - [163, 7, SimpleBlock, 2, 16295179, 5872, null]
+ - [163, 7, SimpleBlock, 2, 16301054, 8042, null]
+ - [163, 7, SimpleBlock, 2, 16309099, 6886, null]
+ - [163, 7, SimpleBlock, 2, 16315988, 6278, null]
+ - [163, 7, SimpleBlock, 2, 16322269, 5524, null]
+ - [163, 7, SimpleBlock, 2, 16327796, 5777, null]
+ - [163, 7, SimpleBlock, 2, 16333576, 4813, null]
+ - [163, 7, SimpleBlock, 2, 16338392, 4026, null]
+ - [163, 7, SimpleBlock, 2, 16342421, 3079, null]
+ - [163, 7, SimpleBlock, 2, 16345503, 2908, null]
+ - [163, 7, SimpleBlock, 2, 16348414, 2809, null]
+ - [163, 7, SimpleBlock, 2, 16351226, 5585, null]
+ - [163, 7, SimpleBlock, 2, 16356814, 2952, null]
+ - [163, 7, SimpleBlock, 2, 16359769, 2981, null]
+ - [163, 7, SimpleBlock, 2, 16362753, 3155, null]
+ - [163, 7, SimpleBlock, 2, 16365911, 3321, null]
+ - [163, 7, SimpleBlock, 2, 16369235, 3586, null]
+ - [163, 7, SimpleBlock, 2, 16372824, 5779, null]
+ - [163, 7, SimpleBlock, 2, 16378606, 3776, null]
+ - [163, 7, SimpleBlock, 2, 16382385, 4020, null]
+ - [163, 7, SimpleBlock, 2, 16386408, 4418, null]
+ - [163, 7, SimpleBlock, 2, 16390829, 5383, null]
+ - [163, 7, SimpleBlock, 2, 16396215, 5295, null]
+ - [163, 7, SimpleBlock, 2, 16401513, 6085, null]
+ - [163, 7, SimpleBlock, 2, 16407601, 6943, null]
+ - [163, 7, SimpleBlock, 2, 16414547, 7869, null]
+ - [163, 7, SimpleBlock, 2, 16422419, 8274, null]
+ - [163, 7, SimpleBlock, 2, 16430696, 7703, null]
+ - [163, 7, SimpleBlock, 2, 16438402, 5969, null]
+ - [163, 7, SimpleBlock, 2, 16444374, 7035, null]
+ - [163, 7, SimpleBlock, 2, 16451412, 7128, null]
+ - [163, 7, SimpleBlock, 2, 16458543, 6985, null]
+ - [163, 7, SimpleBlock, 2, 16465531, 6926, null]
+ - [163, 7, SimpleBlock, 2, 16472460, 5872, null]
+ - [163, 7, SimpleBlock, 2, 16478335, 6447, null]
+ - [163, 7, SimpleBlock, 2, 16484785, 5798, null]
+ - [163, 7, SimpleBlock, 2, 16490586, 5291, null]
+ - [163, 7, SimpleBlock, 2, 16495880, 5001, null]
+ - [163, 7, SimpleBlock, 2, 16500884, 4734, null]
+ - [163, 7, SimpleBlock, 2, 16505621, 7311, null]
+ - [163, 7, SimpleBlock, 2, 16512935, 4435, null]
+ - [163, 7, SimpleBlock, 2, 16517373, 6151, null]
+ - [163, 7, SimpleBlock, 2, 16523527, 5456, null]
+ - [163, 7, SimpleBlock, 2, 16528986, 4818, null]
+ - [163, 7, SimpleBlock, 2, 16533807, 5462, null]
+ - [163, 7, SimpleBlock, 2, 16539272, 6159, null]
+ - [163, 7, SimpleBlock, 2, 16545434, 5465, null]
+ - [163, 7, SimpleBlock, 2, 16550902, 5201, null]
+ - [163, 7, SimpleBlock, 2, 16556106, 5002, null]
+ - [163, 7, SimpleBlock, 2, 16561111, 5430, null]
+ - [163, 7, SimpleBlock, 2, 16566544, 5970, null]
+ - [163, 7, SimpleBlock, 2, 16572517, 6146, null]
+ - [163, 7, SimpleBlock, 2, 16578666, 6690, null]
+ - [163, 7, SimpleBlock, 2, 16585359, 7086, null]
+ - [163, 7, SimpleBlock, 2, 16592448, 8078, null]
+ - [163, 7, SimpleBlock, 2, 16600529, 8723, null]
+ - [163, 7, SimpleBlock, 2, 16609255, 6067, null]
+ - [163, 7, SimpleBlock, 2, 16615325, 9113, null]
+ - [163, 7, SimpleBlock, 2, 16624441, 9253, null]
+ - [163, 7, SimpleBlock, 2, 16633697, 10193, null]
+ - [163, 7, SimpleBlock, 2, 16643893, 9354, null]
+ - [163, 7, SimpleBlock, 2, 16653250, 3948, null]
+ - [163, 7, SimpleBlock, 2, 16657201, 9131, null]
+ - [163, 7, SimpleBlock, 2, 16666335, 8881, null]
+ - [163, 7, SimpleBlock, 2, 16675219, 7845, null]
+ - [163, 7, SimpleBlock, 2, 16683067, 7036, null]
+ - - 524531317
+ - 6
+ - Cluster
+ - 1
+ - 16690110
+ - 778801
+ - - [231, 1, Timecode, 2, 16690112, 2, 53333]
+ - [163, 7, SimpleBlock, 2, 16690117, 772, null]
+ - [163, 7, SimpleBlock, 2, 16690892, 6162, null]
+ - [163, 7, SimpleBlock, 2, 16697058, 73939, null]
+ - [163, 7, SimpleBlock, 2, 16771000, 2203, null]
+ - [163, 7, SimpleBlock, 2, 16773206, 2982, null]
+ - [163, 7, SimpleBlock, 2, 16776191, 3662, null]
+ - [163, 7, SimpleBlock, 2, 16779856, 4237, null]
+ - [163, 7, SimpleBlock, 2, 16784096, 5968, null]
+ - [163, 7, SimpleBlock, 2, 16790067, 4508, null]
+ - [163, 7, SimpleBlock, 2, 16794578, 4745, null]
+ - [163, 7, SimpleBlock, 2, 16799326, 4972, null]
+ - [163, 7, SimpleBlock, 2, 16804301, 5076, null]
+ - [163, 7, SimpleBlock, 2, 16809380, 6063, null]
+ - [163, 7, SimpleBlock, 2, 16815446, 5439, null]
+ - [163, 7, SimpleBlock, 2, 16820888, 5560, null]
+ - [163, 7, SimpleBlock, 2, 16826451, 5676, null]
+ - [163, 7, SimpleBlock, 2, 16832130, 5844, null]
+ - [163, 7, SimpleBlock, 2, 16837977, 6061, null]
+ - [163, 7, SimpleBlock, 2, 16844041, 5969, null]
+ - [163, 7, SimpleBlock, 2, 16850013, 6557, null]
+ - [163, 7, SimpleBlock, 2, 16856573, 7227, null]
+ - [163, 7, SimpleBlock, 2, 16863803, 7725, null]
+ - [163, 7, SimpleBlock, 2, 16871531, 8407, null]
+ - [163, 7, SimpleBlock, 2, 16879941, 5582, null]
+ - [163, 7, SimpleBlock, 2, 16885526, 8767, null]
+ - [163, 7, SimpleBlock, 2, 16894296, 9382, null]
+ - [163, 7, SimpleBlock, 2, 16903681, 9861, null]
+ - [163, 7, SimpleBlock, 2, 16913545, 10355, null]
+ - [163, 7, SimpleBlock, 2, 16923903, 9733, null]
+ - [163, 7, SimpleBlock, 2, 16933639, 5873, null]
+ - [163, 7, SimpleBlock, 2, 16939515, 9873, null]
+ - [163, 7, SimpleBlock, 2, 16949391, 9813, null]
+ - [163, 7, SimpleBlock, 2, 16959207, 9508, null]
+ - [163, 7, SimpleBlock, 2, 16968718, 11810, null]
+ - [163, 7, SimpleBlock, 2, 16980531, 12852, null]
+ - [163, 7, SimpleBlock, 2, 16993386, 5393, null]
+ - [163, 7, SimpleBlock, 2, 16998782, 11068, null]
+ - [163, 7, SimpleBlock, 2, 17009853, 10499, null]
+ - [163, 7, SimpleBlock, 2, 17020355, 10353, null]
+ - [163, 7, SimpleBlock, 2, 17030711, 9915, null]
+ - [163, 7, SimpleBlock, 2, 17040629, 5873, null]
+ - [163, 7, SimpleBlock, 2, 17046505, 9921, null]
+ - [163, 7, SimpleBlock, 2, 17056429, 9995, null]
+ - [163, 7, SimpleBlock, 2, 17066427, 10146, null]
+ - [163, 7, SimpleBlock, 2, 17076576, 10535, null]
+ - [163, 7, SimpleBlock, 2, 17087114, 10775, null]
+ - [163, 7, SimpleBlock, 2, 17097892, 5873, null]
+ - [163, 7, SimpleBlock, 2, 17103768, 11200, null]
+ - [163, 7, SimpleBlock, 2, 17114971, 12237, null]
+ - [163, 7, SimpleBlock, 2, 17127211, 12523, null]
+ - [163, 7, SimpleBlock, 2, 17139737, 12799, null]
+ - [163, 7, SimpleBlock, 2, 17152539, 6353, null]
+ - [163, 7, SimpleBlock, 2, 17158895, 12844, null]
+ - [163, 7, SimpleBlock, 2, 17171742, 13331, null]
+ - [163, 7, SimpleBlock, 2, 17185076, 13494, null]
+ - [163, 7, SimpleBlock, 2, 17198573, 13391, null]
+ - [163, 7, SimpleBlock, 2, 17211967, 13210, null]
+ - [163, 7, SimpleBlock, 2, 17225180, 5776, null]
+ - [163, 7, SimpleBlock, 2, 17230959, 12707, null]
+ - [163, 7, SimpleBlock, 2, 17243669, 12771, null]
+ - [163, 7, SimpleBlock, 2, 17256443, 12524, null]
+ - [163, 7, SimpleBlock, 2, 17268970, 12340, null]
+ - [163, 7, SimpleBlock, 2, 17281313, 12283, null]
+ - [163, 7, SimpleBlock, 2, 17293599, 5297, null]
+ - [163, 7, SimpleBlock, 2, 17298899, 12150, null]
+ - [163, 7, SimpleBlock, 2, 17311052, 12123, null]
+ - [163, 7, SimpleBlock, 2, 17323178, 11543, null]
+ - [163, 7, SimpleBlock, 2, 17334724, 10955, null]
+ - [163, 7, SimpleBlock, 2, 17345682, 5487, null]
+ - [163, 7, SimpleBlock, 2, 17351172, 10655, null]
+ - [163, 7, SimpleBlock, 2, 17361830, 10831, null]
+ - [163, 7, SimpleBlock, 2, 17372664, 11824, null]
+ - [163, 7, SimpleBlock, 2, 17384491, 12199, null]
+ - [163, 7, SimpleBlock, 2, 17396693, 10612, null]
+ - [163, 7, SimpleBlock, 2, 17407308, 6162, null]
+ - [163, 7, SimpleBlock, 2, 17413473, 10118, null]
+ - [163, 7, SimpleBlock, 2, 17423594, 9704, null]
+ - [163, 7, SimpleBlock, 2, 17433301, 8930, null]
+ - [163, 7, SimpleBlock, 2, 17442234, 8418, null]
+ - [163, 7, SimpleBlock, 2, 17450655, 676, null]
+ - [163, 7, SimpleBlock, 2, 17451334, 8595, null]
+ - [163, 7, SimpleBlock, 2, 17459932, 8979, null]
+ - - 524531317
+ - 6
+ - Cluster
+ - 1
+ - 17468918
+ - 1159873
+ - - [231, 1, Timecode, 2, 17468920, 2, 56083]
+ - [163, 7, SimpleBlock, 2, 17468925, 676, null]
+ - [163, 7, SimpleBlock, 2, 17469604, 5777, null]
+ - [163, 7, SimpleBlock, 2, 17475385, 22461, null]
+ - [163, 7, SimpleBlock, 2, 17497849, 2509, null]
+ - [163, 7, SimpleBlock, 2, 17500361, 1485, null]
+ - [163, 7, SimpleBlock, 2, 17501849, 320, null]
+ - [163, 7, SimpleBlock, 2, 17502172, 6258, null]
+ - [163, 7, SimpleBlock, 2, 17508433, 245, null]
+ - [163, 7, SimpleBlock, 2, 17508681, 248, null]
+ - [163, 7, SimpleBlock, 2, 17508932, 233, null]
+ - [163, 7, SimpleBlock, 2, 17509168, 218, null]
+ - [163, 7, SimpleBlock, 2, 17509389, 238, null]
+ - [163, 7, SimpleBlock, 2, 17509630, 6353, null]
+ - [163, 7, SimpleBlock, 2, 17515986, 232, null]
+ - [163, 7, SimpleBlock, 2, 17516221, 224, null]
+ - [163, 7, SimpleBlock, 2, 17516448, 235, null]
+ - [163, 7, SimpleBlock, 2, 17516686, 330, null]
+ - [163, 7, SimpleBlock, 2, 17517019, 6545, null]
+ - [163, 7, SimpleBlock, 2, 17523567, 414, null]
+ - [163, 7, SimpleBlock, 2, 17523984, 499, null]
+ - [163, 7, SimpleBlock, 2, 17524486, 554, null]
+ - [163, 7, SimpleBlock, 2, 17525043, 614, null]
+ - [163, 7, SimpleBlock, 2, 17525660, 598, null]
+ - [163, 7, SimpleBlock, 2, 17526261, 5487, null]
+ - [163, 7, SimpleBlock, 2, 17531751, 640, null]
+ - [163, 7, SimpleBlock, 2, 17532394, 771, null]
+ - [163, 7, SimpleBlock, 2, 17533168, 715, null]
+ - [163, 7, SimpleBlock, 2, 17533886, 643, null]
+ - [163, 7, SimpleBlock, 2, 17534532, 686, null]
+ - [163, 7, SimpleBlock, 2, 17535221, 5779, null]
+ - [163, 7, SimpleBlock, 2, 17541003, 763, null]
+ - [163, 7, SimpleBlock, 2, 17541769, 1464, null]
+ - [163, 7, SimpleBlock, 2, 17543236, 1514, null]
+ - [163, 7, SimpleBlock, 2, 17544753, 1618, null]
+ - [163, 7, SimpleBlock, 2, 17546374, 5969, null]
+ - [163, 7, SimpleBlock, 2, 17552346, 1804, null]
+ - [163, 7, SimpleBlock, 2, 17554153, 1848, null]
+ - [163, 7, SimpleBlock, 2, 17556004, 1926, null]
+ - [163, 7, SimpleBlock, 2, 17557933, 1630, null]
+ - [163, 7, SimpleBlock, 2, 17559566, 1113, null]
+ - [163, 7, SimpleBlock, 2, 17560682, 5295, null]
+ - [163, 7, SimpleBlock, 2, 17565980, 1004, null]
+ - [163, 7, SimpleBlock, 2, 17566987, 1093, null]
+ - [163, 7, SimpleBlock, 2, 17568083, 1153, null]
+ - [163, 7, SimpleBlock, 2, 17569239, 1172, null]
+ - [163, 7, SimpleBlock, 2, 17570414, 6354, null]
+ - [163, 7, SimpleBlock, 2, 17576771, 1405, null]
+ - [163, 7, SimpleBlock, 2, 17578179, 2397, null]
+ - [163, 7, SimpleBlock, 2, 17580579, 2799, null]
+ - [163, 7, SimpleBlock, 2, 17583381, 3471, null]
+ - [163, 7, SimpleBlock, 2, 17586855, 3841, null]
+ - [163, 7, SimpleBlock, 2, 17590699, 5779, null]
+ - [163, 7, SimpleBlock, 2, 17596481, 4169, null]
+ - [163, 7, SimpleBlock, 2, 17600653, 4518, null]
+ - [163, 7, SimpleBlock, 2, 17605174, 4830, null]
+ - [163, 7, SimpleBlock, 2, 17610007, 5034, null]
+ - [163, 7, SimpleBlock, 2, 17615044, 4699, null]
+ - [163, 7, SimpleBlock, 2, 17619746, 5776, null]
+ - [163, 7, SimpleBlock, 2, 17625525, 3001, null]
+ - [163, 7, SimpleBlock, 2, 17628529, 2180, null]
+ - [163, 7, SimpleBlock, 2, 17630712, 2302, null]
+ - [163, 7, SimpleBlock, 2, 17633017, 2170, null]
+ - [163, 7, SimpleBlock, 2, 17635190, 6062, null]
+ - [163, 7, SimpleBlock, 2, 17641255, 2343, null]
+ - [163, 7, SimpleBlock, 2, 17643601, 2435, null]
+ - [163, 7, SimpleBlock, 2, 17646039, 2472, null]
+ - [163, 7, SimpleBlock, 2, 17648514, 2495, null]
+ - [163, 7, SimpleBlock, 2, 17651012, 2687, null]
+ - [163, 7, SimpleBlock, 2, 17653702, 5393, null]
+ - [163, 7, SimpleBlock, 2, 17659098, 2627, null]
+ - [163, 7, SimpleBlock, 2, 17661728, 2740, null]
+ - [163, 7, SimpleBlock, 2, 17664471, 2819, null]
+ - [163, 7, SimpleBlock, 2, 17667293, 2973, null]
+ - [163, 7, SimpleBlock, 2, 17670269, 3204, null]
+ - [163, 7, SimpleBlock, 2, 17673476, 6255, null]
+ - [163, 7, SimpleBlock, 2, 17679734, 3227, null]
+ - [163, 7, SimpleBlock, 2, 17682964, 3096, null]
+ - [163, 7, SimpleBlock, 2, 17686063, 2755, null]
+ - [163, 7, SimpleBlock, 2, 17688821, 2414, null]
+ - [163, 7, SimpleBlock, 2, 17691238, 5966, null]
+ - [163, 7, SimpleBlock, 2, 17697207, 2199, null]
+ - [163, 7, SimpleBlock, 2, 17699409, 1988, null]
+ - [163, 7, SimpleBlock, 2, 17701400, 1875, null]
+ - [163, 7, SimpleBlock, 2, 17703278, 1877, null]
+ - [163, 7, SimpleBlock, 2, 17705158, 1855, null]
+ - [163, 7, SimpleBlock, 2, 17707016, 5872, null]
+ - [163, 7, SimpleBlock, 2, 17712891, 1753, null]
+ - [163, 7, SimpleBlock, 2, 17714647, 1698, null]
+ - [163, 7, SimpleBlock, 2, 17716348, 1681, null]
+ - [163, 7, SimpleBlock, 2, 17718032, 1668, null]
+ - [163, 7, SimpleBlock, 2, 17719703, 6449, null]
+ - [163, 7, SimpleBlock, 2, 17726155, 1643, null]
+ - [163, 7, SimpleBlock, 2, 17727801, 1573, null]
+ - [163, 7, SimpleBlock, 2, 17729377, 1510, null]
+ - [163, 7, SimpleBlock, 2, 17730890, 1414, null]
+ - [163, 7, SimpleBlock, 2, 17732307, 1290, null]
+ - [163, 7, SimpleBlock, 2, 17733600, 6066, null]
+ - [163, 7, SimpleBlock, 2, 17739669, 1199, null]
+ - [163, 7, SimpleBlock, 2, 17740871, 1170, null]
+ - [163, 7, SimpleBlock, 2, 17742044, 1056, null]
+ - [163, 7, SimpleBlock, 2, 17743103, 914, null]
+ - [163, 7, SimpleBlock, 2, 17744020, 895, null]
+ - [163, 7, SimpleBlock, 2, 17744918, 6256, null]
+ - [163, 7, SimpleBlock, 2, 17751177, 772, null]
+ - [163, 7, SimpleBlock, 2, 17751952, 686, null]
+ - [163, 7, SimpleBlock, 2, 17752641, 801, null]
+ - [163, 7, SimpleBlock, 2, 17753445, 810, null]
+ - [163, 7, SimpleBlock, 2, 17754258, 5201, null]
+ - [163, 7, SimpleBlock, 2, 17759462, 816, null]
+ - [163, 7, SimpleBlock, 2, 17760281, 773, null]
+ - [163, 7, SimpleBlock, 2, 17761057, 767, null]
+ - [163, 7, SimpleBlock, 2, 17761827, 819, null]
+ - [163, 7, SimpleBlock, 2, 17762649, 878, null]
+ - [163, 7, SimpleBlock, 2, 17763530, 5777, null]
+ - [163, 7, SimpleBlock, 2, 17769310, 1042, null]
+ - [163, 7, SimpleBlock, 2, 17770355, 1207, null]
+ - [163, 7, SimpleBlock, 2, 17771565, 1260, null]
+ - [163, 7, SimpleBlock, 2, 17772828, 1224, null]
+ - [163, 7, SimpleBlock, 2, 17774055, 5679, null]
+ - [163, 7, SimpleBlock, 2, 17779737, 1156, null]
+ - [163, 7, SimpleBlock, 2, 17780896, 1212, null]
+ - [163, 7, SimpleBlock, 2, 17782111, 1231, null]
+ - [163, 7, SimpleBlock, 2, 17783345, 1228, null]
+ - [163, 7, SimpleBlock, 2, 17784576, 1295, null]
+ - [163, 7, SimpleBlock, 2, 17785874, 5010, null]
+ - [163, 7, SimpleBlock, 2, 17790887, 1319, null]
+ - [163, 7, SimpleBlock, 2, 17792209, 1331, null]
+ - [163, 7, SimpleBlock, 2, 17793543, 1360, null]
+ - [163, 7, SimpleBlock, 2, 17794906, 1380, null]
+ - [163, 7, SimpleBlock, 2, 17796289, 1470, null]
+ - [163, 7, SimpleBlock, 2, 17797762, 5968, null]
+ - [163, 7, SimpleBlock, 2, 17803733, 1471, null]
+ - [163, 7, SimpleBlock, 2, 17805207, 1209, null]
+ - [163, 7, SimpleBlock, 2, 17806419, 1172, null]
+ - [163, 7, SimpleBlock, 2, 17807594, 1246, null]
+ - [163, 7, SimpleBlock, 2, 17808843, 5874, null]
+ - [163, 7, SimpleBlock, 2, 17814721, 29320, null]
+ - [163, 7, SimpleBlock, 2, 17844044, 4031, null]
+ - [163, 7, SimpleBlock, 2, 17848078, 3115, null]
+ - [163, 7, SimpleBlock, 2, 17851196, 2426, null]
+ - [163, 7, SimpleBlock, 2, 17853625, 3597, null]
+ - [163, 7, SimpleBlock, 2, 17857225, 5586, null]
+ - [163, 7, SimpleBlock, 2, 17862814, 4146, null]
+ - [163, 7, SimpleBlock, 2, 17866963, 4167, null]
+ - [163, 7, SimpleBlock, 2, 17871133, 3878, null]
+ - [163, 7, SimpleBlock, 2, 17875014, 2803, null]
+ - [163, 7, SimpleBlock, 2, 17877820, 772, null]
+ - [163, 7, SimpleBlock, 2, 17878595, 2519, null]
+ - [163, 7, SimpleBlock, 2, 17881117, 5586, null]
+ - [163, 7, SimpleBlock, 2, 17886706, 3475, null]
+ - [163, 7, SimpleBlock, 2, 17890184, 3964, null]
+ - [163, 7, SimpleBlock, 2, 17894151, 3802, null]
+ - [163, 7, SimpleBlock, 2, 17897956, 3434, null]
+ - [163, 7, SimpleBlock, 2, 17901393, 2679, null]
+ - [163, 7, SimpleBlock, 2, 17904075, 5489, null]
+ - [163, 7, SimpleBlock, 2, 17909567, 2362, null]
+ - [163, 7, SimpleBlock, 2, 17911932, 2821, null]
+ - [163, 7, SimpleBlock, 2, 17914756, 3440, null]
+ - [163, 7, SimpleBlock, 2, 17918199, 3659, null]
+ - [163, 7, SimpleBlock, 2, 17921861, 6737, null]
+ - [163, 7, SimpleBlock, 2, 17928601, 3649, null]
+ - [163, 7, SimpleBlock, 2, 17932253, 2521, null]
+ - [163, 7, SimpleBlock, 2, 17934777, 1893, null]
+ - [163, 7, SimpleBlock, 2, 17936673, 2836, null]
+ - [163, 7, SimpleBlock, 2, 17939512, 3377, null]
+ - [163, 7, SimpleBlock, 2, 17942892, 5392, null]
+ - [163, 7, SimpleBlock, 2, 17948287, 3391, null]
+ - [163, 7, SimpleBlock, 2, 17951681, 3300, null]
+ - [163, 7, SimpleBlock, 2, 17954984, 2321, null]
+ - [163, 7, SimpleBlock, 2, 17957308, 1850, null]
+ - [163, 7, SimpleBlock, 2, 17959161, 5585, null]
+ - [163, 7, SimpleBlock, 2, 17964749, 2256, null]
+ - [163, 7, SimpleBlock, 2, 17967008, 2635, null]
+ - [163, 7, SimpleBlock, 2, 17969646, 2856, null]
+ - [163, 7, SimpleBlock, 2, 17972505, 2837, null]
+ - [163, 7, SimpleBlock, 2, 17975345, 2829, null]
+ - [163, 7, SimpleBlock, 2, 17978177, 6256, null]
+ - [163, 7, SimpleBlock, 2, 17984436, 2566, null]
+ - [163, 7, SimpleBlock, 2, 17987005, 2308, null]
+ - [163, 7, SimpleBlock, 2, 17989316, 2064, null]
+ - [163, 7, SimpleBlock, 2, 17991383, 2007, null]
+ - [163, 7, SimpleBlock, 2, 17993393, 2166, null]
+ - [163, 7, SimpleBlock, 2, 17995562, 5393, null]
+ - [163, 7, SimpleBlock, 2, 18000958, 1772, null]
+ - [163, 7, SimpleBlock, 2, 18002733, 1579, null]
+ - [163, 7, SimpleBlock, 2, 18004315, 1467, null]
+ - [163, 7, SimpleBlock, 2, 18005785, 1362, null]
+ - [163, 7, SimpleBlock, 2, 18007150, 5776, null]
+ - [163, 7, SimpleBlock, 2, 18012929, 1411, null]
+ - [163, 7, SimpleBlock, 2, 18014343, 1553, null]
+ - [163, 7, SimpleBlock, 2, 18015899, 1874, null]
+ - [163, 7, SimpleBlock, 2, 18017776, 2222, null]
+ - [163, 7, SimpleBlock, 2, 18020001, 2455, null]
+ - [163, 7, SimpleBlock, 2, 18022459, 5298, null]
+ - [163, 7, SimpleBlock, 2, 18027760, 2969, null]
+ - [163, 7, SimpleBlock, 2, 18030732, 2986, null]
+ - [163, 7, SimpleBlock, 2, 18033721, 2874, null]
+ - [163, 7, SimpleBlock, 2, 18036598, 3500, null]
+ - [163, 7, SimpleBlock, 2, 18040101, 5299, null]
+ - [163, 7, SimpleBlock, 2, 18045403, 5436, null]
+ - [163, 7, SimpleBlock, 2, 18050842, 5687, null]
+ - [163, 7, SimpleBlock, 2, 18056532, 4865, null]
+ - [163, 7, SimpleBlock, 2, 18061400, 4047, null]
+ - [163, 7, SimpleBlock, 2, 18065450, 4424, null]
+ - [163, 7, SimpleBlock, 2, 18069877, 5776, null]
+ - [163, 7, SimpleBlock, 2, 18075656, 4828, null]
+ - [163, 7, SimpleBlock, 2, 18080487, 5030, null]
+ - [163, 7, SimpleBlock, 2, 18085520, 4587, null]
+ - [163, 7, SimpleBlock, 2, 18090110, 3823, null]
+ - [163, 7, SimpleBlock, 2, 18093936, 3266, null]
+ - [163, 7, SimpleBlock, 2, 18097205, 5969, null]
+ - [163, 7, SimpleBlock, 2, 18103177, 2943, null]
+ - [163, 7, SimpleBlock, 2, 18106123, 2733, null]
+ - [163, 7, SimpleBlock, 2, 18108859, 2523, null]
+ - [163, 7, SimpleBlock, 2, 18111385, 2499, null]
+ - [163, 7, SimpleBlock, 2, 18113887, 6162, null]
+ - [163, 7, SimpleBlock, 2, 18120052, 2530, null]
+ - [163, 7, SimpleBlock, 2, 18122585, 2361, null]
+ - [163, 7, SimpleBlock, 2, 18124949, 2255, null]
+ - [163, 7, SimpleBlock, 2, 18127207, 2069, null]
+ - [163, 7, SimpleBlock, 2, 18129279, 1886, null]
+ - [163, 7, SimpleBlock, 2, 18131168, 5874, null]
+ - [163, 7, SimpleBlock, 2, 18137045, 1818, null]
+ - [163, 7, SimpleBlock, 2, 18138866, 1694, null]
+ - [163, 7, SimpleBlock, 2, 18140563, 1791, null]
+ - [163, 7, SimpleBlock, 2, 18142357, 1640, null]
+ - [163, 7, SimpleBlock, 2, 18144000, 5679, null]
+ - [163, 7, SimpleBlock, 2, 18149682, 1477, null]
+ - [163, 7, SimpleBlock, 2, 18151163, 60499, null]
+ - [163, 7, SimpleBlock, 2, 18211665, 2228, null]
+ - [163, 7, SimpleBlock, 2, 18213896, 2038, null]
+ - [163, 7, SimpleBlock, 2, 18215937, 1445, null]
+ - [163, 7, SimpleBlock, 2, 18217385, 5871, null]
+ - [163, 7, SimpleBlock, 2, 18223259, 1144, null]
+ - [163, 7, SimpleBlock, 2, 18224406, 966, null]
+ - [163, 7, SimpleBlock, 2, 18225375, 1554, null]
+ - [163, 7, SimpleBlock, 2, 18226932, 2149, null]
+ - [163, 7, SimpleBlock, 2, 18229084, 2465, null]
+ - [163, 7, SimpleBlock, 2, 18231552, 5872, null]
+ - [163, 7, SimpleBlock, 2, 18237427, 2573, null]
+ - [163, 7, SimpleBlock, 2, 18240003, 2777, null]
+ - [163, 7, SimpleBlock, 2, 18242783, 2284, null]
+ - [163, 7, SimpleBlock, 2, 18245070, 1929, null]
+ - [163, 7, SimpleBlock, 2, 18247002, 6066, null]
+ - [163, 7, SimpleBlock, 2, 18253071, 1909, null]
+ - [163, 7, SimpleBlock, 2, 18254983, 1855, null]
+ - [163, 7, SimpleBlock, 2, 18256841, 1852, null]
+ - [163, 7, SimpleBlock, 2, 18258696, 1983, null]
+ - [163, 7, SimpleBlock, 2, 18260682, 1894, null]
+ - [163, 7, SimpleBlock, 2, 18262579, 5583, null]
+ - [163, 7, SimpleBlock, 2, 18268165, 2739, null]
+ - [163, 7, SimpleBlock, 2, 18270907, 3644, null]
+ - [163, 7, SimpleBlock, 2, 18274554, 4230, null]
+ - [163, 7, SimpleBlock, 2, 18278787, 3657, null]
+ - [163, 7, SimpleBlock, 2, 18282447, 3713, null]
+ - [163, 7, SimpleBlock, 2, 18286163, 5680, null]
+ - [163, 7, SimpleBlock, 2, 18291846, 4481, null]
+ - [163, 7, SimpleBlock, 2, 18296330, 4123, null]
+ - [163, 7, SimpleBlock, 2, 18300456, 3651, null]
+ - [163, 7, SimpleBlock, 2, 18304110, 3533, null]
+ - [163, 7, SimpleBlock, 2, 18307646, 5681, null]
+ - [163, 7, SimpleBlock, 2, 18313330, 4339, null]
+ - [163, 7, SimpleBlock, 2, 18317672, 5237, null]
+ - [163, 7, SimpleBlock, 2, 18322912, 5918, null]
+ - [163, 7, SimpleBlock, 2, 18328833, 5993, null]
+ - [163, 7, SimpleBlock, 2, 18334829, 4395, null]
+ - [163, 7, SimpleBlock, 2, 18339227, 6066, null]
+ - [163, 7, SimpleBlock, 2, 18345296, 4935, null]
+ - [163, 7, SimpleBlock, 2, 18350234, 6037, null]
+ - [163, 7, SimpleBlock, 2, 18356274, 6409, null]
+ - [163, 7, SimpleBlock, 2, 18362686, 6297, null]
+ - [163, 7, SimpleBlock, 2, 18368986, 5585, null]
+ - [163, 7, SimpleBlock, 2, 18374574, 5557, null]
+ - [163, 7, SimpleBlock, 2, 18380134, 3974, null]
+ - [163, 7, SimpleBlock, 2, 18384111, 3783, null]
+ - [163, 7, SimpleBlock, 2, 18387897, 5331, null]
+ - [163, 7, SimpleBlock, 2, 18393231, 6651, null]
+ - [163, 7, SimpleBlock, 2, 18399885, 6256, null]
+ - [163, 7, SimpleBlock, 2, 18406144, 7153, null]
+ - [163, 7, SimpleBlock, 2, 18413300, 6988, null]
+ - [163, 7, SimpleBlock, 2, 18420291, 5408, null]
+ - [163, 7, SimpleBlock, 2, 18425702, 5607, null]
+ - [163, 7, SimpleBlock, 2, 18431312, 6685, null]
+ - [163, 7, SimpleBlock, 2, 18438000, 5585, null]
+ - [163, 7, SimpleBlock, 2, 18443588, 6995, null]
+ - [163, 7, SimpleBlock, 2, 18450586, 7063, null]
+ - [163, 7, SimpleBlock, 2, 18457652, 7432, null]
+ - [163, 7, SimpleBlock, 2, 18465087, 6784, null]
+ - [163, 7, SimpleBlock, 2, 18471874, 6257, null]
+ - [163, 7, SimpleBlock, 2, 18478134, 5355, null]
+ - [163, 7, SimpleBlock, 2, 18483492, 7557, null]
+ - [163, 7, SimpleBlock, 2, 18491052, 8650, null]
+ - [163, 7, SimpleBlock, 2, 18499705, 8923, null]
+ - [163, 7, SimpleBlock, 2, 18508631, 8857, null]
+ - [163, 7, SimpleBlock, 2, 18517491, 772, null]
+ - [163, 7, SimpleBlock, 2, 18518266, 5487, null]
+ - [163, 7, SimpleBlock, 2, 18523756, 7441, null]
+ - [163, 7, SimpleBlock, 2, 18531200, 8855, null]
+ - [163, 7, SimpleBlock, 2, 18540058, 9921, null]
+ - [163, 7, SimpleBlock, 2, 18549982, 10005, null]
+ - [163, 7, SimpleBlock, 2, 18559990, 10304, null]
+ - [163, 7, SimpleBlock, 2, 18570297, 5199, null]
+ - [163, 7, SimpleBlock, 2, 18575499, 10952, null]
+ - [163, 7, SimpleBlock, 2, 18586454, 11010, null]
+ - [163, 7, SimpleBlock, 2, 18597467, 9866, null]
+ - [163, 7, SimpleBlock, 2, 18607336, 10617, null]
+ - [163, 7, SimpleBlock, 2, 18617956, 10835, null]
+ - - 524531317
+ - 6
+ - Cluster
+ - 1
+ - 18628799
+ - 2103666
+ - - [231, 1, Timecode, 2, 18628801, 3, 66500]
+ - [163, 7, SimpleBlock, 2, 18628807, 772, null]
+ - [163, 7, SimpleBlock, 2, 18629582, 5776, null]
+ - [163, 7, SimpleBlock, 2, 18635362, 36579, null]
+ - [163, 7, SimpleBlock, 2, 18671944, 10731, null]
+ - [163, 7, SimpleBlock, 2, 18682678, 10066, null]
+ - [163, 7, SimpleBlock, 2, 18692747, 10462, null]
+ - [163, 7, SimpleBlock, 2, 18703212, 5680, null]
+ - [163, 7, SimpleBlock, 2, 18708895, 10053, null]
+ - [163, 7, SimpleBlock, 2, 18718951, 9561, null]
+ - [163, 7, SimpleBlock, 2, 18728515, 9702, null]
+ - [163, 7, SimpleBlock, 2, 18738220, 9853, null]
+ - [163, 7, SimpleBlock, 2, 18748076, 10160, null]
+ - [163, 7, SimpleBlock, 2, 18758239, 5777, null]
+ - [163, 7, SimpleBlock, 2, 18764019, 10507, null]
+ - [163, 7, SimpleBlock, 2, 18774529, 9258, null]
+ - [163, 7, SimpleBlock, 2, 18783790, 9531, null]
+ - [163, 7, SimpleBlock, 2, 18793324, 9334, null]
+ - [163, 7, SimpleBlock, 2, 18802661, 5971, null]
+ - [163, 7, SimpleBlock, 2, 18808635, 9543, null]
+ - [163, 7, SimpleBlock, 2, 18818181, 9583, null]
+ - [163, 7, SimpleBlock, 2, 18827767, 9467, null]
+ - [163, 7, SimpleBlock, 2, 18837237, 8785, null]
+ - [163, 7, SimpleBlock, 2, 18846025, 8852, null]
+ - [163, 7, SimpleBlock, 2, 18854880, 5585, null]
+ - [163, 7, SimpleBlock, 2, 18860468, 8937, null]
+ - [163, 7, SimpleBlock, 2, 18869408, 9268, null]
+ - [163, 7, SimpleBlock, 2, 18878679, 9540, null]
+ - [163, 7, SimpleBlock, 2, 18888222, 9647, null]
+ - [163, 7, SimpleBlock, 2, 18897872, 10183, null]
+ - [163, 7, SimpleBlock, 2, 18908058, 6063, null]
+ - [163, 7, SimpleBlock, 2, 18914124, 10189, null]
+ - [163, 7, SimpleBlock, 2, 18924316, 10411, null]
+ - [163, 7, SimpleBlock, 2, 18934730, 10327, null]
+ - [163, 7, SimpleBlock, 2, 18945060, 8746, null]
+ - [163, 7, SimpleBlock, 2, 18953809, 5778, null]
+ - [163, 7, SimpleBlock, 2, 18959590, 6640, null]
+ - [163, 7, SimpleBlock, 2, 18966233, 4282, null]
+ - [163, 7, SimpleBlock, 2, 18970518, 4188, null]
+ - [163, 7, SimpleBlock, 2, 18974709, 4033, null]
+ - [163, 7, SimpleBlock, 2, 18978745, 3879, null]
+ - [163, 7, SimpleBlock, 2, 18982627, 6161, null]
+ - [163, 7, SimpleBlock, 2, 18988791, 3886, null]
+ - [163, 7, SimpleBlock, 2, 18992680, 3964, null]
+ - [163, 7, SimpleBlock, 2, 18996647, 4541, null]
+ - [163, 7, SimpleBlock, 2, 19001191, 6260, null]
+ - [163, 7, SimpleBlock, 2, 19007454, 6063, null]
+ - [163, 7, SimpleBlock, 2, 19013520, 6518, null]
+ - [163, 7, SimpleBlock, 2, 19020041, 7033, null]
+ - [163, 7, SimpleBlock, 2, 19027077, 7616, null]
+ - [163, 7, SimpleBlock, 2, 19034696, 8183, null]
+ - [163, 7, SimpleBlock, 2, 19042882, 8734, null]
+ - [163, 7, SimpleBlock, 2, 19051619, 6067, null]
+ - [163, 7, SimpleBlock, 2, 19057689, 9116, null]
+ - [163, 7, SimpleBlock, 2, 19066808, 9011, null]
+ - [163, 7, SimpleBlock, 2, 19075822, 8607, null]
+ - [163, 7, SimpleBlock, 2, 19084432, 6087, null]
+ - [163, 7, SimpleBlock, 2, 19090522, 4136, null]
+ - [163, 7, SimpleBlock, 2, 19094661, 6061, null]
+ - [163, 7, SimpleBlock, 2, 19100725, 4215, null]
+ - [163, 7, SimpleBlock, 2, 19104943, 8273, null]
+ - [163, 7, SimpleBlock, 2, 19113219, 8929, null]
+ - [163, 7, SimpleBlock, 2, 19122151, 8522, null]
+ - [163, 7, SimpleBlock, 2, 19130676, 5682, null]
+ - [163, 7, SimpleBlock, 2, 19136361, 8505, null]
+ - [163, 7, SimpleBlock, 2, 19144869, 8264, null]
+ - [163, 7, SimpleBlock, 2, 19153136, 8036, null]
+ - [163, 7, SimpleBlock, 2, 19161175, 8376, null]
+ - [163, 7, SimpleBlock, 2, 19169554, 7908, null]
+ - [163, 7, SimpleBlock, 2, 19177465, 6255, null]
+ - [163, 7, SimpleBlock, 2, 19183723, 8350, null]
+ - [163, 7, SimpleBlock, 2, 19192076, 10471, null]
+ - [163, 7, SimpleBlock, 2, 19202550, 12007, null]
+ - [163, 7, SimpleBlock, 2, 19214560, 12700, null]
+ - [163, 7, SimpleBlock, 2, 19227263, 6449, null]
+ - [163, 7, SimpleBlock, 2, 19233715, 13211, null]
+ - [163, 7, SimpleBlock, 2, 19246929, 13350, null]
+ - [163, 7, SimpleBlock, 2, 19260282, 12895, null]
+ - [163, 7, SimpleBlock, 2, 19273180, 11656, null]
+ - [163, 7, SimpleBlock, 2, 19284839, 10411, null]
+ - [163, 7, SimpleBlock, 2, 19295253, 6451, null]
+ - [163, 7, SimpleBlock, 2, 19301707, 9994, null]
+ - [163, 7, SimpleBlock, 2, 19311704, 9523, null]
+ - [163, 7, SimpleBlock, 2, 19321230, 9295, null]
+ - [163, 7, SimpleBlock, 2, 19330528, 8371, null]
+ - [163, 7, SimpleBlock, 2, 19338902, 7294, null]
+ - [163, 7, SimpleBlock, 2, 19346199, 6161, null]
+ - [163, 7, SimpleBlock, 2, 19352363, 6261, null]
+ - [163, 7, SimpleBlock, 2, 19358628, 36475, null]
+ - [163, 7, SimpleBlock, 2, 19395106, 1597, null]
+ - [163, 7, SimpleBlock, 2, 19396706, 1054, null]
+ - [163, 7, SimpleBlock, 2, 19397763, 5967, null]
+ - [163, 7, SimpleBlock, 2, 19403733, 639, null]
+ - [163, 7, SimpleBlock, 2, 19404375, 654, null]
+ - [163, 7, SimpleBlock, 2, 19405032, 649, null]
+ - [163, 7, SimpleBlock, 2, 19405684, 671, null]
+ - [163, 7, SimpleBlock, 2, 19406358, 546, null]
+ - [163, 7, SimpleBlock, 2, 19406907, 6257, null]
+ - [163, 7, SimpleBlock, 2, 19413167, 564, null]
+ - [163, 7, SimpleBlock, 2, 19413734, 551, null]
+ - [163, 7, SimpleBlock, 2, 19414288, 644, null]
+ - [163, 7, SimpleBlock, 2, 19414935, 691, null]
+ - [163, 7, SimpleBlock, 2, 19415629, 559, null]
+ - [163, 7, SimpleBlock, 2, 19416191, 5872, null]
+ - [163, 7, SimpleBlock, 2, 19422066, 619, null]
+ - [163, 7, SimpleBlock, 2, 19422688, 773, null]
+ - [163, 7, SimpleBlock, 2, 19423464, 853, null]
+ - [163, 7, SimpleBlock, 2, 19424320, 816, null]
+ - [163, 7, SimpleBlock, 2, 19425139, 5488, null]
+ - [163, 7, SimpleBlock, 2, 19430630, 760, null]
+ - [163, 7, SimpleBlock, 2, 19431393, 977, null]
+ - [163, 7, SimpleBlock, 2, 19432373, 729, null]
+ - [163, 7, SimpleBlock, 2, 19433105, 915, null]
+ - [163, 7, SimpleBlock, 2, 19434023, 803, null]
+ - [163, 7, SimpleBlock, 2, 19434829, 5967, null]
+ - [163, 7, SimpleBlock, 2, 19440799, 629, null]
+ - [163, 7, SimpleBlock, 2, 19441431, 971, null]
+ - [163, 7, SimpleBlock, 2, 19442405, 704, null]
+ - [163, 7, SimpleBlock, 2, 19443112, 899, null]
+ - [163, 7, SimpleBlock, 2, 19444014, 5584, null]
+ - [163, 7, SimpleBlock, 2, 19449601, 814, null]
+ - [163, 7, SimpleBlock, 2, 19450418, 724, null]
+ - [163, 7, SimpleBlock, 2, 19451145, 950, null]
+ - [163, 7, SimpleBlock, 2, 19452098, 770, null]
+ - [163, 7, SimpleBlock, 2, 19452871, 973, null]
+ - [163, 7, SimpleBlock, 2, 19453847, 5776, null]
+ - [163, 7, SimpleBlock, 2, 19459626, 901, null]
+ - [163, 7, SimpleBlock, 2, 19460530, 719, null]
+ - [163, 7, SimpleBlock, 2, 19461252, 947, null]
+ - [163, 7, SimpleBlock, 2, 19462202, 662, null]
+ - [163, 7, SimpleBlock, 2, 19462867, 836, null]
+ - [163, 7, SimpleBlock, 2, 19463706, 6160, null]
+ - [163, 7, SimpleBlock, 2, 19469869, 723, null]
+ - [163, 7, SimpleBlock, 2, 19470595, 592, null]
+ - [163, 7, SimpleBlock, 2, 19471190, 837, null]
+ - [163, 7, SimpleBlock, 2, 19472030, 607, null]
+ - [163, 7, SimpleBlock, 2, 19472640, 5681, null]
+ - [163, 7, SimpleBlock, 2, 19478324, 784, null]
+ - [163, 7, SimpleBlock, 2, 19479111, 699, null]
+ - [163, 7, SimpleBlock, 2, 19479813, 517, null]
+ - [163, 7, SimpleBlock, 2, 19480333, 787, null]
+ - [163, 7, SimpleBlock, 2, 19481123, 554, null]
+ - [163, 7, SimpleBlock, 2, 19481680, 6065, null]
+ - [163, 7, SimpleBlock, 2, 19487749, 48229, null]
+ - [163, 7, SimpleBlock, 2, 19535981, 1795, null]
+ - [163, 7, SimpleBlock, 2, 19537779, 1433, null]
+ - [163, 7, SimpleBlock, 2, 19539215, 829, null]
+ - [163, 7, SimpleBlock, 2, 19540047, 772, null]
+ - [163, 7, SimpleBlock, 2, 19540822, 647, null]
+ - [163, 7, SimpleBlock, 2, 19541472, 5680, null]
+ - [163, 7, SimpleBlock, 2, 19547155, 757, null]
+ - [163, 7, SimpleBlock, 2, 19547915, 771, null]
+ - [163, 7, SimpleBlock, 2, 19548689, 699, null]
+ - [163, 7, SimpleBlock, 2, 19549391, 793, null]
+ - [163, 7, SimpleBlock, 2, 19550187, 904, null]
+ - [163, 7, SimpleBlock, 2, 19551094, 5872, null]
+ - [163, 7, SimpleBlock, 2, 19556969, 862, null]
+ - [163, 7, SimpleBlock, 2, 19557834, 1178, null]
+ - [163, 7, SimpleBlock, 2, 19559015, 1055, null]
+ - [163, 7, SimpleBlock, 2, 19560073, 1098, null]
+ - [163, 7, SimpleBlock, 2, 19561174, 5774, null]
+ - [163, 7, SimpleBlock, 2, 19566951, 1242, null]
+ - [163, 7, SimpleBlock, 2, 19568196, 1272, null]
+ - [163, 7, SimpleBlock, 2, 19569471, 1209, null]
+ - [163, 7, SimpleBlock, 2, 19570683, 1540, null]
+ - [163, 7, SimpleBlock, 2, 19572226, 1541, null]
+ - [163, 7, SimpleBlock, 2, 19573770, 5680, null]
+ - [163, 7, SimpleBlock, 2, 19579453, 1583, null]
+ - [163, 7, SimpleBlock, 2, 19581039, 1724, null]
+ - [163, 7, SimpleBlock, 2, 19582766, 1947, null]
+ - [163, 7, SimpleBlock, 2, 19584716, 1810, null]
+ - [163, 7, SimpleBlock, 2, 19586529, 5394, null]
+ - [163, 7, SimpleBlock, 2, 19591926, 1931, null]
+ - [163, 7, SimpleBlock, 2, 19593860, 1872, null]
+ - [163, 7, SimpleBlock, 2, 19595735, 1897, null]
+ - [163, 7, SimpleBlock, 2, 19597635, 1731, null]
+ - [163, 7, SimpleBlock, 2, 19599369, 1852, null]
+ - [163, 7, SimpleBlock, 2, 19601224, 5393, null]
+ - [163, 7, SimpleBlock, 2, 19606620, 1761, null]
+ - [163, 7, SimpleBlock, 2, 19608384, 1845, null]
+ - [163, 7, SimpleBlock, 2, 19610232, 1851, null]
+ - [163, 7, SimpleBlock, 2, 19612086, 1824, null]
+ - [163, 7, SimpleBlock, 2, 19613913, 1842, null]
+ - [163, 7, SimpleBlock, 2, 19615758, 5295, null]
+ - [163, 7, SimpleBlock, 2, 19621056, 1767, null]
+ - [163, 7, SimpleBlock, 2, 19622826, 1774, null]
+ - [163, 7, SimpleBlock, 2, 19624603, 1688, null]
+ - [163, 7, SimpleBlock, 2, 19626294, 1683, null]
+ - [163, 7, SimpleBlock, 2, 19627980, 5584, null]
+ - [163, 7, SimpleBlock, 2, 19633567, 1458, null]
+ - [163, 7, SimpleBlock, 2, 19635028, 1661, null]
+ - [163, 7, SimpleBlock, 2, 19636692, 1559, null]
+ - [163, 7, SimpleBlock, 2, 19638254, 1429, null]
+ - [163, 7, SimpleBlock, 2, 19639686, 1566, null]
+ - [163, 7, SimpleBlock, 2, 19641255, 5294, null]
+ - [163, 7, SimpleBlock, 2, 19646552, 1519, null]
+ - [163, 7, SimpleBlock, 2, 19648074, 1356, null]
+ - [163, 7, SimpleBlock, 2, 19649433, 1292, null]
+ - [163, 7, SimpleBlock, 2, 19650728, 1177, null]
+ - [163, 7, SimpleBlock, 2, 19651908, 5488, null]
+ - [163, 7, SimpleBlock, 2, 19657399, 1208, null]
+ - [163, 7, SimpleBlock, 2, 19658610, 1022, null]
+ - [163, 7, SimpleBlock, 2, 19659635, 943, null]
+ - [163, 7, SimpleBlock, 2, 19660581, 804, null]
+ - [163, 7, SimpleBlock, 2, 19661388, 783, null]
+ - [163, 7, SimpleBlock, 2, 19662174, 5393, null]
+ - [163, 7, SimpleBlock, 2, 19667570, 856, null]
+ - [163, 7, SimpleBlock, 2, 19668430, 28957, null]
+ - [163, 7, SimpleBlock, 2, 19697390, 3375, null]
+ - [163, 7, SimpleBlock, 2, 19700768, 3741, null]
+ - [163, 7, SimpleBlock, 2, 19704512, 3590, null]
+ - [163, 7, SimpleBlock, 2, 19708105, 6545, null]
+ - [163, 7, SimpleBlock, 2, 19714653, 3476, null]
+ - [163, 7, SimpleBlock, 2, 19718132, 3331, null]
+ - [163, 7, SimpleBlock, 2, 19721466, 3305, null]
+ - [163, 7, SimpleBlock, 2, 19724774, 3474, null]
+ - [163, 7, SimpleBlock, 2, 19728251, 6063, null]
+ - [163, 7, SimpleBlock, 2, 19734317, 3784, null]
+ - [163, 7, SimpleBlock, 2, 19738104, 3987, null]
+ - [163, 7, SimpleBlock, 2, 19742094, 4201, null]
+ - [163, 7, SimpleBlock, 2, 19746298, 3595, null]
+ - [163, 7, SimpleBlock, 2, 19749896, 3209, null]
+ - [163, 7, SimpleBlock, 2, 19753108, 6256, null]
+ - [163, 7, SimpleBlock, 2, 19759367, 3317, null]
+ - [163, 7, SimpleBlock, 2, 19762687, 3767, null]
+ - [163, 7, SimpleBlock, 2, 19766457, 3811, null]
+ - [163, 7, SimpleBlock, 2, 19770271, 3951, null]
+ - [163, 7, SimpleBlock, 2, 19774225, 6546, null]
+ - [163, 7, SimpleBlock, 2, 19780774, 3884, null]
+ - [163, 7, SimpleBlock, 2, 19784661, 3807, null]
+ - [163, 7, SimpleBlock, 2, 19788471, 4301, null]
+ - [163, 7, SimpleBlock, 2, 19792775, 5472, null]
+ - [163, 7, SimpleBlock, 2, 19798250, 6931, null]
+ - [163, 7, SimpleBlock, 2, 19805184, 5968, null]
+ - [163, 7, SimpleBlock, 2, 19811155, 8261, null]
+ - [163, 7, SimpleBlock, 2, 19819419, 9448, null]
+ - [163, 7, SimpleBlock, 2, 19828870, 9747, null]
+ - [163, 7, SimpleBlock, 2, 19838620, 10939, null]
+ - [163, 7, SimpleBlock, 2, 19849562, 11581, null]
+ - [163, 7, SimpleBlock, 2, 19861146, 6257, null]
+ - [163, 7, SimpleBlock, 2, 19867406, 12226, null]
+ - [163, 7, SimpleBlock, 2, 19879635, 12386, null]
+ - [163, 7, SimpleBlock, 2, 19892024, 12288, null]
+ - [163, 7, SimpleBlock, 2, 19904315, 12342, null]
+ - [163, 7, SimpleBlock, 2, 19916660, 6066, null]
+ - [163, 7, SimpleBlock, 2, 19922729, 12606, null]
+ - [163, 7, SimpleBlock, 2, 19935338, 12670, null]
+ - [163, 7, SimpleBlock, 2, 19948011, 14314, null]
+ - [163, 7, SimpleBlock, 2, 19962328, 14351, null]
+ - [163, 7, SimpleBlock, 2, 19976682, 14816, null]
+ - [163, 7, SimpleBlock, 2, 19991501, 6065, null]
+ - [163, 7, SimpleBlock, 2, 19997569, 14730, null]
+ - [163, 7, SimpleBlock, 2, 20012302, 14115, null]
+ - [163, 7, SimpleBlock, 2, 20026420, 12301, null]
+ - [163, 7, SimpleBlock, 2, 20038724, 9600, null]
+ - [163, 7, SimpleBlock, 2, 20048327, 7875, null]
+ - [163, 7, SimpleBlock, 2, 20056205, 6258, null]
+ - [163, 7, SimpleBlock, 2, 20062466, 7045, null]
+ - [163, 7, SimpleBlock, 2, 20069514, 5980, null]
+ - [163, 7, SimpleBlock, 2, 20075497, 5767, null]
+ - [163, 7, SimpleBlock, 2, 20081267, 5984, null]
+ - [163, 7, SimpleBlock, 2, 20087254, 5874, null]
+ - [163, 7, SimpleBlock, 2, 20093131, 6340, null]
+ - [163, 7, SimpleBlock, 2, 20099474, 6302, null]
+ - [163, 7, SimpleBlock, 2, 20105779, 6276, null]
+ - [163, 7, SimpleBlock, 2, 20112058, 6033, null]
+ - [163, 7, SimpleBlock, 2, 20118094, 5439, null]
+ - [163, 7, SimpleBlock, 2, 20123536, 5873, null]
+ - [163, 7, SimpleBlock, 2, 20129413, 60579, null]
+ - [163, 7, SimpleBlock, 2, 20189995, 7029, null]
+ - [163, 7, SimpleBlock, 2, 20197027, 9336, null]
+ - [163, 7, SimpleBlock, 2, 20206366, 10592, null]
+ - [163, 7, SimpleBlock, 2, 20216961, 5490, null]
+ - [163, 7, SimpleBlock, 2, 20222454, 12788, null]
+ - [163, 7, SimpleBlock, 2, 20235245, 13783, null]
+ - [163, 7, SimpleBlock, 2, 20249031, 14231, null]
+ - [163, 7, SimpleBlock, 2, 20263265, 15363, null]
+ - [163, 7, SimpleBlock, 2, 20278632, 17133, null]
+ - [163, 7, SimpleBlock, 2, 20295768, 5296, null]
+ - [163, 7, SimpleBlock, 2, 20301068, 17470, null]
+ - [163, 7, SimpleBlock, 2, 20318541, 14994, null]
+ - [163, 7, SimpleBlock, 2, 20333538, 14941, null]
+ - [163, 7, SimpleBlock, 2, 20348482, 14520, null]
+ - [163, 7, SimpleBlock, 2, 20363005, 15205, null]
+ - [163, 7, SimpleBlock, 2, 20378213, 5201, null]
+ - [163, 7, SimpleBlock, 2, 20383417, 15907, null]
+ - [163, 7, SimpleBlock, 2, 20399328, 16652, null]
+ - [163, 7, SimpleBlock, 2, 20415984, 17361, null]
+ - [163, 7, SimpleBlock, 2, 20433349, 17414, null]
+ - [163, 7, SimpleBlock, 2, 20450766, 5296, null]
+ - [163, 7, SimpleBlock, 2, 20456066, 17770, null]
+ - [163, 7, SimpleBlock, 2, 20473840, 17844, null]
+ - [163, 7, SimpleBlock, 2, 20491688, 16842, null]
+ - [163, 7, SimpleBlock, 2, 20508534, 16488, null]
+ - [163, 7, SimpleBlock, 2, 20525026, 16936, null]
+ - [163, 7, SimpleBlock, 2, 20541965, 676, null]
+ - [163, 7, SimpleBlock, 2, 20542644, 5870, null]
+ - [163, 7, SimpleBlock, 2, 20548518, 17904, null]
+ - [163, 7, SimpleBlock, 2, 20566426, 17748, null]
+ - [163, 7, SimpleBlock, 2, 20584178, 17327, null]
+ - [163, 7, SimpleBlock, 2, 20601509, 16804, null]
+ - [163, 7, SimpleBlock, 2, 20618317, 16609, null]
+ - [163, 7, SimpleBlock, 2, 20634929, 4620, null]
+ - [163, 7, SimpleBlock, 2, 20639553, 17718, null]
+ - [163, 7, SimpleBlock, 2, 20657275, 19234, null]
+ - [163, 7, SimpleBlock, 2, 20676513, 18749, null]
+ - [163, 7, SimpleBlock, 2, 20695266, 18574, null]
+ - [163, 7, SimpleBlock, 2, 20713844, 18621, null]
+ - - 524531317
+ - 6
+ - Cluster
+ - 1
+ - 20732473
+ - 2606864
+ - - [231, 1, Timecode, 2, 20732475, 3, 76917]
+ - [163, 7, SimpleBlock, 2, 20732481, 676, null]
+ - [163, 7, SimpleBlock, 2, 20733160, 5296, null]
+ - [163, 7, SimpleBlock, 2, 20738460, 57870, null]
+ - [163, 7, SimpleBlock, 2, 20796333, 14161, null]
+ - [163, 7, SimpleBlock, 2, 20810497, 16283, null]
+ - [163, 7, SimpleBlock, 2, 20826784, 18643, null]
+ - [163, 7, SimpleBlock, 2, 20845430, 5584, null]
+ - [163, 7, SimpleBlock, 2, 20851018, 20094, null]
+ - [163, 7, SimpleBlock, 2, 20871116, 20936, null]
+ - [163, 7, SimpleBlock, 2, 20892056, 21547, null]
+ - [163, 7, SimpleBlock, 2, 20913607, 21831, null]
+ - [163, 7, SimpleBlock, 2, 20935442, 21619, null]
+ - [163, 7, SimpleBlock, 2, 20957064, 5967, null]
+ - [163, 7, SimpleBlock, 2, 20963035, 21935, null]
+ - [163, 7, SimpleBlock, 2, 20984974, 21975, null]
+ - [163, 7, SimpleBlock, 2, 21006953, 21595, null]
+ - [163, 7, SimpleBlock, 2, 21028552, 22352, null]
+ - [163, 7, SimpleBlock, 2, 21050907, 5971, null]
+ - [163, 7, SimpleBlock, 2, 21056882, 21241, null]
+ - [163, 7, SimpleBlock, 2, 21078127, 20004, null]
+ - [163, 7, SimpleBlock, 2, 21098135, 18589, null]
+ - [163, 7, SimpleBlock, 2, 21116728, 18088, null]
+ - [163, 7, SimpleBlock, 2, 21134820, 18065, null]
+ - [163, 7, SimpleBlock, 2, 21152888, 5680, null]
+ - [163, 7, SimpleBlock, 2, 21158572, 18783, null]
+ - [163, 7, SimpleBlock, 2, 21177359, 19388, null]
+ - [163, 7, SimpleBlock, 2, 21196751, 19874, null]
+ - [163, 7, SimpleBlock, 2, 21216629, 20500, null]
+ - [163, 7, SimpleBlock, 2, 21237133, 20363, null]
+ - [163, 7, SimpleBlock, 2, 21257499, 5683, null]
+ - [163, 7, SimpleBlock, 2, 21263186, 20383, null]
+ - [163, 7, SimpleBlock, 2, 21283573, 19364, null]
+ - [163, 7, SimpleBlock, 2, 21302941, 19084, null]
+ - [163, 7, SimpleBlock, 2, 21322029, 19510, null]
+ - [163, 7, SimpleBlock, 2, 21341542, 5779, null]
+ - [163, 7, SimpleBlock, 2, 21347325, 19655, null]
+ - [163, 7, SimpleBlock, 2, 21366984, 19150, null]
+ - [163, 7, SimpleBlock, 2, 21386138, 18882, null]
+ - [163, 7, SimpleBlock, 2, 21405024, 18431, null]
+ - [163, 7, SimpleBlock, 2, 21423459, 18037, null]
+ - [163, 7, SimpleBlock, 2, 21441499, 5393, null]
+ - [163, 7, SimpleBlock, 2, 21446896, 18017, null]
+ - [163, 7, SimpleBlock, 2, 21464917, 17810, null]
+ - [163, 7, SimpleBlock, 2, 21482731, 17480, null]
+ - [163, 7, SimpleBlock, 2, 21500215, 17218, null]
+ - [163, 7, SimpleBlock, 2, 21517436, 5968, null]
+ - [163, 7, SimpleBlock, 2, 21523408, 16822, null]
+ - [163, 7, SimpleBlock, 2, 21540234, 16542, null]
+ - [163, 7, SimpleBlock, 2, 21556779, 16000, null]
+ - [163, 7, SimpleBlock, 2, 21572783, 17251, null]
+ - [163, 7, SimpleBlock, 2, 21590038, 18254, null]
+ - [163, 7, SimpleBlock, 2, 21608295, 6451, null]
+ - [163, 7, SimpleBlock, 2, 21614750, 19178, null]
+ - [163, 7, SimpleBlock, 2, 21633932, 19684, null]
+ - [163, 7, SimpleBlock, 2, 21653620, 20370, null]
+ - [163, 7, SimpleBlock, 2, 21673994, 21172, null]
+ - [163, 7, SimpleBlock, 2, 21695170, 21607, null]
+ - [163, 7, SimpleBlock, 2, 21716780, 6162, null]
+ - [163, 7, SimpleBlock, 2, 21722946, 21284, null]
+ - [163, 7, SimpleBlock, 2, 21744234, 19858, null]
+ - [163, 7, SimpleBlock, 2, 21764096, 20103, null]
+ - [163, 7, SimpleBlock, 2, 21784203, 20348, null]
+ - [163, 7, SimpleBlock, 2, 21804554, 5775, null]
+ - [163, 7, SimpleBlock, 2, 21810333, 20136, null]
+ - [163, 7, SimpleBlock, 2, 21830473, 18094, null]
+ - [163, 7, SimpleBlock, 2, 21848571, 17128, null]
+ - [163, 7, SimpleBlock, 2, 21865703, 16497, null]
+ - [163, 7, SimpleBlock, 2, 21882204, 16452, null]
+ - [163, 7, SimpleBlock, 2, 21898659, 6161, null]
+ - [163, 7, SimpleBlock, 2, 21904824, 17164, null]
+ - [163, 7, SimpleBlock, 2, 21921992, 17846, null]
+ - [163, 7, SimpleBlock, 2, 21939842, 18903, null]
+ - [163, 7, SimpleBlock, 2, 21958749, 19244, null]
+ - [163, 7, SimpleBlock, 2, 21977996, 5874, null]
+ - [163, 7, SimpleBlock, 2, 21983874, 18813, null]
+ - [163, 7, SimpleBlock, 2, 22002691, 18540, null]
+ - [163, 7, SimpleBlock, 2, 22021235, 17596, null]
+ - [163, 7, SimpleBlock, 2, 22038835, 17222, null]
+ - [163, 7, SimpleBlock, 2, 22056061, 17986, null]
+ - [163, 7, SimpleBlock, 2, 22074050, 5871, null]
+ - [163, 7, SimpleBlock, 2, 22079925, 19782, null]
+ - [163, 7, SimpleBlock, 2, 22099711, 20933, null]
+ - [163, 7, SimpleBlock, 2, 22120648, 20789, null]
+ - [163, 7, SimpleBlock, 2, 22141441, 19381, null]
+ - [163, 7, SimpleBlock, 2, 22160826, 17254, null]
+ - [163, 7, SimpleBlock, 2, 22178083, 3660, null]
+ - [163, 7, SimpleBlock, 2, 22181746, 15440, null]
+ - [163, 7, SimpleBlock, 2, 22197189, 14453, null]
+ - [163, 7, SimpleBlock, 2, 22211645, 13030, null]
+ - [163, 7, SimpleBlock, 2, 22224678, 5682, null]
+ - [163, 7, SimpleBlock, 2, 22230363, 11253, null]
+ - [163, 7, SimpleBlock, 2, 22241620, 32873, null]
+ - [163, 7, SimpleBlock, 2, 22274496, 6159, null]
+ - [163, 7, SimpleBlock, 2, 22280658, 7281, null]
+ - [163, 7, SimpleBlock, 2, 22287942, 5585, null]
+ - [163, 7, SimpleBlock, 2, 22293530, 7922, null]
+ - [163, 7, SimpleBlock, 2, 22301455, 7943, null]
+ - [163, 7, SimpleBlock, 2, 22309401, 7994, null]
+ - [163, 7, SimpleBlock, 2, 22317398, 7958, null]
+ - [163, 7, SimpleBlock, 2, 22325359, 8355, null]
+ - [163, 7, SimpleBlock, 2, 22333717, 5777, null]
+ - [163, 7, SimpleBlock, 2, 22339497, 8199, null]
+ - [163, 7, SimpleBlock, 2, 22347699, 8535, null]
+ - [163, 7, SimpleBlock, 2, 22356237, 8831, null]
+ - [163, 7, SimpleBlock, 2, 22365071, 9248, null]
+ - [163, 7, SimpleBlock, 2, 22374322, 5680, null]
+ - [163, 7, SimpleBlock, 2, 22380005, 9220, null]
+ - [163, 7, SimpleBlock, 2, 22389228, 9075, null]
+ - [163, 7, SimpleBlock, 2, 22398306, 9125, null]
+ - [163, 7, SimpleBlock, 2, 22407434, 9290, null]
+ - [163, 7, SimpleBlock, 2, 22416727, 9547, null]
+ - [163, 7, SimpleBlock, 2, 22426277, 5969, null]
+ - [163, 7, SimpleBlock, 2, 22432249, 9837, null]
+ - [163, 7, SimpleBlock, 2, 22442089, 10391, null]
+ - [163, 7, SimpleBlock, 2, 22452483, 10332, null]
+ - [163, 7, SimpleBlock, 2, 22462818, 10531, null]
+ - [163, 7, SimpleBlock, 2, 22473352, 10458, null]
+ - [163, 7, SimpleBlock, 2, 22483813, 5872, null]
+ - [163, 7, SimpleBlock, 2, 22489688, 10067, null]
+ - [163, 7, SimpleBlock, 2, 22499758, 10130, null]
+ - [163, 7, SimpleBlock, 2, 22509891, 9976, null]
+ - [163, 7, SimpleBlock, 2, 22519870, 10480, null]
+ - [163, 7, SimpleBlock, 2, 22530353, 6162, null]
+ - [163, 7, SimpleBlock, 2, 22536518, 11192, null]
+ - [163, 7, SimpleBlock, 2, 22547713, 10621, null]
+ - [163, 7, SimpleBlock, 2, 22558337, 10069, null]
+ - [163, 7, SimpleBlock, 2, 22568409, 10229, null]
+ - [163, 7, SimpleBlock, 2, 22578641, 10237, null]
+ - [163, 7, SimpleBlock, 2, 22588881, 6162, null]
+ - [163, 7, SimpleBlock, 2, 22595046, 10671, null]
+ - [163, 7, SimpleBlock, 2, 22605720, 10010, null]
+ - [163, 7, SimpleBlock, 2, 22615733, 9164, null]
+ - [163, 7, SimpleBlock, 2, 22624900, 8803, null]
+ - [163, 7, SimpleBlock, 2, 22633706, 6255, null]
+ - [163, 7, SimpleBlock, 2, 22639964, 8877, null]
+ - [163, 7, SimpleBlock, 2, 22648844, 8534, null]
+ - [163, 7, SimpleBlock, 2, 22657381, 7929, null]
+ - [163, 7, SimpleBlock, 2, 22665313, 8015, null]
+ - [163, 7, SimpleBlock, 2, 22673331, 8173, null]
+ - [163, 7, SimpleBlock, 2, 22681507, 6065, null]
+ - [163, 7, SimpleBlock, 2, 22687575, 7984, null]
+ - [163, 7, SimpleBlock, 2, 22695562, 7944, null]
+ - [163, 7, SimpleBlock, 2, 22703509, 8216, null]
+ - [163, 7, SimpleBlock, 2, 22711728, 8112, null]
+ - [163, 7, SimpleBlock, 2, 22719843, 7928, null]
+ - [163, 7, SimpleBlock, 2, 22727774, 6354, null]
+ - [163, 7, SimpleBlock, 2, 22734131, 7622, null]
+ - [163, 7, SimpleBlock, 2, 22741756, 8231, null]
+ - [163, 7, SimpleBlock, 2, 22749990, 8319, null]
+ - [163, 7, SimpleBlock, 2, 22758312, 7903, null]
+ - [163, 7, SimpleBlock, 2, 22766218, 6063, null]
+ - [163, 7, SimpleBlock, 2, 22772284, 8053, null]
+ - [163, 7, SimpleBlock, 2, 22780340, 8385, null]
+ - [163, 7, SimpleBlock, 2, 22788728, 8056, null]
+ - [163, 7, SimpleBlock, 2, 22796787, 8148, null]
+ - [163, 7, SimpleBlock, 2, 22804938, 8115, null]
+ - [163, 7, SimpleBlock, 2, 22813056, 6065, null]
+ - [163, 7, SimpleBlock, 2, 22819124, 8290, null]
+ - [163, 7, SimpleBlock, 2, 22827417, 7985, null]
+ - [163, 7, SimpleBlock, 2, 22835405, 8685, null]
+ - [163, 7, SimpleBlock, 2, 22844093, 9660, null]
+ - [163, 7, SimpleBlock, 2, 22853756, 5874, null]
+ - [163, 7, SimpleBlock, 2, 22859633, 10373, null]
+ - [163, 7, SimpleBlock, 2, 22870009, 9892, null]
+ - [163, 7, SimpleBlock, 2, 22879904, 9637, null]
+ - [163, 7, SimpleBlock, 2, 22889544, 9721, null]
+ - [163, 7, SimpleBlock, 2, 22899268, 9824, null]
+ - [163, 7, SimpleBlock, 2, 22909095, 5874, null]
+ - [163, 7, SimpleBlock, 2, 22914972, 9921, null]
+ - [163, 7, SimpleBlock, 2, 22924896, 8744, null]
+ - [163, 7, SimpleBlock, 2, 22933643, 7978, null]
+ - [163, 7, SimpleBlock, 2, 22941624, 7648, null]
+ - [163, 7, SimpleBlock, 2, 22949275, 7314, null]
+ - [163, 7, SimpleBlock, 2, 22956592, 5969, null]
+ - [163, 7, SimpleBlock, 2, 22962564, 7531, null]
+ - [163, 7, SimpleBlock, 2, 22970098, 7095, null]
+ - [163, 7, SimpleBlock, 2, 22977196, 6685, null]
+ - [163, 7, SimpleBlock, 2, 22983884, 6591, null]
+ - [163, 7, SimpleBlock, 2, 22990478, 5776, null]
+ - [163, 7, SimpleBlock, 2, 22996257, 6599, null]
+ - [163, 7, SimpleBlock, 2, 23002859, 5858, null]
+ - [163, 7, SimpleBlock, 2, 23008720, 5549, null]
+ - [163, 7, SimpleBlock, 2, 23014272, 4881, null]
+ - [163, 7, SimpleBlock, 2, 23019156, 4488, null]
+ - [163, 7, SimpleBlock, 2, 23023647, 5968, null]
+ - [163, 7, SimpleBlock, 2, 23029618, 4335, null]
+ - [163, 7, SimpleBlock, 2, 23033956, 4173, null]
+ - [163, 7, SimpleBlock, 2, 23038132, 3829, null]
+ - [163, 7, SimpleBlock, 2, 23041964, 3577, null]
+ - [163, 7, SimpleBlock, 2, 23045544, 3307, null]
+ - [163, 7, SimpleBlock, 2, 23048854, 5969, null]
+ - [163, 7, SimpleBlock, 2, 23054826, 3053, null]
+ - [163, 7, SimpleBlock, 2, 23057882, 2805, null]
+ - [163, 7, SimpleBlock, 2, 23060690, 2487, null]
+ - [163, 7, SimpleBlock, 2, 23063180, 2142, null]
+ - [163, 7, SimpleBlock, 2, 23065325, 6257, null]
+ - [163, 7, SimpleBlock, 2, 23071585, 1832, null]
+ - [163, 7, SimpleBlock, 2, 23073420, 1453, null]
+ - [163, 7, SimpleBlock, 2, 23074876, 1818, null]
+ - [163, 7, SimpleBlock, 2, 23076697, 1981, null]
+ - [163, 7, SimpleBlock, 2, 23078681, 3420, null]
+ - [163, 7, SimpleBlock, 2, 23082104, 5587, null]
+ - [163, 7, SimpleBlock, 2, 23087694, 3332, null]
+ - [163, 7, SimpleBlock, 2, 23091029, 2137, null]
+ - [163, 7, SimpleBlock, 2, 23093169, 2342, null]
+ - [163, 7, SimpleBlock, 2, 23095514, 3714, null]
+ - [163, 7, SimpleBlock, 2, 23099231, 4914, null]
+ - [163, 7, SimpleBlock, 2, 23104148, 2562, null]
+ - [163, 7, SimpleBlock, 2, 23106713, 3019, null]
+ - [163, 7, SimpleBlock, 2, 23109735, 2231, null]
+ - [163, 7, SimpleBlock, 2, 23111969, 1798, null]
+ - [163, 7, SimpleBlock, 2, 23113770, 1799, null]
+ - [163, 7, SimpleBlock, 2, 23115572, 5775, null]
+ - [163, 7, SimpleBlock, 2, 23121350, 1502, null]
+ - [163, 7, SimpleBlock, 2, 23122855, 1437, null]
+ - [163, 7, SimpleBlock, 2, 23124295, 1266, null]
+ - [163, 7, SimpleBlock, 2, 23125564, 1279, null]
+ - [163, 7, SimpleBlock, 2, 23126846, 1276, null]
+ - [163, 7, SimpleBlock, 2, 23128125, 4816, null]
+ - [163, 7, SimpleBlock, 2, 23132944, 1198, null]
+ - [163, 7, SimpleBlock, 2, 23134145, 1034, null]
+ - [163, 7, SimpleBlock, 2, 23135182, 876, null]
+ - [163, 7, SimpleBlock, 2, 23136061, 721, null]
+ - [163, 7, SimpleBlock, 2, 23136785, 4719, null]
+ - [163, 7, SimpleBlock, 2, 23141507, 541, null]
+ - [163, 7, SimpleBlock, 2, 23142051, 616, null]
+ - [163, 7, SimpleBlock, 2, 23142670, 618, null]
+ - [163, 7, SimpleBlock, 2, 23143291, 597, null]
+ - [163, 7, SimpleBlock, 2, 23143891, 440, null]
+ - [163, 7, SimpleBlock, 2, 23144334, 4527, null]
+ - [163, 7, SimpleBlock, 2, 23148864, 435, null]
+ - [163, 7, SimpleBlock, 2, 23149302, 424, null]
+ - [163, 7, SimpleBlock, 2, 23149729, 432, null]
+ - [163, 7, SimpleBlock, 2, 23150164, 417, null]
+ - [163, 7, SimpleBlock, 2, 23150584, 580, null]
+ - [163, 7, SimpleBlock, 2, 23151167, 405, null]
+ - [163, 7, SimpleBlock, 2, 23151575, 4525, null]
+ - [163, 7, SimpleBlock, 2, 23156103, 413, null]
+ - [163, 7, SimpleBlock, 2, 23156519, 411, null]
+ - [163, 7, SimpleBlock, 2, 23156933, 437, null]
+ - [163, 7, SimpleBlock, 2, 23157373, 422, null]
+ - [163, 7, SimpleBlock, 2, 23157798, 437, null]
+ - [163, 7, SimpleBlock, 2, 23158238, 4613, null]
+ - [163, 7, SimpleBlock, 2, 23162854, 458, null]
+ - [163, 7, SimpleBlock, 2, 23163315, 479, null]
+ - [163, 7, SimpleBlock, 2, 23163797, 478, null]
+ - [163, 7, SimpleBlock, 2, 23164278, 477, null]
+ - [163, 7, SimpleBlock, 2, 23164758, 4613, null]
+ - [163, 7, SimpleBlock, 2, 23169374, 489, null]
+ - [163, 7, SimpleBlock, 2, 23169866, 550, null]
+ - [163, 7, SimpleBlock, 2, 23170419, 541, null]
+ - [163, 7, SimpleBlock, 2, 23170963, 549, null]
+ - [163, 7, SimpleBlock, 2, 23171515, 583, null]
+ - [163, 7, SimpleBlock, 2, 23172101, 4613, null]
+ - [163, 7, SimpleBlock, 2, 23176717, 629, null]
+ - [163, 7, SimpleBlock, 2, 23177349, 728, null]
+ - [163, 7, SimpleBlock, 2, 23178080, 756, null]
+ - [163, 7, SimpleBlock, 2, 23178839, 1142, null]
+ - [163, 7, SimpleBlock, 2, 23179984, 5009, null]
+ - [163, 7, SimpleBlock, 2, 23184996, 1271, null]
+ - [163, 7, SimpleBlock, 2, 23186270, 1824, null]
+ - [163, 7, SimpleBlock, 2, 23188097, 3163, null]
+ - [163, 7, SimpleBlock, 2, 23191263, 3939, null]
+ - [163, 7, SimpleBlock, 2, 23195205, 4206, null]
+ - [163, 7, SimpleBlock, 2, 23199414, 5106, null]
+ - [163, 7, SimpleBlock, 2, 23204523, 4020, null]
+ - [163, 7, SimpleBlock, 2, 23208546, 4034, null]
+ - [163, 7, SimpleBlock, 2, 23212583, 4006, null]
+ - [163, 7, SimpleBlock, 2, 23216592, 3598, null]
+ - [163, 7, SimpleBlock, 2, 23220193, 3320, null]
+ - [163, 7, SimpleBlock, 2, 23223516, 5011, null]
+ - [163, 7, SimpleBlock, 2, 23228530, 2597, null]
+ - [163, 7, SimpleBlock, 2, 23231130, 2306, null]
+ - [163, 7, SimpleBlock, 2, 23233439, 1768, null]
+ - [163, 7, SimpleBlock, 2, 23235210, 1621, null]
+ - [163, 7, SimpleBlock, 2, 23236834, 4913, null]
+ - [163, 7, SimpleBlock, 2, 23241750, 1419, null]
+ - [163, 7, SimpleBlock, 2, 23243172, 1361, null]
+ - [163, 7, SimpleBlock, 2, 23244536, 1402, null]
+ - [163, 7, SimpleBlock, 2, 23245941, 1566, null]
+ - [163, 7, SimpleBlock, 2, 23247510, 1694, null]
+ - [163, 7, SimpleBlock, 2, 23249207, 4914, null]
+ - [163, 7, SimpleBlock, 2, 23254124, 2301, null]
+ - [163, 7, SimpleBlock, 2, 23256428, 2605, null]
+ - [163, 7, SimpleBlock, 2, 23259036, 2604, null]
+ - [163, 7, SimpleBlock, 2, 23261643, 2765, null]
+ - [163, 7, SimpleBlock, 2, 23264411, 4816, null]
+ - [163, 7, SimpleBlock, 2, 23269230, 2631, null]
+ - [163, 7, SimpleBlock, 2, 23271864, 2608, null]
+ - [163, 7, SimpleBlock, 2, 23274475, 2650, null]
+ - [163, 7, SimpleBlock, 2, 23277128, 3690, null]
+ - [163, 7, SimpleBlock, 2, 23280821, 4659, null]
+ - [163, 7, SimpleBlock, 2, 23285483, 5586, null]
+ - [163, 7, SimpleBlock, 2, 23291072, 5402, null]
+ - [163, 7, SimpleBlock, 2, 23296477, 5586, null]
+ - [163, 7, SimpleBlock, 2, 23302066, 4918, null]
+ - [163, 7, SimpleBlock, 2, 23306987, 3245, null]
+ - [163, 7, SimpleBlock, 2, 23310235, 3349, null]
+ - [163, 7, SimpleBlock, 2, 23313587, 5393, null]
+ - [163, 7, SimpleBlock, 2, 23318983, 3421, null]
+ - [163, 7, SimpleBlock, 2, 23322407, 3249, null]
+ - [163, 7, SimpleBlock, 2, 23325659, 2877, null]
+ - [163, 7, SimpleBlock, 2, 23328539, 2659, null]
+ - [163, 7, SimpleBlock, 2, 23331201, 2410, null]
+ - [163, 7, SimpleBlock, 2, 23333614, 2300, null]
+ - [163, 7, SimpleBlock, 2, 23335917, 1891, null]
+ - [163, 7, SimpleBlock, 2, 23337811, 1526, null]
diff --git a/lib/enzyme/tests/test_mkv.py b/lib/enzyme/tests/test_mkv.py
new file mode 100644
index 0000000000000000000000000000000000000000..2403661e5296203f8d2fb52a5e38ef13cae2b68d
--- /dev/null
+++ b/lib/enzyme/tests/test_mkv.py
@@ -0,0 +1,607 @@
+# -*- coding: utf-8 -*-
+from datetime import timedelta, datetime
+from enzyme.mkv import MKV, VIDEO_TRACK, AUDIO_TRACK, SUBTITLE_TRACK
+import io
+import os.path
+import requests
+import unittest
+import zipfile
+
+
+# Test directory
+TEST_DIR = os.path.join(os.path.dirname(__file__), os.path.splitext(__file__)[0])
+
+
+def setUpModule():
+ if not os.path.exists(TEST_DIR):
+ r = requests.get('http://downloads.sourceforge.net/project/matroska/test_files/matroska_test_w1_1.zip')
+ with zipfile.ZipFile(io.BytesIO(r.content), 'r') as f:
+ f.extractall(TEST_DIR)
+
+
+class MKVTestCase(unittest.TestCase):
+ def test_test1(self):
+ with io.open(os.path.join(TEST_DIR, 'test1.mkv'), 'rb') as stream:
+ mkv = MKV(stream)
+ # info
+ self.assertTrue(mkv.info.title is None)
+ self.assertTrue(mkv.info.duration == timedelta(minutes=1, seconds=27, milliseconds=336))
+ self.assertTrue(mkv.info.date_utc == datetime(2010, 8, 21, 7, 23, 3))
+ self.assertTrue(mkv.info.muxing_app == 'libebml2 v0.10.0 + libmatroska2 v0.10.1')
+ self.assertTrue(mkv.info.writing_app == 'mkclean 0.5.5 ru from libebml v1.0.0 + libmatroska v1.0.0 + mkvmerge v4.1.1 (\'Bouncin\' Back\') built on Jul 3 2010 22:54:08')
+ # video track
+ self.assertTrue(len(mkv.video_tracks) == 1)
+ self.assertTrue(mkv.video_tracks[0].type == VIDEO_TRACK)
+ self.assertTrue(mkv.video_tracks[0].number == 1)
+ self.assertTrue(mkv.video_tracks[0].name is None)
+ self.assertTrue(mkv.video_tracks[0].language == 'und')
+ self.assertTrue(mkv.video_tracks[0].enabled == True)
+ self.assertTrue(mkv.video_tracks[0].default == True)
+ self.assertTrue(mkv.video_tracks[0].forced == False)
+ self.assertTrue(mkv.video_tracks[0].lacing == False)
+ self.assertTrue(mkv.video_tracks[0].codec_id == 'V_MS/VFW/FOURCC')
+ self.assertTrue(mkv.video_tracks[0].codec_name is None)
+ self.assertTrue(mkv.video_tracks[0].width == 854)
+ self.assertTrue(mkv.video_tracks[0].height == 480)
+ self.assertTrue(mkv.video_tracks[0].interlaced == False)
+ self.assertTrue(mkv.video_tracks[0].stereo_mode is None)
+ self.assertTrue(mkv.video_tracks[0].crop == {})
+ self.assertTrue(mkv.video_tracks[0].display_width is None)
+ self.assertTrue(mkv.video_tracks[0].display_height is None)
+ self.assertTrue(mkv.video_tracks[0].display_unit is None)
+ self.assertTrue(mkv.video_tracks[0].aspect_ratio_type is None)
+ # audio track
+ self.assertTrue(len(mkv.audio_tracks) == 1)
+ self.assertTrue(mkv.audio_tracks[0].type == AUDIO_TRACK)
+ self.assertTrue(mkv.audio_tracks[0].number == 2)
+ self.assertTrue(mkv.audio_tracks[0].name is None)
+ self.assertTrue(mkv.audio_tracks[0].language == 'und')
+ self.assertTrue(mkv.audio_tracks[0].enabled == True)
+ self.assertTrue(mkv.audio_tracks[0].default == True)
+ self.assertTrue(mkv.audio_tracks[0].forced == False)
+ self.assertTrue(mkv.audio_tracks[0].lacing == True)
+ self.assertTrue(mkv.audio_tracks[0].codec_id == 'A_MPEG/L3')
+ self.assertTrue(mkv.audio_tracks[0].codec_name is None)
+ self.assertTrue(mkv.audio_tracks[0].sampling_frequency == 48000.0)
+ self.assertTrue(mkv.audio_tracks[0].channels == 2)
+ self.assertTrue(mkv.audio_tracks[0].output_sampling_frequency is None)
+ self.assertTrue(mkv.audio_tracks[0].bit_depth is None)
+ # subtitle track
+ self.assertTrue(len(mkv.subtitle_tracks) == 0)
+ # chapters
+ self.assertTrue(len(mkv.chapters) == 0)
+ # tags
+ self.assertTrue(len(mkv.tags) == 1)
+ self.assertTrue(len(mkv.tags[0].simpletags) == 3)
+ self.assertTrue(mkv.tags[0].simpletags[0].name == 'TITLE')
+ self.assertTrue(mkv.tags[0].simpletags[0].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[0].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[0].string == 'Big Buck Bunny - test 1')
+ self.assertTrue(mkv.tags[0].simpletags[0].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[1].name == 'DATE_RELEASED')
+ self.assertTrue(mkv.tags[0].simpletags[1].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[1].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[1].string == '2010')
+ self.assertTrue(mkv.tags[0].simpletags[1].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[2].name == 'COMMENT')
+ self.assertTrue(mkv.tags[0].simpletags[2].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[2].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[2].string == 'Matroska Validation File1, basic MPEG4.2 and MP3 with only SimpleBlock')
+ self.assertTrue(mkv.tags[0].simpletags[2].binary is None)
+
+ def test_test2(self):
+ with io.open(os.path.join(TEST_DIR, 'test2.mkv'), 'rb') as stream:
+ mkv = MKV(stream)
+ # info
+ self.assertTrue(mkv.info.title is None)
+ self.assertTrue(mkv.info.duration == timedelta(seconds=47, milliseconds=509))
+ self.assertTrue(mkv.info.date_utc == datetime(2011, 6, 2, 12, 45, 20))
+ self.assertTrue(mkv.info.muxing_app == 'libebml2 v0.21.0 + libmatroska2 v0.22.1')
+ self.assertTrue(mkv.info.writing_app == 'mkclean 0.8.3 ru from libebml2 v0.10.0 + libmatroska2 v0.10.1 + mkclean 0.5.5 ru from libebml v1.0.0 + libmatroska v1.0.0 + mkvmerge v4.1.1 (\'Bouncin\' Back\') built on Jul 3 2010 22:54:08')
+ # video track
+ self.assertTrue(len(mkv.video_tracks) == 1)
+ self.assertTrue(mkv.video_tracks[0].type == VIDEO_TRACK)
+ self.assertTrue(mkv.video_tracks[0].number == 1)
+ self.assertTrue(mkv.video_tracks[0].name is None)
+ self.assertTrue(mkv.video_tracks[0].language == 'und')
+ self.assertTrue(mkv.video_tracks[0].enabled == True)
+ self.assertTrue(mkv.video_tracks[0].default == True)
+ self.assertTrue(mkv.video_tracks[0].forced == False)
+ self.assertTrue(mkv.video_tracks[0].lacing == False)
+ self.assertTrue(mkv.video_tracks[0].codec_id == 'V_MPEG4/ISO/AVC')
+ self.assertTrue(mkv.video_tracks[0].codec_name is None)
+ self.assertTrue(mkv.video_tracks[0].width == 1024)
+ self.assertTrue(mkv.video_tracks[0].height == 576)
+ self.assertTrue(mkv.video_tracks[0].interlaced == False)
+ self.assertTrue(mkv.video_tracks[0].stereo_mode is None)
+ self.assertTrue(mkv.video_tracks[0].crop == {})
+ self.assertTrue(mkv.video_tracks[0].display_width == 1354)
+ self.assertTrue(mkv.video_tracks[0].display_height is None)
+ self.assertTrue(mkv.video_tracks[0].display_unit is None)
+ self.assertTrue(mkv.video_tracks[0].aspect_ratio_type is None)
+ # audio track
+ self.assertTrue(len(mkv.audio_tracks) == 1)
+ self.assertTrue(mkv.audio_tracks[0].type == AUDIO_TRACK)
+ self.assertTrue(mkv.audio_tracks[0].number == 2)
+ self.assertTrue(mkv.audio_tracks[0].name is None)
+ self.assertTrue(mkv.audio_tracks[0].language == 'und')
+ self.assertTrue(mkv.audio_tracks[0].enabled == True)
+ self.assertTrue(mkv.audio_tracks[0].default == True)
+ self.assertTrue(mkv.audio_tracks[0].forced == False)
+ self.assertTrue(mkv.audio_tracks[0].lacing == True)
+ self.assertTrue(mkv.audio_tracks[0].codec_id == 'A_AAC')
+ self.assertTrue(mkv.audio_tracks[0].codec_name is None)
+ self.assertTrue(mkv.audio_tracks[0].sampling_frequency == 48000.0)
+ self.assertTrue(mkv.audio_tracks[0].channels == 2)
+ self.assertTrue(mkv.audio_tracks[0].output_sampling_frequency is None)
+ self.assertTrue(mkv.audio_tracks[0].bit_depth is None)
+ # subtitle track
+ self.assertTrue(len(mkv.subtitle_tracks) == 0)
+ # chapters
+ self.assertTrue(len(mkv.chapters) == 0)
+ # tags
+ self.assertTrue(len(mkv.tags) == 1)
+ self.assertTrue(len(mkv.tags[0].simpletags) == 3)
+ self.assertTrue(mkv.tags[0].simpletags[0].name == 'TITLE')
+ self.assertTrue(mkv.tags[0].simpletags[0].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[0].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[0].string == 'Elephant Dream - test 2')
+ self.assertTrue(mkv.tags[0].simpletags[0].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[1].name == 'DATE_RELEASED')
+ self.assertTrue(mkv.tags[0].simpletags[1].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[1].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[1].string == '2010')
+ self.assertTrue(mkv.tags[0].simpletags[1].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[2].name == 'COMMENT')
+ self.assertTrue(mkv.tags[0].simpletags[2].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[2].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[2].string == 'Matroska Validation File 2, 100,000 timecode scale, odd aspect ratio, and CRC-32. Codecs are AVC and AAC')
+ self.assertTrue(mkv.tags[0].simpletags[2].binary is None)
+
+ def test_test3(self):
+ with io.open(os.path.join(TEST_DIR, 'test3.mkv'), 'rb') as stream:
+ mkv = MKV(stream)
+ # info
+ self.assertTrue(mkv.info.title is None)
+ self.assertTrue(mkv.info.duration == timedelta(seconds=49, milliseconds=64))
+ self.assertTrue(mkv.info.date_utc == datetime(2010, 8, 21, 21, 43, 25))
+ self.assertTrue(mkv.info.muxing_app == 'libebml2 v0.11.0 + libmatroska2 v0.10.1')
+ self.assertTrue(mkv.info.writing_app == 'mkclean 0.5.5 ro from libebml v1.0.0 + libmatroska v1.0.0 + mkvmerge v4.1.1 (\'Bouncin\' Back\') built on Jul 3 2010 22:54:08')
+ # video track
+ self.assertTrue(len(mkv.video_tracks) == 1)
+ self.assertTrue(mkv.video_tracks[0].type == VIDEO_TRACK)
+ self.assertTrue(mkv.video_tracks[0].number == 1)
+ self.assertTrue(mkv.video_tracks[0].name is None)
+ self.assertTrue(mkv.video_tracks[0].language == 'und')
+ self.assertTrue(mkv.video_tracks[0].enabled == True)
+ self.assertTrue(mkv.video_tracks[0].default == True)
+ self.assertTrue(mkv.video_tracks[0].forced == False)
+ self.assertTrue(mkv.video_tracks[0].lacing == False)
+ self.assertTrue(mkv.video_tracks[0].codec_id == 'V_MPEG4/ISO/AVC')
+ self.assertTrue(mkv.video_tracks[0].codec_name is None)
+ self.assertTrue(mkv.video_tracks[0].width == 1024)
+ self.assertTrue(mkv.video_tracks[0].height == 576)
+ self.assertTrue(mkv.video_tracks[0].interlaced == False)
+ self.assertTrue(mkv.video_tracks[0].stereo_mode is None)
+ self.assertTrue(mkv.video_tracks[0].crop == {})
+ self.assertTrue(mkv.video_tracks[0].display_width is None)
+ self.assertTrue(mkv.video_tracks[0].display_height is None)
+ self.assertTrue(mkv.video_tracks[0].display_unit is None)
+ self.assertTrue(mkv.video_tracks[0].aspect_ratio_type is None)
+ # audio track
+ self.assertTrue(len(mkv.audio_tracks) == 1)
+ self.assertTrue(mkv.audio_tracks[0].type == AUDIO_TRACK)
+ self.assertTrue(mkv.audio_tracks[0].number == 2)
+ self.assertTrue(mkv.audio_tracks[0].name is None)
+ self.assertTrue(mkv.audio_tracks[0].language is None)
+ self.assertTrue(mkv.audio_tracks[0].enabled == True)
+ self.assertTrue(mkv.audio_tracks[0].default == True)
+ self.assertTrue(mkv.audio_tracks[0].forced == False)
+ self.assertTrue(mkv.audio_tracks[0].lacing == True)
+ self.assertTrue(mkv.audio_tracks[0].codec_id == 'A_MPEG/L3')
+ self.assertTrue(mkv.audio_tracks[0].codec_name is None)
+ self.assertTrue(mkv.audio_tracks[0].sampling_frequency == 48000.0)
+ self.assertTrue(mkv.audio_tracks[0].channels == 2)
+ self.assertTrue(mkv.audio_tracks[0].output_sampling_frequency is None)
+ self.assertTrue(mkv.audio_tracks[0].bit_depth is None)
+ # subtitle track
+ self.assertTrue(len(mkv.subtitle_tracks) == 0)
+ # chapters
+ self.assertTrue(len(mkv.chapters) == 0)
+ # tags
+ self.assertTrue(len(mkv.tags) == 1)
+ self.assertTrue(len(mkv.tags[0].simpletags) == 3)
+ self.assertTrue(mkv.tags[0].simpletags[0].name == 'TITLE')
+ self.assertTrue(mkv.tags[0].simpletags[0].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[0].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[0].string == 'Elephant Dream - test 3')
+ self.assertTrue(mkv.tags[0].simpletags[0].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[1].name == 'DATE_RELEASED')
+ self.assertTrue(mkv.tags[0].simpletags[1].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[1].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[1].string == '2010')
+ self.assertTrue(mkv.tags[0].simpletags[1].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[2].name == 'COMMENT')
+ self.assertTrue(mkv.tags[0].simpletags[2].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[2].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[2].string == 'Matroska Validation File 3, header stripping on the video track and no SimpleBlock')
+ self.assertTrue(mkv.tags[0].simpletags[2].binary is None)
+
+ def test_test5(self):
+ with io.open(os.path.join(TEST_DIR, 'test5.mkv'), 'rb') as stream:
+ mkv = MKV(stream)
+ # info
+ self.assertTrue(mkv.info.title is None)
+ self.assertTrue(mkv.info.duration == timedelta(seconds=46, milliseconds=665))
+ self.assertTrue(mkv.info.date_utc == datetime(2010, 8, 21, 18, 6, 43))
+ self.assertTrue(mkv.info.muxing_app == 'libebml v1.0.0 + libmatroska v1.0.0')
+ self.assertTrue(mkv.info.writing_app == 'mkvmerge v4.0.0 (\'The Stars were mine\') built on Jun 6 2010 16:18:42')
+ # video track
+ self.assertTrue(len(mkv.video_tracks) == 1)
+ self.assertTrue(mkv.video_tracks[0].type == VIDEO_TRACK)
+ self.assertTrue(mkv.video_tracks[0].number == 1)
+ self.assertTrue(mkv.video_tracks[0].name is None)
+ self.assertTrue(mkv.video_tracks[0].language == 'und')
+ self.assertTrue(mkv.video_tracks[0].enabled == True)
+ self.assertTrue(mkv.video_tracks[0].default == True)
+ self.assertTrue(mkv.video_tracks[0].forced == False)
+ self.assertTrue(mkv.video_tracks[0].lacing == False)
+ self.assertTrue(mkv.video_tracks[0].codec_id == 'V_MPEG4/ISO/AVC')
+ self.assertTrue(mkv.video_tracks[0].codec_name is None)
+ self.assertTrue(mkv.video_tracks[0].width == 1024)
+ self.assertTrue(mkv.video_tracks[0].height == 576)
+ self.assertTrue(mkv.video_tracks[0].interlaced == False)
+ self.assertTrue(mkv.video_tracks[0].stereo_mode is None)
+ self.assertTrue(mkv.video_tracks[0].crop == {})
+ self.assertTrue(mkv.video_tracks[0].display_width == 1024)
+ self.assertTrue(mkv.video_tracks[0].display_height == 576)
+ self.assertTrue(mkv.video_tracks[0].display_unit is None)
+ self.assertTrue(mkv.video_tracks[0].aspect_ratio_type is None)
+ # audio tracks
+ self.assertTrue(len(mkv.audio_tracks) == 2)
+ self.assertTrue(mkv.audio_tracks[0].type == AUDIO_TRACK)
+ self.assertTrue(mkv.audio_tracks[0].number == 2)
+ self.assertTrue(mkv.audio_tracks[0].name is None)
+ self.assertTrue(mkv.audio_tracks[0].language == 'und')
+ self.assertTrue(mkv.audio_tracks[0].enabled == True)
+ self.assertTrue(mkv.audio_tracks[0].default == True)
+ self.assertTrue(mkv.audio_tracks[0].forced == False)
+ self.assertTrue(mkv.audio_tracks[0].lacing == True)
+ self.assertTrue(mkv.audio_tracks[0].codec_id == 'A_AAC')
+ self.assertTrue(mkv.audio_tracks[0].codec_name is None)
+ self.assertTrue(mkv.audio_tracks[0].sampling_frequency == 48000.0)
+ self.assertTrue(mkv.audio_tracks[0].channels == 2)
+ self.assertTrue(mkv.audio_tracks[0].output_sampling_frequency is None)
+ self.assertTrue(mkv.audio_tracks[0].bit_depth is None)
+ self.assertTrue(mkv.audio_tracks[1].type == AUDIO_TRACK)
+ self.assertTrue(mkv.audio_tracks[1].number == 10)
+ self.assertTrue(mkv.audio_tracks[1].name == 'Commentary')
+ self.assertTrue(mkv.audio_tracks[1].language is None)
+ self.assertTrue(mkv.audio_tracks[1].enabled == True)
+ self.assertTrue(mkv.audio_tracks[1].default == False)
+ self.assertTrue(mkv.audio_tracks[1].forced == False)
+ self.assertTrue(mkv.audio_tracks[1].lacing == True)
+ self.assertTrue(mkv.audio_tracks[1].codec_id == 'A_AAC')
+ self.assertTrue(mkv.audio_tracks[1].codec_name is None)
+ self.assertTrue(mkv.audio_tracks[1].sampling_frequency == 22050.0)
+ self.assertTrue(mkv.audio_tracks[1].channels == 1)
+ self.assertTrue(mkv.audio_tracks[1].output_sampling_frequency == 44100.0)
+ self.assertTrue(mkv.audio_tracks[1].bit_depth is None)
+ # subtitle track
+ self.assertTrue(len(mkv.subtitle_tracks) == 8)
+ self.assertTrue(mkv.subtitle_tracks[0].type == SUBTITLE_TRACK)
+ self.assertTrue(mkv.subtitle_tracks[0].number == 3)
+ self.assertTrue(mkv.subtitle_tracks[0].name is None)
+ self.assertTrue(mkv.subtitle_tracks[0].language is None)
+ self.assertTrue(mkv.subtitle_tracks[0].enabled == True)
+ self.assertTrue(mkv.subtitle_tracks[0].default == True)
+ self.assertTrue(mkv.subtitle_tracks[0].forced == False)
+ self.assertTrue(mkv.subtitle_tracks[0].lacing == False)
+ self.assertTrue(mkv.subtitle_tracks[0].codec_id == 'S_TEXT/UTF8')
+ self.assertTrue(mkv.subtitle_tracks[0].codec_name is None)
+ self.assertTrue(mkv.subtitle_tracks[1].type == SUBTITLE_TRACK)
+ self.assertTrue(mkv.subtitle_tracks[1].number == 4)
+ self.assertTrue(mkv.subtitle_tracks[1].name is None)
+ self.assertTrue(mkv.subtitle_tracks[1].language == 'hun')
+ self.assertTrue(mkv.subtitle_tracks[1].enabled == True)
+ self.assertTrue(mkv.subtitle_tracks[1].default == False)
+ self.assertTrue(mkv.subtitle_tracks[1].forced == False)
+ self.assertTrue(mkv.subtitle_tracks[1].lacing == False)
+ self.assertTrue(mkv.subtitle_tracks[1].codec_id == 'S_TEXT/UTF8')
+ self.assertTrue(mkv.subtitle_tracks[1].codec_name is None)
+ self.assertTrue(mkv.subtitle_tracks[2].type == SUBTITLE_TRACK)
+ self.assertTrue(mkv.subtitle_tracks[2].number == 5)
+ self.assertTrue(mkv.subtitle_tracks[2].name is None)
+ self.assertTrue(mkv.subtitle_tracks[2].language == 'ger')
+ self.assertTrue(mkv.subtitle_tracks[2].enabled == True)
+ self.assertTrue(mkv.subtitle_tracks[2].default == False)
+ self.assertTrue(mkv.subtitle_tracks[2].forced == False)
+ self.assertTrue(mkv.subtitle_tracks[2].lacing == False)
+ self.assertTrue(mkv.subtitle_tracks[2].codec_id == 'S_TEXT/UTF8')
+ self.assertTrue(mkv.subtitle_tracks[2].codec_name is None)
+ self.assertTrue(mkv.subtitle_tracks[3].type == SUBTITLE_TRACK)
+ self.assertTrue(mkv.subtitle_tracks[3].number == 6)
+ self.assertTrue(mkv.subtitle_tracks[3].name is None)
+ self.assertTrue(mkv.subtitle_tracks[3].language == 'fre')
+ self.assertTrue(mkv.subtitle_tracks[3].enabled == True)
+ self.assertTrue(mkv.subtitle_tracks[3].default == False)
+ self.assertTrue(mkv.subtitle_tracks[3].forced == False)
+ self.assertTrue(mkv.subtitle_tracks[3].lacing == False)
+ self.assertTrue(mkv.subtitle_tracks[3].codec_id == 'S_TEXT/UTF8')
+ self.assertTrue(mkv.subtitle_tracks[3].codec_name is None)
+ self.assertTrue(mkv.subtitle_tracks[4].type == SUBTITLE_TRACK)
+ self.assertTrue(mkv.subtitle_tracks[4].number == 8)
+ self.assertTrue(mkv.subtitle_tracks[4].name is None)
+ self.assertTrue(mkv.subtitle_tracks[4].language == 'spa')
+ self.assertTrue(mkv.subtitle_tracks[4].enabled == True)
+ self.assertTrue(mkv.subtitle_tracks[4].default == False)
+ self.assertTrue(mkv.subtitle_tracks[4].forced == False)
+ self.assertTrue(mkv.subtitle_tracks[4].lacing == False)
+ self.assertTrue(mkv.subtitle_tracks[4].codec_id == 'S_TEXT/UTF8')
+ self.assertTrue(mkv.subtitle_tracks[4].codec_name is None)
+ self.assertTrue(mkv.subtitle_tracks[5].type == SUBTITLE_TRACK)
+ self.assertTrue(mkv.subtitle_tracks[5].number == 9)
+ self.assertTrue(mkv.subtitle_tracks[5].name is None)
+ self.assertTrue(mkv.subtitle_tracks[5].language == 'ita')
+ self.assertTrue(mkv.subtitle_tracks[5].enabled == True)
+ self.assertTrue(mkv.subtitle_tracks[5].default == False)
+ self.assertTrue(mkv.subtitle_tracks[5].forced == False)
+ self.assertTrue(mkv.subtitle_tracks[5].lacing == False)
+ self.assertTrue(mkv.subtitle_tracks[5].codec_id == 'S_TEXT/UTF8')
+ self.assertTrue(mkv.subtitle_tracks[5].codec_name is None)
+ self.assertTrue(mkv.subtitle_tracks[6].type == SUBTITLE_TRACK)
+ self.assertTrue(mkv.subtitle_tracks[6].number == 11)
+ self.assertTrue(mkv.subtitle_tracks[6].name is None)
+ self.assertTrue(mkv.subtitle_tracks[6].language == 'jpn')
+ self.assertTrue(mkv.subtitle_tracks[6].enabled == True)
+ self.assertTrue(mkv.subtitle_tracks[6].default == False)
+ self.assertTrue(mkv.subtitle_tracks[6].forced == False)
+ self.assertTrue(mkv.subtitle_tracks[6].lacing == False)
+ self.assertTrue(mkv.subtitle_tracks[6].codec_id == 'S_TEXT/UTF8')
+ self.assertTrue(mkv.subtitle_tracks[6].codec_name is None)
+ self.assertTrue(mkv.subtitle_tracks[7].type == SUBTITLE_TRACK)
+ self.assertTrue(mkv.subtitle_tracks[7].number == 7)
+ self.assertTrue(mkv.subtitle_tracks[7].name is None)
+ self.assertTrue(mkv.subtitle_tracks[7].language == 'und')
+ self.assertTrue(mkv.subtitle_tracks[7].enabled == True)
+ self.assertTrue(mkv.subtitle_tracks[7].default == False)
+ self.assertTrue(mkv.subtitle_tracks[7].forced == False)
+ self.assertTrue(mkv.subtitle_tracks[7].lacing == False)
+ self.assertTrue(mkv.subtitle_tracks[7].codec_id == 'S_TEXT/UTF8')
+ self.assertTrue(mkv.subtitle_tracks[7].codec_name is None)
+ # chapters
+ self.assertTrue(len(mkv.chapters) == 0)
+ # tags
+ self.assertTrue(len(mkv.tags) == 1)
+ self.assertTrue(len(mkv.tags[0].simpletags) == 3)
+ self.assertTrue(mkv.tags[0].simpletags[0].name == 'TITLE')
+ self.assertTrue(mkv.tags[0].simpletags[0].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[0].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[0].string == 'Big Buck Bunny - test 8')
+ self.assertTrue(mkv.tags[0].simpletags[0].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[1].name == 'DATE_RELEASED')
+ self.assertTrue(mkv.tags[0].simpletags[1].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[1].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[1].string == '2010')
+ self.assertTrue(mkv.tags[0].simpletags[1].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[2].name == 'COMMENT')
+ self.assertTrue(mkv.tags[0].simpletags[2].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[2].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[2].string == 'Matroska Validation File 8, secondary audio commentary track, misc subtitle tracks')
+ self.assertTrue(mkv.tags[0].simpletags[2].binary is None)
+
+ def test_test6(self):
+ with io.open(os.path.join(TEST_DIR, 'test6.mkv'), 'rb') as stream:
+ mkv = MKV(stream)
+ # info
+ self.assertTrue(mkv.info.title is None)
+ self.assertTrue(mkv.info.duration == timedelta(seconds=87, milliseconds=336))
+ self.assertTrue(mkv.info.date_utc == datetime(2010, 8, 21, 16, 31, 55))
+ self.assertTrue(mkv.info.muxing_app == 'libebml2 v0.10.1 + libmatroska2 v0.10.1')
+ self.assertTrue(mkv.info.writing_app == 'mkclean 0.5.5 r from libebml v1.0.0 + libmatroska v1.0.0 + mkvmerge v4.0.0 (\'The Stars were mine\') built on Jun 6 2010 16:18:42')
+ # video track
+ self.assertTrue(len(mkv.video_tracks) == 1)
+ self.assertTrue(mkv.video_tracks[0].type == VIDEO_TRACK)
+ self.assertTrue(mkv.video_tracks[0].number == 1)
+ self.assertTrue(mkv.video_tracks[0].name is None)
+ self.assertTrue(mkv.video_tracks[0].language == 'und')
+ self.assertTrue(mkv.video_tracks[0].enabled == True)
+ self.assertTrue(mkv.video_tracks[0].default == False)
+ self.assertTrue(mkv.video_tracks[0].forced == False)
+ self.assertTrue(mkv.video_tracks[0].lacing == False)
+ self.assertTrue(mkv.video_tracks[0].codec_id == 'V_MS/VFW/FOURCC')
+ self.assertTrue(mkv.video_tracks[0].codec_name is None)
+ self.assertTrue(mkv.video_tracks[0].width == 854)
+ self.assertTrue(mkv.video_tracks[0].height == 480)
+ self.assertTrue(mkv.video_tracks[0].interlaced == False)
+ self.assertTrue(mkv.video_tracks[0].stereo_mode is None)
+ self.assertTrue(mkv.video_tracks[0].crop == {})
+ self.assertTrue(mkv.video_tracks[0].display_width is None)
+ self.assertTrue(mkv.video_tracks[0].display_height is None)
+ self.assertTrue(mkv.video_tracks[0].display_unit is None)
+ self.assertTrue(mkv.video_tracks[0].aspect_ratio_type is None)
+ # audio track
+ self.assertTrue(len(mkv.audio_tracks) == 1)
+ self.assertTrue(mkv.audio_tracks[0].type == AUDIO_TRACK)
+ self.assertTrue(mkv.audio_tracks[0].number == 2)
+ self.assertTrue(mkv.audio_tracks[0].name is None)
+ self.assertTrue(mkv.audio_tracks[0].language == 'und')
+ self.assertTrue(mkv.audio_tracks[0].enabled == True)
+ self.assertTrue(mkv.audio_tracks[0].default == False)
+ self.assertTrue(mkv.audio_tracks[0].forced == False)
+ self.assertTrue(mkv.audio_tracks[0].lacing == True)
+ self.assertTrue(mkv.audio_tracks[0].codec_id == 'A_MPEG/L3')
+ self.assertTrue(mkv.audio_tracks[0].codec_name is None)
+ self.assertTrue(mkv.audio_tracks[0].sampling_frequency == 48000.0)
+ self.assertTrue(mkv.audio_tracks[0].channels == 2)
+ self.assertTrue(mkv.audio_tracks[0].output_sampling_frequency is None)
+ self.assertTrue(mkv.audio_tracks[0].bit_depth is None)
+ # subtitle track
+ self.assertTrue(len(mkv.subtitle_tracks) == 0)
+ # chapters
+ self.assertTrue(len(mkv.chapters) == 0)
+ # tags
+ self.assertTrue(len(mkv.tags) == 1)
+ self.assertTrue(len(mkv.tags[0].simpletags) == 3)
+ self.assertTrue(mkv.tags[0].simpletags[0].name == 'TITLE')
+ self.assertTrue(mkv.tags[0].simpletags[0].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[0].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[0].string == 'Big Buck Bunny - test 6')
+ self.assertTrue(mkv.tags[0].simpletags[0].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[1].name == 'DATE_RELEASED')
+ self.assertTrue(mkv.tags[0].simpletags[1].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[1].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[1].string == '2010')
+ self.assertTrue(mkv.tags[0].simpletags[1].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[2].name == 'COMMENT')
+ self.assertTrue(mkv.tags[0].simpletags[2].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[2].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[2].string == 'Matroska Validation File 6, random length to code the size of Clusters and Blocks, no Cues for seeking')
+ self.assertTrue(mkv.tags[0].simpletags[2].binary is None)
+
+ def test_test7(self):
+ with io.open(os.path.join(TEST_DIR, 'test7.mkv'), 'rb') as stream:
+ mkv = MKV(stream)
+ # info
+ self.assertTrue(mkv.info.title is None)
+ self.assertTrue(mkv.info.duration == timedelta(seconds=37, milliseconds=43))
+ self.assertTrue(mkv.info.date_utc == datetime(2010, 8, 21, 17, 0, 23))
+ self.assertTrue(mkv.info.muxing_app == 'libebml2 v0.10.1 + libmatroska2 v0.10.1')
+ self.assertTrue(mkv.info.writing_app == 'mkclean 0.5.5 r from libebml v1.0.0 + libmatroska v1.0.0 + mkvmerge v4.0.0 (\'The Stars were mine\') built on Jun 6 2010 16:18:42')
+ # video track
+ self.assertTrue(len(mkv.video_tracks) == 1)
+ self.assertTrue(mkv.video_tracks[0].type == VIDEO_TRACK)
+ self.assertTrue(mkv.video_tracks[0].number == 1)
+ self.assertTrue(mkv.video_tracks[0].name is None)
+ self.assertTrue(mkv.video_tracks[0].language == 'und')
+ self.assertTrue(mkv.video_tracks[0].enabled == True)
+ self.assertTrue(mkv.video_tracks[0].default == False)
+ self.assertTrue(mkv.video_tracks[0].forced == False)
+ self.assertTrue(mkv.video_tracks[0].lacing == False)
+ self.assertTrue(mkv.video_tracks[0].codec_id == 'V_MPEG4/ISO/AVC')
+ self.assertTrue(mkv.video_tracks[0].codec_name is None)
+ self.assertTrue(mkv.video_tracks[0].width == 1024)
+ self.assertTrue(mkv.video_tracks[0].height == 576)
+ self.assertTrue(mkv.video_tracks[0].interlaced == False)
+ self.assertTrue(mkv.video_tracks[0].stereo_mode is None)
+ self.assertTrue(mkv.video_tracks[0].crop == {})
+ self.assertTrue(mkv.video_tracks[0].display_width is None)
+ self.assertTrue(mkv.video_tracks[0].display_height is None)
+ self.assertTrue(mkv.video_tracks[0].display_unit is None)
+ self.assertTrue(mkv.video_tracks[0].aspect_ratio_type is None)
+ # audio track
+ self.assertTrue(len(mkv.audio_tracks) == 1)
+ self.assertTrue(mkv.audio_tracks[0].type == AUDIO_TRACK)
+ self.assertTrue(mkv.audio_tracks[0].number == 2)
+ self.assertTrue(mkv.audio_tracks[0].name is None)
+ self.assertTrue(mkv.audio_tracks[0].language == 'und')
+ self.assertTrue(mkv.audio_tracks[0].enabled == True)
+ self.assertTrue(mkv.audio_tracks[0].default == False)
+ self.assertTrue(mkv.audio_tracks[0].forced == False)
+ self.assertTrue(mkv.audio_tracks[0].lacing == True)
+ self.assertTrue(mkv.audio_tracks[0].codec_id == 'A_AAC')
+ self.assertTrue(mkv.audio_tracks[0].codec_name is None)
+ self.assertTrue(mkv.audio_tracks[0].sampling_frequency == 48000.0)
+ self.assertTrue(mkv.audio_tracks[0].channels == 2)
+ self.assertTrue(mkv.audio_tracks[0].output_sampling_frequency is None)
+ self.assertTrue(mkv.audio_tracks[0].bit_depth is None)
+ # subtitle track
+ self.assertTrue(len(mkv.subtitle_tracks) == 0)
+ # chapters
+ self.assertTrue(len(mkv.chapters) == 0)
+ # tags
+ self.assertTrue(len(mkv.tags) == 1)
+ self.assertTrue(len(mkv.tags[0].simpletags) == 3)
+ self.assertTrue(mkv.tags[0].simpletags[0].name == 'TITLE')
+ self.assertTrue(mkv.tags[0].simpletags[0].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[0].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[0].string == 'Big Buck Bunny - test 7')
+ self.assertTrue(mkv.tags[0].simpletags[0].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[1].name == 'DATE_RELEASED')
+ self.assertTrue(mkv.tags[0].simpletags[1].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[1].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[1].string == '2010')
+ self.assertTrue(mkv.tags[0].simpletags[1].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[2].name == 'COMMENT')
+ self.assertTrue(mkv.tags[0].simpletags[2].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[2].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[2].string == 'Matroska Validation File 7, junk elements are present at the beggining or end of clusters, the parser should skip it. There is also a damaged element at 451418')
+ self.assertTrue(mkv.tags[0].simpletags[2].binary is None)
+
+ def test_test8(self):
+ with io.open(os.path.join(TEST_DIR, 'test8.mkv'), 'rb') as stream:
+ mkv = MKV(stream)
+ # info
+ self.assertTrue(mkv.info.title is None)
+ self.assertTrue(mkv.info.duration == timedelta(seconds=47, milliseconds=341))
+ self.assertTrue(mkv.info.date_utc == datetime(2010, 8, 21, 17, 22, 14))
+ self.assertTrue(mkv.info.muxing_app == 'libebml2 v0.10.1 + libmatroska2 v0.10.1')
+ self.assertTrue(mkv.info.writing_app == 'mkclean 0.5.5 r from libebml v1.0.0 + libmatroska v1.0.0 + mkvmerge v4.0.0 (\'The Stars were mine\') built on Jun 6 2010 16:18:42')
+ # video track
+ self.assertTrue(len(mkv.video_tracks) == 1)
+ self.assertTrue(mkv.video_tracks[0].type == VIDEO_TRACK)
+ self.assertTrue(mkv.video_tracks[0].number == 1)
+ self.assertTrue(mkv.video_tracks[0].name is None)
+ self.assertTrue(mkv.video_tracks[0].language == 'und')
+ self.assertTrue(mkv.video_tracks[0].enabled == True)
+ self.assertTrue(mkv.video_tracks[0].default == False)
+ self.assertTrue(mkv.video_tracks[0].forced == False)
+ self.assertTrue(mkv.video_tracks[0].lacing == False)
+ self.assertTrue(mkv.video_tracks[0].codec_id == 'V_MPEG4/ISO/AVC')
+ self.assertTrue(mkv.video_tracks[0].codec_name is None)
+ self.assertTrue(mkv.video_tracks[0].width == 1024)
+ self.assertTrue(mkv.video_tracks[0].height == 576)
+ self.assertTrue(mkv.video_tracks[0].interlaced == False)
+ self.assertTrue(mkv.video_tracks[0].stereo_mode is None)
+ self.assertTrue(mkv.video_tracks[0].crop == {})
+ self.assertTrue(mkv.video_tracks[0].display_width is None)
+ self.assertTrue(mkv.video_tracks[0].display_height is None)
+ self.assertTrue(mkv.video_tracks[0].display_unit is None)
+ self.assertTrue(mkv.video_tracks[0].aspect_ratio_type is None)
+ # audio track
+ self.assertTrue(len(mkv.audio_tracks) == 1)
+ self.assertTrue(mkv.audio_tracks[0].type == AUDIO_TRACK)
+ self.assertTrue(mkv.audio_tracks[0].number == 2)
+ self.assertTrue(mkv.audio_tracks[0].name is None)
+ self.assertTrue(mkv.audio_tracks[0].language == 'und')
+ self.assertTrue(mkv.audio_tracks[0].enabled == True)
+ self.assertTrue(mkv.audio_tracks[0].default == False)
+ self.assertTrue(mkv.audio_tracks[0].forced == False)
+ self.assertTrue(mkv.audio_tracks[0].lacing == True)
+ self.assertTrue(mkv.audio_tracks[0].codec_id == 'A_AAC')
+ self.assertTrue(mkv.audio_tracks[0].codec_name is None)
+ self.assertTrue(mkv.audio_tracks[0].sampling_frequency == 48000.0)
+ self.assertTrue(mkv.audio_tracks[0].channels == 2)
+ self.assertTrue(mkv.audio_tracks[0].output_sampling_frequency is None)
+ self.assertTrue(mkv.audio_tracks[0].bit_depth is None)
+ # subtitle track
+ self.assertTrue(len(mkv.subtitle_tracks) == 0)
+ # chapters
+ self.assertTrue(len(mkv.chapters) == 0)
+ # tags
+ self.assertTrue(len(mkv.tags) == 1)
+ self.assertTrue(len(mkv.tags[0].simpletags) == 3)
+ self.assertTrue(mkv.tags[0].simpletags[0].name == 'TITLE')
+ self.assertTrue(mkv.tags[0].simpletags[0].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[0].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[0].string == 'Big Buck Bunny - test 8')
+ self.assertTrue(mkv.tags[0].simpletags[0].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[1].name == 'DATE_RELEASED')
+ self.assertTrue(mkv.tags[0].simpletags[1].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[1].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[1].string == '2010')
+ self.assertTrue(mkv.tags[0].simpletags[1].binary is None)
+ self.assertTrue(mkv.tags[0].simpletags[2].name == 'COMMENT')
+ self.assertTrue(mkv.tags[0].simpletags[2].default == True)
+ self.assertTrue(mkv.tags[0].simpletags[2].language == 'und')
+ self.assertTrue(mkv.tags[0].simpletags[2].string == 'Matroska Validation File 8, audio missing between timecodes 6.019s and 6.360s')
+ self.assertTrue(mkv.tags[0].simpletags[2].binary is None)
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.TestLoader().loadTestsFromTestCase(MKVTestCase))
+ return suite
+
+if __name__ == '__main__':
+ unittest.TextTestRunner().run(suite())
diff --git a/lib/enzyme/tests/test_parsers.py b/lib/enzyme/tests/test_parsers.py
new file mode 100644
index 0000000000000000000000000000000000000000..0fa320ce013b6767e8e70ad537c01499f45968ed
--- /dev/null
+++ b/lib/enzyme/tests/test_parsers.py
@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+from enzyme.parsers import ebml
+import io
+import os.path
+import requests
+import unittest
+import yaml
+import zipfile
+
+
+# Test directory
+TEST_DIR = os.path.join(os.path.dirname(__file__), os.path.splitext(__file__)[0])
+
+# EBML validation directory
+EBML_VALIDATION_DIR = os.path.join(os.path.dirname(__file__), 'parsers', 'ebml')
+
+
+def setUpModule():
+ if not os.path.exists(TEST_DIR):
+ r = requests.get('http://downloads.sourceforge.net/project/matroska/test_files/matroska_test_w1_1.zip')
+ with zipfile.ZipFile(io.BytesIO(r.content), 'r') as f:
+ f.extractall(TEST_DIR)
+
+
+class EBMLTestCase(unittest.TestCase):
+ def setUp(self):
+ self.stream = io.open(os.path.join(TEST_DIR, 'test1.mkv'), 'rb')
+ with io.open(os.path.join(EBML_VALIDATION_DIR, 'test1.mkv.yml'), 'r') as yml:
+ self.validation = yaml.safe_load(yml)
+ self.specs = ebml.get_matroska_specs()
+
+ def tearDown(self):
+ self.stream.close()
+
+ def check_element(self, element_id, element_type, element_name, element_level, element_position, element_size, element_data, element,
+ ignore_element_types=None, ignore_element_names=None, max_level=None):
+ """Recursively check an element"""
+ # base
+ self.assertTrue(element.id == element_id)
+ self.assertTrue(element.type == element_type)
+ self.assertTrue(element.name == element_name)
+ self.assertTrue(element.level == element_level)
+ self.assertTrue(element.position == element_position)
+ self.assertTrue(element.size == element_size)
+ # Element
+ if not isinstance(element_data, list):
+ self.assertTrue(type(element) == ebml.Element)
+ if element_type != ebml.BINARY:
+ self.assertTrue(element.data == element_data)
+ return
+ # MasterElement
+ if ignore_element_types is not None: # filter validation on element types
+ element_data = [e for e in element_data if e[1] not in ignore_element_types]
+ if ignore_element_names is not None: # filter validation on element names
+ element_data = [e for e in element_data if e[2] not in ignore_element_names]
+ if element.level == max_level: # special check when maximum level is reached
+ self.assertTrue(element.data is None)
+ return
+ self.assertTrue(len(element.data) == len(element_data))
+ for i in range(len(element.data)):
+ self.check_element(element_data[i][0], element_data[i][1], element_data[i][2], element_data[i][3],
+ element_data[i][4], element_data[i][5], element_data[i][6], element.data[i], ignore_element_types,
+ ignore_element_names, max_level)
+
+ def test_parse_full(self):
+ result = ebml.parse(self.stream, self.specs)
+ self.assertTrue(len(result) == len(self.validation))
+ for i in range(len(self.validation)):
+ self.check_element(self.validation[i][0], self.validation[i][1], self.validation[i][2], self.validation[i][3],
+ self.validation[i][4], self.validation[i][5], self.validation[i][6], result[i])
+
+ def test_parse_ignore_element_types(self):
+ ignore_element_types = [ebml.INTEGER, ebml.BINARY]
+ result = ebml.parse(self.stream, self.specs, ignore_element_types=ignore_element_types)
+ self.validation = [e for e in self.validation if e[1] not in ignore_element_types]
+ self.assertTrue(len(result) == len(self.validation))
+ for i in range(len(self.validation)):
+ self.check_element(self.validation[i][0], self.validation[i][1], self.validation[i][2], self.validation[i][3],
+ self.validation[i][4], self.validation[i][5], self.validation[i][6], result[i], ignore_element_types=ignore_element_types)
+
+ def test_parse_ignore_element_names(self):
+ ignore_element_names = ['EBML', 'SimpleBlock']
+ result = ebml.parse(self.stream, self.specs, ignore_element_names=ignore_element_names)
+ self.validation = [e for e in self.validation if e[2] not in ignore_element_names]
+ self.assertTrue(len(result) == len(self.validation))
+ for i in range(len(self.validation)):
+ self.check_element(self.validation[i][0], self.validation[i][1], self.validation[i][2], self.validation[i][3],
+ self.validation[i][4], self.validation[i][5], self.validation[i][6], result[i], ignore_element_names=ignore_element_names)
+
+ def test_parse_max_level(self):
+ max_level = 3
+ result = ebml.parse(self.stream, self.specs, max_level=max_level)
+ self.validation = [e for e in self.validation if e[3] <= max_level]
+ self.assertTrue(len(result) == len(self.validation))
+ for i in range(len(self.validation)):
+ self.check_element(self.validation[i][0], self.validation[i][1], self.validation[i][2], self.validation[i][3],
+ self.validation[i][4], self.validation[i][5], self.validation[i][6], result[i], max_level=max_level)
+
+
+def generate_yml(filename, specs):
+ """Generate a validation file for the test video"""
+ def _to_builtin(elements):
+ """Recursively convert elements to built-in types"""
+ result = []
+ for e in elements:
+ if isinstance(e, ebml.MasterElement):
+ result.append((e.id, e.type, e.name, e.level, e.position, e.size, _to_builtin(e.data)))
+ else:
+ result.append((e.id, e.type, e.name, e.level, e.position, e.size, None if isinstance(e.data, io.BytesIO) else e.data))
+ return result
+ video = io.open(os.path.join(TEST_DIR, filename), 'rb')
+ yml = io.open(os.path.join(EBML_VALIDATION_DIR, filename + '.yml'), 'w')
+ yaml.safe_dump(_to_builtin(ebml.parse(video, specs)), yml)
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.TestLoader().loadTestsFromTestCase(EBMLTestCase))
+ return suite
+
+if __name__ == '__main__':
+ unittest.TextTestRunner().run(suite())
diff --git a/lib/fanart/core.py b/lib/fanart/core.py
index 14a6975b3e015769ba63b3a601d7c6304701b96d..6b3af96d468a04db16daa19e7be25eb1bc08b5b5 100644
--- a/lib/fanart/core.py
+++ b/lib/fanart/core.py
@@ -1,4 +1,4 @@
-from lib import requests
+import requests
import fanart
from fanart.errors import RequestFanartError, ResponseFanartError
diff --git a/lib/fanart/items.py b/lib/fanart/items.py
index deca27d9d6bdb45eebd70a37cb717fe592ebbc5d..778e1a1b2ba5e936061deb03e05423468277f4c3 100644
--- a/lib/fanart/items.py
+++ b/lib/fanart/items.py
@@ -1,6 +1,6 @@
import json
import os
-from lib import requests
+import requests
from fanart.core import Request
from fanart.immutable import Immutable
diff --git a/lib/guessit/ISO-3166-1_utf8.txt b/lib/guessit/ISO-3166-1_utf8.txt
deleted file mode 100644
index 7022040d91cffda6e191b7a7db0cac71200f3bd3..0000000000000000000000000000000000000000
--- a/lib/guessit/ISO-3166-1_utf8.txt
+++ /dev/null
@@ -1,249 +0,0 @@
-Afghanistan|AF|AFG|004|ISO 3166-2:AF
-Åland Islands|AX|ALA|248|ISO 3166-2:AX
-Albania|AL|ALB|008|ISO 3166-2:AL
-Algeria|DZ|DZA|012|ISO 3166-2:DZ
-American Samoa|AS|ASM|016|ISO 3166-2:AS
-Andorra|AD|AND|020|ISO 3166-2:AD
-Angola|AO|AGO|024|ISO 3166-2:AO
-Anguilla|AI|AIA|660|ISO 3166-2:AI
-Antarctica|AQ|ATA|010|ISO 3166-2:AQ
-Antigua and Barbuda|AG|ATG|028|ISO 3166-2:AG
-Argentina|AR|ARG|032|ISO 3166-2:AR
-Armenia|AM|ARM|051|ISO 3166-2:AM
-Aruba|AW|ABW|533|ISO 3166-2:AW
-Australia|AU|AUS|036|ISO 3166-2:AU
-Austria|AT|AUT|040|ISO 3166-2:AT
-Azerbaijan|AZ|AZE|031|ISO 3166-2:AZ
-Bahamas|BS|BHS|044|ISO 3166-2:BS
-Bahrain|BH|BHR|048|ISO 3166-2:BH
-Bangladesh|BD|BGD|050|ISO 3166-2:BD
-Barbados|BB|BRB|052|ISO 3166-2:BB
-Belarus|BY|BLR|112|ISO 3166-2:BY
-Belgium|BE|BEL|056|ISO 3166-2:BE
-Belize|BZ|BLZ|084|ISO 3166-2:BZ
-Benin|BJ|BEN|204|ISO 3166-2:BJ
-Bermuda|BM|BMU|060|ISO 3166-2:BM
-Bhutan|BT|BTN|064|ISO 3166-2:BT
-Bolivia, Plurinational State of|BO|BOL|068|ISO 3166-2:BO
-Bonaire, Sint Eustatius and Saba|BQ|BES|535|ISO 3166-2:BQ
-Bosnia and Herzegovina|BA|BIH|070|ISO 3166-2:BA
-Botswana|BW|BWA|072|ISO 3166-2:BW
-Bouvet Island|BV|BVT|074|ISO 3166-2:BV
-Brazil|BR|BRA|076|ISO 3166-2:BR
-British Indian Ocean Territory|IO|IOT|086|ISO 3166-2:IO
-Brunei Darussalam|BN|BRN|096|ISO 3166-2:BN
-Bulgaria|BG|BGR|100|ISO 3166-2:BG
-Burkina Faso|BF|BFA|854|ISO 3166-2:BF
-Burundi|BI|BDI|108|ISO 3166-2:BI
-Cambodia|KH|KHM|116|ISO 3166-2:KH
-Cameroon|CM|CMR|120|ISO 3166-2:CM
-Canada|CA|CAN|124|ISO 3166-2:CA
-Cape Verde|CV|CPV|132|ISO 3166-2:CV
-Cayman Islands|KY|CYM|136|ISO 3166-2:KY
-Central African Republic|CF|CAF|140|ISO 3166-2:CF
-Chad|TD|TCD|148|ISO 3166-2:TD
-Chile|CL|CHL|152|ISO 3166-2:CL
-China|CN|CHN|156|ISO 3166-2:CN
-Christmas Island|CX|CXR|162|ISO 3166-2:CX
-Cocos (Keeling) Islands|CC|CCK|166|ISO 3166-2:CC
-Colombia|CO|COL|170|ISO 3166-2:CO
-Comoros|KM|COM|174|ISO 3166-2:KM
-Congo|CG|COG|178|ISO 3166-2:CG
-Congo, the Democratic Republic of the|CD|COD|180|ISO 3166-2:CD
-Cook Islands|CK|COK|184|ISO 3166-2:CK
-Costa Rica|CR|CRI|188|ISO 3166-2:CR
-Côte d'Ivoire|CI|CIV|384|ISO 3166-2:CI
-Croatia|HR|HRV|191|ISO 3166-2:HR
-Cuba|CU|CUB|192|ISO 3166-2:CU
-Curaçao|CW|CUW|531|ISO 3166-2:CW
-Cyprus|CY|CYP|196|ISO 3166-2:CY
-Czech Republic|CZ|CZE|203|ISO 3166-2:CZ
-Denmark|DK|DNK|208|ISO 3166-2:DK
-Djibouti|DJ|DJI|262|ISO 3166-2:DJ
-Dominica|DM|DMA|212|ISO 3166-2:DM
-Dominican Republic|DO|DOM|214|ISO 3166-2:DO
-Ecuador|EC|ECU|218|ISO 3166-2:EC
-Egypt|EG|EGY|818|ISO 3166-2:EG
-El Salvador|SV|SLV|222|ISO 3166-2:SV
-Equatorial Guinea|GQ|GNQ|226|ISO 3166-2:GQ
-Eritrea|ER|ERI|232|ISO 3166-2:ER
-Estonia|EE|EST|233|ISO 3166-2:EE
-Ethiopia|ET|ETH|231|ISO 3166-2:ET
-Falkland Islands (Malvinas|FK|FLK|238|ISO 3166-2:FK
-Faroe Islands|FO|FRO|234|ISO 3166-2:FO
-Fiji|FJ|FJI|242|ISO 3166-2:FJ
-Finland|FI|FIN|246|ISO 3166-2:FI
-France|FR|FRA|250|ISO 3166-2:FR
-French Guiana|GF|GUF|254|ISO 3166-2:GF
-French Polynesia|PF|PYF|258|ISO 3166-2:PF
-French Southern Territories|TF|ATF|260|ISO 3166-2:TF
-Gabon|GA|GAB|266|ISO 3166-2:GA
-Gambia|GM|GMB|270|ISO 3166-2:GM
-Georgia|GE|GEO|268|ISO 3166-2:GE
-Germany|DE|DEU|276|ISO 3166-2:DE
-Ghana|GH|GHA|288|ISO 3166-2:GH
-Gibraltar|GI|GIB|292|ISO 3166-2:GI
-Greece|GR|GRC|300|ISO 3166-2:GR
-Greenland|GL|GRL|304|ISO 3166-2:GL
-Grenada|GD|GRD|308|ISO 3166-2:GD
-Guadeloupe|GP|GLP|312|ISO 3166-2:GP
-Guam|GU|GUM|316|ISO 3166-2:GU
-Guatemala|GT|GTM|320|ISO 3166-2:GT
-Guernsey|GG|GGY|831|ISO 3166-2:GG
-Guinea|GN|GIN|324|ISO 3166-2:GN
-Guinea-Bissau|GW|GNB|624|ISO 3166-2:GW
-Guyana|GY|GUY|328|ISO 3166-2:GY
-Haiti|HT|HTI|332|ISO 3166-2:HT
-Heard Island and McDonald Islands|HM|HMD|334|ISO 3166-2:HM
-Holy See (Vatican City State|VA|VAT|336|ISO 3166-2:VA
-Honduras|HN|HND|340|ISO 3166-2:HN
-Hong Kong|HK|HKG|344|ISO 3166-2:HK
-Hungary|HU|HUN|348|ISO 3166-2:HU
-Iceland|IS|ISL|352|ISO 3166-2:IS
-India|IN|IND|356|ISO 3166-2:IN
-Indonesia|ID|IDN|360|ISO 3166-2:ID
-Iran, Islamic Republic of|IR|IRN|364|ISO 3166-2:IR
-Iraq|IQ|IRQ|368|ISO 3166-2:IQ
-Ireland|IE|IRL|372|ISO 3166-2:IE
-Isle of Man|IM|IMN|833|ISO 3166-2:IM
-Israel|IL|ISR|376|ISO 3166-2:IL
-Italy|IT|ITA|380|ISO 3166-2:IT
-Jamaica|JM|JAM|388|ISO 3166-2:JM
-Japan|JP|JPN|392|ISO 3166-2:JP
-Jersey|JE|JEY|832|ISO 3166-2:JE
-Jordan|JO|JOR|400|ISO 3166-2:JO
-Kazakhstan|KZ|KAZ|398|ISO 3166-2:KZ
-Kenya|KE|KEN|404|ISO 3166-2:KE
-Kiribati|KI|KIR|296|ISO 3166-2:KI
-Korea, Democratic People's Republic of|KP|PRK|408|ISO 3166-2:KP
-Korea, Republic of|KR|KOR|410|ISO 3166-2:KR
-Kuwait|KW|KWT|414|ISO 3166-2:KW
-Kyrgyzstan|KG|KGZ|417|ISO 3166-2:KG
-Lao People's Democratic Republic|LA|LAO|418|ISO 3166-2:LA
-Latvia|LV|LVA|428|ISO 3166-2:LV
-Lebanon|LB|LBN|422|ISO 3166-2:LB
-Lesotho|LS|LSO|426|ISO 3166-2:LS
-Liberia|LR|LBR|430|ISO 3166-2:LR
-Libya|LY|LBY|434|ISO 3166-2:LY
-Liechtenstein|LI|LIE|438|ISO 3166-2:LI
-Lithuania|LT|LTU|440|ISO 3166-2:LT
-Luxembourg|LU|LUX|442|ISO 3166-2:LU
-Macao|MO|MAC|446|ISO 3166-2:MO
-Macedonia, the former Yugoslav Republic of|MK|MKD|807|ISO 3166-2:MK
-Madagascar|MG|MDG|450|ISO 3166-2:MG
-Malawi|MW|MWI|454|ISO 3166-2:MW
-Malaysia|MY|MYS|458|ISO 3166-2:MY
-Maldives|MV|MDV|462|ISO 3166-2:MV
-Mali|ML|MLI|466|ISO 3166-2:ML
-Malta|MT|MLT|470|ISO 3166-2:MT
-Marshall Islands|MH|MHL|584|ISO 3166-2:MH
-Martinique|MQ|MTQ|474|ISO 3166-2:MQ
-Mauritania|MR|MRT|478|ISO 3166-2:MR
-Mauritius|MU|MUS|480|ISO 3166-2:MU
-Mayotte|YT|MYT|175|ISO 3166-2:YT
-Mexico|MX|MEX|484|ISO 3166-2:MX
-Micronesia, Federated States of|FM|FSM|583|ISO 3166-2:FM
-Moldova, Republic of|MD|MDA|498|ISO 3166-2:MD
-Monaco|MC|MCO|492|ISO 3166-2:MC
-Mongolia|MN|MNG|496|ISO 3166-2:MN
-Montenegro|ME|MNE|499|ISO 3166-2:ME
-Montserrat|MS|MSR|500|ISO 3166-2:MS
-Morocco|MA|MAR|504|ISO 3166-2:MA
-Mozambique|MZ|MOZ|508|ISO 3166-2:MZ
-Myanmar|MM|MMR|104|ISO 3166-2:MM
-Namibia|NA|NAM|516|ISO 3166-2:NA
-Nauru|NR|NRU|520|ISO 3166-2:NR
-Nepal|NP|NPL|524|ISO 3166-2:NP
-Netherlands|NL|NLD|528|ISO 3166-2:NL
-New Caledonia|NC|NCL|540|ISO 3166-2:NC
-New Zealand|NZ|NZL|554|ISO 3166-2:NZ
-Nicaragua|NI|NIC|558|ISO 3166-2:NI
-Niger|NE|NER|562|ISO 3166-2:NE
-Nigeria|NG|NGA|566|ISO 3166-2:NG
-Niue|NU|NIU|570|ISO 3166-2:NU
-Norfolk Island|NF|NFK|574|ISO 3166-2:NF
-Northern Mariana Islands|MP|MNP|580|ISO 3166-2:MP
-Norway|NO|NOR|578|ISO 3166-2:NO
-Oman|OM|OMN|512|ISO 3166-2:OM
-Pakistan|PK|PAK|586|ISO 3166-2:PK
-Palau|PW|PLW|585|ISO 3166-2:PW
-Palestinian Territory, Occupied|PS|PSE|275|ISO 3166-2:PS
-Panama|PA|PAN|591|ISO 3166-2:PA
-Papua New Guinea|PG|PNG|598|ISO 3166-2:PG
-Paraguay|PY|PRY|600|ISO 3166-2:PY
-Peru|PE|PER|604|ISO 3166-2:PE
-Philippines|PH|PHL|608|ISO 3166-2:PH
-Pitcairn|PN|PCN|612|ISO 3166-2:PN
-Poland|PL|POL|616|ISO 3166-2:PL
-Portugal|PT|PRT|620|ISO 3166-2:PT
-Puerto Rico|PR|PRI|630|ISO 3166-2:PR
-Qatar|QA|QAT|634|ISO 3166-2:QA
-Réunion|RE|REU|638|ISO 3166-2:RE
-Romania|RO|ROU|642|ISO 3166-2:RO
-Russian Federation|RU|RUS|643|ISO 3166-2:RU
-Rwanda|RW|RWA|646|ISO 3166-2:RW
-Saint Barthélemy|BL|BLM|652|ISO 3166-2:BL
-Saint Helena, Ascension and Tristan da Cunha|SH|SHN|654|ISO 3166-2:SH
-Saint Kitts and Nevis|KN|KNA|659|ISO 3166-2:KN
-Saint Lucia|LC|LCA|662|ISO 3166-2:LC
-Saint Martin (French part|MF|MAF|663|ISO 3166-2:MF
-Saint Pierre and Miquelon|PM|SPM|666|ISO 3166-2:PM
-Saint Vincent and the Grenadines|VC|VCT|670|ISO 3166-2:VC
-Samoa|WS|WSM|882|ISO 3166-2:WS
-San Marino|SM|SMR|674|ISO 3166-2:SM
-Sao Tome and Principe|ST|STP|678|ISO 3166-2:ST
-Saudi Arabia|SA|SAU|682|ISO 3166-2:SA
-Senegal|SN|SEN|686|ISO 3166-2:SN
-Serbia|RS|SRB|688|ISO 3166-2:RS
-Seychelles|SC|SYC|690|ISO 3166-2:SC
-Sierra Leone|SL|SLE|694|ISO 3166-2:SL
-Singapore|SG|SGP|702|ISO 3166-2:SG
-Sint Maarten (Dutch part|SX|SXM|534|ISO 3166-2:SX
-Slovakia|SK|SVK|703|ISO 3166-2:SK
-Slovenia|SI|SVN|705|ISO 3166-2:SI
-Solomon Islands|SB|SLB|090|ISO 3166-2:SB
-Somalia|SO|SOM|706|ISO 3166-2:SO
-South Africa|ZA|ZAF|710|ISO 3166-2:ZA
-South Georgia and the South Sandwich Islands|GS|SGS|239|ISO 3166-2:GS
-South Sudan|SS|SSD|728|ISO 3166-2:SS
-Spain|ES|ESP|724|ISO 3166-2:ES
-Sri Lanka|LK|LKA|144|ISO 3166-2:LK
-Sudan|SD|SDN|729|ISO 3166-2:SD
-Suriname|SR|SUR|740|ISO 3166-2:SR
-Svalbard and Jan Mayen|SJ|SJM|744|ISO 3166-2:SJ
-Swaziland|SZ|SWZ|748|ISO 3166-2:SZ
-Sweden|SE|SWE|752|ISO 3166-2:SE
-Switzerland|CH|CHE|756|ISO 3166-2:CH
-Syrian Arab Republic|SY|SYR|760|ISO 3166-2:SY
-Taiwan, Province of China|TW|TWN|158|ISO 3166-2:TW
-Tajikistan|TJ|TJK|762|ISO 3166-2:TJ
-Tanzania, United Republic of|TZ|TZA|834|ISO 3166-2:TZ
-Thailand|TH|THA|764|ISO 3166-2:TH
-Timor-Leste|TL|TLS|626|ISO 3166-2:TL
-Togo|TG|TGO|768|ISO 3166-2:TG
-Tokelau|TK|TKL|772|ISO 3166-2:TK
-Tonga|TO|TON|776|ISO 3166-2:TO
-Trinidad and Tobago|TT|TTO|780|ISO 3166-2:TT
-Tunisia|TN|TUN|788|ISO 3166-2:TN
-Turkey|TR|TUR|792|ISO 3166-2:TR
-Turkmenistan|TM|TKM|795|ISO 3166-2:TM
-Turks and Caicos Islands|TC|TCA|796|ISO 3166-2:TC
-Tuvalu|TV|TUV|798|ISO 3166-2:TV
-Uganda|UG|UGA|800|ISO 3166-2:UG
-Ukraine|UA|UKR|804|ISO 3166-2:UA
-United Arab Emirates|AE|ARE|784|ISO 3166-2:AE
-United Kingdom|GB|GBR|826|ISO 3166-2:GB
-United States|US|USA|840|ISO 3166-2:US
-United States Minor Outlying Islands|UM|UMI|581|ISO 3166-2:UM
-Uruguay|UY|URY|858|ISO 3166-2:UY
-Uzbekistan|UZ|UZB|860|ISO 3166-2:UZ
-Vanuatu|VU|VUT|548|ISO 3166-2:VU
-Venezuela, Bolivarian Republic of|VE|VEN|862|ISO 3166-2:VE
-Viet Nam|VN|VNM|704|ISO 3166-2:VN
-Virgin Islands, British|VG|VGB|092|ISO 3166-2:VG
-Virgin Islands, U.S|VI|VIR|850|ISO 3166-2:VI
-Wallis and Futuna|WF|WLF|876|ISO 3166-2:WF
-Western Sahara|EH|ESH|732|ISO 3166-2:EH
-Yemen|YE|YEM|887|ISO 3166-2:YE
-Zambia|ZM|ZMB|894|ISO 3166-2:ZM
-Zimbabwe|ZW|ZWE|716|ISO 3166-2:ZW
diff --git a/lib/guessit/ISO-639-2_utf-8.txt b/lib/guessit/ISO-639-2_utf-8.txt
deleted file mode 100644
index 2961d219f391d7cc2ccdb943cd3de4597298d7dd..0000000000000000000000000000000000000000
--- a/lib/guessit/ISO-639-2_utf-8.txt
+++ /dev/null
@@ -1,485 +0,0 @@
-aar||aa|Afar|afar
-abk||ab|Abkhazian|abkhaze
-ace|||Achinese|aceh
-ach|||Acoli|acoli
-ada|||Adangme|adangme
-ady|||Adyghe; Adygei|adyghé
-afa|||Afro-Asiatic languages|afro-asiatiques, langues
-afh|||Afrihili|afrihili
-afr||af|Afrikaans|afrikaans
-ain|||Ainu|aïnou
-aka||ak|Akan|akan
-akk|||Akkadian|akkadien
-alb|sqi|sq|Albanian|albanais
-ale|||Aleut|aléoute
-alg|||Algonquian languages|algonquines, langues
-alt|||Southern Altai|altai du Sud
-amh||am|Amharic|amharique
-ang|||English, Old (ca.450-1100)|anglo-saxon (ca.450-1100)
-anp|||Angika|angika
-apa|||Apache languages|apaches, langues
-ara||ar|Arabic|arabe
-arc|||Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)|araméen d'empire (700-300 BCE)
-arg||an|Aragonese|aragonais
-arm|hye|hy|Armenian|arménien
-arn|||Mapudungun; Mapuche|mapudungun; mapuche; mapuce
-arp|||Arapaho|arapaho
-art|||Artificial languages|artificielles, langues
-arw|||Arawak|arawak
-asm||as|Assamese|assamais
-ast|||Asturian; Bable; Leonese; Asturleonese|asturien; bable; léonais; asturoléonais
-ath|||Athapascan languages|athapascanes, langues
-aus|||Australian languages|australiennes, langues
-ava||av|Avaric|avar
-ave||ae|Avestan|avestique
-awa|||Awadhi|awadhi
-aym||ay|Aymara|aymara
-aze||az|Azerbaijani|azéri
-bad|||Banda languages|banda, langues
-bai|||Bamileke languages|bamiléké, langues
-bak||ba|Bashkir|bachkir
-bal|||Baluchi|baloutchi
-bam||bm|Bambara|bambara
-ban|||Balinese|balinais
-baq|eus|eu|Basque|basque
-bas|||Basa|basa
-bat|||Baltic languages|baltes, langues
-bej|||Beja; Bedawiyet|bedja
-bel||be|Belarusian|biélorusse
-bem|||Bemba|bemba
-ben||bn|Bengali|bengali
-ber|||Berber languages|berbères, langues
-bho|||Bhojpuri|bhojpuri
-bih||bh|Bihari languages|langues biharis
-bik|||Bikol|bikol
-bin|||Bini; Edo|bini; edo
-bis||bi|Bislama|bichlamar
-bla|||Siksika|blackfoot
-bnt|||Bantu (Other)|bantoues, autres langues
-bos||bs|Bosnian|bosniaque
-bra|||Braj|braj
-bre||br|Breton|breton
-btk|||Batak languages|batak, langues
-bua|||Buriat|bouriate
-bug|||Buginese|bugi
-bul||bg|Bulgarian|bulgare
-bur|mya|my|Burmese|birman
-byn|||Blin; Bilin|blin; bilen
-cad|||Caddo|caddo
-cai|||Central American Indian languages|amérindiennes de L'Amérique centrale, langues
-car|||Galibi Carib|karib; galibi; carib
-cat||ca|Catalan; Valencian|catalan; valencien
-cau|||Caucasian languages|caucasiennes, langues
-ceb|||Cebuano|cebuano
-cel|||Celtic languages|celtiques, langues; celtes, langues
-cha||ch|Chamorro|chamorro
-chb|||Chibcha|chibcha
-che||ce|Chechen|tchétchène
-chg|||Chagatai|djaghataï
-chi|zho|zh|Chinese|chinois
-chk|||Chuukese|chuuk
-chm|||Mari|mari
-chn|||Chinook jargon|chinook, jargon
-cho|||Choctaw|choctaw
-chp|||Chipewyan; Dene Suline|chipewyan
-chr|||Cherokee|cherokee
-chu||cu|Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic|slavon d'église; vieux slave; slavon liturgique; vieux bulgare
-chv||cv|Chuvash|tchouvache
-chy|||Cheyenne|cheyenne
-cmc|||Chamic languages|chames, langues
-cop|||Coptic|copte
-cor||kw|Cornish|cornique
-cos||co|Corsican|corse
-cpe|||Creoles and pidgins, English based|créoles et pidgins basés sur l'anglais
-cpf|||Creoles and pidgins, French-based |créoles et pidgins basés sur le français
-cpp|||Creoles and pidgins, Portuguese-based |créoles et pidgins basés sur le portugais
-cre||cr|Cree|cree
-crh|||Crimean Tatar; Crimean Turkish|tatar de Crimé
-crp|||Creoles and pidgins |créoles et pidgins
-csb|||Kashubian|kachoube
-cus|||Cushitic languages|couchitiques, langues
-cze|ces|cs|Czech|tchèque
-dak|||Dakota|dakota
-dan||da|Danish|danois
-dar|||Dargwa|dargwa
-day|||Land Dayak languages|dayak, langues
-del|||Delaware|delaware
-den|||Slave (Athapascan)|esclave (athapascan)
-dgr|||Dogrib|dogrib
-din|||Dinka|dinka
-div||dv|Divehi; Dhivehi; Maldivian|maldivien
-doi|||Dogri|dogri
-dra|||Dravidian languages|dravidiennes, langues
-dsb|||Lower Sorbian|bas-sorabe
-dua|||Duala|douala
-dum|||Dutch, Middle (ca.1050-1350)|néerlandais moyen (ca. 1050-1350)
-dut|nld|nl|Dutch; Flemish|néerlandais; flamand
-dyu|||Dyula|dioula
-dzo||dz|Dzongkha|dzongkha
-efi|||Efik|efik
-egy|||Egyptian (Ancient)|égyptien
-eka|||Ekajuk|ekajuk
-elx|||Elamite|élamite
-eng||en|English|anglais
-enm|||English, Middle (1100-1500)|anglais moyen (1100-1500)
-epo||eo|Esperanto|espéranto
-est||et|Estonian|estonien
-ewe||ee|Ewe|éwé
-ewo|||Ewondo|éwondo
-fan|||Fang|fang
-fao||fo|Faroese|féroïen
-fat|||Fanti|fanti
-fij||fj|Fijian|fidjien
-fil|||Filipino; Pilipino|filipino; pilipino
-fin||fi|Finnish|finnois
-fiu|||Finno-Ugrian languages|finno-ougriennes, langues
-fon|||Fon|fon
-fre|fra|fr|French|français
-frm|||French, Middle (ca.1400-1600)|français moyen (1400-1600)
-fro|||French, Old (842-ca.1400)|français ancien (842-ca.1400)
-frr|||Northern Frisian|frison septentrional
-frs|||Eastern Frisian|frison oriental
-fry||fy|Western Frisian|frison occidental
-ful||ff|Fulah|peul
-fur|||Friulian|frioulan
-gaa|||Ga|ga
-gay|||Gayo|gayo
-gba|||Gbaya|gbaya
-gem|||Germanic languages|germaniques, langues
-geo|kat|ka|Georgian|géorgien
-ger|deu|de|German|allemand
-gez|||Geez|guèze
-gil|||Gilbertese|kiribati
-gla||gd|Gaelic; Scottish Gaelic|gaélique; gaélique écossais
-gle||ga|Irish|irlandais
-glg||gl|Galician|galicien
-glv||gv|Manx|manx; mannois
-gmh|||German, Middle High (ca.1050-1500)|allemand, moyen haut (ca. 1050-1500)
-goh|||German, Old High (ca.750-1050)|allemand, vieux haut (ca. 750-1050)
-gon|||Gondi|gond
-gor|||Gorontalo|gorontalo
-got|||Gothic|gothique
-grb|||Grebo|grebo
-grc|||Greek, Ancient (to 1453)|grec ancien (jusqu'à 1453)
-gre|ell|el|Greek, Modern (1453-)|grec moderne (après 1453)
-grn||gn|Guarani|guarani
-gsw|||Swiss German; Alemannic; Alsatian|suisse alémanique; alémanique; alsacien
-guj||gu|Gujarati|goudjrati
-gwi|||Gwich'in|gwich'in
-hai|||Haida|haida
-hat||ht|Haitian; Haitian Creole|haïtien; créole haïtien
-hau||ha|Hausa|haoussa
-haw|||Hawaiian|hawaïen
-heb||he|Hebrew|hébreu
-her||hz|Herero|herero
-hil|||Hiligaynon|hiligaynon
-him|||Himachali languages; Western Pahari languages|langues himachalis; langues paharis occidentales
-hin||hi|Hindi|hindi
-hit|||Hittite|hittite
-hmn|||Hmong; Mong|hmong
-hmo||ho|Hiri Motu|hiri motu
-hrv||hr|Croatian|croate
-hsb|||Upper Sorbian|haut-sorabe
-hun||hu|Hungarian|hongrois
-hup|||Hupa|hupa
-iba|||Iban|iban
-ibo||ig|Igbo|igbo
-ice|isl|is|Icelandic|islandais
-ido||io|Ido|ido
-iii||ii|Sichuan Yi; Nuosu|yi de Sichuan
-ijo|||Ijo languages|ijo, langues
-iku||iu|Inuktitut|inuktitut
-ile||ie|Interlingue; Occidental|interlingue
-ilo|||Iloko|ilocano
-ina||ia|Interlingua (International Auxiliary Language Association)|interlingua (langue auxiliaire internationale)
-inc|||Indic languages|indo-aryennes, langues
-ind||id|Indonesian|indonésien
-ine|||Indo-European languages|indo-européennes, langues
-inh|||Ingush|ingouche
-ipk||ik|Inupiaq|inupiaq
-ira|||Iranian languages|iraniennes, langues
-iro|||Iroquoian languages|iroquoises, langues
-ita||it|Italian|italien
-jav||jv|Javanese|javanais
-jbo|||Lojban|lojban
-jpn||ja|Japanese|japonais
-jpr|||Judeo-Persian|judéo-persan
-jrb|||Judeo-Arabic|judéo-arabe
-kaa|||Kara-Kalpak|karakalpak
-kab|||Kabyle|kabyle
-kac|||Kachin; Jingpho|kachin; jingpho
-kal||kl|Kalaallisut; Greenlandic|groenlandais
-kam|||Kamba|kamba
-kan||kn|Kannada|kannada
-kar|||Karen languages|karen, langues
-kas||ks|Kashmiri|kashmiri
-kau||kr|Kanuri|kanouri
-kaw|||Kawi|kawi
-kaz||kk|Kazakh|kazakh
-kbd|||Kabardian|kabardien
-kha|||Khasi|khasi
-khi|||Khoisan languages|khoïsan, langues
-khm||km|Central Khmer|khmer central
-kho|||Khotanese; Sakan|khotanais; sakan
-kik||ki|Kikuyu; Gikuyu|kikuyu
-kin||rw|Kinyarwanda|rwanda
-kir||ky|Kirghiz; Kyrgyz|kirghiz
-kmb|||Kimbundu|kimbundu
-kok|||Konkani|konkani
-kom||kv|Komi|kom
-kon||kg|Kongo|kongo
-kor||ko|Korean|coréen
-kos|||Kosraean|kosrae
-kpe|||Kpelle|kpellé
-krc|||Karachay-Balkar|karatchai balkar
-krl|||Karelian|carélien
-kro|||Kru languages|krou, langues
-kru|||Kurukh|kurukh
-kua||kj|Kuanyama; Kwanyama|kuanyama; kwanyama
-kum|||Kumyk|koumyk
-kur||ku|Kurdish|kurde
-kut|||Kutenai|kutenai
-lad|||Ladino|judéo-espagnol
-lah|||Lahnda|lahnda
-lam|||Lamba|lamba
-lao||lo|Lao|lao
-lat||la|Latin|latin
-lav||lv|Latvian|letton
-lez|||Lezghian|lezghien
-lim||li|Limburgan; Limburger; Limburgish|limbourgeois
-lin||ln|Lingala|lingala
-lit||lt|Lithuanian|lituanien
-lol|||Mongo|mongo
-loz|||Lozi|lozi
-ltz||lb|Luxembourgish; Letzeburgesch|luxembourgeois
-lua|||Luba-Lulua|luba-lulua
-lub||lu|Luba-Katanga|luba-katanga
-lug||lg|Ganda|ganda
-lui|||Luiseno|luiseno
-lun|||Lunda|lunda
-luo|||Luo (Kenya and Tanzania)|luo (Kenya et Tanzanie)
-lus|||Lushai|lushai
-mac|mkd|mk|Macedonian|macédonien
-mad|||Madurese|madourais
-mag|||Magahi|magahi
-mah||mh|Marshallese|marshall
-mai|||Maithili|maithili
-mak|||Makasar|makassar
-mal||ml|Malayalam|malayalam
-man|||Mandingo|mandingue
-mao|mri|mi|Maori|maori
-map|||Austronesian languages|austronésiennes, langues
-mar||mr|Marathi|marathe
-mas|||Masai|massaï
-may|msa|ms|Malay|malais
-mdf|||Moksha|moksa
-mdr|||Mandar|mandar
-men|||Mende|mendé
-mga|||Irish, Middle (900-1200)|irlandais moyen (900-1200)
-mic|||Mi'kmaq; Micmac|mi'kmaq; micmac
-min|||Minangkabau|minangkabau
-mis|||Uncoded languages|langues non codées
-mkh|||Mon-Khmer languages|môn-khmer, langues
-mlg||mg|Malagasy|malgache
-mlt||mt|Maltese|maltais
-mnc|||Manchu|mandchou
-mni|||Manipuri|manipuri
-mno|||Manobo languages|manobo, langues
-moh|||Mohawk|mohawk
-mon||mn|Mongolian|mongol
-mos|||Mossi|moré
-mul|||Multiple languages|multilingue
-mun|||Munda languages|mounda, langues
-mus|||Creek|muskogee
-mwl|||Mirandese|mirandais
-mwr|||Marwari|marvari
-myn|||Mayan languages|maya, langues
-myv|||Erzya|erza
-nah|||Nahuatl languages|nahuatl, langues
-nai|||North American Indian languages|nord-amérindiennes, langues
-nap|||Neapolitan|napolitain
-nau||na|Nauru|nauruan
-nav||nv|Navajo; Navaho|navaho
-nbl||nr|Ndebele, South; South Ndebele|ndébélé du Sud
-nde||nd|Ndebele, North; North Ndebele|ndébélé du Nord
-ndo||ng|Ndonga|ndonga
-nds|||Low German; Low Saxon; German, Low; Saxon, Low|bas allemand; bas saxon; allemand, bas; saxon, bas
-nep||ne|Nepali|népalais
-new|||Nepal Bhasa; Newari|nepal bhasa; newari
-nia|||Nias|nias
-nic|||Niger-Kordofanian languages|nigéro-kordofaniennes, langues
-niu|||Niuean|niué
-nno||nn|Norwegian Nynorsk; Nynorsk, Norwegian|norvégien nynorsk; nynorsk, norvégien
-nob||nb|Bokmål, Norwegian; Norwegian Bokmål|norvégien bokmål
-nog|||Nogai|nogaï; nogay
-non|||Norse, Old|norrois, vieux
-nor||no|Norwegian|norvégien
-nqo|||N'Ko|n'ko
-nso|||Pedi; Sepedi; Northern Sotho|pedi; sepedi; sotho du Nord
-nub|||Nubian languages|nubiennes, langues
-nwc|||Classical Newari; Old Newari; Classical Nepal Bhasa|newari classique
-nya||ny|Chichewa; Chewa; Nyanja|chichewa; chewa; nyanja
-nym|||Nyamwezi|nyamwezi
-nyn|||Nyankole|nyankolé
-nyo|||Nyoro|nyoro
-nzi|||Nzima|nzema
-oci||oc|Occitan (post 1500); Provençal|occitan (après 1500); provençal
-oji||oj|Ojibwa|ojibwa
-ori||or|Oriya|oriya
-orm||om|Oromo|galla
-osa|||Osage|osage
-oss||os|Ossetian; Ossetic|ossète
-ota|||Turkish, Ottoman (1500-1928)|turc ottoman (1500-1928)
-oto|||Otomian languages|otomi, langues
-paa|||Papuan languages|papoues, langues
-pag|||Pangasinan|pangasinan
-pal|||Pahlavi|pahlavi
-pam|||Pampanga; Kapampangan|pampangan
-pan||pa|Panjabi; Punjabi|pendjabi
-pap|||Papiamento|papiamento
-pau|||Palauan|palau
-peo|||Persian, Old (ca.600-400 B.C.)|perse, vieux (ca. 600-400 av. J.-C.)
-per|fas|fa|Persian|persan
-phi|||Philippine languages|philippines, langues
-phn|||Phoenician|phénicien
-pli||pi|Pali|pali
-pol||pl|Polish|polonais
-pon|||Pohnpeian|pohnpei
-por||pt|Portuguese|portugais
-pra|||Prakrit languages|prâkrit, langues
-pro|||Provençal, Old (to 1500)|provençal ancien (jusqu'à 1500)
-pus||ps|Pushto; Pashto|pachto
-qaa-qtz|||Reserved for local use|réservée à l'usage local
-que||qu|Quechua|quechua
-raj|||Rajasthani|rajasthani
-rap|||Rapanui|rapanui
-rar|||Rarotongan; Cook Islands Maori|rarotonga; maori des îles Cook
-roa|||Romance languages|romanes, langues
-roh||rm|Romansh|romanche
-rom|||Romany|tsigane
-rum|ron|ro|Romanian; Moldavian; Moldovan|roumain; moldave
-run||rn|Rundi|rundi
-rup|||Aromanian; Arumanian; Macedo-Romanian|aroumain; macédo-roumain
-rus||ru|Russian|russe
-sad|||Sandawe|sandawe
-sag||sg|Sango|sango
-sah|||Yakut|iakoute
-sai|||South American Indian (Other)|indiennes d'Amérique du Sud, autres langues
-sal|||Salishan languages|salishennes, langues
-sam|||Samaritan Aramaic|samaritain
-san||sa|Sanskrit|sanskrit
-sas|||Sasak|sasak
-sat|||Santali|santal
-scn|||Sicilian|sicilien
-sco|||Scots|écossais
-sel|||Selkup|selkoupe
-sem|||Semitic languages|sémitiques, langues
-sga|||Irish, Old (to 900)|irlandais ancien (jusqu'à 900)
-sgn|||Sign Languages|langues des signes
-shn|||Shan|chan
-sid|||Sidamo|sidamo
-sin||si|Sinhala; Sinhalese|singhalais
-sio|||Siouan languages|sioux, langues
-sit|||Sino-Tibetan languages|sino-tibétaines, langues
-sla|||Slavic languages|slaves, langues
-slo|slk|sk|Slovak|slovaque
-slv||sl|Slovenian|slovène
-sma|||Southern Sami|sami du Sud
-sme||se|Northern Sami|sami du Nord
-smi|||Sami languages|sames, langues
-smj|||Lule Sami|sami de Lule
-smn|||Inari Sami|sami d'Inari
-smo||sm|Samoan|samoan
-sms|||Skolt Sami|sami skolt
-sna||sn|Shona|shona
-snd||sd|Sindhi|sindhi
-snk|||Soninke|soninké
-sog|||Sogdian|sogdien
-som||so|Somali|somali
-son|||Songhai languages|songhai, langues
-sot||st|Sotho, Southern|sotho du Sud
-spa||es|Spanish; Castilian|espagnol; castillan
-srd||sc|Sardinian|sarde
-srn|||Sranan Tongo|sranan tongo
-srp||sr|Serbian|serbe
-srr|||Serer|sérère
-ssa|||Nilo-Saharan languages|nilo-sahariennes, langues
-ssw||ss|Swati|swati
-suk|||Sukuma|sukuma
-sun||su|Sundanese|soundanais
-sus|||Susu|soussou
-sux|||Sumerian|sumérien
-swa||sw|Swahili|swahili
-swe||sv|Swedish|suédois
-syc|||Classical Syriac|syriaque classique
-syr|||Syriac|syriaque
-tah||ty|Tahitian|tahitien
-tai|||Tai languages|tai, langues
-tam||ta|Tamil|tamoul
-tat||tt|Tatar|tatar
-tel||te|Telugu|télougou
-tem|||Timne|temne
-ter|||Tereno|tereno
-tet|||Tetum|tetum
-tgk||tg|Tajik|tadjik
-tgl||tl|Tagalog|tagalog
-tha||th|Thai|thaï
-tib|bod|bo|Tibetan|tibétain
-tig|||Tigre|tigré
-tir||ti|Tigrinya|tigrigna
-tiv|||Tiv|tiv
-tkl|||Tokelau|tokelau
-tlh|||Klingon; tlhIngan-Hol|klingon
-tli|||Tlingit|tlingit
-tmh|||Tamashek|tamacheq
-tog|||Tonga (Nyasa)|tonga (Nyasa)
-ton||to|Tonga (Tonga Islands)|tongan (Îles Tonga)
-tpi|||Tok Pisin|tok pisin
-tsi|||Tsimshian|tsimshian
-tsn||tn|Tswana|tswana
-tso||ts|Tsonga|tsonga
-tuk||tk|Turkmen|turkmène
-tum|||Tumbuka|tumbuka
-tup|||Tupi languages|tupi, langues
-tur||tr|Turkish|turc
-tut|||Altaic languages|altaïques, langues
-tvl|||Tuvalu|tuvalu
-twi||tw|Twi|twi
-tyv|||Tuvinian|touva
-udm|||Udmurt|oudmourte
-uga|||Ugaritic|ougaritique
-uig||ug|Uighur; Uyghur|ouïgour
-ukr||uk|Ukrainian|ukrainien
-umb|||Umbundu|umbundu
-und|||Undetermined|indéterminée
-urd||ur|Urdu|ourdou
-uzb||uz|Uzbek|ouszbek
-vai|||Vai|vaï
-ven||ve|Venda|venda
-vie||vi|Vietnamese|vietnamien
-vol||vo|Volapük|volapük
-vot|||Votic|vote
-wak|||Wakashan languages|wakashanes, langues
-wal|||Walamo|walamo
-war|||Waray|waray
-was|||Washo|washo
-wel|cym|cy|Welsh|gallois
-wen|||Sorbian languages|sorabes, langues
-wln||wa|Walloon|wallon
-wol||wo|Wolof|wolof
-xal|||Kalmyk; Oirat|kalmouk; oïrat
-xho||xh|Xhosa|xhosa
-yao|||Yao|yao
-yap|||Yapese|yapois
-yid||yi|Yiddish|yiddish
-yor||yo|Yoruba|yoruba
-ypk|||Yupik languages|yupik, langues
-zap|||Zapotec|zapotèque
-zbl|||Blissymbols; Blissymbolics; Bliss|symboles Bliss; Bliss
-zen|||Zenaga|zenaga
-zha||za|Zhuang; Chuang|zhuang; chuang
-znd|||Zande languages|zandé, langues
-zul||zu|Zulu|zoulou
-zun|||Zuni|zuni
-zxx|||No linguistic content; Not applicable|pas de contenu linguistique; non applicable
-zza|||Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki|zaza; dimili; dimli; kirdki; kirmanjki; zazaki
\ No newline at end of file
diff --git a/lib/guessit/__init__.py b/lib/guessit/__init__.py
index 30478c4a3a59c6ff38a72f65a981f0b2997e8dd7..c8b6e9065c014e93ec668e3ee4dbe0772588014d 100644
--- a/lib/guessit/__init__.py
+++ b/lib/guessit/__init__.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,21 +18,22 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from .__version__ import __version__
-__version__ = '0.7.dev0'
__all__ = ['Guess', 'Language',
'guess_file_info', 'guess_video_info',
- 'guess_movie_info', 'guess_episode_info']
+ 'guess_movie_info', 'guess_episode_info',
+ 'default_options']
# Do python3 detection before importing any other module, to be sure that
# it will then always be available
# with code from http://lucumr.pocoo.org/2011/1/22/forwards-compatible-python/
import sys
-
-if sys.version_info[0] >= 3:
- PY3 = True
+if sys.version_info[0] >= 3: # pragma: no cover
+ PY2, PY3 = False, True
unicode_text_type = str
native_text_type = str
base_text_type = str
@@ -45,14 +46,13 @@ if sys.version_info[0] >= 3:
class UnicodeMixin(object):
__str__ = lambda x: x.__unicode__()
-
import binascii
def to_hex(x):
return binascii.hexlify(x).decode('utf-8')
-else:
- PY3 = False
+else: # pragma: no cover
+ PY2, PY3 = True, False
__all__ = [str(s) for s in __all__] # fix imports for python2
unicode_text_type = unicode
native_text_type = str
@@ -61,6 +61,8 @@ else:
def u(x):
if isinstance(x, str):
return x.decode('utf-8')
+ if isinstance(x, list):
+ return [u(s) for s in x]
return unicode(x)
def s(x):
@@ -80,11 +82,17 @@ else:
def to_hex(x):
return x.encode('hex')
-from guessit.guess import Guess, merge_all
+ range = xrange
+
+
+from guessit.guess import Guess, smart_merge
from guessit.language import Language
from guessit.matcher import IterativeMatcher
-from guessit.textutils import clean_string
+from guessit.textutils import clean_default, is_camel, from_camel
+import babelfish
+import os.path
import logging
+from copy import deepcopy
log = logging.getLogger(__name__)
@@ -98,138 +106,207 @@ h = NullHandler()
log.addHandler(h)
-def _guess_filename(filename, filetype):
- def find_nodes(tree, props):
- """Yields all nodes containing any of the given props."""
- if isinstance(props, base_text_type):
- props = [props]
- for node in tree.nodes():
- if any(prop in node.guess for prop in props):
- yield node
-
- def warning(title):
- log.warning('%s, guesses: %s - %s' % (title, m.nice_string(), m2.nice_string()))
- return m
-
- mtree = IterativeMatcher(filename, filetype=filetype)
-
- # if there are multiple possible years found, we assume the first one is
- # part of the title, reparse the tree taking this into account
- years = set(n.value for n in find_nodes(mtree.match_tree, 'year'))
- if len(years) >= 2:
- mtree = IterativeMatcher(filename, filetype=filetype,
- opts=['skip_first_year'])
-
- m = mtree.matched()
-
- if 'language' not in m and 'subtitleLanguage' not in m:
- return m
-
- # if we found some language, make sure we didn't cut a title or sth...
- mtree2 = IterativeMatcher(filename, filetype=filetype,
- opts=['nolanguage', 'nocountry'])
- m2 = mtree2.matched()
-
- if m.get('title') is None:
- return m
-
- if m.get('title') != m2.get('title'):
- title = next(find_nodes(mtree.match_tree, 'title'))
- title2 = next(find_nodes(mtree2.match_tree, 'title'))
-
- langs = list(find_nodes(mtree.match_tree, ['language', 'subtitleLanguage']))
- if not langs:
- return warning('A weird error happened with language detection')
-
- # find the language that is likely more relevant
- for lng in langs:
- if lng.value in title2.value:
- # if the language was detected as part of a potential title,
- # look at this one in particular
- lang = lng
- break
- else:
- # pick the first one if we don't have a better choice
- lang = langs[0]
-
-
- # language code are rarely part of a title, and those
- # should be handled by the Language exceptions anyway
- if len(lang.value) <= 3:
- return m
-
-
- # if filetype is subtitle and the language appears last, just before
- # the extension, then it is likely a subtitle language
- parts = clean_string(title.root.value).split()
- if (m['type'] in ['moviesubtitle', 'episodesubtitle']):
- if lang.value in parts and (parts.index(lang.value) == len(parts) - 2):
- return m
-
- # if the language was in the middle of the other potential title,
- # keep the other title (eg: The Italian Job), except if it is at the
- # very beginning, in which case we consider it an error
- if m2['title'].startswith(lang.value):
- return m
- elif lang.value in title2.value:
- return m2
-
- # if a node is in an explicit group, then the correct title is probably
- # the other one
- if title.root.node_at(title.node_idx[:2]).is_explicit():
- return m2
- elif title2.root.node_at(title2.node_idx[:2]).is_explicit():
- return m
-
- return warning('Not sure of the title because of the language position')
-
- return m
-
-
-def guess_file_info(filename, filetype, info=None):
+def _guess_filename(filename, options=None, **kwargs):
+ mtree = _build_filename_mtree(filename, options=options, **kwargs)
+ if options.get('split_camel'):
+ _add_camel_properties(mtree, options=options)
+ return mtree.matched()
+
+
+def _build_filename_mtree(filename, options=None, **kwargs):
+ mtree = IterativeMatcher(filename, options=options, **kwargs)
+ second_pass_options = mtree.second_pass_options
+ if second_pass_options:
+ log.debug("Running 2nd pass")
+ merged_options = dict(options)
+ merged_options.update(second_pass_options)
+ mtree = IterativeMatcher(filename, options=merged_options, **kwargs)
+ return mtree
+
+
+def _add_camel_properties(mtree, options=None, **kwargs):
+ prop = 'title' if mtree.matched().get('type') != 'episode' else 'series'
+ value = mtree.matched().get(prop)
+ _guess_camel_string(mtree, value, options=options, skip_title=False, **kwargs)
+
+ for leaf in mtree.match_tree.unidentified_leaves():
+ value = leaf.value
+ _guess_camel_string(mtree, value, options=options, skip_title=True, **kwargs)
+
+
+def _guess_camel_string(mtree, string, options=None, skip_title=False, **kwargs):
+ if string and is_camel(string):
+ log.debug('"%s" is camel cased. Try to detect more properties.' % (string,))
+ uncameled_value = from_camel(string)
+ merged_options = dict(options)
+ if 'type' in mtree.match_tree.info:
+ current_type = mtree.match_tree.info.get('type')
+ if current_type and current_type != 'unknown':
+ merged_options['type'] = current_type
+ camel_tree = _build_filename_mtree(uncameled_value, options=merged_options, name_only=True, skip_title=skip_title, **kwargs)
+ if len(camel_tree.matched()) > 0:
+ mtree.matched().update(camel_tree.matched())
+ return True
+ return False
+
+
+def guess_video_metadata(filename):
+ """Gets the video metadata properties out of a given file. The file needs to
+ exist on the filesystem to be able to be analyzed. An empty guess is
+ returned otherwise.
+
+ You need to have the Enzyme python package installed for this to work."""
+ result = Guess()
+
+ def found(prop, value):
+ result[prop] = value
+ log.debug('Found with enzyme %s: %s' % (prop, value))
+
+ # first get the size of the file, in bytes
+ try:
+ size = os.stat(filename).st_size
+ found('fileSize', size)
+
+ except Exception as e:
+ log.error('Cannot get video file size: %s' % e)
+ # file probably does not exist, we might as well return now
+ return result
+
+ # then get additional metadata from the file using enzyme, if available
+ try:
+ import enzyme
+
+ with open(filename) as f:
+ mkv = enzyme.MKV(f)
+
+ found('duration', mkv.info.duration.total_seconds())
+
+ if mkv.video_tracks:
+ video_track = mkv.video_tracks[0]
+
+ # resolution
+ if video_track.height in (480, 720, 1080):
+ if video_track.interlaced:
+ found('screenSize', '%di' % video_track.height)
+ else:
+ found('screenSize', '%dp' % video_track.height)
+ else:
+ # TODO: do we want this?
+ #found('screenSize', '%dx%d' % (video_track.width, video_track.height))
+ pass
+
+ # video codec
+ if video_track.codec_id == 'V_MPEG4/ISO/AVC':
+ found('videoCodec', 'h264')
+ elif video_track.codec_id == 'V_MPEG4/ISO/SP':
+ found('videoCodec', 'DivX')
+ elif video_track.codec_id == 'V_MPEG4/ISO/ASP':
+ found('videoCodec', 'XviD')
+
+ else:
+ log.warning('MKV has no video track')
+
+ if mkv.audio_tracks:
+ audio_track = mkv.audio_tracks[0]
+ # audio codec
+ if audio_track.codec_id == 'A_AC3':
+ found('audioCodec', 'AC3')
+ elif audio_track.codec_id == 'A_DTS':
+ found('audioCodec', 'DTS')
+ elif audio_track.codec_id == 'A_AAC':
+ found('audioCodec', 'AAC')
+ else:
+ log.warning('MKV has no audio track')
+
+ if mkv.subtitle_tracks:
+ embedded_subtitle_languages = set()
+ for st in mkv.subtitle_tracks:
+ try:
+ if st.language:
+ lang = babelfish.Language.fromalpha3b(st.language)
+ elif st.name:
+ lang = babelfish.Language.fromname(st.name)
+ else:
+ lang = babelfish.Language('und')
+
+ except babelfish.Error:
+ lang = babelfish.Language('und')
+
+ embedded_subtitle_languages.add(lang)
+
+ found('subtitleLanguage', embedded_subtitle_languages)
+ else:
+ log.debug('MKV has no subtitle track')
+
+ return result
+
+ except ImportError:
+ log.error('Cannot get video file metadata, missing dependency: enzyme')
+ log.error('Please install it from PyPI, by doing eg: pip install enzyme')
+ return result
+
+ except IOError as e:
+ log.error('Could not open file: %s' % filename)
+ log.error('Make sure it exists and is available for reading on the filesystem')
+ log.error('Error: %s' % e)
+ return result
+
+ except enzyme.Error as e:
+ log.error('Cannot guess video file metadata')
+ log.error('enzyme.Error while reading file: %s' % filename)
+ log.error('Error: %s' % e)
+ return result
+
+default_options = {}
+
+
+def guess_file_info(filename, info=None, options=None, **kwargs):
"""info can contain the names of the various plugins, such as 'filename' to
detect filename info, or 'hash_md5' to get the md5 hash of the file.
- >>> guess_file_info('tests/dummy.srt', 'autodetect', info = ['hash_md5', 'hash_sha1'])
- {'hash_md5': 'e781de9b94ba2753a8e2945b2c0a123d', 'hash_sha1': 'bfd18e2f4e5d59775c2bc14d80f56971891ed620'}
+ >>> testfile = os.path.join(os.path.dirname(__file__), 'test/dummy.srt')
+ >>> g = guess_file_info(testfile, info = ['hash_md5', 'hash_sha1'])
+ >>> g['hash_md5'], g['hash_sha1']
+ ('64de6b5893cac24456c46a935ef9c359', 'a703fc0fa4518080505809bf562c6fc6f7b3c98c')
"""
+ info = info or 'filename'
+ options = options or {}
+ if default_options:
+ merged_options = deepcopy(default_options)
+ merged_options.update(options)
+ options = merged_options
+
result = []
hashers = []
# Force unicode as soon as possible
filename = u(filename)
- if info is None:
- info = ['filename']
-
if isinstance(info, base_text_type):
info = [info]
for infotype in info:
if infotype == 'filename':
- result.append(_guess_filename(filename, filetype))
+ result.append(_guess_filename(filename, options, **kwargs))
elif infotype == 'hash_mpc':
from guessit.hash_mpc import hash_file
-
try:
- result.append(Guess({'hash_mpc': hash_file(filename)},
+ result.append(Guess({infotype: hash_file(filename)},
confidence=1.0))
except Exception as e:
log.warning('Could not compute MPC-style hash because: %s' % e)
elif infotype == 'hash_ed2k':
from guessit.hash_ed2k import hash_file
-
try:
- result.append(Guess({'hash_ed2k': hash_file(filename)},
+ result.append(Guess({infotype: hash_file(filename)},
confidence=1.0))
except Exception as e:
log.warning('Could not compute ed2k hash because: %s' % e)
elif infotype.startswith('hash_'):
import hashlib
-
hashname = infotype[5:]
try:
hasher = getattr(hashlib, hashname)()
@@ -237,6 +314,11 @@ def guess_file_info(filename, filetype, info=None):
except AttributeError:
log.warning('Could not compute %s hash because it is not available from python\'s hashlib module' % hashname)
+ elif infotype == 'video':
+ g = guess_video_metadata(filename)
+ if g:
+ result.append(g)
+
else:
log.warning('Invalid infotype: %s' % infotype)
@@ -259,24 +341,18 @@ def guess_file_info(filename, filetype, info=None):
except Exception as e:
log.warning('Could not compute hash because: %s' % e)
- result = merge_all(result)
-
- # last minute adjustments
-
- # if country is in the guessed properties, make it part of the filename
- if 'series' in result and 'country' in result:
- result['series'] += ' (%s)' % result['country'].alpha2.upper()
+ result = smart_merge(result)
return result
-def guess_video_info(filename, info=None):
- return guess_file_info(filename, 'autodetect', info)
+def guess_video_info(filename, info=None, options=None, **kwargs):
+ return guess_file_info(filename, info=info, options=options, type='video', **kwargs)
-def guess_movie_info(filename, info=None):
- return guess_file_info(filename, 'movie', info)
+def guess_movie_info(filename, info=None, options=None, **kwargs):
+ return guess_file_info(filename, info=info, options=options, type='movie', **kwargs)
-def guess_episode_info(filename, info=None):
- return guess_file_info(filename, 'episode', info)
+def guess_episode_info(filename, info=None, options=None, **kwargs):
+ return guess_file_info(filename, info=info, options=options, type='episode', **kwargs)
diff --git a/lib/guessit/__main__.py b/lib/guessit/__main__.py
index 31a2f74a6b78398e6e0b17b79a77f755e61dfb5d..a44b7d5c8ed1c9a7ab2bedf79f3486018bdcc15b 100644
--- a/lib/guessit/__main__.py
+++ b/lib/guessit/__main__.py
@@ -1,8 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,98 +19,267 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from __future__ import print_function
-from guessit import u
-from guessit import slogging, guess_file_info
-from optparse import OptionParser
+from __future__ import absolute_import, division, print_function, unicode_literals
+from collections import defaultdict
import logging
+import os
+from guessit import PY2, u, guess_file_info
+from guessit.options import get_opts
+from guessit.__version__ import __version__
-def detect_filename(filename, filetype, info=['filename']):
+
+def guess_file(filename, info='filename', options=None, **kwargs):
+ options = options or {}
filename = u(filename)
- print('For:', filename)
- print('GuessIt found:', guess_file_info(filename, filetype, info).nice_string())
+ if not options.get('yaml') and not options.get('show_property'):
+ print('For:', filename)
+ guess = guess_file_info(filename, info, options, **kwargs)
+
+ if not options.get('unidentified'):
+ try:
+ del guess['unidentified']
+ except KeyError:
+ pass
+
+ if options.get('show_property'):
+ print(guess.get(options.get('show_property'), ''))
+ return
+
+ if options.get('yaml'):
+ import yaml
+ for k, v in guess.items():
+ if isinstance(v, list) and len(v) == 1:
+ guess[k] = v[0]
+ ystr = yaml.safe_dump({filename: dict(guess)}, default_flow_style=False, allow_unicode=True)
+ i = 0
+ for yline in ystr.splitlines():
+ if i == 0:
+ print("? " + yline[:-1])
+ elif i == 1:
+ print(":" + yline[1:])
+ else:
+ print(yline)
+ i += 1
+ return
+ print('GuessIt found:', guess.nice_string(options.get('advanced')))
+
+
+def _supported_properties():
+ all_properties = defaultdict(list)
+ transformers_properties = []
+
+ from guessit.plugins import transformers
+ for transformer in transformers.all_transformers():
+ supported_properties = transformer.supported_properties()
+ transformers_properties.append((transformer, supported_properties))
+
+ if isinstance(supported_properties, dict):
+ for property_name, possible_values in supported_properties.items():
+ all_properties[property_name].extend(possible_values)
+ else:
+ for property_name in supported_properties:
+ all_properties[property_name] # just make sure it exists
+
+ return all_properties, transformers_properties
-def run_demo(episodes=True, movies=True):
+def display_transformers():
+ print('GuessIt transformers:')
+ _, transformers_properties = _supported_properties()
+ for transformer, _ in transformers_properties:
+ print('[@] %s (%s)' % (transformer.name, transformer.priority))
+
+
+def display_properties(options):
+ values = options.values
+ transformers = options.transformers
+ name_only = options.name_only
+
+ print('GuessIt properties:')
+ all_properties, transformers_properties = _supported_properties()
+ if name_only:
+ # the 'container' property does not apply when using the --name-only
+ # option
+ del all_properties['container']
+
+ if transformers:
+ for transformer, properties_list in transformers_properties:
+ print('[@] %s (%s)' % (transformer.name, transformer.priority))
+ for property_name in properties_list:
+ property_values = all_properties.get(property_name)
+ print(' [+] %s' % (property_name,))
+ if property_values and values:
+ _display_property_values(property_name, indent=4)
+ else:
+ properties_list = sorted(all_properties.keys())
+ for property_name in properties_list:
+ property_values = all_properties.get(property_name)
+ print(' [+] %s' % (property_name,))
+ if property_values and values:
+ _display_property_values(property_name, indent=4)
+
+
+def _display_property_values(property_name, indent=2):
+ all_properties, _ = _supported_properties()
+ property_values = all_properties.get(property_name)
+ for property_value in property_values:
+ print(indent * ' ' + '[!] %s' % (property_value,))
+
+
+def run_demo(episodes=True, movies=True, options=None):
# NOTE: tests should not be added here but rather in the tests/ folder
# this is just intended as a quick example
if episodes:
- testeps = [ 'Series/Californication/Season 2/Californication.2x05.Vaginatown.HDTV.XviD-0TV.[tvu.org.ru].avi',
- 'Series/dexter/Dexter.5x02.Hello,.Bandit.ENG.-.sub.FR.HDTV.XviD-AlFleNi-TeaM.[tvu.org.ru].avi',
- 'Series/Treme/Treme.1x03.Right.Place,.Wrong.Time.HDTV.XviD-NoTV.[tvu.org.ru].avi',
- 'Series/Duckman/Duckman - 101 (01) - 20021107 - I, Duckman.avi',
- 'Series/Duckman/Duckman - S1E13 Joking The Chicken (unedited).avi',
- 'Series/Simpsons/The_simpsons_s13e18_-_i_am_furious_yellow.mpg',
- 'Series/Simpsons/Saison 12 Français/Simpsons,.The.12x08.A.Bas.Le.Sergent.Skinner.FR.[tvu.org.ru].avi',
- 'Series/Dr._Slump_-_002_DVB-Rip_Catalan_by_kelf.avi',
- 'Series/Kaamelott/Kaamelott - Livre V - Second Volet - HD 704x396 Xvid 2 pass - Son 5.1 - TntRip by Slurm.avi'
- ]
+ testeps = ['Series/Californication/Season 2/Californication.2x05.Vaginatown.HDTV.XviD-0TV.[tvu.org.ru].avi',
+ 'Series/dexter/Dexter.5x02.Hello,.Bandit.ENG.-.sub.FR.HDTV.XviD-AlFleNi-TeaM.[tvu.org.ru].avi',
+ 'Series/Treme/Treme.1x03.Right.Place,.Wrong.Time.HDTV.XviD-NoTV.[tvu.org.ru].avi',
+ 'Series/Duckman/Duckman - 101 (01) - 20021107 - I, Duckman.avi',
+ 'Series/Duckman/Duckman - S1E13 Joking The Chicken (unedited).avi',
+ 'Series/Simpsons/The_simpsons_s13e18_-_i_am_furious_yellow.mpg',
+ 'Series/Simpsons/Saison 12 Français/Simpsons,.The.12x08.A.Bas.Le.Sergent.Skinner.FR.[tvu.org.ru].avi',
+ 'Series/Dr._Slump_-_002_DVB-Rip_Catalan_by_kelf.avi',
+ 'Series/Kaamelott/Kaamelott - Livre V - Second Volet - HD 704x396 Xvid 2 pass - Son 5.1 - TntRip by Slurm.avi']
for f in testeps:
- print('-'*80)
- detect_filename(f, filetype='episode')
-
+ print('-' * 80)
+ guess_file(f, options=options, type='episode')
if movies:
- testmovies = [ 'Movies/Fear and Loathing in Las Vegas (1998)/Fear.and.Loathing.in.Las.Vegas.720p.HDDVD.DTS.x264-ESiR.mkv',
- 'Movies/El Dia de la Bestia (1995)/El.dia.de.la.bestia.DVDrip.Spanish.DivX.by.Artik[SEDG].avi',
- 'Movies/Blade Runner (1982)/Blade.Runner.(1982).(Director\'s.Cut).CD1.DVDRip.XviD.AC3-WAF.avi',
- 'Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD.mkv',
- 'Movies/Sin City (BluRay) (2005)/Sin.City.2005.BDRip.720p.x264.AC3-SEPTiC.mkv',
- 'Movies/Borat (2006)/Borat.(2006).R5.PROPER.REPACK.DVDRip.XviD-PUKKA.avi', # FIXME: PROPER and R5 get overwritten
- '[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv', # FIXME: title gets overwritten
- 'Battle Royale (2000)/Battle.Royale.(Batoru.Rowaiaru).(2000).(Special.Edition).CD1of2.DVDRiP.XviD-[ZeaL].avi',
- 'Movies/Brazil (1985)/Brazil_Criterion_Edition_(1985).CD2.English.srt',
- 'Movies/Persepolis (2007)/[XCT] Persepolis [H264+Aac-128(Fr-Eng)+ST(Fr-Eng)+Ind].mkv',
- 'Movies/Toy Story (1995)/Toy Story [HDTV 720p English-Spanish].mkv',
- 'Movies/Pirates of the Caribbean: The Curse of the Black Pearl (2003)/Pirates.Of.The.Carribean.DC.2003.iNT.DVDRip.XviD.AC3-NDRT.CD1.avi',
- 'Movies/Office Space (1999)/Office.Space.[Dual-DVDRip].[Spanish-English].[XviD-AC3-AC3].[by.Oswald].avi',
- 'Movies/The NeverEnding Story (1984)/The.NeverEnding.Story.1.1984.DVDRip.AC3.Xvid-Monteque.avi',
- 'Movies/Juno (2007)/Juno KLAXXON.avi',
- 'Movies/Chat noir, chat blanc (1998)/Chat noir, Chat blanc - Emir Kusturica (VO - VF - sub FR - Chapters).mkv',
- 'Movies/Wild Zero (2000)/Wild.Zero.DVDivX-EPiC.srt',
- 'Movies/El Bosque Animado (1987)/El.Bosque.Animado.[Jose.Luis.Cuerda.1987].[Xvid-Dvdrip-720x432].avi',
- 'testsmewt_bugs/movies/Baraka_Edition_Collector.avi'
- ]
+ testmovies = ['Movies/Fear and Loathing in Las Vegas (1998)/Fear.and.Loathing.in.Las.Vegas.720p.HDDVD.DTS.x264-ESiR.mkv',
+ 'Movies/El Dia de la Bestia (1995)/El.dia.de.la.bestia.DVDrip.Spanish.DivX.by.Artik[SEDG].avi',
+ 'Movies/Blade Runner (1982)/Blade.Runner.(1982).(Director\'s.Cut).CD1.DVDRip.XviD.AC3-WAF.avi',
+ 'Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD.mkv',
+ 'Movies/Sin City (BluRay) (2005)/Sin.City.2005.BDRip.720p.x264.AC3-SEPTiC.mkv',
+ 'Movies/Borat (2006)/Borat.(2006).R5.PROPER.REPACK.DVDRip.XviD-PUKKA.avi',
+ '[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv',
+ 'Battle Royale (2000)/Battle.Royale.(Batoru.Rowaiaru).(2000).(Special.Edition).CD1of2.DVDRiP.XviD-[ZeaL].avi',
+ 'Movies/Brazil (1985)/Brazil_Criterion_Edition_(1985).CD2.English.srt',
+ 'Movies/Persepolis (2007)/[XCT] Persepolis [H264+Aac-128(Fr-Eng)+ST(Fr-Eng)+Ind].mkv',
+ 'Movies/Toy Story (1995)/Toy Story [HDTV 720p English-Spanish].mkv',
+ 'Movies/Pirates of the Caribbean: The Curse of the Black Pearl (2003)/Pirates.Of.The.Carribean.DC.2003.iNT.DVDRip.XviD.AC3-NDRT.CD1.avi',
+ 'Movies/Office Space (1999)/Office.Space.[Dual-DVDRip].[Spanish-English].[XviD-AC3-AC3].[by.Oswald].avi',
+ 'Movies/The NeverEnding Story (1984)/The.NeverEnding.Story.1.1984.DVDRip.AC3.Xvid-Monteque.avi',
+ 'Movies/Juno (2007)/Juno KLAXXON.avi',
+ 'Movies/Chat noir, chat blanc (1998)/Chat noir, Chat blanc - Emir Kusturica (VO - VF - sub FR - Chapters).mkv',
+ 'Movies/Wild Zero (2000)/Wild.Zero.DVDivX-EPiC.srt',
+ 'Movies/El Bosque Animado (1987)/El.Bosque.Animado.[Jose.Luis.Cuerda.1987].[Xvid-Dvdrip-720x432].avi',
+ 'testsmewt_bugs/movies/Baraka_Edition_Collector.avi'
+ ]
for f in testmovies:
- print('-'*80)
- detect_filename(f, filetype = 'movie')
-
-
-def main():
- slogging.setupLogging()
-
- parser = OptionParser(usage = 'usage: %prog [options] file1 [file2...]')
- parser.add_option('-v', '--verbose', action='store_true', dest='verbose', default=False,
- help = 'display debug output')
- parser.add_option('-i', '--info', dest = 'info', default = 'filename',
- help = 'the desired information type: filename, hash_mpc or a hash from python\'s '
- 'hashlib module, such as hash_md5, hash_sha1, ...; or a list of any of '
- 'them, comma-separated')
- parser.add_option('-t', '--type', dest = 'filetype', default = 'autodetect',
- help = 'the suggested file type: movie, episode or autodetect')
- parser.add_option('-d', '--demo', action='store_true', dest='demo', default=False,
- help = 'run a few builtin tests instead of analyzing a file')
-
- options, args = parser.parse_args()
+ print('-' * 80)
+ guess_file(f, options=options, type='movie')
+
+
+def submit_bug(filename, options):
+ import requests # only import when needed
+ from requests.exceptions import RequestException
+
+ try:
+ opts = dict((k, v) for k, v in options.__dict__.items()
+ if v and k != 'submit_bug')
+
+ r = requests.post('http://localhost:5000/bugs', {'filename': filename,
+ 'version': __version__,
+ 'options': str(opts)})
+ if r.status_code == 200:
+ print('Successfully submitted file: %s' % r.text)
+ else:
+ print('Could not submit bug at the moment, please try again later.')
+
+ except RequestException as e:
+ print('Could not submit bug at the moment, please try again later.')
+
+
+def main(args=None, setup_logging=True):
+ if setup_logging:
+ from guessit import slogging
+ slogging.setup_logging()
+
+ if PY2: # pragma: no cover
+ import codecs
+ import locale
+ import sys
+
+ # see http://bugs.python.org/issue2128
+ if os.name == 'nt':
+ for i, a in enumerate(sys.argv):
+ sys.argv[i] = a.decode(locale.getpreferredencoding())
+
+ # see https://github.com/wackou/guessit/issues/43
+ # and http://stackoverflow.com/questions/4545661/unicodedecodeerror-when-redirecting-to-file
+ # Wrap sys.stdout into a StreamWriter to allow writing unicode.
+ sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
+
+ # Needed for guessit.plugins.transformers.reload() to be called.
+ from guessit.plugins import transformers
+
+ if args:
+ options = get_opts().parse_args(args)
+ else: # pragma: no cover
+ options = get_opts().parse_args()
if options.verbose:
- logging.getLogger('guessit').setLevel(logging.DEBUG)
+ logging.getLogger().setLevel(logging.DEBUG)
+
+ help_required = True
+ if options.properties or options.values:
+ display_properties(options)
+ help_required = False
+ elif options.transformers:
+ display_transformers()
+ help_required = False
if options.demo:
- run_demo(episodes=True, movies=True)
- else:
- if args:
- for filename in args:
- detect_filename(filename,
- filetype = options.filetype,
- info = options.info.split(','))
+ run_demo(episodes=True, movies=True, options=vars(options))
+ help_required = False
+ if options.version:
+ print('+-------------------------------------------------------+')
+ print('+ GuessIt ' + __version__ + (28-len(__version__)) * ' ' + '+')
+ print('+-------------------------------------------------------+')
+ print('| Please report any bug or feature request at |')
+ print('| https://github.com/wackou/guessit/issues. |')
+ print('+-------------------------------------------------------+')
+ help_required = False
+
+ if options.yaml:
+ try:
+ import yaml, babelfish
+ def default_representer(dumper, data):
+ return dumper.represent_str(str(data))
+ yaml.SafeDumper.add_representer(babelfish.Language, default_representer)
+ yaml.SafeDumper.add_representer(babelfish.Country, default_representer)
+ except ImportError: # pragma: no cover
+ print('PyYAML not found. Using default output.')
+
+ filenames = []
+ if options.filename:
+ filenames.extend(options.filename)
+ if options.input_file:
+ input_file = open(options.input_file, 'r')
+ try:
+ filenames.extend([line.strip() for line in input_file.readlines()])
+ finally:
+ input_file.close()
+
+ filenames = filter(lambda f: f, filenames)
+
+ if filenames:
+ if options.submit_bug:
+ for filename in filenames:
+ help_required = False
+ submit_bug(filename, options)
else:
- parser.print_help()
+ for filename in filenames:
+ help_required = False
+ guess_file(filename,
+ info=options.info.split(','),
+ options=vars(options))
+
+ if help_required: # pragma: no cover
+ get_opts().print_help()
if __name__ == '__main__':
main()
diff --git a/lib/guessit/__version__.py b/lib/guessit/__version__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b4dd8f8ec51d5d0c9b82fb2b1b3abbc07dc60317
--- /dev/null
+++ b/lib/guessit/__version__.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+__version__ = '0.10.4.dev0'
diff --git a/lib/guessit/containers.py b/lib/guessit/containers.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab97e6f1600e6df71aecce67eb6b9a2bc6793fea
--- /dev/null
+++ b/lib/guessit/containers.py
@@ -0,0 +1,775 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import types
+
+from .patterns import compile_pattern, sep
+from . import base_text_type
+from .guess import Guess
+
+
+def _get_span(prop, match):
+ """Retrieves span for a match"""
+ if not prop.global_span and match.re.groups:
+ start = None
+ end = None
+ for i in range(1, match.re.groups + 1):
+ span = match.span(i)
+ if start is None or span[0] < start:
+ start = span[0]
+ if end is None or span[1] > end:
+ end = span[1]
+ return start, end
+ else:
+ return match.span()
+
+
+def _trim_span(span, value, blanks = sep):
+ start, end = span
+
+ for i in range(0, len(value)):
+ if value[i] in blanks:
+ start += 1
+ else:
+ break
+
+ for i in reversed(range(0, len(value))):
+ if value[i] in blanks:
+ end -= 1
+ else:
+ break
+ if end <= start:
+ return -1, -1
+ return start, end
+
+
+def _get_groups(compiled_re):
+ """
+ Retrieves groups from re
+
+ :return: list of group names
+ """
+ if compiled_re.groups:
+ indexgroup = {}
+ for k, i in compiled_re.groupindex.items():
+ indexgroup[i] = k
+ ret = []
+ for i in range(1, compiled_re.groups + 1):
+ ret.append(indexgroup.get(i, i))
+ return ret
+ else:
+ return [None]
+
+
+class NoValidator(object):
+ @staticmethod
+ def validate(prop, string, node, match, entry_start, entry_end):
+ return True
+
+
+class LeftValidator(object):
+ """Make sure our match is starting by separator, or by another entry"""
+
+ @staticmethod
+ def validate(prop, string, node, match, entry_start, entry_end):
+ span = _get_span(prop, match)
+ span = _trim_span(span, string[span[0]:span[1]])
+ start, end = span
+
+ sep_start = start <= 0 or string[start - 1] in sep
+ start_by_other = start in entry_end
+ if not sep_start and not start_by_other:
+ return False
+ return True
+
+
+class RightValidator(object):
+ """Make sure our match is ended by separator, or by another entry"""
+
+ @staticmethod
+ def validate(prop, string, node, match, entry_start, entry_end):
+ span = _get_span(prop, match)
+ span = _trim_span(span, string[span[0]:span[1]])
+ start, end = span
+
+ sep_end = end >= len(string) or string[end] in sep
+ end_by_other = end in entry_start
+ if not sep_end and not end_by_other:
+ return False
+ return True
+
+
+class ChainedValidator(object):
+ def __init__(self, *validators):
+ self._validators = validators
+
+ def validate(self, prop, string, node, match, entry_start, entry_end):
+ for validator in self._validators:
+ if not validator.validate(prop, string, node, match, entry_start, entry_end):
+ return False
+ return True
+
+
+class SameKeyValidator(object):
+ def __init__(self, validator_function):
+ self.validator_function = validator_function
+
+ def validate(self, prop, string, node, match, entry_start, entry_end):
+ for key in prop.keys:
+ for same_value_leaf in node.root.leaves_containing(key):
+ ret = self.validator_function(same_value_leaf, key, prop, string, node, match, entry_start, entry_end)
+ if ret is not None:
+ return ret
+ return True
+
+
+class OnlyOneValidator(SameKeyValidator):
+ def __init__(self):
+ super(OnlyOneValidator, self).__init__(lambda same_value_leaf, key, prop, string, node, match, entry_start, entry_end: False)
+
+
+class DefaultValidator(object):
+ """Make sure our match is surrounded by separators, or by another entry"""
+ def validate(self, prop, string, node, match, entry_start, entry_end):
+ span = _get_span(prop, match)
+ span = _trim_span(span, string[span[0]:span[1]])
+ start, end = span
+
+ sep_start = start <= 0 or string[start - 1] in sep
+ sep_end = end >= len(string) or string[end] in sep
+ start_by_other = start in entry_end
+ end_by_other = end in entry_start
+ if (sep_start or start_by_other) and (sep_end or end_by_other):
+ return True
+ return False
+
+
+class FunctionValidator(object):
+ def __init__(self, function):
+ self.function = function
+
+ def validate(self, prop, string, node, match, entry_start, entry_end):
+ return self.function(prop, string, node, match, entry_start, entry_end)
+
+
+class FormatterValidator(object):
+ def __init__(self, group_name=None, formatted_validator=None):
+ self.group_name = group_name
+ self.formatted_validator = formatted_validator
+
+ def validate(self, prop, string, node, match, entry_start, entry_end):
+ if self.group_name:
+ formatted = prop.format(match.group(self.group_name), self.group_name)
+ else:
+ formatted = prop.format(match.group())
+ if self.formatted_validator:
+ return self.formatted_validator(formatted)
+ else:
+ return formatted
+
+
+def _get_positions(prop, string, node, match, entry_start, entry_end):
+ span = match.span()
+ start = span[0]
+ end = span[1]
+
+ at_start = True
+ at_end = True
+
+ while start > 0:
+ start -= 1
+ if string[start] not in sep:
+ at_start = False
+ break
+ while end < len(string) - 1:
+ end += 1
+ if string[end] not in sep:
+ at_end = False
+ break
+ return at_start, at_end
+
+
+class WeakValidator(DefaultValidator):
+ """Make sure our match is surrounded by separators and is the first or last element in the string"""
+ def validate(self, prop, string, node, match, entry_start, entry_end):
+ if super(WeakValidator, self).validate(prop, string, node, match, entry_start, entry_end):
+ at_start, at_end = _get_positions(prop, string, node, match, entry_start, entry_end)
+ return at_start or at_end
+ return False
+
+
+class NeighborValidator(DefaultValidator):
+ """Make sure the node is next another one"""
+ def validate(self, prop, string, node, match, entry_start, entry_end):
+ at_start, at_end = _get_positions(prop, string, node, match, entry_start, entry_end)
+
+ if at_start:
+ previous_leaf = node.root.previous_leaf(node)
+ if previous_leaf is not None:
+ return True
+
+ if at_end:
+ next_leaf = node.root.next_leaf(node)
+ if next_leaf is not None:
+ return True
+
+ return False
+
+
+class LeavesValidator(DefaultValidator):
+ def __init__(self, lambdas=None, previous_lambdas=None, next_lambdas=None, both_side=False, default_=True):
+ self.previous_lambdas = previous_lambdas if previous_lambdas is not None else []
+ self.next_lambdas = next_lambdas if next_lambdas is not None else []
+ if lambdas:
+ self.previous_lambdas.extend(lambdas)
+ self.next_lambdas.extend(lambdas)
+ self.both_side = both_side
+ self.default_ = default_
+
+ """Make sure our match is surrounded by separators and validates defined lambdas"""
+ def validate(self, prop, string, node, match, entry_start, entry_end):
+ if self.default_:
+ super_ret = super(LeavesValidator, self).validate(prop, string, node, match, entry_start, entry_end)
+ else:
+ super_ret = True
+ if not super_ret:
+ return False
+
+ previous_ = self._validate_previous(prop, string, node, match, entry_start, entry_end)
+ next_ = self._validate_next(prop, string, node, match, entry_start, entry_end)
+
+ if previous_ is None and next_ is None:
+ return super_ret
+ if self.both_side:
+ return previous_ and next_
+ else:
+ return previous_ or next_
+
+ def _validate_previous(self, prop, string, node, match, entry_start, entry_end):
+ if self.previous_lambdas:
+ for leaf in node.root.previous_leaves(node):
+ for lambda_ in self.previous_lambdas:
+ ret = self._check_rule(lambda_, leaf)
+ if ret is not None:
+ return ret
+ return False
+
+ def _validate_next(self, prop, string, node, match, entry_start, entry_end):
+ if self.next_lambdas:
+ for leaf in node.root.next_leaves(node):
+ for lambda_ in self.next_lambdas:
+ ret = self._check_rule(lambda_, leaf)
+ if ret is not None:
+ return ret
+ return False
+
+ @staticmethod
+ def _check_rule(lambda_, previous_leaf):
+ return lambda_(previous_leaf)
+
+
+class _Property:
+ """Represents a property configuration."""
+ def __init__(self, keys=None, pattern=None, canonical_form=None, canonical_from_pattern=True, confidence=1.0, enhance=True, global_span=False, validator=DefaultValidator(), formatter=None, disabler=None, confidence_lambda=None):
+ """
+ :param keys: Keys of the property (format, screenSize, ...)
+ :type keys: string
+ :param canonical_form: Unique value of the property (DVD, 720p, ...)
+ :type canonical_form: string
+ :param pattern: Regexp pattern
+ :type pattern: string
+ :param confidence: confidence
+ :type confidence: float
+ :param enhance: enhance the pattern
+ :type enhance: boolean
+ :param global_span: if True, the whole match span will used to create the Guess.
+ Else, the span from the capturing groups will be used.
+ :type global_span: boolean
+ :param validator: Validator to use
+ :type validator: :class:`DefaultValidator`
+ :param formatter: Formater to use
+ :type formatter: function
+ """
+ if isinstance(keys, list):
+ self.keys = keys
+ elif isinstance(keys, base_text_type):
+ self.keys = [keys]
+ else:
+ self.keys = []
+ self.canonical_form = canonical_form
+ if pattern is not None:
+ self.pattern = pattern
+ else:
+ self.pattern = canonical_form
+ if self.canonical_form is None and canonical_from_pattern:
+ self.canonical_form = self.pattern
+ self.compiled = compile_pattern(self.pattern, enhance=enhance)
+ for group_name in _get_groups(self.compiled):
+ if isinstance(group_name, base_text_type) and not group_name in self.keys:
+ self.keys.append(group_name)
+ if not self.keys:
+ raise ValueError("No property key is defined")
+ self.confidence = confidence
+ self.confidence_lambda = confidence_lambda
+ self.global_span = global_span
+ self.validator = validator
+ self.formatter = formatter
+ self.disabler = disabler
+
+ def disabled(self, options):
+ if self.disabler:
+ return self.disabler(options)
+ return False
+
+ def format(self, value, group_name=None):
+ """Retrieves the final value from re group match value"""
+ formatter = None
+ if isinstance(self.formatter, dict):
+ formatter = self.formatter.get(group_name)
+ if formatter is None and group_name is not None:
+ formatter = self.formatter.get(None)
+ else:
+ formatter = self.formatter
+ if isinstance(formatter, types.FunctionType):
+ return formatter(value)
+ elif formatter is not None:
+ return formatter.format(value)
+ return value
+
+ def __repr__(self):
+ return "%s: %s" % (self.keys, self.canonical_form if self.canonical_form else self.pattern)
+
+
+class PropertiesContainer(object):
+ def __init__(self, **kwargs):
+ self._properties = []
+ self.default_property_kwargs = kwargs
+
+ def unregister_property(self, name, *canonical_forms):
+ """Unregister a property canonical forms
+
+ If canonical_forms are specified, only those values will be unregistered
+
+ :param name: Property name to unregister
+ :type name: string
+ :param canonical_forms: Values to unregister
+ :type canonical_forms: varargs of string
+ """
+ _properties = [prop for prop in self._properties if prop.name == name and (not canonical_forms or prop.canonical_form in canonical_forms)]
+
+ def register_property(self, name, *patterns, **property_params):
+ """Register property with defined canonical form and patterns.
+
+ :param name: name of the property (format, screenSize, ...)
+ :type name: string
+ :param patterns: regular expression patterns to register for the property canonical_form
+ :type patterns: varargs of string
+ """
+ properties = []
+ for pattern in patterns:
+ params = dict(self.default_property_kwargs)
+ params.update(property_params)
+ if isinstance(pattern, dict):
+ params.update(pattern)
+ prop = _Property(name, **params)
+ else:
+ prop = _Property(name, pattern, **params)
+ self._properties.append(prop)
+ properties.append(prop)
+ return properties
+
+ def register_canonical_properties(self, name, *canonical_forms, **property_params):
+ """Register properties from their canonical forms.
+
+ :param name: name of the property (releaseGroup, ...)
+ :type name: string
+ :param canonical_forms: values of the property ('ESiR', 'WAF', 'SEPTiC', ...)
+ :type canonical_forms: varargs of strings
+ """
+ properties = []
+ for canonical_form in canonical_forms:
+ params = dict(property_params)
+ params['canonical_form'] = canonical_form
+ properties.extend(self.register_property(name, canonical_form, **property_params))
+ return properties
+
+ def unregister_all_properties(self):
+ """Unregister all defined properties"""
+ self._properties.clear()
+
+ def find_properties(self, string, node, options, name=None, validate=True, re_match=False, sort=True, multiple=False):
+ """Find all distinct properties for given string
+
+ If no capturing group is defined in the property, value will be grabbed from the entire match.
+
+ If one ore more unnamed capturing group is defined in the property, first capturing group will be used.
+
+ If named capturing group are defined in the property, they will be returned as property key.
+
+ If validate, found properties will be validated by their defined validator
+
+ If re_match, re.match will be used instead of re.search.
+
+ if sort, found properties will be sorted from longer match to shorter match.
+
+ If multiple is False and multiple values are found for the same property, the more confident one will be returned.
+
+ If multiple is False and multiple values are found for the same property and the same confidence, the longer will be returned.
+
+ :param string: input string
+ :type string: string
+
+ :param node: current node of the matching tree
+ :type node: :class:`guessit.matchtree.MatchTree`
+
+ :param name: name of property to find
+ :type name: string
+
+ :param re_match: use re.match instead of re.search
+ :type re_match: bool
+
+ :param multiple: Allows multiple property values to be returned
+ :type multiple: bool
+
+ :return: found properties
+ :rtype: list of tuples (:class:`_Property`, match, list of tuples (property_name, tuple(value_start, value_end)))
+
+ :see: `_Property`
+ :see: `register_property`
+ :see: `register_canonical_properties`
+ """
+ entry_start = {}
+ entry_end = {}
+
+ entries = []
+ duplicate_matches = {}
+
+ ret = []
+
+ if not string.strip():
+ return ret
+
+ # search all properties
+ for prop in self.get_properties(name):
+ if not prop.disabled(options):
+ valid_match = None
+ if re_match:
+ match = prop.compiled.match(string)
+ if match:
+ entries.append((prop, match))
+ else:
+ matches = list(prop.compiled.finditer(string))
+ duplicate_matches[prop] = matches
+ for match in matches:
+ entries.append((prop, match))
+
+ for prop, match in entries:
+ # compute confidence
+ if prop.confidence_lambda:
+ computed_confidence = prop.confidence_lambda(match)
+ if computed_confidence is not None:
+ prop.confidence = computed_confidence
+
+ if validate:
+ # compute entries start and ends
+ for prop, match in entries:
+ start, end = _get_span(prop, match)
+
+ if start not in entry_start:
+ entry_start[start] = [prop]
+ else:
+ entry_start[start].append(prop)
+
+ if end not in entry_end:
+ entry_end[end] = [prop]
+ else:
+ entry_end[end].append(prop)
+
+ # remove invalid values
+ while True:
+ invalid_entries = []
+ for entry in entries:
+ prop, match = entry
+ if not prop.validator.validate(prop, string, node, match, entry_start, entry_end):
+ invalid_entries.append(entry)
+ if not invalid_entries:
+ break
+ for entry in invalid_entries:
+ prop, match = entry
+ entries.remove(entry)
+ prop_duplicate_matches = duplicate_matches.get(prop)
+ if prop_duplicate_matches:
+ prop_duplicate_matches.remove(match)
+ invalid_span = _get_span(prop, match)
+ start = invalid_span[0]
+ end = invalid_span[1]
+ entry_start[start].remove(prop)
+ if not entry_start.get(start):
+ del entry_start[start]
+ entry_end[end].remove(prop)
+ if not entry_end.get(end):
+ del entry_end[end]
+
+ for prop, prop_duplicate_matches in duplicate_matches.items():
+ # Keeping the last valid match.
+ # Needed for the.100.109.hdtv-lol.mp4
+ for duplicate_match in prop_duplicate_matches[:-1]:
+ entries.remove((prop, duplicate_match))
+
+ if multiple:
+ ret = entries
+ else:
+ # keep only best match if multiple values where found
+ entries_dict = {}
+ for entry in entries:
+ for key in prop.keys:
+ if key not in entries_dict:
+ entries_dict[key] = []
+ entries_dict[key].append(entry)
+
+ for key_entries in entries_dict.values():
+ if multiple:
+ for entry in key_entries:
+ ret.append(entry)
+ else:
+ best_ret = {}
+
+ best_prop, best_match = None, None
+ if len(key_entries) == 1:
+ best_prop, best_match = key_entries[0]
+ else:
+ for prop, match in key_entries:
+ start, end = _get_span(prop, match)
+ if not best_prop or \
+ best_prop.confidence < best_prop.confidence or \
+ best_prop.confidence == best_prop.confidence and \
+ best_match.span()[1] - best_match.span()[0] < match.span()[1] - match.span()[0]:
+ best_prop, best_match = prop, match
+
+ best_ret[best_prop] = best_match
+
+ for prop, match in best_ret.items():
+ ret.append((prop, match))
+
+ if sort:
+ def _sorting(x):
+ _, x_match = x
+ x_start, x_end = x_match.span()
+ return x_start - x_end
+
+ ret.sort(key=_sorting)
+
+ return ret
+
+ def as_guess(self, found_properties, input=None, filter_=None, sep_replacement=None, multiple=False, *args, **kwargs):
+ if filter_ is None:
+ filter_ = lambda property, *args, **kwargs: True
+ guesses = [] if multiple else None
+ for prop, match in found_properties:
+ first_key = None
+ for key in prop.keys:
+ # First property key will be used as base for effective name
+ if isinstance(key, base_text_type):
+ if first_key is None:
+ first_key = key
+ break
+ property_name = first_key if first_key else None
+ span = _get_span(prop, match)
+ guess = Guess(confidence=prop.confidence, input=input, span=span, prop=property_name)
+ groups = _get_groups(match.re)
+ for group_name in groups:
+ name = group_name if isinstance(group_name, base_text_type) else property_name if property_name not in groups else None
+ if name:
+ value = self._effective_prop_value(prop, group_name, input, match.span(group_name) if group_name else match.span(), sep_replacement)
+ if not value is None:
+ is_string = isinstance(value, base_text_type)
+ if not is_string or is_string and value: # Keep non empty strings and other defined objects
+ if isinstance(value, dict):
+ for k, v in value.items():
+ if k is None:
+ k = name
+ guess[k] = v
+ else:
+ if name in guess:
+ if not isinstance(guess[name], list):
+ guess[name] = [guess[name]]
+ guess[name].append(value)
+ else:
+ guess[name] = value
+ if group_name:
+ guess.metadata(prop).span = match.span(group_name)
+ if filter_(guess):
+ if multiple:
+ guesses.append(guess)
+ else:
+ return guess
+ return guesses
+
+ @staticmethod
+ def _effective_prop_value(prop, group_name, input=None, span=None, sep_replacement=None):
+ if prop.canonical_form:
+ return prop.canonical_form
+ if input is None:
+ return None
+ value = input
+ if span is not None:
+ value = value[span[0]:span[1]]
+ value = input[span[0]:span[1]] if input else None
+ if sep_replacement:
+ for sep_char in sep:
+ value = value.replace(sep_char, sep_replacement)
+ if value:
+ value = prop.format(value, group_name)
+ return value
+
+ def get_properties(self, name=None, canonical_form=None):
+ """Retrieve properties
+
+ :return: Properties
+ :rtype: generator
+ """
+ for prop in self._properties:
+ if (name is None or name in prop.keys) and (canonical_form is None or prop.canonical_form == canonical_form):
+ yield prop
+
+ def get_supported_properties(self):
+ supported_properties = {}
+ for prop in self.get_properties():
+ for k in prop.keys:
+ values = supported_properties.get(k)
+ if not values:
+ values = set()
+ supported_properties[k] = values
+ if prop.canonical_form:
+ values.add(prop.canonical_form)
+ return supported_properties
+
+
+class QualitiesContainer():
+ def __init__(self):
+ self._qualities = {}
+
+ def register_quality(self, name, canonical_form, rating):
+ """Register a quality rating.
+
+ :param name: Name of the property
+ :type name: string
+ :param canonical_form: Value of the property
+ :type canonical_form: string
+ :param rating: Estimated quality rating for the property
+ :type rating: int
+ """
+ property_qualities = self._qualities.get(name)
+
+ if property_qualities is None:
+ property_qualities = {}
+ self._qualities[name] = property_qualities
+
+ property_qualities[canonical_form] = rating
+
+ def unregister_quality(self, name, *canonical_forms):
+ """Unregister quality ratings for given property name.
+
+ If canonical_forms are specified, only those values will be unregistered
+
+ :param name: Name of the property
+ :type name: string
+ :param canonical_forms: Value of the property
+ :type canonical_forms: string
+ """
+ if not canonical_forms:
+ if name in self._qualities:
+ del self._qualities[name]
+ else:
+ property_qualities = self._qualities.get(name)
+ if property_qualities is not None:
+ for property_canonical_form in canonical_forms:
+ if property_canonical_form in property_qualities:
+ del property_qualities[property_canonical_form]
+ if not property_qualities:
+ del self._qualities[name]
+
+ def clear_qualities(self,):
+ """Unregister all defined quality ratings.
+ """
+ self._qualities.clear()
+
+ def rate_quality(self, guess, *props):
+ """Rate the quality of guess.
+
+ :param guess: Guess to rate
+ :type guess: :class:`guessit.guess.Guess`
+ :param props: Properties to include in the rating. if empty, rating will be performed for all guess properties.
+ :type props: varargs of string
+
+ :return: Quality of the guess. The higher, the better.
+ :rtype: int
+ """
+ rate = 0
+ if not props:
+ props = guess.keys()
+ for prop in props:
+ prop_value = guess.get(prop)
+ prop_qualities = self._qualities.get(prop)
+ if prop_value is not None and prop_qualities is not None:
+ rate += prop_qualities.get(prop_value, 0)
+ return rate
+
+ def best_quality_properties(self, props, *guesses):
+ """Retrieve the best quality guess, based on given properties
+
+ :param props: Properties to include in the rating
+ :type props: list of strings
+ :param guesses: Guesses to rate
+ :type guesses: :class:`guessit.guess.Guess`
+
+ :return: Best quality guess from all passed guesses
+ :rtype: :class:`guessit.guess.Guess`
+ """
+ best_guess = None
+ best_rate = None
+ for guess in guesses:
+ rate = self.rate_quality(guess, *props)
+ if best_rate is None or best_rate < rate:
+ best_rate = rate
+ best_guess = guess
+ return best_guess
+
+ def best_quality(self, *guesses):
+ """Retrieve the best quality guess.
+
+ :param guesses: Guesses to rate
+ :type guesses: :class:`guessit.guess.Guess`
+
+ :return: Best quality guess from all passed guesses
+ :rtype: :class:`guessit.guess.Guess`
+ """
+ best_guess = None
+ best_rate = None
+ for guess in guesses:
+ rate = self.rate_quality(guess)
+ if best_rate is None or best_rate < rate:
+ best_rate = rate
+ best_guess = guess
+ return best_guess
+
diff --git a/lib/guessit/country.py b/lib/guessit/country.py
deleted file mode 100644
index c5a59f1ec136380f5529f2a1509ab6131e25f43e..0000000000000000000000000000000000000000
--- a/lib/guessit/country.py
+++ /dev/null
@@ -1,112 +0,0 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
-#
-# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
-#
-# GuessIt is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# GuessIt is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
-#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-
-from __future__ import unicode_literals
-from guessit import UnicodeMixin, base_text_type, u
-from guessit.fileutils import load_file_in_same_dir
-import logging
-
-__all__ = [ 'Country' ]
-
-log = logging.getLogger(__name__)
-
-
-# parsed from http://en.wikipedia.org/wiki/ISO_3166-1
-#
-# Description of the fields:
-# "An English name, an alpha-2 code (when given),
-# an alpha-3 code (when given), a numeric code, and an ISO 31666-2 code
-# are all separated by pipe (|) characters."
-_iso3166_contents = load_file_in_same_dir(__file__, 'ISO-3166-1_utf8.txt')
-
-country_matrix = [ l.strip().split('|')
- for l in _iso3166_contents.strip().split('\n') ]
-
-country_matrix += [ [ 'Unknown', 'un', 'unk', '', '' ],
- [ 'Latin America', '', 'lat', '', '' ]
- ]
-
-country_to_alpha3 = dict((c[0].lower(), c[2].lower()) for c in country_matrix)
-country_to_alpha3.update(dict((c[1].lower(), c[2].lower()) for c in country_matrix))
-country_to_alpha3.update(dict((c[2].lower(), c[2].lower()) for c in country_matrix))
-
-# add here exceptions / non ISO representations
-# Note: remember to put those exceptions in lower-case, they won't work otherwise
-country_to_alpha3.update({ 'latinoamérica': 'lat',
- 'brazilian': 'bra',
- 'españa': 'esp',
- 'uk': 'gbr'
- })
-
-country_alpha3_to_en_name = dict((c[2].lower(), c[0]) for c in country_matrix)
-country_alpha3_to_alpha2 = dict((c[2].lower(), c[1].lower()) for c in country_matrix)
-
-
-
-class Country(UnicodeMixin):
- """This class represents a country.
-
- You can initialize it with pretty much anything, as it knows conversion
- from ISO-3166 2-letter and 3-letter codes, and an English name.
- """
-
- def __init__(self, country, strict=False):
- country = u(country.strip().lower())
- self.alpha3 = country_to_alpha3.get(country)
-
- if self.alpha3 is None and strict:
- msg = 'The given string "%s" could not be identified as a country'
- raise ValueError(msg % country)
-
- if self.alpha3 is None:
- self.alpha3 = 'unk'
-
-
- @property
- def alpha2(self):
- return country_alpha3_to_alpha2[self.alpha3]
-
- @property
- def english_name(self):
- return country_alpha3_to_en_name[self.alpha3]
-
- def __hash__(self):
- return hash(self.alpha3)
-
- def __eq__(self, other):
- if isinstance(other, Country):
- return self.alpha3 == other.alpha3
-
- if isinstance(other, base_text_type):
- try:
- return self == Country(other)
- except ValueError:
- return False
-
- return False
-
- def __ne__(self, other):
- return not self == other
-
- def __unicode__(self):
- return self.english_name
-
- def __repr__(self):
- return 'Country(%s)' % self.english_name
diff --git a/lib/guessit/date.py b/lib/guessit/date.py
index 1101c8c6f44877c60370c9df1f77d7c0b3188c3a..385c508945d70f800aa858c1b59310af5be20547 100644
--- a/lib/guessit/date.py
+++ b/lib/guessit/date.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,15 +18,37 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import datetime
import re
-def valid_year(year):
- return 1920 < year < datetime.date.today().year + 5
+from dateutil import parser
+
+
+_dsep = r'[-/ \.]'
+_dsep_bis = r'[-/ \.x]'
+
+date_regexps = [
+ re.compile('%s(\d{8})%s' % (_dsep, _dsep), re.IGNORECASE),
+ re.compile('%s(\d{6})%s' % (_dsep, _dsep), re.IGNORECASE),
+ re.compile('[^\d](\d{2})%s(\d{1,2})%s(\d{1,2})[^\d]' % (_dsep, _dsep), re.IGNORECASE),
+ re.compile('[^\d](\d{1,2})%s(\d{1,2})%s(\d{2})[^\d]' % (_dsep, _dsep), re.IGNORECASE),
+ re.compile('[^\d](\d{4})%s(\d{1,2})%s(\d{1,2})[^\d]' % (_dsep_bis, _dsep), re.IGNORECASE),
+ re.compile('[^\d](\d{1,2})%s(\d{1,2})%s(\d{4})[^\d]' % (_dsep, _dsep_bis), re.IGNORECASE),
+ re.compile('[^\d](\d{1,2}(?:st|nd|rd|th)?%s(?:[a-z]{3,10})%s\d{4})[^\d]' % (_dsep, _dsep), re.IGNORECASE)]
+
+
+def valid_year(year, today=None):
+ """Check if number is a valid year"""
+ if not today:
+ today = datetime.date.today()
+ return 1920 < year < today.year + 5
+
def search_year(string):
"""Looks for year patterns, and if found return the year and group span.
+
Assumes there are sentinels at the beginning and end of the string that
always allow matching a non-digit delimiting the date.
@@ -34,100 +56,73 @@ def search_year(string):
and now + 5 years, so for instance 2000 would be returned as a valid
year but 1492 would not.
- >>> search_year('in the year 2000...')
- (2000, (12, 16))
+ >>> search_year(' in the year 2000... ')
+ (2000, (13, 17))
- >>> search_year('they arrived in 1492.')
+ >>> search_year(' they arrived in 1492. ')
(None, None)
"""
match = re.search(r'[^0-9]([0-9]{4})[^0-9]', string)
if match:
year = int(match.group(1))
if valid_year(year):
- return (year, match.span(1))
+ return year, match.span(1)
- return (None, None)
+ return None, None
-def search_date(string):
+def search_date(string, year_first=None, day_first=True):
"""Looks for date patterns, and if found return the date and group span.
+
Assumes there are sentinels at the beginning and end of the string that
always allow matching a non-digit delimiting the date.
- >>> search_date('This happened on 2002-04-22.')
- (datetime.date(2002, 4, 22), (17, 27))
+ Year can be defined on two digit only. It will return the nearest possible
+ date from today.
- >>> search_date('And this on 17-06-1998.')
- (datetime.date(1998, 6, 17), (12, 22))
+ >>> search_date(' This happened on 2002-04-22. ')
+ (datetime.date(2002, 4, 22), (18, 28))
- >>> search_date('no date in here')
+ >>> search_date(' And this on 17-06-1998. ')
+ (datetime.date(1998, 6, 17), (13, 23))
+
+ >>> search_date(' no date in here ')
(None, None)
"""
-
- dsep = r'[-/ \.]'
-
- date_rexps = [
- # 20010823
- r'[^0-9]' +
- r'(?P<year>[0-9]{4})' +
- r'(?P<month>[0-9]{2})' +
- r'(?P<day>[0-9]{2})' +
- r'[^0-9]',
-
- # 2001-08-23
- r'[^0-9]' +
- r'(?P<year>[0-9]{4})' + dsep +
- r'(?P<month>[0-9]{2})' + dsep +
- r'(?P<day>[0-9]{2})' +
- r'[^0-9]',
-
- # 23-08-2001
- r'[^0-9]' +
- r'(?P<day>[0-9]{2})' + dsep +
- r'(?P<month>[0-9]{2})' + dsep +
- r'(?P<year>[0-9]{4})' +
- r'[^0-9]',
-
- # 23-08-01
- r'[^0-9]' +
- r'(?P<day>[0-9]{2})' + dsep +
- r'(?P<month>[0-9]{2})' + dsep +
- r'(?P<year>[0-9]{2})' +
- r'[^0-9]',
- ]
-
- for drexp in date_rexps:
- match = re.search(drexp, string)
- if match:
- d = match.groupdict()
- year, month, day = int(d['year']), int(d['month']), int(d['day'])
- # years specified as 2 digits should be adjusted here
- if year < 100:
- if year > (datetime.date.today().year % 100) + 5:
- year = 1900 + year
- else:
- year = 2000 + year
-
+ start, end = None, None
+ match = None
+ for date_re in date_regexps:
+ s = date_re.search(string)
+ if s and (match is None or s.end() - s.start() > len(match)):
+ start, end = s.start(), s.end()
+ if date_re.groups:
+ match = '-'.join(s.groups())
+ else:
+ match = s.group()
+
+ if match is None:
+ return None, None
+
+ today = datetime.date.today()
+
+ # If day_first/year_first is undefined, parse is made using both possible values.
+ yearfirst_opts = [False, True]
+ if year_first is not None:
+ yearfirst_opts = [year_first]
+
+ dayfirst_opts = [True, False]
+ if day_first is not None:
+ dayfirst_opts = [day_first]
+
+ kwargs_list = ({'dayfirst': d, 'yearfirst': y} for d in dayfirst_opts for y in yearfirst_opts)
+ for kwargs in kwargs_list:
+ try:
+ date = parser.parse(match, **kwargs)
+ except (ValueError, TypeError) as e: #see https://bugs.launchpad.net/dateutil/+bug/1247643
date = None
- try:
- date = datetime.date(year, month, day)
- except ValueError:
- try:
- date = datetime.date(year, day, month)
- except ValueError:
- pass
-
- if date is None:
- continue
-
- # check date plausibility
- if not 1900 < date.year < datetime.date.today().year + 5:
- continue
-
- # looks like we have a valid date
- # note: span is [+1,-1] because we don't want to include the
- # non-digit char
- start, end = match.span()
- return (date, (start + 1, end - 1))
+ pass
+ # check date plausibility
+ if date and valid_year(date.year, today=today):
+ return date.date(), (start+1, end-1) #compensate for sentinels
return None, None
diff --git a/lib/guessit/fileutils.py b/lib/guessit/fileutils.py
index 2087c171500896a96ea4e8ee0131a5f98b9bbd19..5c96639baca243a5ab576781d22cc1a285fa5555 100644
--- a/lib/guessit/fileutils.py
+++ b/lib/guessit/fileutils.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,12 +18,14 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import s, u
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import os.path
import zipfile
import io
+from guessit import s, u
+
def split_path(path):
r"""Splits the given path into the list of folders and the filename (or the
@@ -44,17 +46,13 @@ def split_path(path):
result = []
while True:
head, tail = os.path.split(path)
- headlen = len(head)
-
- # on Unix systems, the root folder is '/'
- if head and head == '/'*headlen and tail == '':
- return ['/'] + result
- # on Windows, the root folder is a drive letter (eg: 'C:\') or for shares \\
- if ((headlen == 3 and head[1:] == ':\\') or (headlen == 2 and head == '\\\\')) and tail == '':
- return [head] + result
+ if not head and not tail:
+ return result
- if head == '' and tail == '':
+ if not tail and head == path:
+ # Make sure we won't have an infinite loop.
+ result = [head] + result
return result
# we just split a directory ending with '/', so tail is empty
@@ -70,8 +68,8 @@ def split_path(path):
def file_in_same_dir(ref_file, desired_file):
"""Return the path for a file in the same dir as a given reference file.
- >>> s(file_in_same_dir('~/smewt/smewt.db', 'smewt.settings'))
- '~/smewt/smewt.settings'
+ >>> s(file_in_same_dir('~/smewt/smewt.db', 'smewt.settings')) == os.path.normpath('~/smewt/smewt.settings')
+ True
"""
return os.path.join(*(split_path(ref_file)[:-1] + [desired_file]))
@@ -85,6 +83,6 @@ def load_file_in_same_dir(ref_file, filename):
if p.endswith('.zip'):
zfilename = os.path.join(*path[:i + 1])
zfile = zipfile.ZipFile(zfilename)
- return zfile.read('/'.join(path[i + 1:]))
+ return u(zfile.read('/'.join(path[i + 1:])))
return u(io.open(os.path.join(*path), encoding='utf-8').read())
diff --git a/lib/guessit/guess.py b/lib/guessit/guess.py
index e457feb0b3f9fbe8d23c2b70debfd7a23c9f1685..c8e4720f33a88ed48bdc2391b9b8e007895ce31b 100644
--- a/lib/guessit/guess.py
+++ b/lib/guessit/guess.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,17 +18,125 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import UnicodeMixin, s, u, base_text_type
-from guessit.language import Language
-from guessit.country import Country
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import json
import datetime
import logging
+from guessit import UnicodeMixin, s, u, base_text_type
+from babelfish import Language, Country
+from guessit.textutils import common_words
+
+
log = logging.getLogger(__name__)
+class GuessMetadata(object):
+ """GuessMetadata contains confidence, an input string, span and related property.
+
+ If defined on a property of Guess object, it overrides the object defined as global.
+
+ :param parent: The parent metadata, used for undefined properties in self object
+ :type parent: :class: `GuessMedata`
+ :param confidence: The confidence (from 0.0 to 1.0)
+ :type confidence: number
+ :param input: The input string
+ :type input: string
+ :param span: The input string
+ :type span: tuple (int, int)
+ :param prop: The found property definition
+ :type prop: :class `guessit.containers._Property`
+ """
+ def __init__(self, parent=None, confidence=None, input=None, span=None, prop=None, *args, **kwargs):
+ self.parent = parent
+ if confidence is None and self.parent is None:
+ self._confidence = 1.0
+ else:
+ self._confidence = confidence
+ self._input = input
+ self._span = span
+ self._prop = prop
+
+ @property
+ def confidence(self):
+ """The confidence
+
+ :rtype: int
+ :return: confidence value
+ """
+ return self._confidence if self._confidence is not None else self.parent.confidence if self.parent else None
+
+ @confidence.setter
+ def confidence(self, confidence):
+ self._confidence = confidence
+
+ @property
+ def input(self):
+ """The input
+
+ :rtype: string
+ :return: String used to find this guess value
+ """
+ return self._input if self._input is not None else self.parent.input if self.parent else None
+
+ @input.setter
+ def input(self, input):
+ """The input
+
+ :rtype: string
+ """
+ self._input = input
+
+ @property
+ def span(self):
+ """The span
+
+ :rtype: tuple (int, int)
+ :return: span of input string used to find this guess value
+ """
+ return self._span if self._span is not None else self.parent.span if self.parent else None
+
+ @span.setter
+ def span(self, span):
+ """The span
+
+ :rtype: tuple (int, int)
+ :return: span of input string used to find this guess value
+ """
+ self._span = span
+
+ @property
+ def prop(self):
+ """The property
+
+ :rtype: :class:`_Property`
+ :return: The property
+ """
+ return self._prop if self._prop is not None else self.parent.prop if self.parent else None
+
+ @property
+ def raw(self):
+ """Return the raw information (original match from the string,
+ not the cleaned version) associated with the given property name."""
+ if self.input and self.span:
+ return self.input[self.span[0]:self.span[1]]
+ return None
+
+ def __repr__(self, *args, **kwargs):
+ return object.__repr__(self, *args, **kwargs)
+
+
+def _split_kwargs(**kwargs):
+ metadata_args = {}
+ for prop in dir(GuessMetadata):
+ try:
+ metadata_args[prop] = kwargs.pop(prop)
+ except KeyError:
+ pass
+ return metadata_args, kwargs
+
+
class Guess(UnicodeMixin, dict):
"""A Guess is a dictionary which has an associated confidence for each of
its values.
@@ -37,66 +145,125 @@ class Guess(UnicodeMixin, dict):
simple dict."""
def __init__(self, *args, **kwargs):
- try:
- confidence = kwargs.pop('confidence')
- except KeyError:
- confidence = 0
-
+ metadata_kwargs, kwargs = _split_kwargs(**kwargs)
+ self._global_metadata = GuessMetadata(**metadata_kwargs)
dict.__init__(self, *args, **kwargs)
- self._confidence = {}
+ self._metadata = {}
for prop in self:
- self._confidence[prop] = confidence
-
-
- def to_dict(self):
+ self._metadata[prop] = GuessMetadata(parent=self._global_metadata)
+
+ def rename(self, old_name, new_name):
+ if old_name in self._metadata:
+ metadata = self._metadata[old_name]
+ del self._metadata[old_name]
+ self._metadata[new_name] = metadata
+ if old_name in self:
+ value = self[old_name]
+ del self[old_name]
+ self[new_name] = value
+ return True
+ return False
+
+ def to_dict(self, advanced=False):
+ """Return the guess as a dict containing only base types, ie:
+ where dates, languages, countries, etc. are converted to strings.
+
+ if advanced is True, return the data as a json string containing
+ also the raw information of the properties."""
data = dict(self)
for prop, value in data.items():
if isinstance(value, datetime.date):
data[prop] = value.isoformat()
- elif isinstance(value, (Language, Country, base_text_type)):
+ elif isinstance(value, (UnicodeMixin, base_text_type)):
data[prop] = u(value)
+ elif isinstance(value, (Language, Country)):
+ data[prop] = value.guessit
elif isinstance(value, list):
data[prop] = [u(x) for x in value]
+ if advanced:
+ metadata = self.metadata(prop)
+ prop_data = {'value': data[prop]}
+ if metadata.raw:
+ prop_data['raw'] = metadata.raw
+ if metadata.confidence:
+ prop_data['confidence'] = metadata.confidence
+ data[prop] = prop_data
return data
- def nice_string(self):
- data = self.to_dict()
+ def nice_string(self, advanced=False):
+ """Return a string with the property names and their values,
+ that also displays the associated confidence to each property.
- parts = json.dumps(data, indent=4).split('\n')
- for i, p in enumerate(parts):
- if p[:5] != ' "':
- continue
+ FIXME: doc with param"""
+ if advanced:
+ data = self.to_dict(advanced)
+ return json.dumps(data, indent=4, ensure_ascii=False)
+ else:
+ data = self.to_dict()
- prop = p.split('"')[1]
- parts[i] = (' [%.2f] "' % self.confidence(prop)) + p[5:]
+ parts = json.dumps(data, indent=4, ensure_ascii=False).split('\n')
+ for i, p in enumerate(parts):
+ if p[:5] != ' "':
+ continue
- return '\n'.join(parts)
+ prop = p.split('"')[1]
+ parts[i] = (' [%.2f] "' % self.confidence(prop)) + p[5:]
+
+ return '\n'.join(parts)
def __unicode__(self):
return u(self.to_dict())
- def confidence(self, prop):
- return self._confidence.get(prop, -1)
-
- def set(self, prop, value, confidence=None):
- self[prop] = value
- if confidence is not None:
- self._confidence[prop] = confidence
-
- def set_confidence(self, prop, value):
- self._confidence[prop] = value
+ def metadata(self, prop=None):
+ """Return the metadata associated with the given property name
+
+ If no property name is given, get the global_metadata
+ """
+ if prop is None:
+ return self._global_metadata
+ if prop not in self._metadata:
+ self._metadata[prop] = GuessMetadata(parent=self._global_metadata)
+ return self._metadata[prop]
+
+ def confidence(self, prop=None):
+ return self.metadata(prop).confidence
+
+ def set_confidence(self, prop, confidence):
+ self.metadata(prop).confidence = confidence
+
+ def raw(self, prop):
+ return self.metadata(prop).raw
+
+ def set(self, prop_name, value, *args, **kwargs):
+ if value is None:
+ try:
+ del self[prop_name]
+ except KeyError:
+ pass
+ try:
+ del self._metadata[prop_name]
+ except KeyError:
+ pass
+ else:
+ self[prop_name] = value
+ if 'metadata' in kwargs.keys():
+ self._metadata[prop_name] = kwargs['metadata']
+ else:
+ self._metadata[prop_name] = GuessMetadata(parent=self._global_metadata, *args, **kwargs)
def update(self, other, confidence=None):
dict.update(self, other)
if isinstance(other, Guess):
for prop in other:
- self._confidence[prop] = other.confidence(prop)
-
+ try:
+ self._metadata[prop] = other._metadata[prop]
+ except KeyError:
+ pass
if confidence is not None:
for prop in other:
- self._confidence[prop] = confidence
+ self.set_confidence(prop, confidence)
def update_highest_confidence(self, other):
"""Update this guess with the values from the given one. In case
@@ -106,32 +273,32 @@ class Guess(UnicodeMixin, dict):
raise ValueError('Can only call this function on Guess instances')
for prop in other:
- if prop in self and self.confidence(prop) >= other.confidence(prop):
+ if prop in self and self.metadata(prop).confidence >= other.metadata(prop).confidence:
continue
self[prop] = other[prop]
- self._confidence[prop] = other.confidence(prop)
+ self._metadata[prop] = other.metadata(prop)
def choose_int(g1, g2):
"""Function used by merge_similar_guesses to choose between 2 possible
properties when they are integers."""
- v1, c1 = g1 # value, confidence
+ v1, c1 = g1 # value, confidence
v2, c2 = g2
- if (v1 == v2):
- return (v1, 1 - (1 - c1) * (1 - c2))
+ if v1 == v2:
+ return v1, 1 - (1 - c1) * (1 - c2)
else:
if c1 > c2:
- return (v1, c1 - c2)
+ return v1, c1 - c2
else:
- return (v2, c2 - c1)
+ return v2, c2 - c1
def choose_string(g1, g2):
"""Function used by merge_similar_guesses to choose between 2 possible
properties when they are strings.
- If the 2 strings are similar, or one is contained in the other, the latter is returned
- with an increased confidence.
+ If the 2 strings are similar or have common words longer than 3 letters,
+ the one with highest confidence is returned with an increased confidence.
If the 2 strings are dissimilar, the one with the higher confidence is returned, with
a weaker confidence.
@@ -153,7 +320,7 @@ def choose_string(g1, g2):
('The Simpsons', 0.75)
"""
- v1, c1 = g1 # value, confidence
+ v1, c1 = g1 # value, confidence
v2, c2 = g2
if not v1:
@@ -167,26 +334,30 @@ def choose_string(g1, g2):
combined_prob = 1 - (1 - c1) * (1 - c2)
if v1l == v2l:
- return (v1, combined_prob)
+ return v1, combined_prob
# check for common patterns
elif v1l == 'the ' + v2l:
- return (v1, combined_prob)
+ return v1, combined_prob
elif v2l == 'the ' + v1l:
- return (v2, combined_prob)
-
- # if one string is contained in the other, return the shortest one
- elif v2l in v1l:
- return (v2, combined_prob)
- elif v1l in v2l:
- return (v1, combined_prob)
+ return v2, combined_prob
+
+ # If the 2 strings have common words longer than 3 letters,
+ # return the one with highest confidence.
+ commons = common_words(v1l, v2l)
+ for common_word in commons:
+ if len(common_word) > 3:
+ if c1 >= c2:
+ return v1, combined_prob
+ else:
+ return v2, combined_prob
# in case of conflict, return the one with highest confidence
else:
if c1 > c2:
- return (v1, c1 - c2)
+ return v1, c1 - c2
else:
- return (v2, c2 - c1)
+ return v2, c2 - c1
def _merge_similar_guesses_nocheck(guesses, prop, choose):
@@ -200,17 +371,7 @@ def _merge_similar_guesses_nocheck(guesses, prop, choose):
g1, g2 = similar[0], similar[1]
- other_props = set(g1) & set(g2) - set([prop])
- if other_props:
- log.debug('guess 1: %s' % g1)
- log.debug('guess 2: %s' % g2)
- for prop in other_props:
- if g1[prop] != g2[prop]:
- log.warning('both guesses to be merged have more than one '
- 'different property in common, bailing out...')
- return
-
- # merge all props of s2 into s1, updating the confidence for the
+ # merge only this prop of s2 into s1, updating the confidence for the
# considered property
v1, v2 = g1[prop], g2[prop]
c1, c2 = g1.confidence(prop), g2.confidence(prop)
@@ -222,11 +383,12 @@ def _merge_similar_guesses_nocheck(guesses, prop, choose):
msg = "Updating non-matching property '%s' with confidence %.2f"
log.debug(msg % (prop, new_confidence))
- g2[prop] = new_value
- g2.set_confidence(prop, new_confidence)
+ g1.set(prop, new_value, confidence=new_confidence)
+ g2.pop(prop)
- g1.update(g2)
- guesses.remove(g2)
+ # remove g2 if there are no properties left
+ if not g2.keys():
+ guesses.remove(g2)
def merge_similar_guesses(guesses, prop, choose):
@@ -260,42 +422,53 @@ def merge_all(guesses, append=None):
instead of being merged.
>>> s(merge_all([ Guess({'season': 2}, confidence=0.6),
- ... Guess({'episodeNumber': 13}, confidence=0.8) ]))
- {'season': 2, 'episodeNumber': 13}
+ ... Guess({'episodeNumber': 13}, confidence=0.8) ])
+ ... ) == {'season': 2, 'episodeNumber': 13}
+ True
+
>>> s(merge_all([ Guess({'episodeNumber': 27}, confidence=0.02),
- ... Guess({'season': 1}, confidence=0.2) ]))
- {'season': 1}
+ ... Guess({'season': 1}, confidence=0.2) ])
+ ... ) == {'season': 1}
+ True
>>> s(merge_all([ Guess({'other': 'PROPER'}, confidence=0.8),
... Guess({'releaseGroup': '2HD'}, confidence=0.8) ],
- ... append=['other']))
- {'releaseGroup': '2HD', 'other': ['PROPER']}
-
+ ... append=['other'])
+ ... ) == {'releaseGroup': '2HD', 'other': ['PROPER']}
+ True
"""
+ result = Guess()
if not guesses:
- return Guess()
+ return result
- result = guesses[0]
if append is None:
append = []
- for g in guesses[1:]:
+ for g in guesses:
# first append our appendable properties
for prop in append:
if prop in g:
- result.set(prop, result.get(prop, []) + [g[prop]],
+ if isinstance(g[prop], (list, set)):
+ new_values = result.get(prop, []) + list(g[prop])
+ else:
+ new_values = result.get(prop, []) + [g[prop]]
+
+ result.set(prop, new_values,
# TODO: what to do with confidence here? maybe an
# arithmetic mean...
- confidence=g.confidence(prop))
+ confidence=g.metadata(prop).confidence,
+ input=g.metadata(prop).input,
+ span=g.metadata(prop).span,
+ prop=g.metadata(prop).prop)
del g[prop]
# then merge the remaining ones
dups = set(result) & set(g)
if dups:
- log.warning('duplicate properties %s in merged result...' % [ (result[p], g[p]) for p in dups] )
+ log.debug('duplicate properties %s in merged result...' % [(result[p], g[p]) for p in dups])
result.update_highest_confidence(g)
@@ -311,8 +484,38 @@ def merge_all(guesses, append=None):
if isinstance(value, list):
result[prop] = list(set(value))
else:
- result[prop] = [ value ]
+ result[prop] = [value]
except KeyError:
pass
return result
+
+
+def smart_merge(guesses):
+ """First tries to merge well-known similar properties, and then merges
+ the rest with a merge_all call.
+
+ Should be the function to call in most cases, unless one wants to have more
+ control.
+
+ Warning: this function is destructive, ie: it will merge the list in-place.
+ """
+
+ # 1- try to merge similar information together and give it a higher
+ # confidence
+ for int_part in ('year', 'season', 'episodeNumber'):
+ merge_similar_guesses(guesses, int_part, choose_int)
+
+ for string_part in ('title', 'series', 'container', 'format',
+ 'releaseGroup', 'website', 'audioCodec',
+ 'videoCodec', 'screenSize', 'episodeFormat',
+ 'audioChannels', 'idNumber'):
+ merge_similar_guesses(guesses, string_part, choose_string)
+
+ # 2- merge the rest, potentially discarding information not properly
+ # merged before
+ result = merge_all(guesses,
+ append=['language', 'subtitleLanguage', 'other',
+ 'episodeDetails', 'unidentified'])
+
+ return result
diff --git a/lib/guessit/hash_ed2k.py b/lib/guessit/hash_ed2k.py
index 935d2cbab6e7cfe38d3cad23e106b023b451ca8e..f0e0b567611c2d84776e653656afd2568bbca78e 100644
--- a/lib/guessit/hash_ed2k.py
+++ b/lib/guessit/hash_ed2k.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,17 +18,21 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import s, to_hex
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import hashlib
import os.path
+from functools import reduce
+
+from guessit import s, to_hex
def hash_file(filename):
"""Returns the ed2k hash of a given file.
- >>> s(hash_file('tests/dummy.srt'))
- 'ed2k://|file|dummy.srt|44|1CA0B9DED3473B926AA93A0A546138BB|/'
+ >>> testfile = os.path.join(os.path.dirname(__file__), 'test/dummy.srt')
+ >>> s(hash_file(testfile))
+ 'ed2k://|file|dummy.srt|59|41F58B913AB3973F593BEBA8B8DF6510|/'
"""
return 'ed2k://|file|%s|%d|%s|/' % (os.path.basename(filename),
os.path.getsize(filename),
diff --git a/lib/guessit/hash_mpc.py b/lib/guessit/hash_mpc.py
index 800631b3a22d7031b4e8cd5224e80b201309b70d..fb6c52bdfd20ca78fe73e2b5528c63d802056599 100644
--- a/lib/guessit/hash_mpc.py
+++ b/lib/guessit/hash_mpc.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,7 +18,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import struct
import os
@@ -28,7 +29,7 @@ def hash_file(filename):
http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes
and is licensed under the GPL."""
- longlongformat = 'q' # long long
+ longlongformat = b'q' # long long
bytesize = struct.calcsize(longlongformat)
f = open(filename, "rb")
@@ -39,18 +40,18 @@ def hash_file(filename):
if filesize < 65536 * 2:
raise Exception("SizeError: size is %d, should be > 132K..." % filesize)
- for x in range(65536 / bytesize):
+ for x in range(int(65536 / bytesize)):
buf = f.read(bytesize)
(l_value,) = struct.unpack(longlongformat, buf)
hash_value += l_value
- hash_value = hash_value & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number
+ hash_value &= 0xFFFFFFFFFFFFFFFF # to remain as 64bit number
f.seek(max(0, filesize - 65536), 0)
- for x in range(65536 / bytesize):
+ for x in range(int(65536 / bytesize)):
buf = f.read(bytesize)
(l_value,) = struct.unpack(longlongformat, buf)
hash_value += l_value
- hash_value = hash_value & 0xFFFFFFFFFFFFFFFF
+ hash_value &= 0xFFFFFFFFFFFFFFFF
f.close()
diff --git a/lib/guessit/language.py b/lib/guessit/language.py
index 7f51b2c19575d36dda2e8e491a98f2353db281d8..d67ff66468729d79f3dbdb9e27709c6258611b24 100644
--- a/lib/guessit/language.py
+++ b/lib/guessit/language.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,363 +18,289 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import UnicodeMixin, base_text_type, u, s
-from guessit.fileutils import load_file_in_same_dir
-from guessit.textutils import find_words
-from guessit.country import Country
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import re
import logging
-__all__ = [ 'is_iso_language', 'is_language', 'lang_set', 'Language',
- 'ALL_LANGUAGES', 'ALL_LANGUAGES_NAMES', 'UNDETERMINED',
- 'search_language', 'guess_language' ]
-
-
-log = logging.getLogger(__name__)
-
-
-# downloaded from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
-#
-# Description of the fields:
-# "An alpha-3 (bibliographic) code, an alpha-3 (terminologic) code (when given),
-# an alpha-2 code (when given), an English name, and a French name of a language
-# are all separated by pipe (|) characters."
-_iso639_contents = load_file_in_same_dir(__file__, 'ISO-639-2_utf-8.txt')
-
-# drop the BOM from the beginning of the file
-_iso639_contents = _iso639_contents[1:]
-
-language_matrix = [ l.strip().split('|')
- for l in _iso639_contents.strip().split('\n') ]
-
-
-# update information in the language matrix
-language_matrix += [['mol', '', 'mo', 'Moldavian', 'moldave'],
- ['ass', '', '', 'Assyrian', 'assyrien']]
-
-for lang in language_matrix:
- # remove unused languages that shadow other common ones with a non-official form
- if (lang[2] == 'se' or # Northern Sami shadows Swedish
- lang[2] == 'br'): # Breton shadows Brazilian
- lang[2] = ''
- # add missing information
- if lang[0] == 'und':
- lang[2] = 'un'
- if lang[0] == 'srp':
- lang[1] = 'scc' # from OpenSubtitles
-
-
-lng3 = frozenset(l[0] for l in language_matrix if l[0])
-lng3term = frozenset(l[1] for l in language_matrix if l[1])
-lng2 = frozenset(l[2] for l in language_matrix if l[2])
-lng_en_name = frozenset(lng for l in language_matrix
- for lng in l[3].lower().split('; ') if lng)
-lng_fr_name = frozenset(lng for l in language_matrix
- for lng in l[4].lower().split('; ') if lng)
-lng_all_names = lng3 | lng3term | lng2 | lng_en_name | lng_fr_name
-
-lng3_to_lng3term = dict((l[0], l[1]) for l in language_matrix if l[1])
-lng3term_to_lng3 = dict((l[1], l[0]) for l in language_matrix if l[1])
-
-lng3_to_lng2 = dict((l[0], l[2]) for l in language_matrix if l[2])
-lng2_to_lng3 = dict((l[2], l[0]) for l in language_matrix if l[2])
-
-# we only return the first given english name, hoping it is the most used one
-lng3_to_lng_en_name = dict((l[0], l[3].split('; ')[0])
- for l in language_matrix if l[3])
-lng_en_name_to_lng3 = dict((en_name.lower(), l[0])
- for l in language_matrix if l[3]
- for en_name in l[3].split('; '))
-
-# we only return the first given french name, hoping it is the most used one
-lng3_to_lng_fr_name = dict((l[0], l[4].split('; ')[0])
- for l in language_matrix if l[4])
-lng_fr_name_to_lng3 = dict((fr_name.lower(), l[0])
- for l in language_matrix if l[4]
- for fr_name in l[4].split('; '))
-
-# contains a list of exceptions: strings that should be parsed as a language
-# but which are not in an ISO form
-lng_exceptions = { 'unknown': ('und', None),
- 'inconnu': ('und', None),
- 'unk': ('und', None),
- 'un': ('und', None),
- 'gr': ('gre', None),
- 'greek': ('gre', None),
- 'esp': ('spa', None),
- 'español': ('spa', None),
- 'se': ('swe', None),
- 'po': ('pt', 'br'),
- 'pb': ('pt', 'br'),
- 'pob': ('pt', 'br'),
- 'br': ('pt', 'br'),
- 'brazilian': ('pt', 'br'),
- 'català': ('cat', None),
- 'cz': ('cze', None),
- 'ua': ('ukr', None),
- 'cn': ('chi', None),
- 'chs': ('chi', None),
- 'jp': ('jpn', None),
- 'scr': ('hrv', None)
- }
-
-
-def is_iso_language(language):
- return language.lower() in lng_all_names
-
-def is_language(language):
- return is_iso_language(language) or language in lng_exceptions
-
-def lang_set(languages, strict=False):
- """Return a set of guessit.Language created from their given string
- representation.
-
- if strict is True, then this will raise an exception if any language
- could not be identified.
- """
- return set(Language(l, strict=strict) for l in languages)
-
-
-class Language(UnicodeMixin):
- """This class represents a human language.
-
- You can initialize it with pretty much anything, as it knows conversion
- from ISO-639 2-letter and 3-letter codes, English and French names.
-
- You can also distinguish languages for specific countries, such as
- Portuguese and Brazilian Portuguese.
+from guessit import u
+from guessit.textutils import find_words
- There are various properties on the language object that give you the
- representation of the language for a specific usage, such as .alpha3
- to get the ISO 3-letter code, or .opensubtitles to get the OpenSubtitles
- language code.
+from babelfish import Language, Country
+import babelfish
+from guessit.guess import Guess
- >>> Language('fr')
- Language(French)
- >>> s(Language('eng').french_name)
- 'anglais'
+__all__ = ['Language', 'UNDETERMINED',
+ 'search_language', 'guess_language']
- >>> s(Language('pt(br)').country.english_name)
- 'Brazil'
+log = logging.getLogger(__name__)
- >>> s(Language('Español (Latinoamérica)').country.english_name)
- 'Latin America'
+UNDETERMINED = babelfish.Language('und')
- >>> Language('Spanish (Latin America)') == Language('Español (Latinoamérica)')
- True
+SYN = {('und', None): ['unknown', 'inconnu', 'unk', 'un'],
+ ('ell', None): ['gr', 'greek'],
+ ('spa', None): ['esp', 'español'],
+ ('fra', None): ['français', 'vf', 'vff', 'vfi'],
+ ('swe', None): ['se'],
+ ('por', 'BR'): ['po', 'pb', 'pob', 'br', 'brazilian'],
+ ('cat', None): ['català'],
+ ('ces', None): ['cz'],
+ ('ukr', None): ['ua'],
+ ('zho', None): ['cn'],
+ ('jpn', None): ['jp'],
+ ('hrv', None): ['scr'],
+ ('mul', None): ['multi', 'dl'], # http://scenelingo.wordpress.com/2009/03/24/what-does-dl-mean/
+ }
- >>> s(Language('zz', strict=False).english_name)
- 'Undetermined'
- >>> s(Language('pt(br)').opensubtitles)
- 'pob'
- """
+class GuessitConverter(babelfish.LanguageReverseConverter):
_with_country_regexp = re.compile('(.*)\((.*)\)')
_with_country_regexp2 = re.compile('(.*)-(.*)')
- def __init__(self, language, country=None, strict=False, scheme=None):
- language = u(language.strip().lower())
- with_country = (Language._with_country_regexp.match(language) or
- Language._with_country_regexp2.match(language))
+ def __init__(self):
+ self.guessit_exceptions = {}
+ for (alpha3, country), synlist in SYN.items():
+ for syn in synlist:
+ self.guessit_exceptions[syn.lower()] = (alpha3, country, None)
+
+ @property
+ def codes(self):
+ return (babelfish.language_converters['alpha3b'].codes |
+ babelfish.language_converters['alpha2'].codes |
+ babelfish.language_converters['name'].codes |
+ babelfish.language_converters['opensubtitles'].codes |
+ babelfish.country_converters['name'].codes |
+ frozenset(self.guessit_exceptions.keys()))
+
+ @staticmethod
+ def convert(alpha3, country=None, script=None):
+ return str(babelfish.Language(alpha3, country, script))
+
+ def reverse(self, name):
+ with_country = (GuessitConverter._with_country_regexp.match(name) or
+ GuessitConverter._with_country_regexp2.match(name))
+
+ name = u(name.lower())
if with_country:
- self.lang = Language(with_country.group(1)).lang
- self.country = Country(with_country.group(2))
- return
-
- self.lang = None
- self.country = Country(country) if country else None
-
- # first look for scheme specific languages
- if scheme == 'opensubtitles':
- if language == 'br':
- self.lang = 'bre'
- return
- elif language == 'se':
- self.lang = 'sme'
- return
- elif scheme is not None:
- log.warning('Unrecognized scheme: "%s" - Proceeding with standard one' % scheme)
-
- # look for ISO language codes
- if len(language) == 2:
- self.lang = lng2_to_lng3.get(language)
- elif len(language) == 3:
- self.lang = (language
- if language in lng3
- else lng3term_to_lng3.get(language))
- else:
- self.lang = (lng_en_name_to_lng3.get(language) or
- lng_fr_name_to_lng3.get(language))
+ lang = Language.fromguessit(with_country.group(1).strip())
+ lang.country = babelfish.Country.fromguessit(with_country.group(2).strip())
+ return lang.alpha3, lang.country.alpha2 if lang.country else None, lang.script or None
+
+ # exceptions come first, as they need to override a potential match
+ # with any of the other guessers
+ try:
+ return self.guessit_exceptions[name]
+ except KeyError:
+ pass
+
+ for conv in [babelfish.Language,
+ babelfish.Language.fromalpha3b,
+ babelfish.Language.fromalpha2,
+ babelfish.Language.fromname,
+ babelfish.Language.fromopensubtitles]:
+ try:
+ c = conv(name)
+ return c.alpha3, c.country, c.script
+ except (ValueError, babelfish.LanguageReverseError):
+ pass
- # general language exceptions
- if self.lang is None and language in lng_exceptions:
- lang, country = lng_exceptions[language]
- self.lang = Language(lang).alpha3
- self.country = Country(country) if country else None
+ raise babelfish.LanguageReverseError(name)
- msg = 'The given string "%s" could not be identified as a language' % language
- if self.lang is None and strict:
- raise ValueError(msg)
+babelfish.language_converters['guessit'] = GuessitConverter()
- if self.lang is None:
- log.debug(msg)
- self.lang = 'und'
+COUNTRIES_SYN = {'ES': ['españa'],
+ 'GB': ['UK'],
+ 'BR': ['brazilian', 'bra'],
+ # FIXME: this one is a bit of a stretch, not sure how to do
+ # it properly, though...
+ 'MX': ['Latinoamérica', 'latin america']
+ }
- @property
- def alpha2(self):
- return lng3_to_lng2[self.lang]
- @property
- def alpha3(self):
- return self.lang
+class GuessitCountryConverter(babelfish.CountryReverseConverter):
+ def __init__(self):
+ self.guessit_exceptions = {}
- @property
- def alpha3term(self):
- return lng3_to_lng3term[self.lang]
+ for alpha2, synlist in COUNTRIES_SYN.items():
+ for syn in synlist:
+ self.guessit_exceptions[syn.lower()] = alpha2
@property
- def english_name(self):
- return lng3_to_lng_en_name[self.lang]
-
- @property
- def french_name(self):
- return lng3_to_lng_fr_name[self.lang]
+ def codes(self):
+ return (babelfish.country_converters['name'].codes |
+ frozenset(babelfish.COUNTRIES.values()) |
+ frozenset(self.guessit_exceptions.keys()))
+
+ @staticmethod
+ def convert(alpha2):
+ if alpha2 == 'GB':
+ return 'UK'
+ return str(Country(alpha2))
+
+ def reverse(self, name):
+ # exceptions come first, as they need to override a potential match
+ # with any of the other guessers
+ try:
+ return self.guessit_exceptions[name.lower()]
+ except KeyError:
+ pass
+
+ try:
+ return babelfish.Country(name.upper()).alpha2
+ except ValueError:
+ pass
+
+ for conv in [babelfish.Country.fromname]:
+ try:
+ return conv(name).alpha2
+ except babelfish.CountryReverseError:
+ pass
+
+ raise babelfish.CountryReverseError(name)
+
+
+babelfish.country_converters['guessit'] = GuessitCountryConverter()
+
+
+# list of common words which could be interpreted as languages, but which
+# are far too common to be able to say they represent a language in the
+# middle of a string (where they most likely carry their commmon meaning)
+LNG_COMMON_WORDS = frozenset([
+ # english words
+ 'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to',
+ 'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan',
+ 'fry', 'cop', 'zen', 'gay', 'fat', 'one', 'cherokee', 'got', 'an', 'as',
+ 'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr', 'rum', 'pi', 'bb', 'bt',
+ 'tv', 'aw', 'by', 'md', 'mp', 'cd', 'lt', 'gt', 'in', 'ad', 'ice', 'ay', 'at',
+ # french words
+ 'bas', 'de', 'le', 'son', 'ne', 'ca', 'ce', 'et', 'que',
+ 'mal', 'est', 'vol', 'or', 'mon', 'se', 'je', 'tu', 'me',
+ 'ne', 'ma', 'va', 'au',
+ # japanese words,
+ 'wa', 'ga', 'ao',
+ # spanish words
+ 'la', 'el', 'del', 'por', 'mar', 'al',
+ # other
+ 'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san', 'oss', 'iii',
+ 'vi', 'ben', 'da', 'lt', 'ch',
+ # new from babelfish
+ 'mkv', 'avi', 'dmd', 'the', 'dis', 'cut', 'stv', 'des', 'dia', 'and',
+ 'cab', 'sub', 'mia', 'rim', 'las', 'une', 'par', 'srt', 'ano', 'toy',
+ 'job', 'gag', 'reel', 'www', 'for', 'ayu', 'csi', 'ren', 'moi', 'sur',
+ 'fer', 'fun', 'two', 'big', 'psy', 'air',
+ # movie title
+ 'brazil',
+ # release groups
+ 'bs', # Bosnian
+ 'kz',
+ # countries
+ 'gt', 'lt',
+ # part/pt
+ 'pt'
+ ])
+
+LNG_COMMON_WORDS_STRICT = frozenset(['brazil'])
+
+
+subtitle_prefixes = ['sub', 'subs', 'st', 'vost', 'subforced', 'fansub', 'hardsub']
+subtitle_suffixes = ['subforced', 'fansub', 'hardsub']
+lang_prefixes = ['true']
+
+
+def find_possible_languages(string, allowed_languages=None):
+ """Find possible languages in the string
+
+ :return: list of tuple (property, Language, lang_word, word)
+ """
- @property
- def opensubtitles(self):
- if self.lang == 'por' and self.country and self.country.alpha2 == 'br':
- return 'pob'
- elif self.lang in ['gre', 'srp']:
- return self.alpha3term
- return self.alpha3
+ common_words = None
+ if allowed_languages:
+ common_words = LNG_COMMON_WORDS_STRICT
+ else:
+ common_words = LNG_COMMON_WORDS
+
+ words = find_words(string)
+
+ valid_words = []
+ for word in words:
+ lang_word = word.lower()
+ key = 'language'
+ for prefix in subtitle_prefixes:
+ if lang_word.startswith(prefix):
+ lang_word = lang_word[len(prefix):]
+ key = 'subtitleLanguage'
+ for suffix in subtitle_suffixes:
+ if lang_word.endswith(suffix):
+ lang_word = lang_word[:len(suffix)]
+ key = 'subtitleLanguage'
+ for prefix in lang_prefixes:
+ if lang_word.startswith(prefix):
+ lang_word = lang_word[len(prefix):]
+ if lang_word not in common_words:
+ try:
+ lang = Language.fromguessit(lang_word)
+ if allowed_languages:
+ if lang.name.lower() in allowed_languages or lang.alpha2.lower() in allowed_languages or lang.alpha3.lower() in allowed_languages:
+ valid_words.append((key, lang, lang_word, word))
+ # Keep language with alpha2 equivalent. Others are probably
+ # uncommon languages.
+ elif lang == 'mul' or hasattr(lang, 'alpha2'):
+ valid_words.append((key, lang, lang_word, word))
+ except babelfish.Error:
+ pass
+ return valid_words
+
+
+def search_language(string, allowed_languages=None):
+ """Looks for language patterns, and if found return the language object,
+ its group span and an associated confidence.
- @property
- def tmdb(self):
- if self.country:
- return '%s-%s' % (self.alpha2, self.country.alpha2.upper())
- return self.alpha2
+ you can specify a list of allowed languages using the lang_filter argument,
+ as in lang_filter = [ 'fr', 'eng', 'spanish' ]
- def __hash__(self):
- return hash(self.lang)
+ >>> search_language('movie [en].avi')['language']
+ <Language [en]>
- def __eq__(self, other):
- if isinstance(other, Language):
- return self.lang == other.lang
+ >>> search_language('the zen fat cat and the gay mad men got a new fan', allowed_languages = ['en', 'fr', 'es'])
- if isinstance(other, base_text_type):
- try:
- return self == Language(other)
- except ValueError:
- return False
+ """
- return False
+ if allowed_languages:
+ allowed_languages = set(Language.fromguessit(lang) for lang in allowed_languages)
- def __ne__(self, other):
- return not self == other
+ confidence = 1.0 # for all of them
- def __nonzero__(self):
- return self.lang != 'und'
+ for prop, language, lang, word in find_possible_languages(string, allowed_languages):
+ pos = string.find(word)
+ end = pos + len(word)
- def __unicode__(self):
- if self.country:
- return '%s(%s)' % (self.english_name, self.country.alpha2)
- else:
- return self.english_name
+ # only allow those languages that have a 2-letter code, those that
+ # don't are too esoteric and probably false matches
+ # if language.lang not in lng3_to_lng2:
+ # continue
- def __repr__(self):
- if self.country:
- return 'Language(%s, country=%s)' % (self.english_name, self.country)
+ # confidence depends on alpha2, alpha3, english name, ...
+ if len(lang) == 2:
+ confidence = 0.8
+ elif len(lang) == 3:
+ confidence = 0.9
+ elif prop == 'subtitleLanguage':
+ confidence = 0.6 # Subtitle prefix found with language
else:
- return 'Language(%s)' % self.english_name
+ # Note: we could either be really confident that we found a
+ # language or assume that full language names are too
+ # common words and lower their confidence accordingly
+ confidence = 0.3 # going with the low-confidence route here
+ return Guess({prop: language}, confidence=confidence, input=string, span=(pos, end))
-UNDETERMINED = Language('und')
-ALL_LANGUAGES = frozenset(Language(lng) for lng in lng_all_names) - frozenset([UNDETERMINED])
-ALL_LANGUAGES_NAMES = lng_all_names
-
-def search_language(string, lang_filter=None):
- """Looks for language patterns, and if found return the language object,
- its group span and an associated confidence.
+ return None
- you can specify a list of allowed languages using the lang_filter argument,
- as in lang_filter = [ 'fr', 'eng', 'spanish' ]
-
- >>> search_language('movie [en].avi')
- (Language(English), (7, 9), 0.8)
-
- >>> search_language('the zen fat cat and the gay mad men got a new fan', lang_filter = ['en', 'fr', 'es'])
- (None, None, None)
- """
- # list of common words which could be interpreted as languages, but which
- # are far too common to be able to say they represent a language in the
- # middle of a string (where they most likely carry their commmon meaning)
- lng_common_words = frozenset([
- # english words
- 'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to',
- 'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan',
- 'fry', 'cop', 'zen', 'gay', 'fat', 'cherokee', 'got', 'an', 'as',
- 'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr', 'rum', 'pi',
- # french words
- 'bas', 'de', 'le', 'son', 'vo', 'vf', 'ne', 'ca', 'ce', 'et', 'que',
- 'mal', 'est', 'vol', 'or', 'mon', 'se',
- # spanish words
- 'la', 'el', 'del', 'por', 'mar',
- # other
- 'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san', 'oss', 'iii',
- 'vi', 'ben', 'da', 'lt'
- ])
- sep = r'[](){} \._-+'
-
- if lang_filter:
- lang_filter = lang_set(lang_filter)
-
- slow = ' %s ' % string.lower()
- confidence = 1.0 # for all of them
-
- for lang in set(find_words(slow)) & lng_all_names:
-
- if lang in lng_common_words:
- continue
-
- pos = slow.find(lang)
-
- if pos != -1:
- end = pos + len(lang)
- # make sure our word is always surrounded by separators
- if slow[pos - 1] not in sep or slow[end] not in sep:
- continue
-
- language = Language(slow[pos:end])
- if lang_filter and language not in lang_filter:
- continue
-
- # only allow those languages that have a 2-letter code, those that
- # don't are too esoteric and probably false matches
- if language.lang not in lng3_to_lng2:
- continue
-
- # confidence depends on lng2, lng3, english name, ...
- if len(lang) == 2:
- confidence = 0.8
- elif len(lang) == 3:
- confidence = 0.9
- else:
- # Note: we could either be really confident that we found a
- # language or assume that full language names are too
- # common words and lower their confidence accordingly
- confidence = 0.3 # going with the low-confidence route here
-
- return language, (pos - 1, end - 1), confidence
-
- return None, None, None
-
-
-def guess_language(text):
+def guess_language(text): # pragma: no cover
"""Guess the language in which a body of text is written.
This uses the external guess-language python module, and will fail and return
@@ -382,7 +308,7 @@ def guess_language(text):
"""
try:
from guess_language import guessLanguage
- return Language(guessLanguage(text))
+ return Language.fromguessit(guessLanguage(text))
except ImportError:
log.error('Cannot detect the language of the given text body, missing dependency: guess-language')
diff --git a/lib/guessit/matcher.py b/lib/guessit/matcher.py
index d6c7db311ea048c36c7dfc685f3f65d8d30b1bc2..fc914fce18ff2a8d18e9ee6b04d72702d3ee99ce 100644
--- a/lib/guessit/matcher.py
+++ b/lib/guessit/matcher.py
@@ -1,8 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,145 +19,290 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import PY3, u, base_text_type
-from guessit.matchtree import MatchTree
-from guessit.textutils import normalize_unicode, clean_string
+from __future__ import absolute_import, division, print_function, \
+ unicode_literals
+
import logging
+import inspect
+
+from guessit import PY3, u
+from guessit.transfo import TransformerException
+from guessit.matchtree import MatchTree
+from guessit.textutils import normalize_unicode, clean_default
+from guessit.guess import Guess
log = logging.getLogger(__name__)
class IterativeMatcher(object):
- def __init__(self, filename, filetype='autodetect', opts=None):
- """An iterative matcher tries to match different patterns that appear
- in the filename.
-
- The 'filetype' argument indicates which type of file you want to match.
- If it is 'autodetect', the matcher will try to see whether it can guess
- that the file corresponds to an episode, or otherwise will assume it is
- a movie.
-
- The recognized 'filetype' values are:
- [ autodetect, subtitle, movie, moviesubtitle, episode, episodesubtitle ]
-
-
- The IterativeMatcher works mainly in 2 steps:
-
- First, it splits the filename into a match_tree, which is a tree of groups
- which have a semantic meaning, such as episode number, movie title,
- etc...
-
- The match_tree created looks like the following:
-
- 0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111
- 0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000
- 0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000
- __________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___
- xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc
- [XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv
-
- The first 3 lines indicates the group index in which a char in the
- filename is located. So for instance, x264 is the group (0, 4, 1), and
- it corresponds to a video codec, denoted by the letter'v' in the 4th line.
- (for more info, see guess.matchtree.to_string)
-
-
- Second, it tries to merge all this information into a single object
- containing all the found properties, and does some (basic) conflict
- resolution when they arise.
- """
-
- valid_filetypes = ('autodetect', 'subtitle', 'video',
- 'movie', 'moviesubtitle',
- 'episode', 'episodesubtitle')
- if filetype not in valid_filetypes:
- raise ValueError("filetype needs to be one of %s" % valid_filetypes)
+ """An iterative matcher tries to match different patterns that appear
+ in the filename.
+
+ The ``filetype`` argument indicates which type of file you want to match.
+ If it is undefined, the matcher will try to see whether it can guess
+ that the file corresponds to an episode, or otherwise will assume it is
+ a movie.
+
+ The recognized ``filetype`` values are:
+ ``['subtitle', 'info', 'movie', 'moviesubtitle', 'movieinfo', 'episode',
+ 'episodesubtitle', 'episodeinfo']``
+
+ ``options`` is a dict of options values to be passed to the transformations used
+ by the matcher.
+
+ The IterativeMatcher works mainly in 2 steps:
+
+ First, it splits the filename into a match_tree, which is a tree of groups
+ which have a semantic meaning, such as episode number, movie title,
+ etc...
+
+ The match_tree created looks like the following::
+
+ 0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111
+ 0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000
+ 0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000
+ __________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___
+ xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc
+ [XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv
+
+ The first 3 lines indicates the group index in which a char in the
+ filename is located. So for instance, ``x264`` (in the middle) is the group (0, 4, 1), and
+ it corresponds to a video codec, denoted by the letter ``v`` in the 4th line.
+ (for more info, see guess.matchtree.to_string)
+
+ Second, it tries to merge all this information into a single object
+ containing all the found properties, and does some (basic) conflict
+ resolution when they arise.
+ """
+ def __init__(self, filename, options=None, **kwargs):
+ options = dict(options or {})
+ for k, v in kwargs.items():
+ if k not in options or not options[k]:
+ options[k] = v # options dict has priority over keyword arguments
+ self._validate_options(options)
if not PY3 and not isinstance(filename, unicode):
log.warning('Given filename to matcher is not unicode...')
filename = filename.decode('utf-8')
filename = normalize_unicode(filename)
+ if options and options.get('clean_function'):
+ clean_function = options.get('clean_function')
+ if not hasattr(clean_function, '__call__'):
+ module, function = clean_function.rsplit('.')
+ if not module:
+ module = 'guessit.textutils'
+ clean_function = getattr(__import__(module), function)
+ if not clean_function:
+ log.error('Can\'t find clean function %s. Default will be used.' % options.get('clean_function'))
+ clean_function = clean_default
+ else:
+ clean_function = clean_default
- if opts is None:
- opts = []
- elif isinstance(opts, base_text_type):
- opts = opts.split()
-
- self.match_tree = MatchTree(filename)
+ self.match_tree = MatchTree(filename, clean_function=clean_function)
+ self.options = options
+ self._transfo_calls = []
# sanity check: make sure we don't process a (mostly) empty string
- if clean_string(filename) == '':
+ if clean_function(filename).strip() == '':
return
- mtree = self.match_tree
- mtree.guess.set('type', filetype, confidence=1.0)
-
- def apply_transfo(transfo_name, *args, **kwargs):
- transfo = __import__('guessit.transfo.' + transfo_name,
- globals=globals(), locals=locals(),
- fromlist=['process'], level=0)
- transfo.process(mtree, *args, **kwargs)
+ from guessit.plugins import transformers
+
+ try:
+ mtree = self.match_tree
+ if 'type' in self.options:
+ mtree.guess.set('type', self.options['type'], confidence=0.0)
+
+ # Process
+ for transformer in transformers.all_transformers():
+ disabled = options.get('disabled_transformers')
+ if not disabled or transformer.name not in disabled:
+ self._process(transformer, False)
+
+ # Post-process
+ for transformer in transformers.all_transformers():
+ disabled = options.get('disabled_transformers')
+ if not disabled or transformer.name not in disabled:
+ self._process(transformer, True)
+
+ log.debug('Found match tree:\n%s' % u(mtree))
+ except TransformerException as e:
+ log.debug('An error has occurred in Transformer %s: %s' % (e.transformer, e))
+
+ def _process(self, transformer, post=False):
+
+ if not hasattr(transformer, 'should_process') or transformer.should_process(self.match_tree, self.options):
+ if post:
+ transformer.post_process(self.match_tree, self.options)
+ else:
+ transformer.process(self.match_tree, self.options)
+ self._transfo_calls.append(transformer)
+
+ @property
+ def second_pass_options(self):
+ second_pass_options = {}
+ for transformer in self._transfo_calls:
+ if hasattr(transformer, 'second_pass_options'):
+ transformer_second_pass_options = transformer.second_pass_options(self.match_tree, self.options)
+ if transformer_second_pass_options:
+ second_pass_options.update(transformer_second_pass_options)
+
+ return second_pass_options
+
+ @staticmethod
+ def _validate_options(options):
+ valid_filetypes = ('subtitle', 'info', 'video',
+ 'movie', 'moviesubtitle', 'movieinfo',
+ 'episode', 'episodesubtitle', 'episodeinfo')
+
+ type_ = options.get('type')
+ if type_ and type_ not in valid_filetypes:
+ raise ValueError("filetype needs to be one of %s" % (valid_filetypes,))
- # 1- first split our path into dirs + basename + ext
- apply_transfo('split_path_components')
+ def matched(self):
+ return self.match_tree.matched()
- # 2- guess the file type now (will be useful later)
- apply_transfo('guess_filetype', filetype)
- if mtree.guess['type'] == 'unknown':
- return
- # 3- split each of those into explicit groups (separated by parentheses
- # or square brackets)
- apply_transfo('split_explicit_groups')
-
- # 4- try to match information for specific patterns
- # NOTE: order needs to comply to the following:
- # - website before language (eg: tvu.org.ru vs russian)
- # - language before episodes_rexps
- # - properties before language (eg: he-aac vs hebrew)
- # - release_group before properties (eg: XviD-?? vs xvid)
- if mtree.guess['type'] in ('episode', 'episodesubtitle'):
- strategy = [ 'guess_date', 'guess_website', 'guess_release_group',
- 'guess_properties', 'guess_language',
- 'guess_video_rexps',
- 'guess_episodes_rexps', 'guess_weak_episodes_rexps' ]
- else:
- strategy = [ 'guess_date', 'guess_website', 'guess_release_group',
- 'guess_properties', 'guess_language',
- 'guess_video_rexps' ]
+def build_guess(node, name, value=None, confidence=1.0):
+ guess = Guess({name: node.clean_value if value is None else value}, confidence=confidence)
+ guess.metadata().input = node.value if value is None else value
+ if value is None:
+ left_offset = 0
+ right_offset = 0
- if 'nolanguage' in opts:
- strategy.remove('guess_language')
+ clean_value = node.clean_value
- for name in strategy:
- apply_transfo(name)
+ if clean_value:
+ for i in range(0, len(node.value)):
+ if clean_value[0] == node.value[i]:
+ break
+ left_offset += 1
- # more guessers for both movies and episodes
- apply_transfo('guess_bonus_features')
- apply_transfo('guess_year', skip_first_year=('skip_first_year' in opts))
+ for i in reversed(range(0, len(node.value))):
+ if clean_value[-1] == node.value[i]:
+ break
+ right_offset += 1
- if 'nocountry' not in opts:
- apply_transfo('guess_country')
+ guess.metadata().span = (node.span[0] - node.offset + left_offset, node.span[1] - node.offset - right_offset)
+ return guess
- apply_transfo('guess_idnumber')
+def found_property(node, name, value=None, confidence=1.0, update_guess=True, logger=None):
+ # automatically retrieve the log object from the caller frame
+ if not logger:
+ caller_frame = inspect.stack()[1][0]
+ logger = caller_frame.f_locals['self'].log
+ guess = build_guess(node, name, value, confidence)
+ return found_guess(node, guess, update_guess=update_guess, logger=logger)
- # split into '-' separated subgroups (with required separator chars
- # around the dash)
- apply_transfo('split_on_dash')
- # 5- try to identify the remaining unknown groups by looking at their
- # position relative to other known elements
- if mtree.guess['type'] in ('episode', 'episodesubtitle'):
- apply_transfo('guess_episode_info_from_position')
+def found_guess(node, guess, update_guess=True, logger=None):
+ if node.guess:
+ if update_guess:
+ node.guess.update_highest_confidence(guess)
else:
- apply_transfo('guess_movie_title_from_position')
-
- # 6- perform some post-processing steps
- apply_transfo('post_process')
-
- log.debug('Found match tree:\n%s' % u(mtree))
+ child = node.add_child(guess.metadata().span)
+ child.guess = guess
+ else:
+ node.guess = guess
+ log_found_guess(guess, logger)
+ return node.guess
+
+
+def log_found_guess(guess, logger=None):
+ for k, v in guess.items():
+ (logger or log).debug('Property found: %s=%s (%s) (confidence=%.2f)' %
+ (k, v, guess.raw(k), guess.confidence(k)))
+
+
+def _get_split_spans(node, span):
+ partition_spans = node.get_partition_spans(span)
+ for to_remove_span in partition_spans:
+ if to_remove_span[0] == span[0] and to_remove_span[1] in [span[1], span[1] + 1]:
+ partition_spans.remove(to_remove_span)
+ break
+ return partition_spans
+
+
+class GuessFinder(object):
+ def __init__(self, guess_func, confidence=None, logger=None, options=None):
+ self.guess_func = guess_func
+ self.confidence = confidence
+ self.logger = logger or log
+ self.options = options
+
+ def process_nodes(self, nodes):
+ for node in nodes:
+ self.process_node(node)
+
+ def process_node(self, node, iterative=True, partial_span=None):
+ if partial_span:
+ value = node.value[partial_span[0]:partial_span[1]]
+ else:
+ value = node.value
+ string = ' %s ' % value # add sentinels
- def matched(self):
- return self.match_tree.matched()
+ if not self.options:
+ matcher_result = self.guess_func(string, node)
+ else:
+ matcher_result = self.guess_func(string, node, self.options)
+
+ if matcher_result:
+ if not isinstance(matcher_result, Guess):
+ result, span = matcher_result
+ else:
+ result, span = matcher_result, matcher_result.metadata().span
+
+ if result:
+ # readjust span to compensate for sentinels
+ span = (span[0] - 1, span[1] - 1)
+
+ # readjust span to compensate for partial_span
+ if partial_span:
+ span = (span[0] + partial_span[0], span[1] + partial_span[0])
+
+ partition_spans = None
+ if self.options and 'skip_nodes' in self.options:
+ skip_nodes = self.options.get('skip_nodes')
+ for skip_node in skip_nodes:
+ if skip_node.parent.node_idx == node.node_idx[:len(skip_node.parent.node_idx)] and\
+ skip_node.span == span or\
+ skip_node.span == (span[0] + skip_node.offset, span[1] + skip_node.offset):
+ if partition_spans is None:
+ partition_spans = _get_split_spans(node, skip_node.span)
+ else:
+ new_partition_spans = []
+ for partition_span in partition_spans:
+ tmp_node = MatchTree(value, span=partition_span, parent=node)
+ tmp_partitions_spans = _get_split_spans(tmp_node, skip_node.span)
+ new_partition_spans.extend(tmp_partitions_spans)
+ partition_spans.extend(new_partition_spans)
+
+ if not partition_spans:
+ # restore sentinels compensation
+
+ if isinstance(result, Guess):
+ guess = result
+ else:
+ guess = Guess(result, confidence=self.confidence, input=string, span=span)
+
+ if not iterative:
+ found_guess(node, guess, logger=self.logger)
+ else:
+ absolute_span = (span[0] + node.offset, span[1] + node.offset)
+ node.partition(span)
+ if node.is_leaf():
+ found_guess(node, guess, logger=self.logger)
+ else:
+ found_child = None
+ for child in node.children:
+ if child.span == absolute_span:
+ found_guess(child, guess, logger=self.logger)
+ found_child = child
+ break
+ for child in node.children:
+ if child is not found_child:
+ self.process_node(child)
+ else:
+ for partition_span in partition_spans:
+ self.process_node(node, partial_span=partition_span)
diff --git a/lib/guessit/matchtree.py b/lib/guessit/matchtree.py
index 3195df25ce652ecd083385c71043e0a84f4995b5..c4b3a3d6f822412f5313326f5edb1ac5a4d63c37 100644
--- a/lib/guessit/matchtree.py
+++ b/lib/guessit/matchtree.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,36 +18,88 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import UnicodeMixin, base_text_type, Guess
-from guessit.textutils import clean_string, str_fill
-from guessit.patterns import group_delimiters
-from guessit.guess import (merge_similar_guesses, merge_all,
- choose_int, choose_string)
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import copy
import logging
+import guessit # @UnusedImport needed for doctests
+from guessit import UnicodeMixin, base_text_type
+from guessit.textutils import clean_default, str_fill
+from guessit.patterns import group_delimiters
+from guessit.guess import (smart_merge,
+ Guess)
+
+
log = logging.getLogger(__name__)
class BaseMatchTree(UnicodeMixin):
- """A MatchTree represents the hierarchical split of a string into its
- constituent semantic groups."""
-
- def __init__(self, string='', span=None, parent=None):
+ """A BaseMatchTree is a tree covering the filename, where each
+ node represents a substring in the filename and can have a ``Guess``
+ associated with it that contains the information that has been guessed
+ in this node. Nodes can be further split into subnodes until a proper
+ split has been found.
+
+ Each node has the following attributes:
+ - string = the original string of which this node represents a region
+ - span = a pair of (begin, end) indices delimiting the substring
+ - parent = parent node
+ - children = list of children nodes
+ - guess = Guess()
+
+ BaseMatchTrees are displayed in the following way:
+
+ >>> path = 'Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD.mkv'
+ >>> print(guessit.IterativeMatcher(path).match_tree)
+ 000000 1111111111111111 2222222222222222222222222222222222222222222 333
+ 000000 0000000000111111 0000000000111111222222222222222222222222222 000
+ 011112 011112000011111222222222222222222 000
+ 011112222222222222
+ 0000011112222
+ 01112 0111
+ Movies/__________(____)/Dark.City.(____).DC._____.____.___.____-___.___
+ tttttttttt yyyy yyyy fffff ssss aaa vvvv rrr ccc
+ Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD.mkv
+
+ The last line contains the filename, which you can use a reference.
+ The previous line contains the type of property that has been found.
+ The line before that contains the filename, where all the found groups
+ have been blanked. Basically, what is left on this line are the leftover
+ groups which could not be identified.
+
+ The lines before that indicate the indices of the groups in the tree.
+
+ For instance, the part of the filename 'BDRip' is the leaf with index
+ ``(2, 2, 1)`` (read from top to bottom), and its meaning is 'format'
+ (as shown by the ``f``'s on the last-but-one line).
+ """
+
+ def __init__(self, string='', span=None, parent=None, clean_function=None):
self.string = string
self.span = span or (0, len(string))
self.parent = parent
self.children = []
self.guess = Guess()
+ self._clean_value = None
+ self._clean_function = clean_function or clean_default
@property
def value(self):
+ """Return the substring that this node matches."""
return self.string[self.span[0]:self.span[1]]
@property
def clean_value(self):
- return clean_string(self.value)
+ """Return a cleaned value of the matched substring, with better
+ presentation formatting (punctuation marks removed, duplicate
+ spaces, ...)"""
+ if self._clean_value is None:
+ self._clean_value = self.clean_string(self.value)
+ return self._clean_value
+
+ def clean_string(self, string):
+ return self._clean_function(string)
@property
def offset(self):
@@ -55,6 +107,8 @@ class BaseMatchTree(UnicodeMixin):
@property
def info(self):
+ """Return a dict containing all the info guessed by this node,
+ subnodes included."""
result = dict(self.guess)
for c in self.children:
@@ -64,6 +118,7 @@ class BaseMatchTree(UnicodeMixin):
@property
def root(self):
+ """Return the root node of the tree."""
if not self.parent:
return self
@@ -71,28 +126,43 @@ class BaseMatchTree(UnicodeMixin):
@property
def depth(self):
+ """Return the depth of this node."""
if self.is_leaf():
return 0
return 1 + max(c.depth for c in self.children)
def is_leaf(self):
+ """Return whether this node is a leaf or not."""
return self.children == []
def add_child(self, span):
- child = MatchTree(self.string, span=span, parent=self)
+ """Add a new child node to this node with the given span."""
+ child = MatchTree(self.string, span=span, parent=self, clean_function=self._clean_function)
self.children.append(child)
+ return child
- def partition(self, indices):
+ def get_partition_spans(self, indices):
+ """Return the list of absolute spans for the regions of the original
+ string defined by splitting this node at the given indices (relative
+ to this node)"""
indices = sorted(indices)
if indices[0] != 0:
indices.insert(0, 0)
if indices[-1] != len(self.value):
indices.append(len(self.value))
+ spans = []
for start, end in zip(indices[:-1], indices[1:]):
- self.add_child(span=(self.offset + start,
- self.offset + end))
+ spans.append((self.offset + start,
+ self.offset + end))
+ return spans
+
+ def partition(self, indices):
+ """Partition this node by splitting it at the given indices,
+ relative to this node."""
+ for partition_span in self.get_partition_spans(indices):
+ self.add_child(span=partition_span)
def split_on_components(self, components):
offset = 0
@@ -104,6 +174,7 @@ class BaseMatchTree(UnicodeMixin):
offset = end
def nodes_at_depth(self, depth):
+ """Return all the nodes at a given depth in the tree"""
if depth == 0:
yield self
@@ -113,38 +184,109 @@ class BaseMatchTree(UnicodeMixin):
@property
def node_idx(self):
+ """Return this node's index in the tree, as a tuple.
+ If this node is the root of the tree, then return ()."""
if self.parent is None:
return ()
- return self.parent.node_idx + (self.parent.children.index(self),)
+ return self.parent.node_idx + (self.node_last_idx,)
+
+ @property
+ def node_last_idx(self):
+ if self.parent is None:
+ return None
+ return self.parent.children.index(self)
def node_at(self, idx):
+ """Return the node at the given index in the subtree rooted at
+ this node."""
if not idx:
return self
try:
return self.children[idx[0]].node_at(idx[1:])
- except:
+ except IndexError:
raise ValueError('Non-existent node index: %s' % (idx,))
def nodes(self):
+ """Return all the nodes and subnodes in this tree."""
yield self
for child in self.children:
for node in child.nodes():
yield node
- def _leaves(self):
+ def leaves(self):
+ """Return a generator over all the nodes that are leaves."""
if self.is_leaf():
yield self
else:
for child in self.children:
# pylint: disable=W0212
- for leaf in child._leaves():
+ for leaf in child.leaves():
yield leaf
- def leaves(self):
- return list(self._leaves())
+ def group_node(self):
+ return self._other_group_node(0)
+
+ def previous_group_node(self):
+ return self._other_group_node(-1)
+
+ def next_group_node(self):
+ return self._other_group_node(+1)
+
+ def _other_group_node(self, offset):
+ if len(self.node_idx) > 1:
+ group_idx = self.node_idx[:2]
+ if group_idx[1] + offset >= 0:
+ other_group_idx = (group_idx[0], group_idx[1] + offset)
+ try:
+ other_group_node = self.root.node_at(other_group_idx)
+ return other_group_node
+ except ValueError:
+ pass
+ return None
+
+ def previous_leaf(self, leaf):
+ """Return previous leaf for this node"""
+ return self._other_leaf(leaf, -1)
+
+ def next_leaf(self, leaf):
+ """Return next leaf for this node"""
+ return self._other_leaf(leaf, +1)
+
+ def _other_leaf(self, leaf, offset):
+ leaves = list(self.leaves())
+ index = leaves.index(leaf) + offset
+ if 0 < index < len(leaves):
+ return leaves[index]
+ return None
+
+ def previous_leaves(self, leaf):
+ """Return previous leaves for this node"""
+ leaves = list(self.leaves())
+ index = leaves.index(leaf)
+ if 0 < index < len(leaves):
+ previous_leaves = leaves[:index]
+ previous_leaves.reverse()
+ return previous_leaves
+ return []
+
+ def next_leaves(self, leaf):
+ """Return next leaves for this node"""
+ leaves = list(self.leaves())
+ index = leaves.index(leaf)
+ if 0 < index < len(leaves):
+ return leaves[index + 1:len(leaves)]
+ return []
def to_string(self):
+ """Return a readable string representation of this tree.
+
+ The result is a multi-line string, where the lines are:
+ - line 1 -> N-2: each line contains the nodes at the given depth in the tree
+ - line N-2: original string where all the found groups have been blanked
+ - line N-1: type of property that has been found
+ - line N: the original string, which you can use a reference.
+ """
empty_line = ' ' * len(self.string)
def to_hex(x):
@@ -153,23 +295,27 @@ class BaseMatchTree(UnicodeMixin):
return x
def meaning(result):
- mmap = { 'episodeNumber': 'E',
- 'season': 'S',
- 'extension': 'e',
- 'format': 'f',
- 'language': 'l',
- 'country': 'C',
- 'videoCodec': 'v',
- 'audioCodec': 'a',
- 'website': 'w',
- 'container': 'c',
- 'series': 'T',
- 'title': 't',
- 'date': 'd',
- 'year': 'y',
- 'releaseGroup': 'r',
- 'screenSize': 's'
- }
+ mmap = {'episodeNumber': 'E',
+ 'season': 'S',
+ 'extension': 'e',
+ 'format': 'f',
+ 'language': 'l',
+ 'country': 'C',
+ 'videoCodec': 'v',
+ 'videoProfile': 'v',
+ 'audioCodec': 'a',
+ 'audioProfile': 'a',
+ 'audioChannels': 'a',
+ 'website': 'w',
+ 'container': 'c',
+ 'series': 'T',
+ 'title': 't',
+ 'date': 'd',
+ 'year': 'y',
+ 'releaseGroup': 'r',
+ 'screenSize': 's',
+ 'other': 'o'
+ }
if result is None:
return ' '
@@ -180,7 +326,7 @@ class BaseMatchTree(UnicodeMixin):
return 'x'
- lines = [ empty_line ] * (self.depth + 2) # +2: remaining, meaning
+ lines = [empty_line] * (self.depth + 2) # +2: remaining, meaning
lines[-2] = self.string
for node in self.nodes():
@@ -198,90 +344,84 @@ class BaseMatchTree(UnicodeMixin):
lines.append(self.string)
- return '\n'.join(lines)
+ return '\n'.join(l.rstrip() for l in lines)
def __unicode__(self):
return self.to_string()
+ def __repr__(self):
+ return '<MatchTree: root=%s>' % self.value
+
class MatchTree(BaseMatchTree):
"""The MatchTree contains a few "utility" methods which are not necessary
for the BaseMatchTree, but add a lot of convenience for writing
- higher-level rules."""
+ higher-level rules.
+ """
- def _unidentified_leaves(self,
- valid=lambda leaf: len(leaf.clean_value) >= 2):
- for leaf in self._leaves():
+ def unidentified_leaves(self,
+ valid=lambda leaf: len(leaf.clean_value) > 0):
+ """Return a generator of leaves that are not empty."""
+ for leaf in self.leaves():
if not leaf.guess and valid(leaf):
yield leaf
- def unidentified_leaves(self,
- valid=lambda leaf: len(leaf.clean_value) >= 2):
- return list(self._unidentified_leaves(valid))
-
- def _leaves_containing(self, property_name):
+ def leaves_containing(self, property_name):
+ """Return a generator of leaves that guessed the given property."""
if isinstance(property_name, base_text_type):
- property_name = [ property_name ]
+ property_name = [property_name]
- for leaf in self._leaves():
+ for leaf in self.leaves():
for prop in property_name:
if prop in leaf.guess:
yield leaf
break
- def leaves_containing(self, property_name):
- return list(self._leaves_containing(property_name))
-
def first_leaf_containing(self, property_name):
+ """Return the first leaf containing the given property."""
try:
- return next(self._leaves_containing(property_name))
+ return next(self.leaves_containing(property_name))
except StopIteration:
return None
- def _previous_unidentified_leaves(self, node):
+ def previous_unidentified_leaves(self, node):
+ """Return a generator of non-empty leaves that are before the given
+ node (in the string)."""
node_idx = node.node_idx
- for leaf in self._unidentified_leaves():
+ for leaf in self.unidentified_leaves():
if leaf.node_idx < node_idx:
yield leaf
- def previous_unidentified_leaves(self, node):
- return list(self._previous_unidentified_leaves(node))
-
- def _previous_leaves_containing(self, node, property_name):
+ def previous_leaves_containing(self, node, property_name):
+ """Return a generator of leaves containing the given property that are
+ before the given node (in the string)."""
node_idx = node.node_idx
- for leaf in self._leaves_containing(property_name):
+ for leaf in self.leaves_containing(property_name):
if leaf.node_idx < node_idx:
yield leaf
- def previous_leaves_containing(self, node, property_name):
- return list(self._previous_leaves_containing(node, property_name))
-
def is_explicit(self):
"""Return whether the group was explicitly enclosed by
parentheses/square brackets/etc."""
return (self.value[0] + self.value[-1]) in group_delimiters
def matched(self):
- # we need to make a copy here, as the merge functions work in place and
- # calling them on the match tree would modify it
- parts = [node.guess for node in self.nodes() if node.guess]
- parts = copy.deepcopy(parts)
-
- # 1- try to merge similar information together and give it a higher
- # confidence
- for int_part in ('year', 'season', 'episodeNumber'):
- merge_similar_guesses(parts, int_part, choose_int)
-
- for string_part in ('title', 'series', 'container', 'format',
- 'releaseGroup', 'website', 'audioCodec',
- 'videoCodec', 'screenSize', 'episodeFormat',
- 'audioChannels', 'idNumber'):
- merge_similar_guesses(parts, string_part, choose_string)
-
- # 2- merge the rest, potentially discarding information not properly
- # merged before
- result = merge_all(parts,
- append=['language', 'subtitleLanguage', 'other'])
-
- log.debug('Final result: ' + result.nice_string())
- return result
+ """Return a single guess that contains all the info found in the
+ nodes of this tree, trying to merge properties as good as possible.
+ """
+ if not getattr(self, '_matched_result', None):
+ # we need to make a copy here, as the merge functions work in place and
+ # calling them on the match tree would modify it
+ parts = [copy.copy(node.guess) for node in self.nodes() if node.guess]
+
+ result = smart_merge(parts)
+
+ log.debug('Final result: ' + result.nice_string())
+ self._matched_result = result
+
+ for leaf in self.unidentified_leaves():
+ if 'unidentified' not in self._matched_result:
+ self._matched_result['unidentified'] = []
+ self._matched_result['unidentified'].append(leaf.clean_value)
+
+ return self._matched_result
diff --git a/lib/guessit/options.py b/lib/guessit/options.py
new file mode 100644
index 0000000000000000000000000000000000000000..9b8dc0fb969731e4cbf701f2492d584cb5842ee3
--- /dev/null
+++ b/lib/guessit/options.py
@@ -0,0 +1,69 @@
+from argparse import ArgumentParser
+
+
+def build_opts(transformers=None):
+ opts = ArgumentParser()
+ opts.add_argument(dest='filename', help='Filename or release name to guess', nargs='*')
+
+ naming_opts = opts.add_argument_group("Naming")
+ naming_opts.add_argument('-t', '--type', dest='type', default=None,
+ help='The suggested file type: movie, episode. If undefined, type will be guessed.')
+ naming_opts.add_argument('-n', '--name-only', dest='name_only', action='store_true', default=False,
+ help='Parse files as name only. Disable folder parsing, extension parsing, and file content analysis.')
+ naming_opts.add_argument('-c', '--split-camel', dest='split_camel', action='store_true', default=False,
+ help='Split camel case part of filename.')
+
+ naming_opts.add_argument('-X', '--disabled-transformer', action='append', dest='disabled_transformers',
+ help='Transformer to disable (can be used multiple time)')
+
+ output_opts = opts.add_argument_group("Output")
+ output_opts.add_argument('-v', '--verbose', action='store_true', dest='verbose', default=False,
+ help='Display debug output')
+ output_opts.add_argument('-P', '--show-property', dest='show_property', default=None,
+ help='Display the value of a single property (title, series, videoCodec, year, type ...)'),
+ output_opts.add_argument('-u', '--unidentified', dest='unidentified', action='store_true', default=False,
+ help='Display the unidentified parts.'),
+ output_opts.add_argument('-a', '--advanced', dest='advanced', action='store_true', default=False,
+ help='Display advanced information for filename guesses, as json output')
+ output_opts.add_argument('-y', '--yaml', dest='yaml', action='store_true', default=False,
+ help='Display information for filename guesses as yaml output (like unit-test)')
+ output_opts.add_argument('-f', '--input-file', dest='input_file', default=False,
+ help='Read filenames from an input file.')
+ output_opts.add_argument('-d', '--demo', action='store_true', dest='demo', default=False,
+ help='Run a few builtin tests instead of analyzing a file')
+
+ information_opts = opts.add_argument_group("Information")
+ information_opts.add_argument('-p', '--properties', dest='properties', action='store_true', default=False,
+ help='Display properties that can be guessed.')
+ information_opts.add_argument('-V', '--values', dest='values', action='store_true', default=False,
+ help='Display property values that can be guessed.')
+ information_opts.add_argument('-s', '--transformers', dest='transformers', action='store_true', default=False,
+ help='Display transformers that can be used.')
+ information_opts.add_argument('--version', dest='version', action='store_true', default=False,
+ help='Display the guessit version.')
+
+ webservice_opts = opts.add_argument_group("guessit.io")
+ webservice_opts.add_argument('-b', '--bug', action='store_true', dest='submit_bug', default=False,
+ help='Submit a wrong detection to the guessit.io service')
+
+ other_opts = opts.add_argument_group("Other features")
+ other_opts.add_argument('-i', '--info', dest='info', default='filename',
+ help='The desired information type: filename, video, hash_mpc or a hash from python\'s '
+ 'hashlib module, such as hash_md5, hash_sha1, ...; or a list of any of '
+ 'them, comma-separated')
+
+ if transformers:
+ for transformer in transformers:
+ transformer.register_arguments(opts, naming_opts, output_opts, information_opts, webservice_opts, other_opts)
+
+ return opts, naming_opts, output_opts, information_opts, webservice_opts, other_opts
+_opts, _naming_opts, _output_opts, _information_opts, _webservice_opts, _other_opts = None, None, None, None, None, None
+
+
+def reload(transformers=None):
+ global _opts, _naming_opts, _output_opts, _information_opts, _webservice_opts, _other_opts
+ _opts, _naming_opts, _output_opts, _information_opts, _webservice_opts, _other_opts = build_opts(transformers)
+
+
+def get_opts():
+ return _opts
diff --git a/lib/guessit/patterns.py b/lib/guessit/patterns.py
deleted file mode 100644
index 447e24d9d3da6b88fd20bb656f6d0dafd789be8d..0000000000000000000000000000000000000000
--- a/lib/guessit/patterns.py
+++ /dev/null
@@ -1,244 +0,0 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
-#
-# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
-# Copyright (c) 2011 Ricard Marxer <ricardmp@gmail.com>
-#
-# GuessIt is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# GuessIt is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
-#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-
-from __future__ import unicode_literals
-import re
-
-
-subtitle_exts = [ 'srt', 'idx', 'sub', 'ssa' ]
-
-video_exts = ['3g2', '3gp', '3gp2', 'asf', 'avi', 'divx', 'flv', 'm4v', 'mk2',
- 'mka', 'mkv', 'mov', 'mp4', 'mp4a', 'mpeg', 'mpg', 'ogg', 'ogm',
- 'ogv', 'qt', 'ra', 'ram', 'rm', 'ts', 'wav', 'webm', 'wma', 'wmv']
-
-group_delimiters = [ '()', '[]', '{}' ]
-
-# separator character regexp
-sep = r'[][)(}{+ /\._-]' # regexp art, hehe :D
-
-# character used to represent a deleted char (when matching groups)
-deleted = '_'
-
-# format: [ (regexp, confidence, span_adjust) ]
-episode_rexps = [ # ... Season 2 ...
- (r'season (?P<season>[0-9]+)', 1.0, (0, 0)),
- (r'saison (?P<season>[0-9]+)', 1.0, (0, 0)),
-
- # ... s02e13 ...
- (r'[Ss](?P<season>[0-9]{1,3})[^0-9]?(?P<episodeNumber>(?:-?[eE-][0-9]{1,3})+)[^0-9]', 1.0, (0, -1)),
-
- # ... s03-x02 ... # FIXME: redundant? remove it?
- #(r'[Ss](?P<season>[0-9]{1,3})[^0-9]?(?P<bonusNumber>(?:-?[xX-][0-9]{1,3})+)[^0-9]', 1.0, (0, -1)),
-
- # ... 2x13 ...
- (r'[^0-9](?P<season>[0-9]{1,2})[^0-9]?(?P<episodeNumber>(?:-?[xX][0-9]{1,3})+)[^0-9]', 1.0, (1, -1)),
-
- # ... s02 ...
- #(sep + r's(?P<season>[0-9]{1,2})' + sep, 0.6, (1, -1)),
- (r's(?P<season>[0-9]{1,2})[^0-9]', 0.6, (0, -1)),
-
- # v2 or v3 for some mangas which have multiples rips
- (r'(?P<episodeNumber>[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)),
-
- # ... ep 23 ...
- ('ep' + sep + r'(?P<episodeNumber>[0-9]{1,2})[^0-9]', 0.7, (0, -1)),
-
- # ... e13 ... for a mini-series without a season number
- (sep + r'e(?P<episodeNumber>[0-9]{1,2})' + sep, 0.6, (1, -1))
-
- ]
-
-
-weak_episode_rexps = [ # ... 213 or 0106 ...
- (sep + r'(?P<episodeNumber>[0-9]{2,4})' + sep, (1, -1))
- ]
-
-non_episode_title = [ 'extras', 'rip' ]
-
-
-video_rexps = [ # cd number
- (r'cd ?(?P<cdNumber>[0-9])( ?of ?(?P<cdNumberTotal>[0-9]))?', 1.0, (0, 0)),
- (r'(?P<cdNumberTotal>[1-9]) cds?', 0.9, (0, 0)),
-
- # special editions
- (r'edition' + sep + r'(?P<edition>collector)', 1.0, (0, 0)),
- (r'(?P<edition>collector)' + sep + 'edition', 1.0, (0, 0)),
- (r'(?P<edition>special)' + sep + 'edition', 1.0, (0, 0)),
- (r'(?P<edition>criterion)' + sep + 'edition', 1.0, (0, 0)),
-
- # director's cut
- (r"(?P<edition>director'?s?" + sep + "cut)", 1.0, (0, 0)),
-
- # video size
- (r'(?P<width>[0-9]{3,4})x(?P<height>[0-9]{3,4})', 0.9, (0, 0)),
-
- # website
- (r'(?P<website>www(\.[a-zA-Z0-9]+){2,3})', 0.8, (0, 0)),
-
- # bonusNumber: ... x01 ...
- (r'x(?P<bonusNumber>[0-9]{1,2})', 1.0, (0, 0)),
-
- # filmNumber: ... f01 ...
- (r'f(?P<filmNumber>[0-9]{1,2})', 1.0, (0, 0))
- ]
-
-websites = [ 'tvu.org.ru', 'emule-island.com', 'UsaBit.com', 'www.divx-overnet.com',
- 'sharethefiles.com' ]
-
-unlikely_series = [ 'series' ]
-
-
-# prop_multi is a dict of { property_name: { canonical_form: [ pattern ] } }
-# pattern is a string considered as a regexp, with the addition that dashes are
-# replaced with '([ \.-_])?' which matches more types of separators (or none)
-# note: simpler patterns need to be at the end of the list to not shadow more
-# complete ones, eg: 'AAC' needs to come after 'He-AAC'
-# ie: from most specific to less specific
-prop_multi = { 'format': { 'DVD': [ 'DVD', 'DVD-Rip', 'VIDEO-TS', 'DVDivX' ],
- 'HD-DVD': [ 'HD-(?:DVD)?-Rip', 'HD-DVD' ],
- 'BluRay': [ 'Blu-ray', 'B[DR]Rip' ],
- 'HDTV': [ 'HD-TV' ],
- 'DVB': [ 'DVB-Rip', 'DVB', 'PD-TV' ],
- 'WEBRip': [ 'WEB-Rip' ],
- 'Screener': [ 'DVD-SCR', 'Screener' ],
- 'VHS': [ 'VHS' ],
- 'WEB-DL': [ 'WEB-DL' ] },
-
- 'screenSize': { '480p': [ '480[pi]?' ],
- '720p': [ '720[pi]?' ],
- '1080p': [ '1080[pi]?' ] },
-
- 'videoCodec': { 'XviD': [ 'Xvid' ],
- 'DivX': [ 'DVDivX', 'DivX' ],
- 'h264': [ '[hx]-264' ],
- 'Rv10': [ 'Rv10' ],
- 'Mpeg2': [ 'Mpeg2' ] },
-
- # has nothing to do here (or on filenames for that matter), but some
- # releases use it and it helps to identify release groups, so we adapt
- 'videoApi': { 'DXVA': [ 'DXVA' ] },
-
- 'audioCodec': { 'AC3': [ 'AC3' ],
- 'DTS': [ 'DTS' ],
- 'AAC': [ 'He-AAC', 'AAC-He', 'AAC' ] },
-
- 'audioChannels': { '5.1': [ r'5\.1', 'DD5[\._ ]1', '5ch' ] },
-
- 'episodeFormat': { 'Minisode': [ 'Minisodes?' ] }
-
- }
-
-# prop_single dict of { property_name: [ canonical_form ] }
-prop_single = { 'releaseGroup': [ 'ESiR', 'WAF', 'SEPTiC', r'\[XCT\]', 'iNT', 'PUKKA',
- 'CHD', 'ViTE', 'TLF', 'FLAiTE',
- 'MDX', 'GM4F', 'DVL', 'SVD', 'iLUMiNADOS',
- 'aXXo', 'KLAXXON', 'NoTV', 'ZeaL', 'LOL',
- 'CtrlHD', 'POD', 'WiKi','IMMERSE', 'FQM',
- '2HD', 'CTU', 'HALCYON', 'EbP', 'SiTV',
- 'HDBRiSe', 'AlFleNi-TeaM', 'EVOLVE', '0TV',
- 'TLA', 'NTB', 'ASAP', 'MOMENTUM', 'FoV', 'D-Z0N3',
- 'TrollHD', 'ECI'
- ],
-
- # potentially confusing release group names (they are words)
- 'weakReleaseGroup': [ 'DEiTY', 'FiNaLe', 'UnSeeN', 'KiNGS', 'CLUE', 'DIMENSION',
- 'SAiNTS', 'ARROW', 'EuReKA', 'SiNNERS', 'DiRTY', 'REWARD',
- 'REPTiLE',
- ],
-
- 'other': [ 'PROPER', 'REPACK', 'LIMITED', 'DualAudio', 'Audiofixed', 'R5',
- 'complete', 'classic', # not so sure about these ones, could appear in a title
- 'ws' ] # widescreen
- }
-
-_dash = '-'
-_psep = '[-\. _]?'
-
-def _to_rexp(prop):
- return re.compile(prop.replace(_dash, _psep), re.IGNORECASE)
-
-# properties_rexps dict of { property_name: { canonical_form: [ rexp ] } }
-# containing the rexps compiled from both prop_multi and prop_single
-properties_rexps = dict((type, dict((canonical_form,
- [ _to_rexp(pattern) for pattern in patterns ])
- for canonical_form, patterns in props.items()))
- for type, props in prop_multi.items())
-
-properties_rexps.update(dict((type, dict((canonical_form, [ _to_rexp(canonical_form) ])
- for canonical_form in props))
- for type, props in prop_single.items()))
-
-
-
-def find_properties(string):
- result = []
- for property_name, props in properties_rexps.items():
- # FIXME: this should be done in a more flexible way...
- if property_name in ['weakReleaseGroup']:
- continue
-
- for canonical_form, rexps in props.items():
- for value_rexp in rexps:
- match = value_rexp.search(string)
- if match:
- start, end = match.span()
- # make sure our word is always surrounded by separators
- # note: sep is a regexp, but in this case using it as
- # a char sequence achieves the same goal
- if ((start > 0 and string[start-1] not in sep) or
- (end < len(string) and string[end] not in sep)):
- continue
-
- result.append((property_name, canonical_form, start, end))
- return result
-
-
-property_synonyms = { 'Special Edition': [ 'Special' ],
- 'Collector Edition': [ 'Collector' ],
- 'Criterion Edition': [ 'Criterion' ]
- }
-
-
-def revert_synonyms():
- reverse = {}
-
- for canonical, synonyms in property_synonyms.items():
- for synonym in synonyms:
- reverse[synonym.lower()] = canonical
-
- return reverse
-
-
-reverse_synonyms = revert_synonyms()
-
-
-def canonical_form(string):
- return reverse_synonyms.get(string.lower(), string)
-
-
-def compute_canonical_form(property_name, value):
- """Return the canonical form of a property given its type if it is a valid
- one, None otherwise."""
- for canonical_form, rexps in properties_rexps[property_name].items():
- for rexp in rexps:
- if rexp.match(value):
- return canonical_form
- return None
diff --git a/lib/guessit/patterns/__init__.py b/lib/guessit/patterns/__init__.py
new file mode 100755
index 0000000000000000000000000000000000000000..910024795f26c8c902e1fd8377f6703bee4a7979
--- /dev/null
+++ b/lib/guessit/patterns/__init__.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import re
+
+group_delimiters = ['()', '[]', '{}']
+
+# separator character regexp
+sep = r'[][,)(}:{+ /~/\._-]' # regexp art, hehe :D
+
+_dash = '-'
+_psep = '[\W_]?'
+
+
+def build_or_pattern(patterns, escape=False):
+ """Build a or pattern string from a list of possible patterns
+ """
+ or_pattern = []
+ for pattern in patterns:
+ if not or_pattern:
+ or_pattern.append('(?:')
+ else:
+ or_pattern.append('|')
+ or_pattern.append('(?:%s)' % re.escape(pattern) if escape else pattern)
+ or_pattern.append(')')
+ return ''.join(or_pattern)
+
+
+def compile_pattern(pattern, enhance=True):
+ """Compile and enhance a pattern
+
+ :param pattern: Pattern to compile (regexp).
+ :type pattern: string
+
+ :param pattern: Enhance pattern before compiling.
+ :type pattern: string
+
+ :return: The compiled pattern
+ :rtype: regular expression object
+ """
+ return re.compile(enhance_pattern(pattern) if enhance else pattern, re.IGNORECASE)
+
+
+def enhance_pattern(pattern):
+ """Enhance pattern to match more equivalent values.
+
+ '-' are replaced by '[\W_]?', which matches more types of separators (or none)
+
+ :param pattern: Pattern to enhance (regexp).
+ :type pattern: string
+
+ :return: The enhanced pattern
+ :rtype: string
+ """
+ return pattern.replace(_dash, _psep)
diff --git a/lib/guessit/patterns/extension.py b/lib/guessit/patterns/extension.py
new file mode 100644
index 0000000000000000000000000000000000000000..40a576b677d4cd7a811e9e5f0da67b68731794f2
--- /dev/null
+++ b/lib/guessit/patterns/extension.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
+# Copyright (c) 2011 Ricard Marxer <ricardmp@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+subtitle_exts = ['srt', 'idx', 'sub', 'ssa', 'ass']
+
+info_exts = ['nfo']
+
+video_exts = ['3g2', '3gp', '3gp2', 'asf', 'avi', 'divx', 'flv', 'm4v', 'mk2',
+ 'mka', 'mkv', 'mov', 'mp4', 'mp4a', 'mpeg', 'mpg', 'ogg', 'ogm',
+ 'ogv', 'qt', 'ra', 'ram', 'rm', 'ts', 'wav', 'webm', 'wma', 'wmv',
+ 'iso']
diff --git a/lib/guessit/patterns/numeral.py b/lib/guessit/patterns/numeral.py
new file mode 100644
index 0000000000000000000000000000000000000000..f254c6b823b1909517dc55d1d7a3fbfcb680f5d5
--- /dev/null
+++ b/lib/guessit/patterns/numeral.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import re
+
+digital_numeral = '\d{1,4}'
+
+roman_numeral = "(?=[MCDLXVI]+)M{0,4}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})"
+
+english_word_numeral_list = [
+ 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',
+ 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen', 'twenty'
+]
+
+french_word_numeral_list = [
+ 'zéro', 'un', 'deux', 'trois', 'quatre', 'cinq', 'six', 'sept', 'huit', 'neuf', 'dix',
+ 'onze', 'douze', 'treize', 'quatorze', 'quinze', 'seize', 'dix-sept', 'dix-huit', 'dix-neuf', 'vingt'
+]
+
+french_alt_word_numeral_list = [
+ 'zero', 'une', 'deux', 'trois', 'quatre', 'cinq', 'six', 'sept', 'huit', 'neuf', 'dix',
+ 'onze', 'douze', 'treize', 'quatorze', 'quinze', 'seize', 'dixsept', 'dixhuit', 'dixneuf', 'vingt'
+]
+
+
+def __build_word_numeral(*args, **kwargs):
+ re_ = None
+ for word_list in args:
+ for word in word_list:
+ if not re_:
+ re_ = '(?:(?=\w+)'
+ else:
+ re_ += '|'
+ re_ += word
+ re_ += ')'
+ return re_
+
+word_numeral = __build_word_numeral(english_word_numeral_list, french_word_numeral_list, french_alt_word_numeral_list)
+
+numeral = '(?:' + digital_numeral + '|' + roman_numeral + '|' + word_numeral + ')'
+
+__romanNumeralMap = (
+ ('M', 1000),
+ ('CM', 900),
+ ('D', 500),
+ ('CD', 400),
+ ('C', 100),
+ ('XC', 90),
+ ('L', 50),
+ ('XL', 40),
+ ('X', 10),
+ ('IX', 9),
+ ('V', 5),
+ ('IV', 4),
+ ('I', 1)
+ )
+
+__romanNumeralPattern = re.compile('^' + roman_numeral + '$')
+
+
+def __parse_roman(value):
+ """convert Roman numeral to integer"""
+ if not __romanNumeralPattern.search(value):
+ raise ValueError('Invalid Roman numeral: %s' % value)
+
+ result = 0
+ index = 0
+ for num, integer in __romanNumeralMap:
+ while value[index:index + len(num)] == num:
+ result += integer
+ index += len(num)
+ return result
+
+
+def __parse_word(value):
+ """Convert Word numeral to integer"""
+ for word_list in [english_word_numeral_list, french_word_numeral_list, french_alt_word_numeral_list]:
+ try:
+ return word_list.index(value.lower())
+ except ValueError:
+ pass
+ raise ValueError
+
+
+_clean_re = re.compile('[^\d]*(\d+)[^\d]*')
+
+
+def parse_numeral(value, int_enabled=True, roman_enabled=True, word_enabled=True, clean=True):
+ """Parse a numeric value into integer.
+
+ input can be an integer as a string, a roman numeral or a word
+
+ :param value: Value to parse. Can be an integer, roman numeral or word.
+ :type value: string
+
+ :return: Numeric value, or None if value can't be parsed
+ :rtype: int
+ """
+ if int_enabled:
+ try:
+ if clean:
+ match = _clean_re.match(value)
+ if match:
+ clean_value = match.group(1)
+ return int(clean_value)
+ return int(value)
+ except ValueError:
+ pass
+ if roman_enabled:
+ try:
+ if clean:
+ for word in value.split():
+ try:
+ return __parse_roman(word.upper())
+ except ValueError:
+ pass
+ return __parse_roman(value)
+ except ValueError:
+ pass
+ if word_enabled:
+ try:
+ if clean:
+ for word in value.split():
+ try:
+ return __parse_word(word)
+ except ValueError:
+ pass
+ return __parse_word(value)
+ except ValueError:
+ pass
+ raise ValueError('Invalid numeral: ' + value)
diff --git a/lib/guessit/plugins/__init__.py b/lib/guessit/plugins/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6a63e4e11b04e73347f4fe8ba321e04b2b2679ed
--- /dev/null
+++ b/lib/guessit/plugins/__init__.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
diff --git a/lib/guessit/plugins/transformers.py b/lib/guessit/plugins/transformers.py
new file mode 100644
index 0000000000000000000000000000000000000000..1252dada3a11f34469c772c280959ea55ade8a89
--- /dev/null
+++ b/lib/guessit/plugins/transformers.py
@@ -0,0 +1,222 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+from logging import getLogger
+
+from pkg_resources import EntryPoint
+
+from guessit.options import reload as reload_options
+from stevedore import ExtensionManager
+from stevedore.extension import Extension
+
+log = getLogger(__name__)
+
+
+class Transformer(object): # pragma: no cover
+ def __init__(self, priority=0):
+ self.priority = priority
+ self.log = getLogger(self.name)
+
+ @property
+ def name(self):
+ return self.__class__.__name__
+
+ def supported_properties(self):
+ return {}
+
+ def second_pass_options(self, mtree, options=None):
+ return None
+
+ def should_process(self, mtree, options=None):
+ return True
+
+ def process(self, mtree, options=None):
+ pass
+
+ def post_process(self, mtree, options=None):
+ pass
+
+ def register_arguments(self, opts, naming_opts, output_opts, information_opts, webservice_opts, other_options):
+ pass
+
+ def rate_quality(self, guess, *props):
+ return 0
+
+
+class CustomTransformerExtensionManager(ExtensionManager):
+ def __init__(self, namespace='guessit.transformer', invoke_on_load=True,
+ invoke_args=(), invoke_kwds={}, propagate_map_exceptions=True, on_load_failure_callback=None,
+ verify_requirements=False):
+ super(CustomTransformerExtensionManager, self).__init__(namespace=namespace,
+ invoke_on_load=invoke_on_load,
+ invoke_args=invoke_args,
+ invoke_kwds=invoke_kwds,
+ propagate_map_exceptions=propagate_map_exceptions,
+ on_load_failure_callback=on_load_failure_callback,
+ verify_requirements=verify_requirements)
+
+ @staticmethod
+ def order_extensions(extensions):
+ """Order the loaded transformers
+
+ It should follow those rules
+ - website before language (eg: tvu.org.ru vs russian)
+ - language before episodes_rexps
+ - properties before language (eg: he-aac vs hebrew)
+ - release_group before properties (eg: XviD-?? vs xvid)
+ """
+ extensions.sort(key=lambda ext: -ext.obj.priority)
+ return extensions
+
+ @staticmethod
+ def _load_one_plugin(ep, invoke_on_load, invoke_args, invoke_kwds, verify_requirements=True):
+ if not ep.dist:
+ # `require` argument of ep.load() is deprecated in newer versions of setuptools
+ if hasattr(ep, 'resolve'):
+ plugin = ep.resolve()
+ elif hasattr(ep, '_load'):
+ plugin = ep._load()
+ else:
+ plugin = ep.load(require=False)
+ else:
+ plugin = ep.load()
+ if invoke_on_load:
+ obj = plugin(*invoke_args, **invoke_kwds)
+ else:
+ obj = None
+ return Extension(ep.name, ep, plugin, obj)
+
+ def _load_plugins(self, invoke_on_load, invoke_args, invoke_kwds, verify_requirements):
+ return self.order_extensions(super(CustomTransformerExtensionManager, self)._load_plugins(invoke_on_load, invoke_args, invoke_kwds, verify_requirements))
+
+ def objects(self):
+ return self.map(self._get_obj)
+
+ @staticmethod
+ def _get_obj(ext):
+ return ext.obj
+
+ def object(self, name):
+ try:
+ return self[name].obj
+ except KeyError:
+ return None
+
+ def register_module(self, name=None, module_name=None, attrs=(), entry_point=None):
+ if entry_point:
+ ep = EntryPoint.parse(entry_point)
+ else:
+ ep = EntryPoint(name, module_name, attrs)
+ loaded = self._load_one_plugin(ep, invoke_on_load=True, invoke_args=(), invoke_kwds={})
+ if loaded:
+ self.extensions.append(loaded)
+ self.extensions = self.order_extensions(self.extensions)
+ self._extensions_by_name = None
+
+
+class DefaultTransformerExtensionManager(CustomTransformerExtensionManager):
+ @property
+ def _internal_entry_points(self):
+ return ['split_path_components = guessit.transfo.split_path_components:SplitPathComponents',
+ 'guess_filetype = guessit.transfo.guess_filetype:GuessFiletype',
+ 'split_explicit_groups = guessit.transfo.split_explicit_groups:SplitExplicitGroups',
+ 'guess_date = guessit.transfo.guess_date:GuessDate',
+ 'guess_website = guessit.transfo.guess_website:GuessWebsite',
+ 'guess_release_group = guessit.transfo.guess_release_group:GuessReleaseGroup',
+ 'guess_properties = guessit.transfo.guess_properties:GuessProperties',
+ 'guess_language = guessit.transfo.guess_language:GuessLanguage',
+ 'guess_video_rexps = guessit.transfo.guess_video_rexps:GuessVideoRexps',
+ 'guess_episodes_rexps = guessit.transfo.guess_episodes_rexps:GuessEpisodesRexps',
+ 'guess_weak_episodes_rexps = guessit.transfo.guess_weak_episodes_rexps:GuessWeakEpisodesRexps',
+ 'guess_bonus_features = guessit.transfo.guess_bonus_features:GuessBonusFeatures',
+ 'guess_year = guessit.transfo.guess_year:GuessYear',
+ 'guess_country = guessit.transfo.guess_country:GuessCountry',
+ 'guess_idnumber = guessit.transfo.guess_idnumber:GuessIdnumber',
+ 'split_on_dash = guessit.transfo.split_on_dash:SplitOnDash',
+ 'guess_episode_info_from_position = guessit.transfo.guess_episode_info_from_position:GuessEpisodeInfoFromPosition',
+ 'guess_movie_title_from_position = guessit.transfo.guess_movie_title_from_position:GuessMovieTitleFromPosition',
+ 'guess_episode_details = guessit.transfo.guess_episode_details:GuessEpisodeDetails',
+ 'expected_series = guessit.transfo.expected_series:ExpectedSeries',
+ 'expected_title = guessit.transfo.expected_title:ExpectedTitle',]
+
+ def _find_entry_points(self, namespace):
+ entry_points = {}
+ # Internal entry points
+ if namespace == self.namespace:
+ for internal_entry_point_str in self._internal_entry_points:
+ internal_entry_point = EntryPoint.parse(internal_entry_point_str)
+ entry_points[internal_entry_point.name] = internal_entry_point
+
+ # Package entry points
+ setuptools_entrypoints = super(DefaultTransformerExtensionManager, self)._find_entry_points(namespace)
+ for setuptools_entrypoint in setuptools_entrypoints:
+ entry_points[setuptools_entrypoint.name] = setuptools_entrypoint
+
+ return list(entry_points.values())
+
+_extensions = None
+
+
+def all_transformers():
+ return _extensions.objects()
+
+
+def get_transformer(name):
+ return _extensions.object(name)
+
+
+def add_transformer(name, module_name, class_name):
+ """
+ Add a transformer
+
+ :param name: the name of the transformer. ie: 'guess_regexp_id'
+ :param name: the module name. ie: 'flexget.utils.parsers.transformers.guess_regexp_id'
+ :param class_name: the class name. ie: 'GuessRegexpId'
+ """
+
+ _extensions.register_module(name, module_name, (class_name,))
+
+
+def add_transformer(entry_point):
+ """
+ Add a transformer
+
+ :param entry_point: entry point spec format. ie: 'guess_regexp_id = flexget.utils.parsers.transformers.guess_regexp_id:GuessRegexpId'
+ """
+ _extensions.register_module(entry_point = entry_point)
+
+
+def reload(custom=False):
+ """
+ Reload extension manager with default or custom one.
+ :param custom: if True, custom manager will be used, else default one.
+ Default manager will load default extensions from guessit and setuptools packaging extensions
+ Custom manager will not load default extensions from guessit, using only setuptools packaging extensions.
+ :type custom: boolean
+ """
+ global _extensions
+ if custom:
+ _extensions = CustomTransformerExtensionManager()
+ else:
+ _extensions = DefaultTransformerExtensionManager()
+ reload_options(all_transformers())
+
+reload()
diff --git a/lib/guessit/quality.py b/lib/guessit/quality.py
new file mode 100644
index 0000000000000000000000000000000000000000..870bbdbb477872c7f81fcf962a57ce0df76fe561
--- /dev/null
+++ b/lib/guessit/quality.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.plugins.transformers import all_transformers
+
+
+def best_quality_properties(props, *guesses):
+ """Retrieve the best quality guess, based on given properties
+
+ :param props: Properties to include in the rating
+ :type props: list of strings
+ :param guesses: Guesses to rate
+ :type guesses: :class:`guessit.guess.Guess`
+
+ :return: Best quality guess from all passed guesses
+ :rtype: :class:`guessit.guess.Guess`
+ """
+ best_guess = None
+ best_rate = None
+ for guess in guesses:
+ for transformer in all_transformers():
+ rate = transformer.rate_quality(guess, *props)
+ if best_rate is None or best_rate < rate:
+ best_rate = rate
+ best_guess = guess
+ return best_guess
+
+
+def best_quality(*guesses):
+ """Retrieve the best quality guess.
+
+ :param guesses: Guesses to rate
+ :type guesses: :class:`guessit.guess.Guess`
+
+ :return: Best quality guess from all passed guesses
+ :rtype: :class:`guessit.guess.Guess`
+ """
+ best_guess = None
+ best_rate = None
+ for guess in guesses:
+ for transformer in all_transformers():
+ rate = transformer.rate_quality(guess)
+ if best_rate is None or best_rate < rate:
+ best_rate = rate
+ best_guess = guess
+ return best_guess
diff --git a/lib/guessit/slogging.py b/lib/guessit/slogging.py
index 1e8c650251f4738d963f07a075250c07e62bcac3..00fb80f7a5fa868aede139f04424c5e901e333d7 100644
--- a/lib/guessit/slogging.py
+++ b/lib/guessit/slogging.py
@@ -1,28 +1,28 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
-# Smewt - A smart collection manager
-# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
-# Smewt is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
-# Smewt is distributed in the hope that it will be useful,
+# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# Lesser GNU General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
+# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import logging
import sys
-import os, os.path
-
+import os
GREEN_FONT = "\x1B[0;32m"
YELLOW_FONT = "\x1B[0;33m"
@@ -31,14 +31,15 @@ RED_FONT = "\x1B[0;31m"
RESET_FONT = "\x1B[0m"
-def setupLogging(colored=True, with_time=False, with_thread=False, filename=None):
+def setup_logging(colored=True, with_time=False, with_thread=False, filename=None, with_lineno=False): # pragma: no cover
"""Set up a nice colored logger as the main application logger."""
class SimpleFormatter(logging.Formatter):
def __init__(self, with_time, with_thread):
self.fmt = (('%(asctime)s ' if with_time else '') +
'%(levelname)-8s ' +
- '[%(name)s:%(funcName)s]' +
+ '[%(name)s:%(funcName)s' +
+ (':%(lineno)s' if with_lineno else '') + ']' +
('[%(threadName)s]' if with_thread else '') +
' -- %(message)s')
logging.Formatter.__init__(self, self.fmt)
@@ -47,7 +48,8 @@ def setupLogging(colored=True, with_time=False, with_thread=False, filename=None
def __init__(self, with_time, with_thread):
self.fmt = (('%(asctime)s ' if with_time else '') +
'-CC-%(levelname)-8s ' +
- BLUE_FONT + '[%(name)s:%(funcName)s]' +
+ BLUE_FONT + '[%(name)s:%(funcName)s' +
+ (':%(lineno)s' if with_lineno else '') + ']' +
RESET_FONT + ('[%(threadName)s]' if with_thread else '') +
' -- %(message)s')
diff --git a/lib/guessit/test/1MB b/lib/guessit/test/1MB
new file mode 100644
index 0000000000000000000000000000000000000000..66d50a84dfddf2af162389d19170d62caa342668
Binary files /dev/null and b/lib/guessit/test/1MB differ
diff --git a/lib/guessit/test/__init__.py b/lib/guessit/test/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e190e51f9f7f4382a14332bba823359fa5fbb562
--- /dev/null
+++ b/lib/guessit/test/__init__.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import logging
+
+from guessit.slogging import setup_logging
+
+setup_logging()
+logging.disable(logging.INFO)
diff --git a/lib/guessit/test/autodetect.yaml b/lib/guessit/test/autodetect.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ea17db0caf87f60a1af4b4a65762b549e9dae57e
--- /dev/null
+++ b/lib/guessit/test/autodetect.yaml
@@ -0,0 +1,527 @@
+? Movies/Fear and Loathing in Las Vegas (1998)/Fear.and.Loathing.in.Las.Vegas.720p.HDDVD.DTS.x264-ESiR.mkv
+: type: movie
+ title: Fear and Loathing in Las Vegas
+ year: 1998
+ screenSize: 720p
+ format: HD-DVD
+ audioCodec: DTS
+ videoCodec: h264
+ releaseGroup: ESiR
+
+? Leopard.dmg
+: type: unknown
+ extension: dmg
+
+? Series/Duckman/Duckman - 101 (01) - 20021107 - I, Duckman.avi
+: type: episode
+ series: Duckman
+ season: 1
+ episodeNumber: 1
+ title: I, Duckman
+ date: 2002-11-07
+
+? Series/Neverwhere/Neverwhere.05.Down.Street.[tvu.org.ru].avi
+: type: episode
+ series: Neverwhere
+ episodeNumber: 5
+ title: Down Street
+ website: tvu.org.ru
+
+? Neverwhere.05.Down.Street.[tvu.org.ru].avi
+: type: episode
+ series: Neverwhere
+ episodeNumber: 5
+ title: Down Street
+ website: tvu.org.ru
+
+? Series/Breaking Bad/Minisodes/Breaking.Bad.(Minisodes).01.Good.Cop.Bad.Cop.WEBRip.XviD.avi
+: type: episode
+ series: Breaking Bad
+ episodeFormat: Minisode
+ episodeNumber: 1
+ title: Good Cop Bad Cop
+ format: WEBRip
+ videoCodec: XviD
+
+? Series/Kaamelott/Kaamelott - Livre V - Ep 23 - Le Forfait.avi
+: type: episode
+ series: Kaamelott
+ episodeNumber: 23
+ title: Le Forfait
+
+? Movies/The Doors (1991)/09.03.08.The.Doors.(1991).BDRip.720p.AC3.X264-HiS@SiLUHD-English.[sharethefiles.com].mkv
+: type: movie
+ title: The Doors
+ year: 1991
+ date: 2008-03-09
+ format: BluRay
+ screenSize: 720p
+ audioCodec: AC3
+ videoCodec: h264
+ releaseGroup: HiS@SiLUHD
+ language: english
+ website: sharethefiles.com
+
+? Movies/M.A.S.H. (1970)/MASH.(1970).[Divx.5.02][Dual-Subtitulos][DVDRip].ogm
+: type: movie
+ title: M.A.S.H.
+ year: 1970
+ videoCodec: DivX
+ format: DVD
+
+? the.mentalist.501.hdtv-lol.mp4
+: type: episode
+ series: The Mentalist
+ season: 5
+ episodeNumber: 1
+ format: HDTV
+ releaseGroup: LOL
+
+? the.simpsons.2401.hdtv-lol.mp4
+: type: episode
+ series: The Simpsons
+ season: 24
+ episodeNumber: 1
+ format: HDTV
+ releaseGroup: LOL
+
+? Homeland.S02E01.HDTV.x264-EVOLVE.mp4
+: type: episode
+ series: Homeland
+ season: 2
+ episodeNumber: 1
+ format: HDTV
+ videoCodec: h264
+ releaseGroup: EVOLVE
+
+? /media/Band_of_Brothers-e01-Currahee.mkv
+: type: episode
+ series: Band of Brothers
+ episodeNumber: 1
+ title: Currahee
+
+? /media/Band_of_Brothers-x02-We_Stand_Alone_Together.mkv
+: type: episode
+ series: Band of Brothers
+ bonusNumber: 2
+ bonusTitle: We Stand Alone Together
+
+? /movies/James_Bond-f21-Casino_Royale-x02-Stunts.mkv
+: type: movie
+ title: Casino Royale
+ filmSeries: James Bond
+ filmNumber: 21
+ bonusNumber: 2
+ bonusTitle: Stunts
+
+? /TV Shows/new.girl.117.hdtv-lol.mp4
+: type: episode
+ series: New Girl
+ season: 1
+ episodeNumber: 17
+ format: HDTV
+ releaseGroup: LOL
+
+? The.Office.(US).1x03.Health.Care.HDTV.XviD-LOL.avi
+: type: episode
+ series: The Office (US)
+ country: US
+ season: 1
+ episodeNumber: 3
+ title: Health Care
+ format: HDTV
+ videoCodec: XviD
+ releaseGroup: LOL
+
+? The_Insider-(1999)-x02-60_Minutes_Interview-1996.mp4
+: type: movie
+ title: The Insider
+ year: 1999
+ bonusNumber: 2
+ bonusTitle: 60 Minutes Interview-1996
+
+? OSS_117--Cairo,_Nest_of_Spies.mkv
+: type: movie
+ title: OSS 117--Cairo, Nest of Spies
+
+? Rush.._Beyond_The_Lighted_Stage-x09-Between_Sun_and_Moon-2002_Hartford.mkv
+: type: movie
+ title: Rush Beyond The Lighted Stage
+ bonusNumber: 9
+ bonusTitle: Between Sun and Moon-2002 Hartford
+
+? House.Hunters.International.S56E06.720p.hdtv.x264.mp4
+: type: episode
+ series: House Hunters International
+ season: 56
+ episodeNumber: 6
+ screenSize: 720p
+ format: HDTV
+ videoCodec: h264
+
+? White.House.Down.2013.1080p.BluRay.DTS-HD.MA.5.1.x264-PublicHD.mkv
+: type: movie
+ title: White House Down
+ year: 2013
+ screenSize: 1080p
+ format: BluRay
+ audioCodec: DTS
+ audioProfile: HDMA
+ videoCodec: h264
+ releaseGroup: PublicHD
+ audioChannels: "5.1"
+
+? White.House.Down.2013.1080p.BluRay.DTSHD.MA.5.1.x264-PublicHD.mkv
+: type: movie
+ title: White House Down
+ year: 2013
+ screenSize: 1080p
+ format: BluRay
+ audioCodec: DTS
+ audioProfile: HDMA
+ videoCodec: h264
+ releaseGroup: PublicHD
+ audioChannels: "5.1"
+
+? Hostages.S01E01.Pilot.for.Air.720p.WEB-DL.DD5.1.H.264-NTb.nfo
+: type: episodeinfo
+ series: Hostages
+ title: Pilot for Air
+ season: 1
+ episodeNumber: 1
+ screenSize: 720p
+ format: WEB-DL
+ audioChannels: "5.1"
+ videoCodec: h264
+ audioCodec: DolbyDigital
+ releaseGroup: NTb
+
+? Despicable.Me.2.2013.1080p.BluRay.x264-VeDeTT.nfo
+: type: movieinfo
+ title: Despicable Me 2
+ year: 2013
+ screenSize: 1080p
+ format: BluRay
+ videoCodec: h264
+ releaseGroup: VeDeTT
+
+? Le Cinquieme Commando 1971 SUBFORCED FRENCH DVDRiP XViD AC3 Bandix.mkv
+: type: movie
+ audioCodec: AC3
+ format: DVD
+ releaseGroup: Bandix
+ subtitleLanguage: French
+ title: Le Cinquieme Commando
+ videoCodec: XviD
+ year: 1971
+
+? Le Seigneur des Anneaux - La Communauté de l'Anneau - Version Longue - BDRip.mkv
+: type: movie
+ format: BluRay
+ title: Le Seigneur des Anneaux
+
+? La petite bande (Michel Deville - 1983) VF PAL MP4 x264 AAC.mkv
+: type: movie
+ audioCodec: AAC
+ language: French
+ title: La petite bande
+ videoCodec: h264
+ year: 1983
+ other: PAL
+
+? Retour de Flammes (Gregor Schnitzler 2003) FULL DVD.iso
+: type: movie
+ format: DVD
+ title: Retour de Flammes
+ type: movie
+ year: 2003
+
+? A.Common.Title.Special.2014.avi
+: type: movie
+ year: 2014
+ title: A Common Title Special
+
+? A.Common.Title.2014.Special.avi
+: type: episode
+ year: 2014
+ series: A Common Title
+ title: Special
+ episodeDetails: Special
+
+? A.Common.Title.2014.Special.Edition.avi
+: type: movie
+ year: 2014
+ title: A Common Title
+ edition: Special Edition
+
+? Downton.Abbey.2013.Christmas.Special.HDTV.x264-FoV.mp4
+: type: episode
+ year: 2013
+ series: Downton Abbey
+ title: Christmas Special
+ videoCodec: h264
+ releaseGroup: FoV
+ format: HDTV
+ episodeDetails: Special
+
+? Doctor_Who_2013_Christmas_Special.The_Time_of_The_Doctor.HD
+: options: -n
+ type: episode
+ series: Doctor Who
+ other: HD
+ episodeDetails: Special
+ title: Christmas Special The Time of The Doctor
+ year: 2013
+
+? Doctor Who 2005 50th Anniversary Special The Day of the Doctor 3.avi
+: type: episode
+ series: Doctor Who
+ episodeDetails: Special
+ title: 50th Anniversary Special The Day of the Doctor 3
+ year: 2005
+
+? Robot Chicken S06-Born Again Virgin Christmas Special HDTV x264.avi
+: type: episode
+ series: Robot Chicken
+ format: HDTV
+ season: 6
+ title: Born Again Virgin Christmas Special
+ videoCodec: h264
+ episodeDetails: Special
+
+? Wicked.Tuna.S03E00.Head.To.Tail.Special.HDTV.x264-YesTV
+: options: -n
+ type: episode
+ series: Wicked Tuna
+ title: Head To Tail Special
+ releaseGroup: YesTV
+ season: 3
+ episodeNumber: 0
+ videoCodec: h264
+ format: HDTV
+ episodeDetails: Special
+
+? The.Voice.UK.S03E12.HDTV.x264-C4TV
+: options: -n
+ episodeNumber: 12
+ videoCodec: h264
+ format: HDTV
+ series: The Voice (UK)
+ releaseGroup: C4TV
+ season: 3
+ country: United Kingdom
+ type: episode
+
+? /tmp/star.trek.9/star.trek.9.mkv
+: type: movie
+ title: star trek 9
+
+? star.trek.9.mkv
+: type: movie
+ title: star trek 9
+
+? FlexGet.S01E02.TheName.HDTV.xvid
+: options: -n
+ episodeNumber: 2
+ format: HDTV
+ season: 1
+ series: FlexGet
+ title: TheName
+ type: episode
+ videoCodec: XviD
+
+? FlexGet.S01E02.TheName.HDTV.xvid
+: options: -n
+ episodeNumber: 2
+ format: HDTV
+ season: 1
+ series: FlexGet
+ title: TheName
+ type: episode
+ videoCodec: XviD
+
+? some.series.S03E14.Title.Here.720p
+: options: -n
+ episodeNumber: 14
+ screenSize: 720p
+ season: 3
+ series: some series
+ title: Title Here
+ type: episode
+
+? '[the.group] Some.Series.S03E15.Title.Two.720p'
+: options: -n
+ episodeNumber: 15
+ releaseGroup: the.group
+ screenSize: 720p
+ season: 3
+ series: Some Series
+ title: Title Two
+ type: episode
+
+? 'HD 720p: Some series.S03E16.Title.Three'
+: options: -n
+ episodeNumber: 16
+ other: HD
+ screenSize: 720p
+ season: 3
+ series: Some series
+ title: Title Three
+ type: episode
+
+? Something.Season.2.1of4.Ep.Title.HDTV.torrent
+: episodeCount: 4
+ episodeNumber: 1
+ format: HDTV
+ season: 2
+ series: Something
+ title: Title
+ type: episode
+ extension: torrent
+
+? Show-A (US) - Episode Title S02E09 hdtv
+: options: -n
+ country: US
+ episodeNumber: 9
+ format: HDTV
+ season: 2
+ series: Show-A (US)
+ type: episode
+
+? Jack's.Show.S03E01.blah.1080p
+: options: -n
+ episodeNumber: 1
+ screenSize: 1080p
+ season: 3
+ series: Jack's Show
+ title: blah
+ type: episode
+
+? FlexGet.epic
+: options: -n
+ title: FlexGet epic
+ type: movie
+
+? FlexGet.Apt.1
+: options: -n
+ title: FlexGet Apt 1
+ type: movie
+
+? FlexGet.aptitude
+: options: -n
+ title: FlexGet aptitude
+ type: movie
+
+? FlexGet.Step1
+: options: -n
+ title: FlexGet Step1
+ type: movie
+
+? Movies/El Bosque Animado (1987)/El.Bosque.Animado.[Jose.Luis.Cuerda.1987].[Xvid-Dvdrip-720 * 432].avi
+: format: DVD
+ screenSize: 720x432
+ title: El Bosque Animado
+ videoCodec: XviD
+ year: 1987
+ type: movie
+
+? Movies/El Bosque Animado (1987)/El.Bosque.Animado.[Jose.Luis.Cuerda.1987].[Xvid-Dvdrip-720x432].avi
+: format: DVD
+ screenSize: 720x432
+ title: El Bosque Animado
+ videoCodec: XviD
+ year: 1987
+ type: movie
+
+? 2009.shoot.fruit.chan.multi.dvd9.pal
+: options: -n
+ format: DVD
+ language: mul
+ other: PAL
+ title: shoot fruit chan
+ type: movie
+ year: 2009
+
+? 2009.shoot.fruit.chan.multi.dvd5.pal
+: options: -n
+ format: DVD
+ language: mul
+ other: PAL
+ title: shoot fruit chan
+ type: movie
+ year: 2009
+
+? The.Flash.2014.S01E01.PREAIR.WEBRip.XviD-EVO.avi
+: episodeNumber: 1
+ format: WEBRip
+ other: Preair
+ releaseGroup: EVO
+ season: 1
+ series: The Flash
+ type: episode
+ videoCodec: XviD
+ year: 2014
+
+? Ice.Lake.Rebels.S01E06.Ice.Lake.Games.720p.HDTV.x264-DHD
+: options: -n
+ episodeNumber: 6
+ format: HDTV
+ releaseGroup: DHD
+ screenSize: 720p
+ season: 1
+ series: Ice Lake Rebels
+ title: Ice Lake Games
+ type: episode
+ videoCodec: h264
+
+? The League - S06E10 - Epi Sexy.mkv
+: episodeNumber: 10
+ season: 6
+ series: The League
+ title: Epi Sexy
+ type: episode
+
+? Stay (2005) [1080p]/Stay.2005.1080p.BluRay.x264.YIFY.mp4
+: format: BluRay
+ releaseGroup: YIFY
+ screenSize: 1080p
+ title: Stay
+ type: movie
+ videoCodec: h264
+ year: 2005
+
+? /media/live/A/Anger.Management.S02E82.720p.HDTV.X264-DIMENSION.mkv
+: format: HDTV
+ releaseGroup: DIMENSION
+ screenSize: 720p
+ series: Anger Management
+ type: episode
+ season: 2
+ episodeNumber: 82
+ videoCodec: h264
+
+? "[Figmentos] Monster 34 - At the End of Darkness [781219F1].mkv"
+: type: episode
+ releaseGroup: Figmentos
+ series: Monster
+ episodeNumber: 34
+ title: At the End of Darkness
+ crc32: 781219F1
+
+? Game.of.Thrones.S05E07.720p.HDTV-KILLERS.mkv
+: type: episode
+ episodeNumber: 7
+ format: HDTV
+ releaseGroup: KILLERS
+ screenSize: 720p
+ season: 5
+ series: Game of Thrones
+
+? Game.of.Thrones.S05E07.HDTV.720p-KILLERS.mkv
+: type: episode
+ episodeNumber: 7
+ format: HDTV
+ releaseGroup: KILLERS
+ screenSize: 720p
+ season: 5
+ series: Game of Thrones
diff --git a/lib/guessit/test/dummy.srt b/lib/guessit/test/dummy.srt
new file mode 100644
index 0000000000000000000000000000000000000000..ca4cf8b818ed9ff7a74d00b4bfe8d8a6b2e7602c
--- /dev/null
+++ b/lib/guessit/test/dummy.srt
@@ -0,0 +1 @@
+Just a dummy srt file (used for unittests: do not remove!)
diff --git a/lib/guessit/test/episodes.yaml b/lib/guessit/test/episodes.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7fb092e4b7f36553898d46b596d4f221218c9510
--- /dev/null
+++ b/lib/guessit/test/episodes.yaml
@@ -0,0 +1,1188 @@
+# Dubious tests
+#
+#? "finale "
+#: releaseGroup: FiNaLe
+# extension: ""
+
+
+? Series/Californication/Season 2/Californication.2x05.Vaginatown.HDTV.XviD-0TV.avi
+: series: Californication
+ season: 2
+ episodeNumber: 5
+ title: Vaginatown
+ format: HDTV
+ videoCodec: XviD
+ releaseGroup: 0TV
+
+? Series/dexter/Dexter.5x02.Hello,.Bandit.ENG.-.sub.FR.HDTV.XviD-AlFleNi-TeaM.[tvu.org.ru].avi
+: series: Dexter
+ season: 5
+ episodeNumber: 2
+ title: Hello, Bandit
+ language: English
+ subtitleLanguage: French
+ format: HDTV
+ videoCodec: XviD
+ releaseGroup: AlFleNi-TeaM
+ website: tvu.org.ru
+
+? Series/Treme/Treme.1x03.Right.Place,.Wrong.Time.HDTV.XviD-NoTV.avi
+: series: Treme
+ season: 1
+ episodeNumber: 3
+ title: Right Place, Wrong Time
+ format: HDTV
+ videoCodec: XviD
+ releaseGroup: NoTV
+
+? Series/Duckman/Duckman - 101 (01) - 20021107 - I, Duckman.avi
+: series: Duckman
+ season: 1
+ episodeNumber: 1
+ title: I, Duckman
+ date: 2002-11-07
+
+? Series/Duckman/Duckman - S1E13 Joking The Chicken (unedited).avi
+: series: Duckman
+ season: 1
+ episodeNumber: 13
+ title: Joking The Chicken
+
+? Series/Simpsons/Saison 12 Français/Simpsons,.The.12x08.A.Bas.Le.Sergent.Skinner.FR.avi
+: series: The Simpsons
+ season: 12
+ episodeNumber: 8
+ title: A Bas Le Sergent Skinner
+ language: French
+
+? Series/Futurama/Season 3 (mkv)/[™] Futurama - S03E22 - Le chef de fer à 30% ( 30 Percent Iron Chef ).mkv
+: series: Futurama
+ season: 3
+ episodeNumber: 22
+ title: Le chef de fer à 30%
+
+? Series/The Office/Season 6/The Office - S06xE01.avi
+: series: The Office
+ season: 6
+ episodeNumber: 1
+
+? series/The Office/Season 4/The Office [401] Fun Run.avi
+: series: The Office
+ season: 4
+ episodeNumber: 1
+ title: Fun Run
+
+? Series/Mad Men Season 1 Complete/Mad.Men.S01E01.avi
+: series: Mad Men
+ season: 1
+ episodeNumber: 1
+ other: complete
+
+? series/Psych/Psych S02 Season 2 Complete English DVD/Psych.S02E02.65.Million.Years.Off.avi
+: series: Psych
+ season: 2
+ episodeNumber: 2
+ title: 65 Million Years Off
+ language: english
+ format: DVD
+ other: complete
+
+? series/Psych/Psych S02 Season 2 Complete English DVD/Psych.S02E03.Psy.Vs.Psy.Français.srt
+: series: Psych
+ season: 2
+ episodeNumber: 3
+ title: Psy Vs Psy
+ format: DVD
+ language: English
+ subtitleLanguage: French
+ other: complete
+
+? Series/Pure Laine/Pure.Laine.1x01.Toutes.Couleurs.Unies.FR.(Québec).DVB-Kceb.[tvu.org.ru].avi
+: series: Pure Laine
+ season: 1
+ episodeNumber: 1
+ title: Toutes Couleurs Unies
+ format: DVB
+ releaseGroup: Kceb
+ language: french
+ website: tvu.org.ru
+
+? Series/Pure Laine/2x05 - Pure Laine - Je Me Souviens.avi
+: series: Pure Laine
+ season: 2
+ episodeNumber: 5
+ title: Je Me Souviens
+
+? Series/Tout sur moi/Tout sur moi - S02E02 - Ménage à trois (14-01-2008) [Rip by Ampli].avi
+: series: Tout sur moi
+ season: 2
+ episodeNumber: 2
+ title: Ménage à trois
+ date: 2008-01-14
+
+? The.Mentalist.2x21.18-5-4.ENG.-.sub.FR.HDTV.XviD-AlFleNi-TeaM.[tvu.org.ru].avi
+: series: The Mentalist
+ season: 2
+ episodeNumber: 21
+ title: 18-5-4
+ language: english
+ subtitleLanguage: french
+ format: HDTV
+ videoCodec: Xvid
+ releaseGroup: AlFleNi-TeaM
+ website: tvu.org.ru
+
+? series/__ Incomplete __/Dr Slump (Catalan)/Dr._Slump_-_003_DVB-Rip_Catalan_by_kelf.avi
+: series: Dr Slump
+ episodeNumber: 3
+ format: DVB
+ language: catalan
+
+? series/Ren and Stimpy - Black_hole_[DivX].avi
+: series: Ren and Stimpy
+ title: Black hole
+ videoCodec: DivX
+
+? Series/Walt Disney/Donald.Duck.-.Good.Scouts.[www.bigernie.jump.to].avi
+: series: Donald Duck
+ title: Good Scouts
+ website: www.bigernie.jump.to
+
+? Series/Neverwhere/Neverwhere.05.Down.Street.[tvu.org.ru].avi
+: series: Neverwhere
+ episodeNumber: 5
+ title: Down Street
+ website: tvu.org.ru
+
+? Series/South Park/Season 4/South.Park.4x07.Cherokee.Hair.Tampons.DVDRip.[tvu.org.ru].avi
+: series: South Park
+ season: 4
+ episodeNumber: 7
+ title: Cherokee Hair Tampons
+ format: DVD
+ website: tvu.org.ru
+
+? Series/Kaamelott/Kaamelott - Livre V - Ep 23 - Le Forfait.avi
+: series: Kaamelott
+ episodeNumber: 23
+ title: Le Forfait
+
+? Series/Duckman/Duckman - 110 (10) - 20021218 - Cellar Beware.avi
+: series: Duckman
+ season: 1
+ episodeNumber: 10
+ date: 2002-12-18
+ title: Cellar Beware
+
+? Series/Ren & Stimpy/Ren And Stimpy - Onward & Upward-Adult Party Cartoon.avi
+: series: Ren And Stimpy
+ title: Onward & Upward-Adult Party Cartoon
+
+? Series/Breaking Bad/Minisodes/Breaking.Bad.(Minisodes).01.Good.Cop.Bad.Cop.WEBRip.XviD.avi
+: series: Breaking Bad
+ episodeFormat: Minisode
+ episodeNumber: 1
+ title: Good Cop Bad Cop
+ format: WEBRip
+ videoCodec: XviD
+
+? Series/My Name Is Earl/My.Name.Is.Earl.S01Extras.-.Bad.Karma.DVDRip.XviD.avi
+: series: My Name Is Earl
+ season: 1
+ title: Bad Karma
+ format: DVD
+ episodeDetails: Extras
+ videoCodec: XviD
+
+? series/Freaks And Geeks/Season 1/Episode 4 - Kim Kelly Is My Friend-eng(1).srt
+: series: Freaks And Geeks
+ season: 1
+ episodeNumber: 4
+ title: Kim Kelly Is My Friend
+ language: English
+
+? /mnt/series/The Big Bang Theory/S01/The.Big.Bang.Theory.S01E01.mkv
+: series: The Big Bang Theory
+ season: 1
+ episodeNumber: 1
+
+? /media/Parks_and_Recreation-s03-e01.mkv
+: series: Parks and Recreation
+ season: 3
+ episodeNumber: 1
+
+? /media/Parks_and_Recreation-s03-e02-Flu_Season.mkv
+: series: Parks and Recreation
+ season: 3
+ title: Flu Season
+ episodeNumber: 2
+
+? /media/Parks_and_Recreation-s03-x01.mkv
+: series: Parks and Recreation
+ season: 3
+ bonusNumber: 1
+
+? /media/Parks_and_Recreation-s03-x02-Gag_Reel.mkv
+: series: Parks and Recreation
+ season: 3
+ bonusNumber: 2
+ bonusTitle: Gag Reel
+
+? /media/Band_of_Brothers-e01-Currahee.mkv
+: series: Band of Brothers
+ episodeNumber: 1
+ title: Currahee
+
+? /media/Band_of_Brothers-x02-We_Stand_Alone_Together.mkv
+: series: Band of Brothers
+ bonusNumber: 2
+ bonusTitle: We Stand Alone Together
+
+? /TV Shows/Mad.M-5x9.mkv
+: series: Mad M
+ season: 5
+ episodeNumber: 9
+
+? /TV Shows/new.girl.117.hdtv-lol.mp4
+: series: New Girl
+ season: 1
+ episodeNumber: 17
+ format: HDTV
+ releaseGroup: LOL
+
+? Kaamelott - 5x44x45x46x47x48x49x50.avi
+: series: Kaamelott
+ season: 5
+ episodeNumber: 44
+ episodeList: [44, 45, 46, 47, 48, 49, 50]
+
+? Example S01E01-02.avi
+: series: Example
+ season: 1
+ episodeNumber: 1
+ episodeList: [1, 2]
+
+? Example S01E01E02.avi
+: series: Example
+ season: 1
+ episodeNumber: 1
+ episodeList: [1, 2]
+
+? Series/Baccano!/Baccano!_-_T1_-_Trailer_-_[Ayu](dae8173e).mkv
+: series: Baccano!
+ other: Trailer
+ releaseGroup: Ayu
+ title: T1
+ crc32: dae8173e
+
+? Series/Doctor Who (2005)/Season 06/Doctor Who (2005) - S06E01 - The Impossible Astronaut (1).avi
+: series: Doctor Who
+ year: 2005
+ season: 6
+ episodeNumber: 1
+ title: The Impossible Astronaut
+
+? Parks and Recreation - [04x12] - Ad Campaign.avi
+: series: Parks and Recreation
+ season: 4
+ episodeNumber: 12
+ title: Ad Campaign
+
+? The Sopranos - [05x07] - In Camelot.mp4
+: series: The Sopranos
+ season: 5
+ episodeNumber: 7
+ title: In Camelot
+
+? The.Office.(US).1x03.Health.Care.HDTV.XviD-LOL.avi
+: series: The Office (US)
+ country: US
+ season: 1
+ episodeNumber: 3
+ title: Health Care
+ format: HDTV
+ videoCodec: XviD
+ releaseGroup: LOL
+
+? /Volumes/data-1/Series/Futurama/Season 3/Futurama_-_S03_DVD_Bonus_-_Deleted_Scenes_Part_3.ogm
+: series: Futurama
+ season: 3
+ part: 3
+ other: Bonus
+ title: Deleted Scenes
+ format: DVD
+
+? Ben.and.Kate.S01E02.720p.HDTV.X264-DIMENSION.mkv
+: series: Ben and Kate
+ season: 1
+ episodeNumber: 2
+ screenSize: 720p
+ format: HDTV
+ videoCodec: h264
+ releaseGroup: DIMENSION
+
+? /volume1/TV Series/Drawn Together/Season 1/Drawn Together 1x04 Requiem for a Reality Show.avi
+: series: Drawn Together
+ season: 1
+ episodeNumber: 4
+ title: Requiem for a Reality Show
+
+? Sons.of.Anarchy.S05E06.720p.WEB.DL.DD5.1.H.264-CtrlHD.mkv
+: series: Sons of Anarchy
+ season: 5
+ episodeNumber: 6
+ screenSize: 720p
+ format: WEB-DL
+ audioChannels: "5.1"
+ audioCodec: DolbyDigital
+ videoCodec: h264
+ releaseGroup: CtrlHD
+
+? /media/bdc64bfe-e36f-4af8-b550-e6fd2dfaa507/TV_Shows/Doctor Who (2005)/Saison 6/Doctor Who (2005) - S06E13 - The Wedding of River Song.mkv
+: series: Doctor Who
+ season: 6
+ episodeNumber: 13
+ year: 2005
+ title: The Wedding of River Song
+ idNumber: bdc64bfe-e36f-4af8-b550-e6fd2dfaa507
+
+? /mnt/videos/tvshows/Doctor Who/Season 06/E13 - The Wedding of River Song.mkv
+: series: Doctor Who
+ season: 6
+ episodeNumber: 13
+ title: The Wedding of River Song
+
+? The.Simpsons.S24E03.Adventures.in.Baby-Getting.720p.WEB-DL.DD5.1.H.264-CtrlHD.mkv
+: series: The Simpsons
+ season: 24
+ episodeNumber: 3
+ title: Adventures in Baby-Getting
+ screenSize: 720p
+ format: WEB-DL
+ audioChannels: "5.1"
+ audioCodec: DolbyDigital
+ videoCodec: h264
+ releaseGroup: CtrlHD
+
+? /home/disaster/Videos/TV/Merlin/merlin_2008.5x02.arthurs_bane_part_two.repack.720p_hdtv_x264-fov.mkv
+: series: Merlin
+ season: 5
+ episodeNumber: 2
+ part: 2
+ title: Arthurs bane
+ screenSize: 720p
+ format: HDTV
+ videoCodec: h264
+ releaseGroup: Fov
+ year: 2008
+ other: Proper
+ properCount: 1
+
+? "Da Vinci's Demons - 1x04 - The Magician.mkv"
+: series: "Da Vinci's Demons"
+ season: 1
+ episodeNumber: 4
+ title: The Magician
+
+? CSI.S013E18.Sheltered.720p.WEB-DL.DD5.1.H.264.mkv
+: series: CSI
+ season: 13
+ episodeNumber: 18
+ title: Sheltered
+ screenSize: 720p
+ format: WEB-DL
+ audioChannels: "5.1"
+ audioCodec: DolbyDigital
+ videoCodec: h264
+
+? Game of Thrones S03E06 1080i HDTV DD5.1 MPEG2-TrollHD.ts
+: series: Game of Thrones
+ season: 3
+ episodeNumber: 6
+ screenSize: 1080i
+ format: HDTV
+ audioChannels: "5.1"
+ audioCodec: DolbyDigital
+ videoCodec: MPEG2
+ releaseGroup: TrollHD
+
+? gossip.girl.s01e18.hdtv.xvid-2hd.eng.srt
+: series: gossip girl
+ season: 1
+ episodeNumber: 18
+ format: HDTV
+ videoCodec: XviD
+ releaseGroup: 2HD
+ subtitleLanguage: english
+
+? Wheels.S03E01E02.720p.HDTV.x264-IMMERSE.mkv
+: series: Wheels
+ season: 3
+ episodeNumber: 1
+ episodeList: [1, 2]
+ screenSize: 720p
+ format: HDTV
+ videoCodec: h264
+ releaseGroup: IMMERSE
+
+? Wheels.S03E01-02.720p.HDTV.x264-IMMERSE.mkv
+: series: Wheels
+ season: 3
+ episodeNumber: 1
+ episodeList: [1, 2]
+ screenSize: 720p
+ format: HDTV
+ videoCodec: h264
+ releaseGroup: IMMERSE
+
+? Wheels.S03E01-E02.720p.HDTV.x264-IMMERSE.mkv
+: series: Wheels
+ season: 3
+ episodeNumber: 1
+ episodeList: [1, 2]
+ screenSize: 720p
+ format: HDTV
+ videoCodec: h264
+ releaseGroup: IMMERSE
+
+? Wheels.S03E01-03.720p.HDTV.x264-IMMERSE.mkv
+: series: Wheels
+ season: 3
+ episodeNumber: 1
+ episodeList: [1, 2, 3]
+ screenSize: 720p
+ format: HDTV
+ videoCodec: h264
+ releaseGroup: IMMERSE
+
+? Marvels.Agents.of.S.H.I.E.L.D.S01E06.720p.HDTV.X264-DIMENSION.mkv
+: series: Marvels Agents of S.H.I.E.L.D.
+ season: 1
+ episodeNumber: 6
+ screenSize: 720p
+ format: HDTV
+ videoCodec: h264
+ releaseGroup: DIMENSION
+
+? Marvels.Agents.of.S.H.I.E.L.D..S01E06.720p.HDTV.X264-DIMENSION.mkv
+: series: Marvels Agents of S.H.I.E.L.D.
+ season: 1
+ episodeNumber: 6
+ screenSize: 720p
+ format: HDTV
+ videoCodec: h264
+ releaseGroup: DIMENSION
+
+? Series/Friday Night Lights/Season 1/Friday Night Lights S01E19 - Ch-Ch-Ch-Ch-Changes.avi
+: series: Friday Night Lights
+ season: 1
+ episodeNumber: 19
+ title: Ch-Ch-Ch-Ch-Changes
+
+? Dexter Saison VII FRENCH.BDRip.XviD-MiND.nfo
+: series: Dexter
+ season: 7
+ videoCodec: XviD
+ language: French
+ format: BluRay
+ releaseGroup: MiND
+
+? Dexter Saison sept FRENCH.BDRip.XviD-MiND.nfo
+: series: Dexter
+ season: 7
+ videoCodec: XviD
+ language: French
+ format: BluRay
+ releaseGroup: MiND
+
+? "Pokémon S16 - E29 - 1280*720 HDTV VF.mkv"
+: series: Pokémon
+ format: HDTV
+ language: French
+ season: 16
+ episodeNumber: 29
+ screenSize: 720p
+
+? One.Piece.E576.VOSTFR.720p.HDTV.x264-MARINE-FORD.mkv
+: episodeNumber: 576
+ videoCodec: h264
+ format: HDTV
+ series: One Piece
+ releaseGroup: MARINE-FORD
+ subtitleLanguage: French
+ screenSize: 720p
+
+? Dexter.S08E12.FINAL.MULTi.1080p.BluRay.x264-MiND.mkv
+: videoCodec: h264
+ episodeNumber: 12
+ season: 8
+ format: BluRay
+ series: Dexter
+ other: final
+ language: Multiple languages
+ releaseGroup: MiND
+ screenSize: 1080p
+
+? One Piece - E623 VOSTFR HD [www.manga-ddl-free.com].mkv
+: website: www.manga-ddl-free.com
+ episodeNumber: 623
+ subtitleLanguage: French
+ series: One Piece
+ other: HD
+
+? Falling Skies Saison 1.HDLight.720p.x264.VFF.mkv
+: language: French
+ screenSize: 720p
+ season: 1
+ series: Falling Skies
+ videoCodec: h264
+ other: HDLight
+
+? Sleepy.Hollow.S01E09.720p.WEB-DL.DD5.1.H.264-BP.mkv
+: episodeNumber: 9
+ videoCodec: h264
+ format: WEB-DL
+ series: Sleepy Hollow
+ audioChannels: "5.1"
+ screenSize: 720p
+ season: 1
+ videoProfile: BP
+ audioCodec: DolbyDigital
+
+? Sleepy.Hollow.S01E09.720p.WEB-DL.DD5.1.H.264-BS.mkv
+: episodeNumber: 9
+ videoCodec: h264
+ format: WEB-DL
+ series: Sleepy Hollow
+ audioChannels: "5.1"
+ screenSize: 720p
+ season: 1
+ releaseGroup: BS
+ audioCodec: DolbyDigital
+
+? Battlestar.Galactica.S00.Pilot.FRENCH.DVDRip.XviD-NOTAG.avi
+: series: Battlestar Galactica
+ season: 0
+ title: Pilot
+ episodeDetails: Pilot
+ language: French
+ format: DVD
+ videoCodec: XviD
+ releaseGroup: NOTAG
+
+? The Big Bang Theory S00E00 Unaired Pilot VOSTFR TVRip XviD-VioCs
+: options: -n
+ series: The Big Bang Theory
+ season: 0
+ episodeNumber: 0
+ subtitleLanguage: French
+ format: TV
+ videoCodec: XviD
+ releaseGroup: VioCs
+ episodeDetails: [Unaired, Pilot]
+ title: Unaired Pilot
+
+? The Big Bang Theory S01E00 PROPER Unaired Pilot TVRip XviD-GIGGITY
+: options: -n
+ series: The Big Bang Theory
+ season: 1
+ episodeNumber: 0
+ format: TV
+ videoCodec: XviD
+ releaseGroup: GIGGITY
+ other: proper
+ properCount: 1
+ episodeDetails: [Unaired, Pilot]
+ title: Unaired Pilot
+
+? Pawn.Stars.S2014E18.720p.HDTV.x264-KILLERS
+: options: -n
+ series: Pawn Stars
+ season: 2014
+ year: 2014
+ episodeNumber: 18
+ screenSize: 720p
+ format: HDTV
+ videoCodec: h264
+ releaseGroup: KILLERS
+
+? 2.Broke.Girls.S03E10.480p.HDTV.x264-mSD.mkv
+: series: 2 Broke Girls
+ season: 3
+ episodeNumber: 10
+ screenSize: 480p
+ format: HDTV
+ videoCodec: h264
+ releaseGroup: mSD
+
+? House.of.Cards.2013.S02E03.1080p.NF.WEBRip.DD5.1.x264-NTb.mkv
+: series: House of Cards
+ year: 2013
+ season: 2
+ episodeNumber: 3
+ screenSize: 1080p
+ other: Netflix
+ format: Webrip
+ audioChannels: "5.1"
+ audioCodec: DolbyDigital
+ videoCodec: h264
+ releaseGroup: NTb
+
+? the.100.109.hdtv-lol.mp4
+: series: the 100
+ season: 1
+ episodeNumber: 9
+ format: HDTV
+ releaseGroup: lol
+
+? 03-Criminal.Minds.5x03.Reckoner.ENG.-.sub.FR.HDTV.XviD-STi.[tvu.org.ru].avi
+: series: Criminal Minds
+ language: English
+ subtitleLanguage: French
+ season: 5
+ episodeNumber: 3
+ videoCodec: XviD
+ format: HDTV
+ website: tvu.org.ru
+ releaseGroup: STi
+ title: Reckoner
+
+? 03-Criminal.Minds.avi
+: series: Criminal Minds
+ episodeNumber: 3
+
+? '[Evil-Saizen]_Laughing_Salesman_14_[DVD][1C98686A].mkv'
+: crc32: 1C98686A
+ episodeNumber: 14
+ format: DVD
+ releaseGroup: Evil-Saizen
+ series: Laughing Salesman
+
+? '[Kaylith] Zankyou no Terror - 04 [480p][B4D4514E].mp4'
+: crc32: B4D4514E
+ episodeNumber: 4
+ releaseGroup: Kaylith
+ screenSize: 480p
+ series: Zankyou no Terror
+
+? '[PuyaSubs!] Seirei Tsukai no Blade Dance - 05 [720p][32DD560E].mkv'
+: crc32: 32DD560E
+ episodeNumber: 5
+ releaseGroup: PuyaSubs!
+ screenSize: 720p
+ series: Seirei Tsukai no Blade Dance
+
+? '[Doremi].Happiness.Charge.Precure.27.[1280x720].[DC91581A].mkv'
+: crc32: DC91581A
+ episodeNumber: 27
+ releaseGroup: Doremi
+ screenSize: 720p
+ series: Happiness Charge Precure
+
+? "[Daisei] Free!：Iwatobi Swim Club - 01 ~ (BD 720p 10-bit AAC) [99E8E009].mkv"
+: audioCodec: AAC
+ crc32: 99E8E009
+ episodeNumber: 1
+ format: BluRay
+ releaseGroup: Daisei
+ screenSize: 720p
+ series: Free!：Iwatobi Swim Club
+ videoProfile: 10bit
+
+? '[Tsundere] Boku wa Tomodachi ga Sukunai - 03 [BDRip h264 1920x1080 10bit FLAC][AF0C22CC].mkv'
+: audioCodec: Flac
+ crc32: AF0C22CC
+ episodeNumber: 3
+ format: BluRay
+ releaseGroup: Tsundere
+ screenSize: 1080p
+ series: Boku wa Tomodachi ga Sukunai
+ videoCodec: h264
+ videoProfile: 10bit
+
+? '[t.3.3.d]_Mikakunin_de_Shinkoukei_-_12_[720p][5DDC1352].mkv'
+: crc32: 5DDC1352
+ episodeNumber: 12
+ screenSize: 720p
+ series: Mikakunin de Shinkoukei
+ releaseGroup: t.3.3.d
+
+? '[Anime-Koi] Sabagebu! - 06 [h264-720p][ABB3728A].mkv'
+: crc32: ABB3728A
+ episodeNumber: 6
+ releaseGroup: Anime-Koi
+ screenSize: 720p
+ series: Sabagebu!
+ videoCodec: h264
+
+? '[aprm-Diogo4D] [BD][1080p] Nagi no Asukara 08 [4D102B7C].mkv'
+: crc32: 4D102B7C
+ episodeNumber: 8
+ format: BluRay
+ releaseGroup: aprm-Diogo4D
+ screenSize: 1080p
+ series: Nagi no Asukara
+
+? '[Akindo-SSK] Zankyou no Terror - 05 [720P][Sub_ITA][F5CCE87C].mkv'
+: crc32: F5CCE87C
+ episodeNumber: 5
+ releaseGroup: Akindo-SSK
+ screenSize: 720p
+ series: Zankyou no Terror
+ subtitleLanguage: it
+
+? Naruto Shippuden Episode 366 VOSTFR.avi
+: episodeNumber: 366
+ series: Naruto Shippuden
+ subtitleLanguage: fr
+
+? Naruto Shippuden Episode 366v2 VOSTFR.avi
+: episodeNumber: 366
+ version: 2
+ series: Naruto Shippuden
+ subtitleLanguage: fr
+
+? '[HorribleSubs] Ao Haru Ride - 06 [480p].mkv'
+: episodeNumber: 6
+ releaseGroup: HorribleSubs
+ screenSize: 480p
+ series: Ao Haru Ride
+
+? '[DeadFish] Tari Tari - 01 [BD][720p][AAC].mp4'
+: audioCodec: AAC
+ episodeNumber: 1
+ format: BluRay
+ releaseGroup: DeadFish
+ screenSize: 720p
+ series: Tari Tari
+
+? '[NoobSubs] Sword Art Online II 06 (720p 8bit AAC).mp4'
+: audioCodec: AAC
+ episodeNumber: 6
+ releaseGroup: NoobSubs
+ screenSize: 720p
+ series: Sword Art Online II
+ videoProfile: 8bit
+
+? '[DeadFish] 01 - Tari Tari [BD][720p][AAC].mp4'
+: audioCodec: AAC
+ episodeNumber: 1
+ format: BluRay
+ releaseGroup: DeadFish
+ screenSize: 720p
+ series: Tari Tari
+
+? '[NoobSubs] 06 Sword Art Online II (720p 8bit AAC).mp4'
+: audioCodec: AAC
+ episodeNumber: 6
+ releaseGroup: NoobSubs
+ screenSize: 720p
+ series: Sword Art Online II
+ videoProfile: 8bit
+
+? '[DeadFish] 12 - Tari Tari [BD][720p][AAC].mp4'
+: audioCodec: AAC
+ episodeNumber: 12
+ format: BluRay
+ releaseGroup: DeadFish
+ screenSize: 720p
+ series: Tari Tari
+
+? Something.Season.2.1of4.Ep.Title.HDTV.torrent
+: episodeCount: 4
+ episodeNumber: 1
+ format: HDTV
+ season: 2
+ series: Something
+ title: Title
+ extension: torrent
+
+? Something.Season.2of5.3of9.Ep.Title.HDTV.torrent
+: episodeCount: 9
+ episodeNumber: 3
+ format: HDTV
+ season: 2
+ seasonCount: 5
+ series: Something
+ title: Title
+ extension: torrent
+
+? Something.Other.Season.3of5.Complete.HDTV.torrent
+: format: HDTV
+ other: Complete
+ season: 3
+ seasonCount: 5
+ series: Something Other
+ extension: torrent
+
+? Something.Other.Season.1-3.avi
+: season: 1
+ seasonList:
+ - 1
+ - 2
+ - 3
+ series: Something Other
+
+? Something.Other.Season.1&3.avi
+: season: 1
+ seasonList:
+ - 1
+ - 3
+ series: Something Other
+
+? Something.Other.Season.1&3-1to12ep.avi
+: season: 1
+ seasonList:
+ - 1
+ - 3
+ series: Something Other
+
+? Something.Other.saison 1 2 & 4 a 7.avi
+: season: 1
+ seasonList:
+ - 1
+ - 2
+ - 4
+ - 5
+ - 6
+ - 7
+ series: Something Other
+
+? W2Test.123.HDTV.XViD-FlexGet
+: options: -n
+ episodeNumber: 23
+ season: 1
+ format: HDTV
+ releaseGroup: FlexGet
+ series: W2Test
+ videoCodec: XviD
+
+? W2Test.123.HDTV.XViD-FlexGet
+: options: -n --episode-prefer-number
+ episodeNumber: 123
+ format: HDTV
+ releaseGroup: FlexGet
+ series: W2Test
+ videoCodec: XviD
+
+? FooBar.0307.PDTV-FlexGet
+: options: -n --episode-prefer-number
+ episodeNumber: 7
+ format: DVB
+ releaseGroup: FlexGet
+ season: 3
+ series: FooBar
+
+? FooBar.307.PDTV-FlexGet
+: options: -n --episode-prefer-number
+ episodeNumber: 307
+ format: DVB
+ releaseGroup: FlexGet
+ series: FooBar
+
+? FooBar.07.PDTV-FlexGet
+: options: -n --episode-prefer-number
+ episodeNumber: 7
+ format: DVB
+ releaseGroup: FlexGet
+ series: FooBar
+
+? FooBar.7.PDTV-FlexGet
+: options: -n -t episode --episode-prefer-number
+ episodeNumber: 7
+ format: DVB
+ releaseGroup: FlexGet
+ series: FooBar
+
+? FooBar.0307.PDTV-FlexGet
+: options: -n
+ episodeNumber: 7
+ format: DVB
+ releaseGroup: FlexGet
+ season: 3
+ series: FooBar
+
+? FooBar.307.PDTV-FlexGet
+: options: -n
+ episodeNumber: 7
+ format: DVB
+ releaseGroup: FlexGet
+ season: 3
+ series: FooBar
+
+? FooBar.07.PDTV-FlexGet
+: options: -n
+ episodeNumber: 7
+ format: DVB
+ releaseGroup: FlexGet
+ series: FooBar
+
+? FooBar.07v4.PDTV-FlexGet
+: options: -n
+ episodeNumber: 7
+ version: 4
+ format: DVB
+ releaseGroup: FlexGet
+ series: FooBar
+
+? FooBar.7.PDTV-FlexGet
+: options: -n -t episode
+ format: DVB
+ releaseGroup: FlexGet
+ series: FooBar 7
+
+? FooBar.7v3.PDTV-FlexGet
+: options: -n -t episode
+ episodeNumber: 7
+ version: 3
+ format: DVB
+ releaseGroup: FlexGet
+ series: FooBar
+
+? Test.S02E01.hdtv.real.proper
+: options: -n
+ episodeNumber: 1
+ format: HDTV
+ other: Proper
+ properCount: 2
+ season: 2
+ series: Test
+
+? Real.Test.S02E01.hdtv.proper
+: options: -n
+ episodeNumber: 1
+ format: HDTV
+ other: Proper
+ properCount: 1
+ season: 2
+ series: Real Test
+
+? Test.Real.S02E01.hdtv.proper
+: options: -n
+ episodeNumber: 1
+ format: HDTV
+ other: Proper
+ properCount: 1
+ season: 2
+ series: Test Real
+
+? Test.S02E01.hdtv.proper
+: options: -n
+ episodeNumber: 1
+ format: HDTV
+ other: Proper
+ properCount: 1
+ season: 2
+ series: Test
+
+? Test.S02E01.hdtv.real.repack.proper
+: options: -n
+ episodeNumber: 1
+ format: HDTV
+ other: Proper
+ properCount: 3
+ season: 2
+ series: Test
+
+? Date.Show.03-29-2012.HDTV.XViD-FlexGet
+: options: -n
+ date: 2012-03-29
+ format: HDTV
+ releaseGroup: FlexGet
+ series: Date Show
+ videoCodec: XviD
+
+? Something.1x5.Season.Complete-FlexGet
+: options: -n
+ episodeNumber: 5
+ other: Complete
+ season: 1
+ series: Something
+ releaseGroup: FlexGet
+
+? Something Seasons 1 & 2 - Complete
+: options: -n
+ other: Complete
+ season: 1
+ seasonList:
+ - 1
+ - 2
+ series: Something
+
+? Something Seasons 4 Complete
+: options: -n
+ other: Complete
+ season: 4
+ series: Something
+
+? Something.1xAll.Season.Complete-FlexGet
+: options: -n
+ other: Complete
+ season: 1
+ series: Something
+ releaseGroup: FlexGet
+
+? Something.1xAll-FlexGet
+: options: -n
+ other: Complete
+ season: 1
+ series: Something
+ releaseGroup: FlexGet
+
+? FlexGet.US.S2013E14.Title.Here.720p.HDTV.AAC5.1.x264-NOGRP
+: options: -n
+ audioChannels: '5.1'
+ audioCodec: AAC
+ country: US
+ episodeNumber: 14
+ format: HDTV
+ releaseGroup: NOGRP
+ screenSize: 720p
+ season: 2013
+ series: FlexGet (US)
+ title: Title Here
+ videoCodec: h264
+ year: 2013
+
+? FlexGet.14.of.21.Title.Here.720p.HDTV.AAC5.1.x264-NOGRP
+: options: -n
+ audioChannels: '5.1'
+ audioCodec: AAC
+ episodeCount: 21
+ episodeNumber: 14
+ format: HDTV
+ releaseGroup: NOGRP
+ screenSize: 720p
+ series: FlexGet
+ title: Title Here
+ videoCodec: h264
+
+? FlexGet.Series.2013.14.of.21.Title.Here.720p.HDTV.AAC5.1.x264-NOGRP
+: options: -n
+ audioChannels: '5.1'
+ audioCodec: AAC
+ episodeCount: 21
+ episodeNumber: 14
+ format: HDTV
+ releaseGroup: NOGRP
+ screenSize: 720p
+ season: 2013
+ series: FlexGet
+ title: Title Here
+ videoCodec: h264
+ year: 2013
+
+? Something.S04E05E09
+: options: -n
+ episodeList:
+ - 5
+ - 6
+ - 7
+ - 8
+ - 9
+ episodeNumber: 5
+ season: 4
+ series: Something
+
+? FooBar 360 1080i
+: options: -n -t episode --episode-prefer-number
+ episodeNumber: 360
+ screenSize: 1080i
+ series: FooBar
+
+? FooBar 360 1080i
+: options: -n -t episode
+ episodeNumber: 60
+ season: 3
+ screenSize: 1080i
+ series: FooBar
+
+? FooBar 360
+: options: -n -t episode
+ screenSize: 360p
+ series: FooBar
+
+? BarFood christmas special HDTV
+: options: -n -t episode --expected-series BarFood
+ format: HDTV
+ series: BarFood
+ title: christmas special
+ episodeDetails: Special
+
+? Something.2008x12.13-FlexGet
+: options: -n -t episode
+ series: Something
+ date: 2008-12-13
+ title: FlexGet
+
+? '[Ignored] Test 12'
+: options: -n
+ episodeNumber: 12
+ releaseGroup: Ignored
+ series: Test
+
+? '[FlexGet] Test 12'
+: options: -n
+ episodeNumber: 12
+ releaseGroup: FlexGet
+ series: Test
+
+? Test.13.HDTV-Ignored
+: options: -n
+ episodeNumber: 13
+ format: HDTV
+ releaseGroup: Ignored
+ series: Test
+
+? Test.13.HDTV-Ignored
+: options: -n --expected-series test
+ episodeNumber: 13
+ format: HDTV
+ releaseGroup: Ignored
+ series: Test
+
+? Test.13.HDTV-Ignored
+: series: Test
+ episodeNumber: 13
+ format: HDTV
+ releaseGroup: Ignored
+
+? Test.13.HDTV-Ignored
+: options: -n --expected-group "Name;FlexGet"
+ episodeNumber: 13
+ format: HDTV
+ releaseGroup: Ignored
+ series: Test
+
+? Test.13.HDTV-FlexGet
+: options: -n
+ episodeNumber: 13
+ format: HDTV
+ releaseGroup: FlexGet
+ series: Test
+
+? Test.14.HDTV-Name
+: options: -n
+ episodeNumber: 14
+ format: HDTV
+ releaseGroup: Name
+ series: Test
+
+? Real.Time.With.Bill.Maher.2014.10.31.HDTV.XviD-AFG.avi
+: date: 2014-10-31
+ format: HDTV
+ releaseGroup: AFG
+ series: Real Time With Bill Maher
+ videoCodec: XviD
+
+? Arrow.S03E21.Al.Sah-Him.1080p.WEB-DL.DD5.1.H.264-BS.mkv
+: series: Arrow
+ season: 3
+ episodeNumber: 21
+ title: Al Sah-Him
+ screenSize: 1080p
+ audioCodec: DolbyDigital
+ audioChannels: "5.1"
+ videoCodec: h264
+ releaseGroup: BS
+ format: WEB-DL
diff --git a/lib/guessit/test/guessittest.py b/lib/guessit/test/guessittest.py
new file mode 100644
index 0000000000000000000000000000000000000000..5fafb656717033fc4111bef6794178bdda896db3
--- /dev/null
+++ b/lib/guessit/test/guessittest.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from collections import defaultdict
+from unittest import TestCase, TestLoader
+import shlex
+import logging
+import os
+import sys
+from os.path import *
+
+import babelfish
+import yaml
+
+
+def currentPath():
+ """Returns the path in which the calling file is located."""
+ return dirname(join(os.getcwd(), sys._getframe(1).f_globals['__file__']))
+
+
+def addImportPath(path):
+ """Function that adds the specified path to the import path. The path can be
+ absolute or relative to the calling file."""
+ importPath = abspath(join(currentPath(), path))
+ sys.path = [importPath] + sys.path
+
+log = logging.getLogger(__name__)
+
+from guessit.options import get_opts
+from guessit import base_text_type
+from guessit import *
+from guessit.matcher import *
+from guessit.fileutils import *
+import guessit
+
+
+def allTests(testClass):
+ return TestLoader().loadTestsFromTestCase(testClass)
+
+
+class TestGuessit(TestCase):
+
+ def checkMinimumFieldsCorrect(self, filename, filetype=None, remove_type=True,
+ exclude_files=None):
+ groundTruth = yaml.load(load_file_in_same_dir(__file__, filename))
+
+ def guess_func(string, options=None):
+ return guess_file_info(string, options=options, type=filetype)
+
+ return self.checkFields(groundTruth, guess_func, remove_type, exclude_files)
+
+ def checkFields(self, groundTruth, guess_func, remove_type=True,
+ exclude_files=None):
+ total = 0
+ exclude_files = exclude_files or []
+
+ fails = defaultdict(list)
+ additionals = defaultdict(list)
+
+ for filename, required_fields in groundTruth.items():
+ filename = u(filename)
+ if filename in exclude_files:
+ continue
+
+ log.debug('\n' + '-' * 120)
+ log.info('Guessing information for file: %s' % filename)
+
+ options = required_fields.pop('options') if 'options' in required_fields else None
+
+ if options:
+ args = shlex.split(options)
+ options = get_opts().parse_args(args)
+ options = vars(options)
+ try:
+ found = guess_func(filename, options)
+ except Exception as e:
+ fails[filename].append("An exception has occured in %s: %s" % (filename, e))
+ log.exception("An exception has occured in %s: %s" % (filename, e))
+ continue
+
+ total += 1
+
+ # no need for these in the unittests
+ if remove_type:
+ try:
+ del found['type']
+ except:
+ pass
+ for prop in ('container', 'mimetype', 'unidentified'):
+ if prop in found:
+ del found[prop]
+
+ # props which are list of just 1 elem should be opened for easier writing of the tests
+ for prop in ('language', 'subtitleLanguage', 'other', 'episodeDetails', 'unidentified'):
+ value = found.get(prop, None)
+ if isinstance(value, list) and len(value) == 1:
+ found[prop] = value[0]
+
+ # look for missing properties
+ for prop, value in required_fields.items():
+ if prop not in found:
+ log.debug("Prop '%s' not found in: %s" % (prop, filename))
+ fails[filename].append("'%s' not found in: %s" % (prop, filename))
+ continue
+
+ # if both properties are strings, do a case-insensitive comparison
+ if (isinstance(value, base_text_type) and
+ isinstance(found[prop], base_text_type)):
+ if value.lower() != found[prop].lower():
+ log.debug("Wrong prop value [str] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
+ fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
+
+ elif isinstance(value, list) and isinstance(found[prop], list):
+ if found[prop] and isinstance(found[prop][0], babelfish.Language):
+ # list of languages
+ s1 = set(Language.fromguessit(s) for s in value)
+ s2 = set(found[prop])
+ else:
+ # by default we assume list of strings and do a case-insensitive
+ # comparison on their elements
+ s1 = set(u(s).lower() for s in value)
+ s2 = set(u(s).lower() for s in found[prop])
+
+ if s1 != s2:
+ log.debug("Wrong prop value [list] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
+ fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
+
+ elif isinstance(found[prop], babelfish.Language):
+ try:
+ if babelfish.Language.fromguessit(value) != found[prop]:
+ raise ValueError
+ except:
+ log.debug("Wrong prop value [Language] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
+ fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
+
+ elif isinstance(found[prop], babelfish.Country):
+ try:
+ if babelfish.Country.fromguessit(value) != found[prop]:
+ raise ValueError
+ except:
+ log.debug("Wrong prop value [Country] for '%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
+ fails[filename].append("'%s': expected = '%s' - received = '%s'" % (prop, u(value), u(found[prop])))
+
+
+ # otherwise, just compare their values directly
+ else:
+ if found[prop] != value:
+ log.debug("Wrong prop value for '%s': expected = '%s' [%s] - received = '%s' [%s]" % (prop, u(value), type(value), u(found[prop]), type(found[prop])))
+ fails[filename].append("'%s': expected = '%s' [%s] - received = '%s' [%s]" % (prop, u(value), type(value), u(found[prop]), type(found[prop])))
+
+ # look for additional properties
+ for prop, value in found.items():
+ if prop not in required_fields:
+ log.debug("Found additional info for prop = '%s': '%s'" % (prop, u(value)))
+ additionals[filename].append("'%s': '%s'" % (prop, u(value)))
+
+ correct = total - len(fails)
+ log.info('SUMMARY: Guessed correctly %d out of %d filenames' % (correct, total))
+
+ for failed_entry, failed_properties in fails.items():
+ log.error('---- ' + failed_entry + ' ----')
+ for failed_property in failed_properties:
+ log.error("FAILED: " + failed_property)
+
+ for additional_entry, additional_properties in additionals.items():
+ log.warning('---- ' + additional_entry + ' ----')
+ for additional_property in additional_properties:
+ log.warning("ADDITIONAL: " + additional_property)
+
+ assert correct == total, 'Correct: %d < Total: %d' % (correct, total)
diff --git a/lib/guessit/test/movies.yaml b/lib/guessit/test/movies.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0ea23a745dcb62995091bf72a765e2e90771af81
--- /dev/null
+++ b/lib/guessit/test/movies.yaml
@@ -0,0 +1,761 @@
+
+? Movies/Fear and Loathing in Las Vegas (1998)/Fear.and.Loathing.in.Las.Vegas.720p.HDDVD.DTS.x264-ESiR.mkv
+: title: Fear and Loathing in Las Vegas
+ year: 1998
+ screenSize: 720p
+ format: HD-DVD
+ audioCodec: DTS
+ videoCodec: h264
+ releaseGroup: ESiR
+
+? Movies/El Dia de la Bestia (1995)/El.dia.de.la.bestia.DVDrip.Spanish.DivX.by.Artik[SEDG].avi
+: title: El Dia de la Bestia
+ year: 1995
+ format: DVD
+ language: spanish
+ videoCodec: DivX
+ releaseGroup: Artik[SEDG]
+
+? Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD.mkv
+: title: Dark City
+ year: 1998
+ format: BluRay
+ screenSize: 720p
+ audioCodec: DTS
+ videoCodec: h264
+ releaseGroup: CHD
+
+? Movies/Sin City (BluRay) (2005)/Sin.City.2005.BDRip.720p.x264.AC3-SEPTiC.mkv
+: title: Sin City
+ year: 2005
+ format: BluRay
+ screenSize: 720p
+ videoCodec: h264
+ audioCodec: AC3
+ releaseGroup: SEPTiC
+
+
+? Movies/Borat (2006)/Borat.(2006).R5.PROPER.REPACK.DVDRip.XviD-PUKKA.avi
+: title: Borat
+ year: 2006
+ other: PROPER
+ properCount: 2
+ format: DVD
+ other: [ R5, Proper ]
+ videoCodec: XviD
+ releaseGroup: PUKKA
+
+
+? "[XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv"
+: title: Le Prestige
+ format: DVD
+ videoCodec: h264
+ videoProfile: HP
+ audioCodec: AAC
+ audioProfile: HE
+ language: [ french, english ]
+ subtitleLanguage: [ french, english ]
+ releaseGroup: XCT
+
+? Battle Royale (2000)/Battle.Royale.(Batoru.Rowaiaru).(2000).(Special.Edition).CD1of2.DVDRiP.XviD-[ZeaL].avi
+: title: Battle Royale
+ year: 2000
+ edition: special edition
+ cdNumber: 1
+ cdNumberTotal: 2
+ format: DVD
+ videoCodec: XviD
+ releaseGroup: ZeaL
+
+? Movies/Brazil (1985)/Brazil_Criterion_Edition_(1985).CD2.avi
+: title: Brazil
+ edition: Criterion Edition
+ year: 1985
+ cdNumber: 2
+
+? Movies/Persepolis (2007)/[XCT] Persepolis [H264+Aac-128(Fr-Eng)+ST(Fr-Eng)+Ind].mkv
+: title: Persepolis
+ year: 2007
+ videoCodec: h264
+ audioCodec: AAC
+ language: [ French, English ]
+ subtitleLanguage: [ French, English ]
+ releaseGroup: XCT
+
+? Movies/Toy Story (1995)/Toy Story [HDTV 720p English-Spanish].mkv
+: title: Toy Story
+ year: 1995
+ format: HDTV
+ screenSize: 720p
+ language: [ english, spanish ]
+
+? Movies/Office Space (1999)/Office.Space.[Dual-DVDRip].[Spanish-English].[XviD-AC3-AC3].[by.Oswald].avi
+: title: Office Space
+ year: 1999
+ format: DVD
+ language: [ english, spanish ]
+ videoCodec: XviD
+ audioCodec: AC3
+
+? Movies/Wild Zero (2000)/Wild.Zero.DVDivX-EPiC.avi
+: title: Wild Zero
+ year: 2000
+ videoCodec: DivX
+ releaseGroup: EPiC
+
+? movies/Baraka_Edition_Collector.avi
+: title: Baraka
+ edition: collector edition
+
+? Movies/Blade Runner (1982)/Blade.Runner.(1982).(Director's.Cut).CD1.DVDRip.XviD.AC3-WAF.avi
+: title: Blade Runner
+ year: 1982
+ edition: Director's Cut
+ cdNumber: 1
+ format: DVD
+ videoCodec: XviD
+ audioCodec: AC3
+ releaseGroup: WAF
+
+? movies/American.The.Bill.Hicks.Story.2009.DVDRip.XviD-EPiSODE.[UsaBit.com]/UsaBit.com_esd-americanbh.avi
+: title: American The Bill Hicks Story
+ year: 2009
+ format: DVD
+ videoCodec: XviD
+ releaseGroup: EPiSODE
+ website: UsaBit.com
+
+? movies/Charlie.And.Boots.DVDRip.XviD-TheWretched/wthd-cab.avi
+: title: Charlie And Boots
+ format: DVD
+ videoCodec: XviD
+ releaseGroup: TheWretched
+
+? movies/Steig Larsson Millenium Trilogy (2009) BRrip 720 AAC x264/(1)The Girl With The Dragon Tattoo (2009) BRrip 720 AAC x264.mkv
+: title: The Girl With The Dragon Tattoo
+ filmSeries: Steig Larsson Millenium Trilogy
+ filmNumber: 1
+ year: 2009
+ format: BluRay
+ audioCodec: AAC
+ videoCodec: h264
+ screenSize: 720p
+
+? movies/Greenberg.REPACK.LiMiTED.DVDRip.XviD-ARROW/arw-repack-greenberg.dvdrip.xvid.avi
+: title: Greenberg
+ format: DVD
+ videoCodec: XviD
+ releaseGroup: ARROW
+ other: ['Proper', 'Limited']
+ properCount: 2
+
+? Movies/Fr - Paris 2054, Renaissance (2005) - De Christian Volckman - (Film Divx Science Fiction Fantastique Thriller Policier N&B).avi
+: title: Paris 2054, Renaissance
+ year: 2005
+ language: french
+ videoCodec: DivX
+
+? Movies/[阿维达].Avida.2006.FRENCH.DVDRiP.XViD-PROD.avi
+: title: Avida
+ year: 2006
+ language: french
+ format: DVD
+ videoCodec: XviD
+ releaseGroup: PROD
+
+? Movies/Alice in Wonderland DVDRip.XviD-DiAMOND/dmd-aw.avi
+: title: Alice in Wonderland
+ format: DVD
+ videoCodec: XviD
+ releaseGroup: DiAMOND
+
+? Movies/Ne.Le.Dis.A.Personne.Fr 2 cd/personnea_mp.avi
+: title: Ne Le Dis A Personne
+ language: french
+ cdNumberTotal: 2
+
+? Movies/Bunker Palace Hôtel (Enki Bilal) (1989)/Enki Bilal - Bunker Palace Hotel (Fr Vhs Rip).avi
+: title: Bunker Palace Hôtel
+ year: 1989
+ language: french
+ format: VHS
+
+? Movies/21 (2008)/21.(2008).DVDRip.x264.AC3-FtS.[sharethefiles.com].mkv
+: title: "21"
+ year: 2008
+ format: DVD
+ videoCodec: h264
+ audioCodec: AC3
+ releaseGroup: FtS
+ website: sharethefiles.com
+
+? Movies/9 (2009)/9.2009.Blu-ray.DTS.720p.x264.HDBRiSe.[sharethefiles.com].mkv
+: title: "9"
+ year: 2009
+ format: BluRay
+ audioCodec: DTS
+ screenSize: 720p
+ videoCodec: h264
+ releaseGroup: HDBRiSe
+ website: sharethefiles.com
+
+? Movies/Mamma.Mia.2008.DVDRip.AC3.XviD-CrazyTeam/Mamma.Mia.2008.DVDRip.AC3.XviD-CrazyTeam.avi
+: title: Mamma Mia
+ year: 2008
+ format: DVD
+ audioCodec: AC3
+ videoCodec: XviD
+ releaseGroup: CrazyTeam
+
+? Movies/M.A.S.H. (1970)/MASH.(1970).[Divx.5.02][Dual-Subtitulos][DVDRip].ogm
+: title: M.A.S.H.
+ year: 1970
+ videoCodec: DivX
+ format: DVD
+
+? Movies/The Doors (1991)/09.03.08.The.Doors.(1991).BDRip.720p.AC3.X264-HiS@SiLUHD-English.[sharethefiles.com].mkv
+: title: The Doors
+ year: 1991
+ date: 2008-03-09
+ format: BluRay
+ screenSize: 720p
+ audioCodec: AC3
+ videoCodec: h264
+ releaseGroup: HiS@SiLUHD
+ language: english
+ website: sharethefiles.com
+
+? Movies/The Doors (1991)/08.03.09.The.Doors.(1991).BDRip.720p.AC3.X264-HiS@SiLUHD-English.[sharethefiles.com].mkv
+: options: --date-year-first
+ title: The Doors
+ year: 1991
+ date: 2008-03-09
+ format: BluRay
+ screenSize: 720p
+ audioCodec: AC3
+ videoCodec: h264
+ releaseGroup: HiS@SiLUHD
+ language: english
+ website: sharethefiles.com
+
+? Movies/Ratatouille/video_ts-ratatouille.srt
+: title: Ratatouille
+ format: DVD
+
+? Movies/001 __ A classer/Fantomas se déchaine - Louis de Funès.avi
+: title: Fantomas se déchaine
+
+? Movies/Comme une Image (2004)/Comme.Une.Image.FRENCH.DVDRiP.XViD-NTK.par-www.divx-overnet.com.avi
+: title: Comme une Image
+ year: 2004
+ language: french
+ format: DVD
+ videoCodec: XviD
+ releaseGroup: NTK
+ website: www.divx-overnet.com
+
+? Movies/Fantastic Mr Fox/Fantastic.Mr.Fox.2009.DVDRip.{x264+LC-AAC.5.1}{Fr-Eng}{Sub.Fr-Eng}-™.[sharethefiles.com].mkv
+: title: Fantastic Mr Fox
+ year: 2009
+ format: DVD
+ videoCodec: h264
+ audioCodec: AAC
+ audioProfile: LC
+ audioChannels: "5.1"
+ language: [ french, english ]
+ subtitleLanguage: [ french, english ]
+ website: sharethefiles.com
+
+? Movies/Somewhere.2010.DVDRip.XviD-iLG/i-smwhr.avi
+: title: Somewhere
+ year: 2010
+ format: DVD
+ videoCodec: XviD
+ releaseGroup: iLG
+
+? Movies/Moon_(2009).mkv
+: title: Moon
+ year: 2009
+
+? Movies/Moon_(2009)-x01.mkv
+: title: Moon
+ year: 2009
+ bonusNumber: 1
+
+? Movies/Moon_(2009)-x02-Making_Of.mkv
+: title: Moon
+ year: 2009
+ bonusNumber: 2
+ bonusTitle: Making Of
+
+? movies/James_Bond-f17-Goldeneye.mkv
+: title: Goldeneye
+ filmSeries: James Bond
+ filmNumber: 17
+
+? /movies/James_Bond-f21-Casino_Royale.mkv
+: title: Casino Royale
+ filmSeries: James Bond
+ filmNumber: 21
+
+? /movies/James_Bond-f21-Casino_Royale-x01-Becoming_Bond.mkv
+: title: Casino Royale
+ filmSeries: James Bond
+ filmNumber: 21
+ bonusNumber: 1
+ bonusTitle: Becoming Bond
+
+? /movies/James_Bond-f21-Casino_Royale-x02-Stunts.mkv
+: title: Casino Royale
+ filmSeries: James Bond
+ filmNumber: 21
+ bonusNumber: 2
+ bonusTitle: Stunts
+
+? OSS_117--Cairo,_Nest_of_Spies.mkv
+: title: OSS 117--Cairo, Nest of Spies
+
+? The Godfather Part III.mkv
+: title: The Godfather
+ part: 3
+
+? Foobar Part VI.mkv
+: title: Foobar
+ part: 6
+
+? The_Insider-(1999)-x02-60_Minutes_Interview-1996.mp4
+: title: The Insider
+ year: 1999
+ bonusNumber: 2
+ bonusTitle: 60 Minutes Interview-1996
+
+? Rush.._Beyond_The_Lighted_Stage-x09-Between_Sun_and_Moon-2002_Hartford.mkv
+: title: Rush Beyond The Lighted Stage
+ bonusNumber: 9
+ bonusTitle: Between Sun and Moon-2002 Hartford
+
+? /public/uTorrent/Downloads Finished/Movies/Indiana.Jones.and.the.Temple.of.Doom.1984.HDTV.720p.x264.AC3.5.1-REDµX/Indiana.Jones.and.the.Temple.of.Doom.1984.HDTV.720p.x264.AC3.5.1-REDµX.mkv
+: title: Indiana Jones and the Temple of Doom
+ year: 1984
+ format: HDTV
+ screenSize: 720p
+ videoCodec: h264
+ audioCodec: AC3
+ audioChannels: "5.1"
+ releaseGroup: REDµX
+
+? The.Director’s.Notebook.2006.Blu-Ray.x264.DXVA.720p.AC3-de[42].mkv
+: title: The Director’s Notebook
+ year: 2006
+ format: BluRay
+ videoCodec: h264
+ videoApi: DXVA
+ screenSize: 720p
+ audioCodec: AC3
+ releaseGroup: de[42]
+
+? Movies/Cosmopolis.2012.LiMiTED.720p.BluRay.x264-AN0NYM0US[bb]/ano-cosmo.720p.mkv
+: title: Cosmopolis
+ year: 2012
+ screenSize: 720p
+ videoCodec: h264
+ releaseGroup: AN0NYM0US[bb]
+ format: BluRay
+ other: LIMITED
+
+? movies/La Science des Rêves (2006)/La.Science.Des.Reves.FRENCH.DVDRip.XviD-MP-AceBot.avi
+: title: La Science des Rêves
+ year: 2006
+ format: DVD
+ videoCodec: XviD
+ videoProfile: MP
+ releaseGroup: AceBot
+ language: French
+
+? The_Italian_Job.mkv
+: title: The Italian Job
+
+? The.Rum.Diary.2011.1080p.BluRay.DTS.x264.D-Z0N3.mkv
+: title: The Rum Diary
+ year: 2011
+ screenSize: 1080p
+ format: BluRay
+ videoCodec: h264
+ audioCodec: DTS
+ releaseGroup: D-Z0N3
+
+? Life.Of.Pi.2012.1080p.BluRay.DTS.x264.D-Z0N3.mkv
+: title: Life Of Pi
+ year: 2012
+ screenSize: 1080p
+ format: BluRay
+ videoCodec: h264
+ audioCodec: DTS
+ releaseGroup: D-Z0N3
+
+? The.Kings.Speech.2010.1080p.BluRay.DTS.x264.D Z0N3.mkv
+: title: The Kings Speech
+ year: 2010
+ screenSize: 1080p
+ format: BluRay
+ audioCodec: DTS
+ videoCodec: h264
+ releaseGroup: D Z0N3
+
+? Street.Kings.2008.BluRay.1080p.DTS.x264.dxva EuReKA.mkv
+: title: Street Kings
+ year: 2008
+ format: BluRay
+ screenSize: 1080p
+ audioCodec: DTS
+ videoCodec: h264
+ videoApi: DXVA
+ releaseGroup: EuReKa
+
+? 2001.A.Space.Odyssey.1968.HDDVD.1080p.DTS.x264.dxva EuReKA.mkv
+: title: 2001 A Space Odyssey
+ year: 1968
+ format: HD-DVD
+ screenSize: 1080p
+ audioCodec: DTS
+ videoCodec: h264
+ videoApi: DXVA
+ releaseGroup: EuReKa
+
+? 2012.2009.720p.BluRay.x264.DTS WiKi.mkv
+: title: "2012"
+ year: 2009
+ screenSize: 720p
+ format: BluRay
+ videoCodec: h264
+ audioCodec: DTS
+ releaseGroup: WiKi
+
+? /share/Download/movie/Dead Man Down (2013) BRRiP XViD DD5_1 Custom NLSubs =-_lt Q_o_Q gt-=_/XD607ebb-BRc59935-5155473f-1c5f49/XD607ebb-BRc59935-5155473f-1c5f49.avi
+: title: Dead Man Down
+ year: 2013
+ format: BluRay
+ videoCodec: XviD
+ audioChannels: "5.1"
+ audioCodec: DolbyDigital
+ idNumber: XD607ebb-BRc59935-5155473f-1c5f49
+
+? Pacific.Rim.3D.2013.COMPLETE.BLURAY-PCH.avi
+: title: Pacific Rim
+ year: 2013
+ format: BluRay
+ other:
+ - complete
+ - 3D
+ releaseGroup: PCH
+
+? Immersion.French.2011.STV.READNFO.QC.FRENCH.ENGLISH.NTSC.DVDR.nfo
+: title: Immersion French
+ year: 2011
+ language:
+ - French
+ - English
+ format: DVD
+ other: NTSC
+
+? Immersion.French.2011.STV.READNFO.QC.FRENCH.NTSC.DVDR.nfo
+: title: Immersion French
+ year: 2011
+ language: French
+ format: DVD
+ other: NTSC
+
+? Immersion.French.2011.STV.READNFO.QC.NTSC.DVDR.nfo
+: title: Immersion French
+ year: 2011
+ format: DVD
+ other: NTSC
+
+? French.Immersion.2011.STV.READNFO.QC.ENGLISH.NTSC.DVDR.nfo
+: title: French Immersion
+ year: 2011
+ language: ENGLISH
+ format: DVD
+ other: NTSC
+
+? Howl's_Moving_Castle_(2004)_[720p,HDTV,x264,DTS]-FlexGet.avi
+: videoCodec: h264
+ format: HDTV
+ title: Howl's Moving Castle
+ screenSize: 720p
+ year: 2004
+ audioCodec: DTS
+ releaseGroup: FlexGet
+
+? Pirates de langkasuka.2008.FRENCH.1920X1080.h264.AVC.AsiaRa.mkv
+: screenSize: 1080p
+ year: 2008
+ language: French
+ videoCodec: h264
+ title: Pirates de langkasuka
+ releaseGroup: AsiaRa
+
+? Masala (2013) Telugu Movie HD DVDScr XviD - Exclusive.avi
+: year: 2013
+ videoCodec: XviD
+ title: Masala
+ format: HD-DVD
+ other: screener
+ language: Telugu
+ releaseGroup: Exclusive
+
+? Django Unchained 2012 DVDSCR X264 AAC-P2P.nfo
+: year: 2012
+ other: screener
+ videoCodec: h264
+ title: Django Unchained
+ audioCodec: AAC
+ format: DVD
+ releaseGroup: P2P
+
+? Ejecutiva.En.Apuros(2009).BLURAY.SCR.Xvid.Spanish.LanzamientosD.nfo
+: year: 2009
+ other: screener
+ format: BluRay
+ videoCodec: XviD
+ language: Spanish
+ title: Ejecutiva En Apuros
+
+? Die.Schluempfe.2.German.DL.1080p.BluRay.x264-EXQUiSiTE.mkv
+: title: Die Schluempfe 2
+ format: BluRay
+ language:
+ - Multiple languages
+ - German
+ videoCodec: h264
+ releaseGroup: EXQUiSiTE
+ screenSize: 1080p
+
+? Rocky 1976 French SubForced BRRip x264 AC3-FUNKY.mkv
+: title: Rocky
+ year: 1976
+ subtitleLanguage: French
+ format: BluRay
+ videoCodec: h264
+ audioCodec: AC3
+ releaseGroup: FUNKY
+
+? REDLINE (BD 1080p H264 10bit FLAC) [3xR].mkv
+: title: REDLINE
+ format: BluRay
+ videoCodec: h264
+ videoProfile: 10bit
+ audioCodec: Flac
+ screenSize: 1080p
+
+? The.Lizzie.McGuire.Movie.(2003).HR.DVDRiP.avi
+: title: The Lizzie McGuire Movie
+ year: 2003
+ format: DVD
+ other: HR
+
+? Hua.Mulan.BRRIP.MP4.x264.720p-HR.avi
+: title: Hua Mulan
+ videoCodec: h264
+ format: BluRay
+ screenSize: 720p
+ other: HR
+
+? Dr.Seuss.The.Lorax.2012.DVDRip.LiNE.XviD.AC3.HQ.Hive-CM8.mp4
+: videoCodec: XviD
+ title: Dr Seuss The Lorax
+ format: DVD
+ other: LiNE
+ year: 2012
+ audioCodec: AC3
+ audioProfile: HQ
+ releaseGroup: Hive-CM8
+
+
+? "Star Wars: Episode IV - A New Hope (2004) Special Edition.MKV"
+: title: Star Wars Episode IV
+ year: 2004
+ edition: Special Edition
+
+? Dr.LiNE.The.Lorax.2012.DVDRip.LiNE.XviD.AC3.HQ.Hive-CM8.mp4
+: videoCodec: XviD
+ title: Dr LiNE The Lorax
+ format: DVD
+ other: LiNE
+ year: 2012
+ audioCodec: AC3
+ audioProfile: HQ
+ releaseGroup: Hive-CM8
+
+? Perfect Child-2007-TRUEFRENCH-TVRip.Xvid-h@mster.avi
+: releaseGroup: h@mster
+ title: Perfect Child
+ videoCodec: XviD
+ language: French
+ format: TV
+ year: 2007
+
+? entre.ciel.et.terre.(1994).dvdrip.h264.aac-psypeon.avi
+: audioCodec: AAC
+ format: DVD
+ releaseGroup: psypeon
+ title: entre ciel et terre
+ videoCodec: h264
+ year: 1994
+
+? Yves.Saint.Laurent.2013.FRENCH.DVDSCR.MD.XviD-ViVARiUM.avi
+: format: DVD
+ language: French
+ other: Screener
+ releaseGroup: ViVARiUM
+ title: Yves Saint Laurent
+ videoCodec: XviD
+ year: 2013
+
+? Echec et Mort - Hard to Kill - Steven Seagal Multi 1080p BluRay x264 CCATS.avi
+: format: BluRay
+ language: Multiple languages
+ releaseGroup: CCATS
+ screenSize: 1080p
+ title: Echec et Mort
+ videoCodec: h264
+
+? Paparazzi - Timsit/Lindon (MKV 1080p tvripHD)
+: options: -n
+ title: Paparazzi
+ screenSize: 1080p
+ format: HDTV
+
+? some.movie.720p.bluray.x264-mind
+: options: -n
+ title: some movie
+ screenSize: 720p
+ videoCodec: h264
+ releaseGroup: mind
+ format: BluRay
+
+? Dr LiNE The Lorax 720p h264 BluRay
+: options: -n
+ title: Dr LiNE The Lorax
+ screenSize: 720p
+ videoCodec: h264
+ format: BluRay
+
+? BeatdownFrenchDVDRip.mkv
+: options: -c
+ title: Beatdown
+ language: French
+ format: DVD
+
+? YvesSaintLaurent2013FrenchDVDScrXvid.avi
+: options: -c
+ format: DVD
+ language: French
+ other: Screener
+ title: Yves saint laurent
+ videoCodec: XviD
+ year: 2013
+
+? Elle.s.en.va.720p.mkv
+: screenSize: 720p
+ title: Elle s en va
+
+? FooBar.7.PDTV-FlexGet
+: options: -n
+ format: DVB
+ releaseGroup: FlexGet
+ title: FooBar 7
+
+? h265 - HEVC Riddick Unrated Director Cut French 1080p DTS.mkv
+: audioCodec: DTS
+ edition: Director's cut
+ language: fr
+ screenSize: 1080p
+ title: Riddick Unrated
+ videoCodec: h265
+
+? "[h265 - HEVC] Riddick Unrated Director Cut French [1080p DTS].mkv"
+: audioCodec: DTS
+ edition: Director's cut
+ language: fr
+ screenSize: 1080p
+ title: Riddick Unrated
+ videoCodec: h265
+
+? Barbecue-2014-French-mHD-1080p
+: options: -n
+ language: fr
+ other: mHD
+ screenSize: 1080p
+ title: Barbecue
+ year: 2014
+
+? Underworld Quadrilogie VO+VFF+VFQ 1080p HDlight.x264~Tonyk~Monde Infernal
+: options: -n
+ language:
+ - fr
+ - vo
+ other: HDLight
+ screenSize: 1080p
+ title: Underworld Quadrilogie
+ videoCodec: h264
+
+? A Bout Portant (The Killers).PAL.Multi.DVD-R-KZ
+: options: -n
+ format: DVD
+ language: mul
+ releaseGroup: KZ
+ title: A Bout Portant
+
+? "Mise à Sac (Alain Cavalier, 1967) [Vhs.Rip.Vff]"
+: options: -n
+ format: VHS
+ language: fr
+ title: "Mise à Sac"
+ year: 1967
+
+? A Bout Portant (The Killers).PAL.Multi.DVD-R-KZ
+: options: -n
+ format: DVD
+ other: PAL
+ language: mul
+ releaseGroup: KZ
+ title: A Bout Portant
+
+? Youth.In.Revolt.(Be.Bad).2009.MULTI.1080p.LAME3*92-MEDIOZZ
+: options: -n
+ audioCodec: MP3
+ language: mul
+ releaseGroup: MEDIOZZ
+ screenSize: 1080p
+ title: Youth In Revolt
+ year: 2009
+
+? La Defense Lincoln (The Lincoln Lawyer) 2011 [DVDRIP][Vostfr]
+: options: -n
+ format: DVD
+ subtitleLanguage: fr
+ title: La Defense Lincoln
+ year: 2011
+
+? '[h265 - HEVC] Fight Club French 1080p DTS.'
+: options: -n
+ audioCodec: DTS
+ language: fr
+ screenSize: 1080p
+ title: Fight Club
+ videoCodec: h265
+
+? Love Gourou (Mike Myers) - FR
+: options: -n
+ language: fr
+ title: Love Gourou
+
+? '[h265 - hevc] transformers 2 1080p french ac3 6ch.'
+: options: -n
+ audioChannels: '5.1'
+ audioCodec: AC3
+ language: fr
+ screenSize: 1080p
+ title: transformers 2
+ videoCodec: h265
diff --git a/lib/guessit/test/opensubtitles_languages_2012_05_09.txt b/lib/guessit/test/opensubtitles_languages_2012_05_09.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4a08d9b52014b4d9488b0282934a9b7c496d2142
--- /dev/null
+++ b/lib/guessit/test/opensubtitles_languages_2012_05_09.txt
@@ -0,0 +1,473 @@
+IdSubLanguage ISO639 LanguageName UploadEnabled WebEnabled
+aar aa Afar, afar 0 0
+abk ab Abkhazian 0 0
+ace Achinese 0 0
+ach Acoli 0 0
+ada Adangme 0 0
+ady adyghé 0 0
+afa Afro-Asiatic (Other) 0 0
+afh Afrihili 0 0
+afr af Afrikaans 0 0
+ain Ainu 0 0
+aka ak Akan 0 0
+akk Akkadian 0 0
+alb sq Albanian 1 1
+ale Aleut 0 0
+alg Algonquian languages 0 0
+alt Southern Altai 0 0
+amh am Amharic 0 0
+ang English, Old (ca.450-1100) 0 0
+apa Apache languages 0 0
+ara ar Arabic 1 1
+arc Aramaic 0 0
+arg an Aragonese 0 0
+arm hy Armenian 1 0
+arn Araucanian 0 0
+arp Arapaho 0 0
+art Artificial (Other) 0 0
+arw Arawak 0 0
+asm as Assamese 0 0
+ast Asturian, Bable 0 0
+ath Athapascan languages 0 0
+aus Australian languages 0 0
+ava av Avaric 0 0
+ave ae Avestan 0 0
+awa Awadhi 0 0
+aym ay Aymara 0 0
+aze az Azerbaijani 0 0
+bad Banda 0 0
+bai Bamileke languages 0 0
+bak ba Bashkir 0 0
+bal Baluchi 0 0
+bam bm Bambara 0 0
+ban Balinese 0 0
+baq eu Basque 1 1
+bas Basa 0 0
+bat Baltic (Other) 0 0
+bej Beja 0 0
+bel be Belarusian 0 0
+bem Bemba 0 0
+ben bn Bengali 1 0
+ber Berber (Other) 0 0
+bho Bhojpuri 0 0
+bih bh Bihari 0 0
+bik Bikol 0 0
+bin Bini 0 0
+bis bi Bislama 0 0
+bla Siksika 0 0
+bnt Bantu (Other) 0 0
+bos bs Bosnian 1 0
+bra Braj 0 0
+bre br Breton 1 0
+btk Batak (Indonesia) 0 0
+bua Buriat 0 0
+bug Buginese 0 0
+bul bg Bulgarian 1 1
+bur my Burmese 0 0
+byn Blin 0 0
+cad Caddo 0 0
+cai Central American Indian (Other) 0 0
+car Carib 0 0
+cat ca Catalan 1 1
+cau Caucasian (Other) 0 0
+ceb Cebuano 0 0
+cel Celtic (Other) 0 0
+cha ch Chamorro 0 0
+chb Chibcha 0 0
+che ce Chechen 0 0
+chg Chagatai 0 0
+chi zh Chinese 1 1
+chk Chuukese 0 0
+chm Mari 0 0
+chn Chinook jargon 0 0
+cho Choctaw 0 0
+chp Chipewyan 0 0
+chr Cherokee 0 0
+chu cu Church Slavic 0 0
+chv cv Chuvash 0 0
+chy Cheyenne 0 0
+cmc Chamic languages 0 0
+cop Coptic 0 0
+cor kw Cornish 0 0
+cos co Corsican 0 0
+cpe Creoles and pidgins, English based (Other) 0 0
+cpf Creoles and pidgins, French-based (Other) 0 0
+cpp Creoles and pidgins, Portuguese-based (Other) 0 0
+cre cr Cree 0 0
+crh Crimean Tatar 0 0
+crp Creoles and pidgins (Other) 0 0
+csb Kashubian 0 0
+cus Cushitic (Other)' couchitiques, autres langues 0 0
+cze cs Czech 1 1
+dak Dakota 0 0
+dan da Danish 1 1
+dar Dargwa 0 0
+day Dayak 0 0
+del Delaware 0 0
+den Slave (Athapascan) 0 0
+dgr Dogrib 0 0
+din Dinka 0 0
+div dv Divehi 0 0
+doi Dogri 0 0
+dra Dravidian (Other) 0 0
+dua Duala 0 0
+dum Dutch, Middle (ca.1050-1350) 0 0
+dut nl Dutch 1 1
+dyu Dyula 0 0
+dzo dz Dzongkha 0 0
+efi Efik 0 0
+egy Egyptian (Ancient) 0 0
+eka Ekajuk 0 0
+elx Elamite 0 0
+eng en English 1 1
+enm English, Middle (1100-1500) 0 0
+epo eo Esperanto 1 0
+est et Estonian 1 1
+ewe ee Ewe 0 0
+ewo Ewondo 0 0
+fan Fang 0 0
+fao fo Faroese 0 0
+fat Fanti 0 0
+fij fj Fijian 0 0
+fil Filipino 0 0
+fin fi Finnish 1 1
+fiu Finno-Ugrian (Other) 0 0
+fon Fon 0 0
+fre fr French 1 1
+frm French, Middle (ca.1400-1600) 0 0
+fro French, Old (842-ca.1400) 0 0
+fry fy Frisian 0 0
+ful ff Fulah 0 0
+fur Friulian 0 0
+gaa Ga 0 0
+gay Gayo 0 0
+gba Gbaya 0 0
+gem Germanic (Other) 0 0
+geo ka Georgian 1 1
+ger de German 1 1
+gez Geez 0 0
+gil Gilbertese 0 0
+gla gd Gaelic 0 0
+gle ga Irish 0 0
+glg gl Galician 1 1
+glv gv Manx 0 0
+gmh German, Middle High (ca.1050-1500) 0 0
+goh German, Old High (ca.750-1050) 0 0
+gon Gondi 0 0
+gor Gorontalo 0 0
+got Gothic 0 0
+grb Grebo 0 0
+grc Greek, Ancient (to 1453) 0 0
+ell el Greek 1 1
+grn gn Guarani 0 0
+guj gu Gujarati 0 0
+gwi Gwich´in 0 0
+hai Haida 0 0
+hat ht Haitian 0 0
+hau ha Hausa 0 0
+haw Hawaiian 0 0
+heb he Hebrew 1 1
+her hz Herero 0 0
+hil Hiligaynon 0 0
+him Himachali 0 0
+hin hi Hindi 1 1
+hit Hittite 0 0
+hmn Hmong 0 0
+hmo ho Hiri Motu 0 0
+hrv hr Croatian 1 1
+hun hu Hungarian 1 1
+hup Hupa 0 0
+iba Iban 0 0
+ibo ig Igbo 0 0
+ice is Icelandic 1 1
+ido io Ido 0 0
+iii ii Sichuan Yi 0 0
+ijo Ijo 0 0
+iku iu Inuktitut 0 0
+ile ie Interlingue 0 0
+ilo Iloko 0 0
+ina ia Interlingua (International Auxiliary Language Asso 0 0
+inc Indic (Other) 0 0
+ind id Indonesian 1 1
+ine Indo-European (Other) 0 0
+inh Ingush 0 0
+ipk ik Inupiaq 0 0
+ira Iranian (Other) 0 0
+iro Iroquoian languages 0 0
+ita it Italian 1 1
+jav jv Javanese 0 0
+jpn ja Japanese 1 1
+jpr Judeo-Persian 0 0
+jrb Judeo-Arabic 0 0
+kaa Kara-Kalpak 0 0
+kab Kabyle 0 0
+kac Kachin 0 0
+kal kl Kalaallisut 0 0
+kam Kamba 0 0
+kan kn Kannada 0 0
+kar Karen 0 0
+kas ks Kashmiri 0 0
+kau kr Kanuri 0 0
+kaw Kawi 0 0
+kaz kk Kazakh 1 0
+kbd Kabardian 0 0
+kha Khasi 0 0
+khi Khoisan (Other) 0 0
+khm km Khmer 1 1
+kho Khotanese 0 0
+kik ki Kikuyu 0 0
+kin rw Kinyarwanda 0 0
+kir ky Kirghiz 0 0
+kmb Kimbundu 0 0
+kok Konkani 0 0
+kom kv Komi 0 0
+kon kg Kongo 0 0
+kor ko Korean 1 1
+kos Kosraean 0 0
+kpe Kpelle 0 0
+krc Karachay-Balkar 0 0
+kro Kru 0 0
+kru Kurukh 0 0
+kua kj Kuanyama 0 0
+kum Kumyk 0 0
+kur ku Kurdish 0 0
+kut Kutenai 0 0
+lad Ladino 0 0
+lah Lahnda 0 0
+lam Lamba 0 0
+lao lo Lao 0 0
+lat la Latin 0 0
+lav lv Latvian 1 0
+lez Lezghian 0 0
+lim li Limburgan 0 0
+lin ln Lingala 0 0
+lit lt Lithuanian 1 0
+lol Mongo 0 0
+loz Lozi 0 0
+ltz lb Luxembourgish 1 0
+lua Luba-Lulua 0 0
+lub lu Luba-Katanga 0 0
+lug lg Ganda 0 0
+lui Luiseno 0 0
+lun Lunda 0 0
+luo Luo (Kenya and Tanzania) 0 0
+lus lushai 0 0
+mac mk Macedonian 1 1
+mad Madurese 0 0
+mag Magahi 0 0
+mah mh Marshallese 0 0
+mai Maithili 0 0
+mak Makasar 0 0
+mal ml Malayalam 0 0
+man Mandingo 0 0
+mao mi Maori 0 0
+map Austronesian (Other) 0 0
+mar mr Marathi 0 0
+mas Masai 0 0
+may ms Malay 1 1
+mdf Moksha 0 0
+mdr Mandar 0 0
+men Mende 0 0
+mga Irish, Middle (900-1200) 0 0
+mic Mi'kmaq 0 0
+min Minangkabau 0 0
+mis Miscellaneous languages 0 0
+mkh Mon-Khmer (Other) 0 0
+mlg mg Malagasy 0 0
+mlt mt Maltese 0 0
+mnc Manchu 0 0
+mni Manipuri 0 0
+mno Manobo languages 0 0
+moh Mohawk 0 0
+mol mo Moldavian 0 0
+mon mn Mongolian 1 0
+mos Mossi 0 0
+mwl Mirandese 0 0
+mul Multiple languages 0 0
+mun Munda languages 0 0
+mus Creek 0 0
+mwr Marwari 0 0
+myn Mayan languages 0 0
+myv Erzya 0 0
+nah Nahuatl 0 0
+nai North American Indian 0 0
+nap Neapolitan 0 0
+nau na Nauru 0 0
+nav nv Navajo 0 0
+nbl nr Ndebele, South 0 0
+nde nd Ndebele, North 0 0
+ndo ng Ndonga 0 0
+nds Low German 0 0
+nep ne Nepali 0 0
+new Nepal Bhasa 0 0
+nia Nias 0 0
+nic Niger-Kordofanian (Other) 0 0
+niu Niuean 0 0
+nno nn Norwegian Nynorsk 0 0
+nob nb Norwegian Bokmal 0 0
+nog Nogai 0 0
+non Norse, Old 0 0
+nor no Norwegian 1 1
+nso Northern Sotho 0 0
+nub Nubian languages 0 0
+nwc Classical Newari 0 0
+nya ny Chichewa 0 0
+nym Nyamwezi 0 0
+nyn Nyankole 0 0
+nyo Nyoro 0 0
+nzi Nzima 0 0
+oci oc Occitan 1 1
+oji oj Ojibwa 0 0
+ori or Oriya 0 0
+orm om Oromo 0 0
+osa Osage 0 0
+oss os Ossetian 0 0
+ota Turkish, Ottoman (1500-1928) 0 0
+oto Otomian languages 0 0
+paa Papuan (Other) 0 0
+pag Pangasinan 0 0
+pal Pahlavi 0 0
+pam Pampanga 0 0
+pan pa Panjabi 0 0
+pap Papiamento 0 0
+pau Palauan 0 0
+peo Persian, Old (ca.600-400 B.C.) 0 0
+per fa Persian 1 1
+phi Philippine (Other) 0 0
+phn Phoenician 0 0
+pli pi Pali 0 0
+pol pl Polish 1 1
+pon Pohnpeian 0 0
+por pt Portuguese 1 1
+pra Prakrit languages 0 0
+pro Provençal, Old (to 1500) 0 0
+pus ps Pushto 0 0
+que qu Quechua 0 0
+raj Rajasthani 0 0
+rap Rapanui 0 0
+rar Rarotongan 0 0
+roa Romance (Other) 0 0
+roh rm Raeto-Romance 0 0
+rom Romany 0 0
+run rn Rundi 0 0
+rup Aromanian 0 0
+rus ru Russian 1 1
+sad Sandawe 0 0
+sag sg Sango 0 0
+sah Yakut 0 0
+sai South American Indian (Other) 0 0
+sal Salishan languages 0 0
+sam Samaritan Aramaic 0 0
+san sa Sanskrit 0 0
+sas Sasak 0 0
+sat Santali 0 0
+scc sr Serbian 1 1
+scn Sicilian 0 0
+sco Scots 0 0
+sel Selkup 0 0
+sem Semitic (Other) 0 0
+sga Irish, Old (to 900) 0 0
+sgn Sign Languages 0 0
+shn Shan 0 0
+sid Sidamo 0 0
+sin si Sinhalese 1 1
+sio Siouan languages 0 0
+sit Sino-Tibetan (Other) 0 0
+sla Slavic (Other) 0 0
+slo sk Slovak 1 1
+slv sl Slovenian 1 1
+sma Southern Sami 0 0
+sme se Northern Sami 0 0
+smi Sami languages (Other) 0 0
+smj Lule Sami 0 0
+smn Inari Sami 0 0
+smo sm Samoan 0 0
+sms Skolt Sami 0 0
+sna sn Shona 0 0
+snd sd Sindhi 0 0
+snk Soninke 0 0
+sog Sogdian 0 0
+som so Somali 0 0
+son Songhai 0 0
+sot st Sotho, Southern 0 0
+spa es Spanish 1 1
+srd sc Sardinian 0 0
+srr Serer 0 0
+ssa Nilo-Saharan (Other) 0 0
+ssw ss Swati 0 0
+suk Sukuma 0 0
+sun su Sundanese 0 0
+sus Susu 0 0
+sux Sumerian 0 0
+swa sw Swahili 1 0
+swe sv Swedish 1 1
+syr Syriac 1 0
+tah ty Tahitian 0 0
+tai Tai (Other) 0 0
+tam ta Tamil 0 0
+tat tt Tatar 0 0
+tel te Telugu 0 0
+tem Timne 0 0
+ter Tereno 0 0
+tet Tetum 0 0
+tgk tg Tajik 0 0
+tgl tl Tagalog 1 1
+tha th Thai 1 1
+tib bo Tibetan 0 0
+tig Tigre 0 0
+tir ti Tigrinya 0 0
+tiv Tiv 0 0
+tkl Tokelau 0 0
+tlh Klingon 0 0
+tli Tlingit 0 0
+tmh Tamashek 0 0
+tog Tonga (Nyasa) 0 0
+ton to Tonga (Tonga Islands) 0 0
+tpi Tok Pisin 0 0
+tsi Tsimshian 0 0
+tsn tn Tswana 0 0
+tso ts Tsonga 0 0
+tuk tk Turkmen 0 0
+tum Tumbuka 0 0
+tup Tupi languages 0 0
+tur tr Turkish 1 1
+tut Altaic (Other) 0 0
+tvl Tuvalu 0 0
+twi tw Twi 0 0
+tyv Tuvinian 0 0
+udm Udmurt 0 0
+uga Ugaritic 0 0
+uig ug Uighur 0 0
+ukr uk Ukrainian 1 1
+umb Umbundu 0 0
+und Undetermined 0 0
+urd ur Urdu 1 0
+uzb uz Uzbek 0 0
+vai Vai 0 0
+ven ve Venda 0 0
+vie vi Vietnamese 1 1
+vol vo Volapük 0 0
+vot Votic 0 0
+wak Wakashan languages 0 0
+wal Walamo 0 0
+war Waray 0 0
+was Washo 0 0
+wel cy Welsh 0 0
+wen Sorbian languages 0 0
+wln wa Walloon 0 0
+wol wo Wolof 0 0
+xal Kalmyk 0 0
+xho xh Xhosa 0 0
+yao Yao 0 0
+yap Yapese 0 0
+yid yi Yiddish 0 0
+yor yo Yoruba 0 0
+ypk Yupik languages 0 0
+zap Zapotec 0 0
+zen Zenaga 0 0
+zha za Zhuang 0 0
+znd Zande 0 0
+zul zu Zulu 0 0
+zun Zuni 0 0
+rum ro Romanian 1 1
+pob pb Brazilian 1 1
diff --git a/lib/guessit/test/test_api.py b/lib/guessit/test/test_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..b950e50b41d237587a077fb8249458c4df9985cf
--- /dev/null
+++ b/lib/guessit/test/test_api.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2014 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.test.guessittest import *
+
+
+class TestApi(TestGuessit):
+ def test_api(self):
+ movie_path = 'Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD.mkv'
+
+ movie_info = guessit.guess_movie_info(movie_path)
+ video_info = guessit.guess_video_info(movie_path)
+ episode_info = guessit.guess_episode_info(movie_path)
+ file_info = guessit.guess_file_info(movie_path)
+
+ assert guessit.guess_file_info(movie_path, type='movie') == movie_info
+ assert guessit.guess_file_info(movie_path, type='video') == video_info
+ assert guessit.guess_file_info(movie_path, type='episode') == episode_info
+
+ assert guessit.guess_file_info(movie_path, options={'type': 'movie'}) == movie_info
+ assert guessit.guess_file_info(movie_path, options={'type': 'video'}) == video_info
+ assert guessit.guess_file_info(movie_path, options={'type': 'episode'}) == episode_info
+
+ # kwargs priority other options
+ assert guessit.guess_file_info(movie_path, options={'type': 'episode'}, type='movie') == episode_info
+
+ movie_path_name_only = 'Movies/Dark City (1998)/Dark.City.(1998).DC.BDRip.720p.DTS.X264-CHD'
+ file_info_name_only = guessit.guess_file_info(movie_path_name_only, options={"name_only": True})
+
+ assert 'container' not in file_info_name_only
+ assert 'container' in file_info
diff --git a/lib/guessit/test/test_autodetect.py b/lib/guessit/test/test_autodetect.py
new file mode 100644
index 0000000000000000000000000000000000000000..6be97c097b3dda99d09e7d4811f3daa9042e3ac2
--- /dev/null
+++ b/lib/guessit/test/test_autodetect.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.test.guessittest import *
+
+
+class TestAutoDetect(TestGuessit):
+ def testEmpty(self):
+ result = guessit.guess_file_info('')
+ assert result == {}
+
+ result = guessit.guess_file_info('___-__')
+ assert result == {}
+
+ result = guessit.guess_file_info('__-.avc')
+ assert result == {'type': 'unknown', 'extension': 'avc'}
+
+ def testAutoDetect(self):
+ self.checkMinimumFieldsCorrect(filename='autodetect.yaml',
+ remove_type=False)
diff --git a/lib/guessit/test/test_autodetect_all.py b/lib/guessit/test/test_autodetect_all.py
new file mode 100644
index 0000000000000000000000000000000000000000..3a6ef7312a44f1b31b577ef4bbf10c8968f411c4
--- /dev/null
+++ b/lib/guessit/test/test_autodetect_all.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.test.guessittest import *
+
+IGNORE_EPISODES = []
+IGNORE_MOVIES = []
+
+
+class TestAutoDetectAll(TestGuessit):
+ def testAutoMatcher(self):
+ self.checkMinimumFieldsCorrect(filename='autodetect.yaml',
+ remove_type=False)
+
+ def testAutoMatcherMovies(self):
+ self.checkMinimumFieldsCorrect(filename='movies.yaml',
+ exclude_files=IGNORE_MOVIES)
+
+ def testAutoMatcherEpisodes(self):
+ self.checkMinimumFieldsCorrect(filename='episodes.yaml',
+ exclude_files=IGNORE_EPISODES)
diff --git a/lib/guessit/test/test_episode.py b/lib/guessit/test/test_episode.py
new file mode 100644
index 0000000000000000000000000000000000000000..406cbedc66df1fd3195a1c79ee0ed0b8effb4de2
--- /dev/null
+++ b/lib/guessit/test/test_episode.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.test.guessittest import *
+
+
+class TestEpisode(TestGuessit):
+ def testEpisodes(self):
+ self.checkMinimumFieldsCorrect(filetype='episode',
+ filename='episodes.yaml')
diff --git a/lib/guessit/test/test_hashes.py b/lib/guessit/test/test_hashes.py
new file mode 100644
index 0000000000000000000000000000000000000000..c6a5c356dc96697967976399ace2806ef729f669
--- /dev/null
+++ b/lib/guessit/test/test_hashes.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.test.guessittest import *
+
+
+class TestHashes(TestGuessit):
+ def test_hashes(self):
+ hashes = (
+ ('hash_mpc', '1MB', u'8542ad406c15c8bd'), # TODO: Check if this value is valid
+ ('hash_ed2k', '1MB', u'ed2k://|file|1MB|1048576|AA3CC5552A9931A76B61A41D306735F7|/'), # TODO: Check if this value is valid
+ ('hash_md5', '1MB', u'5d8dcbca8d8ac21766f28797d6c3954c'),
+ ('hash_sha1', '1MB', u'51d2b8f3248d7ee495b7750c8da5aa3b3819de9d'),
+ ('hash_md5', 'dummy.srt', u'64de6b5893cac24456c46a935ef9c359'),
+ ('hash_sha1', 'dummy.srt', u'a703fc0fa4518080505809bf562c6fc6f7b3c98c')
+ )
+
+ for hash_type, filename, expected_value in hashes:
+ guess = guess_file_info(file_in_same_dir(__file__, filename), hash_type)
+ computed_value = guess.get(hash_type)
+ assert expected_value == guess.get(hash_type), \
+ "Invalid %s for %s: %s != %s" % (hash_type, filename, computed_value, expected_value)
diff --git a/lib/guessit/test/test_language.py b/lib/guessit/test/test_language.py
new file mode 100644
index 0000000000000000000000000000000000000000..93ec9d7286d426520ac9ce43fa067224ff5d7671
--- /dev/null
+++ b/lib/guessit/test/test_language.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.test.guessittest import *
+
+
+class TestLanguage(TestGuessit):
+
+ def check_languages(self, languages):
+ for lang1, lang2 in languages.items():
+ assert Language.fromguessit(lang1) == Language.fromguessit(lang2)
+
+ def test_addic7ed(self):
+ languages = {'English': 'en',
+ 'English (US)': 'en-US',
+ 'English (UK)': 'en-UK',
+ 'Italian': 'it',
+ 'Portuguese': 'pt',
+ 'Portuguese (Brazilian)': 'pt-BR',
+ 'Romanian': 'ro',
+ 'Español (Latinoamérica)': 'es-MX',
+ 'Español (España)': 'es-ES',
+ 'Spanish (Latin America)': 'es-MX',
+ 'Español': 'es',
+ 'Spanish': 'es',
+ 'Spanish (Spain)': 'es-ES',
+ 'French': 'fr',
+ 'Greek': 'el',
+ 'Arabic': 'ar',
+ 'German': 'de',
+ 'Croatian': 'hr',
+ 'Indonesian': 'id',
+ 'Hebrew': 'he',
+ 'Russian': 'ru',
+ 'Turkish': 'tr',
+ 'Swedish': 'se',
+ 'Czech': 'cs',
+ 'Dutch': 'nl',
+ 'Hungarian': 'hu',
+ 'Norwegian': 'no',
+ 'Polish': 'pl',
+ 'Persian': 'fa'}
+
+ self.check_languages(languages)
+
+ def test_subswiki(self):
+ languages = {'English (US)': 'en-US', 'English (UK)': 'en-UK', 'English': 'en',
+ 'French': 'fr', 'Brazilian': 'po', 'Portuguese': 'pt',
+ 'Español (Latinoamérica)': 'es-MX', 'Español (España)': 'es-ES',
+ 'Español': 'es', 'Italian': 'it', 'Català': 'ca'}
+
+ self.check_languages(languages)
+
+ def test_tvsubtitles(self):
+ languages = {'English': 'en', 'Español': 'es', 'French': 'fr', 'German': 'de',
+ 'Brazilian': 'br', 'Russian': 'ru', 'Ukrainian': 'ua', 'Italian': 'it',
+ 'Greek': 'gr', 'Arabic': 'ar', 'Hungarian': 'hu', 'Polish': 'pl',
+ 'Turkish': 'tr', 'Dutch': 'nl', 'Portuguese': 'pt', 'Swedish': 'sv',
+ 'Danish': 'da', 'Finnish': 'fi', 'Korean': 'ko', 'Chinese': 'cn',
+ 'Japanese': 'jp', 'Bulgarian': 'bg', 'Czech': 'cz', 'Romanian': 'ro'}
+
+ self.check_languages(languages)
+
+ def test_opensubtitles(self):
+ opensubtitles_langfile = file_in_same_dir(__file__, 'opensubtitles_languages_2012_05_09.txt')
+ for l in [u(l).strip() for l in io.open(opensubtitles_langfile, encoding='utf-8')][1:]:
+ idlang, alpha2, _, upload_enabled, web_enabled = l.strip().split('\t')
+ # do not test languages that are too esoteric / not widely available
+ if int(upload_enabled) and int(web_enabled):
+ # check that we recognize the opensubtitles language code correctly
+ # and that we are able to output this code from a language
+ assert idlang == Language.fromguessit(idlang).opensubtitles
+ if alpha2:
+ # check we recognize the opensubtitles 2-letter code correctly
+ self.check_languages({idlang: alpha2})
+
+ def test_tmdb(self):
+ # examples from http://api.themoviedb.org/2.1/language-tags
+ for lang in ['en-US', 'en-CA', 'es-MX', 'fr-PF']:
+ assert lang == str(Language.fromguessit(lang))
+
+ def test_subtitulos(self):
+ languages = {'English (US)': 'en-US', 'English (UK)': 'en-UK', 'English': 'en',
+ 'French': 'fr', 'Brazilian': 'po', 'Portuguese': 'pt',
+ 'Español (Latinoamérica)': 'es-MX', 'Español (España)': 'es-ES',
+ 'Español': 'es', 'Italian': 'it', 'Català': 'ca'}
+
+ self.check_languages(languages)
+
+ def test_thesubdb(self):
+ languages = {'af': 'af', 'cs': 'cs', 'da': 'da', 'de': 'de', 'en': 'en', 'es': 'es', 'fi': 'fi',
+ 'fr': 'fr', 'hu': 'hu', 'id': 'id', 'it': 'it', 'la': 'la', 'nl': 'nl', 'no': 'no',
+ 'oc': 'oc', 'pl': 'pl', 'pt': 'pt', 'ro': 'ro', 'ru': 'ru', 'sl': 'sl', 'sr': 'sr',
+ 'sv': 'sv', 'tr': 'tr'}
+
+ self.check_languages(languages)
+
+ def test_exceptions(self):
+ assert Language.fromguessit('br') == Language.fromguessit('pt(br)')
+ assert Language.fromguessit('unknown') == Language.fromguessit('und')
diff --git a/lib/guessit/test/test_main.py b/lib/guessit/test/test_main.py
new file mode 100644
index 0000000000000000000000000000000000000000..05ae6c4ff7aeaab856937a9d2bccdcfb6537ab37
--- /dev/null
+++ b/lib/guessit/test/test_main.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2014 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.test.guessittest import *
+from guessit.fileutils import file_in_same_dir
+from guessit import PY2
+from guessit import __main__
+
+if PY2:
+ from StringIO import StringIO
+else:
+ from io import StringIO
+
+
+class TestMain(TestGuessit):
+ def setUp(self):
+ self._stdout = sys.stdout
+ string_out = StringIO()
+ sys.stdout = string_out
+
+ def tearDown(self):
+ sys.stdout = self._stdout
+
+ def test_list_properties(self):
+ __main__.main(["-p"], False)
+ __main__.main(["-V"], False)
+
+ def test_list_transformers(self):
+ __main__.main(["--transformers"], False)
+ __main__.main(["-V", "--transformers"], False)
+
+ def test_demo(self):
+ __main__.main(["-d"], False)
+
+ def test_filename(self):
+ __main__.main(["A.Movie.2014.avi"], False)
+ __main__.main(["A.Movie.2014.avi", "A.2nd.Movie.2014.avi"], False)
+ __main__.main(["-y", "A.Movie.2014.avi"], False)
+ __main__.main(["-a", "A.Movie.2014.avi"], False)
+ __main__.main(["-v", "A.Movie.2014.avi"], False)
+ __main__.main(["-t", "movie", "A.Movie.2014.avi"], False)
+ __main__.main(["-t", "episode", "A.Serie.S02E06.avi"], False)
+ __main__.main(["-i", "hash_mpc", file_in_same_dir(__file__, "1MB")], False)
+ __main__.main(["-i", "hash_md5", file_in_same_dir(__file__, "1MB")], False)
diff --git a/lib/guessit/test/test_matchtree.py b/lib/guessit/test/test_matchtree.py
new file mode 100644
index 0000000000000000000000000000000000000000..7dcded18db1e59063313e58c5f58d9bb1224994d
--- /dev/null
+++ b/lib/guessit/test/test_matchtree.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.test.guessittest import *
+
+from guessit.transfo.guess_release_group import GuessReleaseGroup
+from guessit.transfo.guess_properties import GuessProperties
+from guessit.matchtree import BaseMatchTree
+
+keywords = yaml.load("""
+
+? Xvid PROPER
+: videoCodec: Xvid
+ other: PROPER
+
+? PROPER-Xvid
+: videoCodec: Xvid
+ other: PROPER
+
+""")
+
+
+def guess_info(string, options=None):
+ mtree = MatchTree(string)
+ GuessReleaseGroup().process(mtree, options)
+ GuessProperties().process(mtree, options)
+ return mtree.matched()
+
+
+class TestMatchTree(TestGuessit):
+ def test_base_tree(self):
+ t = BaseMatchTree('One Two Three(Three) Four')
+ t.partition((3, 7, 20))
+ leaves = list(t.leaves())
+
+ assert leaves[0].span == (0, 3)
+
+ assert 'One' == leaves[0].value
+ assert ' Two' == leaves[1].value
+ assert ' Three(Three)' == leaves[2].value
+ assert ' Four' == leaves[3].value
+
+ leaves[2].partition((1, 6, 7, 12))
+ three_leaves = list(leaves[2].leaves())
+
+ assert 'Three' == three_leaves[1].value
+ assert 'Three' == three_leaves[3].value
+
+ leaves = list(t.leaves())
+
+ assert len(leaves) == 8
+
+ assert leaves[5] == three_leaves[3]
+
+ assert t.previous_leaf(leaves[5]) == leaves[4]
+ assert t.next_leaf(leaves[5]) == leaves[6]
+
+ assert t.next_leaves(leaves[5]) == [leaves[6], leaves[7]]
+ assert t.previous_leaves(leaves[5]) == [leaves[4], leaves[3], leaves[2], leaves[1], leaves[0]]
+
+ assert t.next_leaf(leaves[7]) is None
+ assert t.previous_leaf(leaves[0]) is None
+
+ assert t.next_leaves(leaves[7]) == []
+ assert t.previous_leaves(leaves[0]) == []
+
+ def test_match(self):
+ self.checkFields(keywords, guess_info)
diff --git a/lib/guessit/test/test_movie.py b/lib/guessit/test/test_movie.py
new file mode 100644
index 0000000000000000000000000000000000000000..c0e28de420298f1c3f13f9a7e3a6047ac215922f
--- /dev/null
+++ b/lib/guessit/test/test_movie.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.test.guessittest import *
+
+
+class TestMovie(TestGuessit):
+ def testMovies(self):
+ self.checkMinimumFieldsCorrect(filetype='movie',
+ filename='movies.yaml')
diff --git a/lib/guessit/test/test_quality.py b/lib/guessit/test/test_quality.py
new file mode 100644
index 0000000000000000000000000000000000000000..c9bdb641958ec62726dcf7a381897780d89bfcb6
--- /dev/null
+++ b/lib/guessit/test/test_quality.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.quality import best_quality, best_quality_properties
+from guessit.containers import QualitiesContainer
+from guessit.test.guessittest import *
+
+
+class TestQuality(TestGuessit):
+ def test_container(self):
+ container = QualitiesContainer()
+
+ container.register_quality('color', 'red', 10)
+ container.register_quality('color', 'orange', 20)
+ container.register_quality('color', 'green', 30)
+
+ container.register_quality('context', 'sun', 100)
+ container.register_quality('context', 'sea', 200)
+ container.register_quality('context', 'sex', 300)
+
+ g1 = Guess()
+ g1['color'] = 'red'
+
+ g2 = Guess()
+ g2['color'] = 'green'
+
+ g3 = Guess()
+ g3['color'] = 'orange'
+
+ q3 = container.rate_quality(g3)
+ assert q3 == 20, "ORANGE should be rated 20. Don't ask why!"
+
+ q1 = container.rate_quality(g1)
+ q2 = container.rate_quality(g2)
+
+ assert q2 > q1, "GREEN should be greater than RED. Don't ask why!"
+
+ g1['context'] = 'sex'
+ g2['context'] = 'sun'
+
+ q1 = container.rate_quality(g1)
+ q2 = container.rate_quality(g2)
+
+ assert q1 > q2, "SEX should be greater than SUN. Don't ask why!"
+
+ assert container.best_quality(g1, g2) == g1, "RED&SEX should be better than GREEN&SUN. Don't ask why!"
+
+ assert container.best_quality_properties(['color'], g1, g2) == g2, \
+ "GREEN should be better than RED. Don't ask why!"
+
+ assert container.best_quality_properties(['context'], g1, g2) == g1, \
+ "SEX should be better than SUN. Don't ask why!"
+
+ q1 = container.rate_quality(g1, 'color')
+ q2 = container.rate_quality(g2, 'color')
+
+ assert q2 > q1, "GREEN should be greater than RED. Don't ask why!"
+
+ container.unregister_quality('context', 'sex')
+ container.unregister_quality('context', 'sun')
+
+ q1 = container.rate_quality(g1)
+ q2 = container.rate_quality(g2)
+
+ assert q2 > q1, "GREEN&SUN should be greater than RED&SEX. Don't ask why!"
+
+ g3['context'] = 'sea'
+ container.unregister_quality('context', 'sea')
+
+ q3 = container.rate_quality(g3, 'context')
+ assert q3 == 0, "Context should be unregistered."
+
+ container.unregister_quality('color')
+ q3 = container.rate_quality(g3, 'color')
+
+ assert q3 == 0, "Color should be unregistered."
+
+ container.clear_qualities()
+
+ q1 = container.rate_quality(g1)
+ q2 = container.rate_quality(g2)
+
+ assert q1 == q2 == 0, "Empty quality container should rate each guess to 0"
+
+ def test_quality_transformers(self):
+ guess_720p = guessit.guess_file_info("2012.2009.720p.BluRay.x264.DTS WiKi.mkv")
+ guess_1080p = guessit.guess_file_info("2012.2009.1080p.BluRay.x264.MP3 WiKi.mkv")
+
+ assert 'audioCodec' in guess_720p, "audioCodec should be present"
+ assert 'audioCodec' in guess_1080p, "audioCodec should be present"
+ assert 'screenSize' in guess_720p, "screenSize should be present"
+ assert 'screenSize' in guess_1080p, "screenSize should be present"
+
+ best_quality_guess = best_quality(guess_720p, guess_1080p)
+
+ assert guess_1080p == best_quality_guess, "1080p+MP3 is not the best global quality"
+
+ best_quality_guess = best_quality_properties(['screenSize'], guess_720p, guess_1080p)
+
+ assert guess_1080p == best_quality_guess, "1080p is not the best screenSize"
+
+ best_quality_guess = best_quality_properties(['audioCodec'], guess_720p, guess_1080p)
+
+ assert guess_720p == best_quality_guess, "DTS is not the best audioCodec"
diff --git a/lib/guessit/test/test_utils.py b/lib/guessit/test/test_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..93fcccd6613d5adf6926084e19d045c626cc5387
--- /dev/null
+++ b/lib/guessit/test/test_utils.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from datetime import date, timedelta
+
+from guessit.test.guessittest import *
+
+from guessit.fileutils import split_path
+from guessit.textutils import strip_brackets, str_replace, str_fill, from_camel, is_camel,\
+ levenshtein, reorder_title
+from guessit import PY2
+from guessit.date import search_date, search_year
+
+
+class TestUtils(TestGuessit):
+ def test_splitpath(self):
+ alltests = {False: {'/usr/bin/smewt': ['/', 'usr', 'bin', 'smewt'],
+ 'relative_path/to/my_folder/': ['relative_path', 'to', 'my_folder'],
+ '//some/path': ['//', 'some', 'path'],
+ '//some//path': ['//', 'some', 'path'],
+ '///some////path': ['///', 'some', 'path']
+
+ },
+ True: {'C:\\Program Files\\Smewt\\smewt.exe': ['C:\\', 'Program Files', 'Smewt', 'smewt.exe'],
+ 'Documents and Settings\\User\\config': ['Documents and Settings', 'User', 'config'],
+ 'C:\\Documents and Settings\\User\\config': ['C:\\', 'Documents and Settings', 'User', 'config'],
+ # http://bugs.python.org/issue19945
+ '\\\\netdrive\\share': ['\\\\', 'netdrive', 'share'] if PY2 else ['\\\\netdrive\\share'],
+ '\\\\netdrive\\share\\folder': ['\\\\', 'netdrive', 'share', 'folder'] if PY2 else ['\\\\netdrive\\share\\', 'folder'],
+ }
+ }
+ tests = alltests[sys.platform == 'win32']
+ for path, split in tests.items():
+ assert split == split_path(path)
+
+ def test_strip_brackets(self):
+ allTests = (('', ''),
+ ('[test]', 'test'),
+ ('{test2}', 'test2'),
+ ('(test3)', 'test3'),
+ ('(test4]', '(test4]'),
+ )
+
+ for i, e in allTests:
+ assert e == strip_brackets(i)
+
+ def test_levenshtein(self):
+ assert levenshtein("abcdef ghijk lmno", "abcdef ghijk lmno") == 0
+ assert levenshtein("abcdef ghijk lmnop", "abcdef ghijk lmno") == 1
+ assert levenshtein("abcdef ghijk lmno", "abcdef ghijk lmn") == 1
+ assert levenshtein("abcdef ghijk lmno", "abcdef ghijk lmnp") == 1
+ assert levenshtein("abcdef ghijk lmno", "abcdef ghijk lmnq") == 1
+ assert levenshtein("cbcdef ghijk lmno", "abcdef ghijk lmnq") == 2
+ assert levenshtein("cbcdef ghihk lmno", "abcdef ghijk lmnq") == 3
+
+ def test_reorder_title(self):
+ assert reorder_title("Simpsons, The") == "The Simpsons"
+ assert reorder_title("Simpsons,The") == "The Simpsons"
+ assert reorder_title("Simpsons,Les", articles=('the', 'le', 'la', 'les')) == "Les Simpsons"
+ assert reorder_title("Simpsons, Les", articles=('the', 'le', 'la', 'les')) == "Les Simpsons"
+
+ def test_camel(self):
+ assert "" == from_camel("")
+
+ assert "Hello world" == str_replace("Hello World", 6, 'w')
+ assert "Hello *****" == str_fill("Hello World", (6, 11), '*')
+
+ assert "This is camel" == from_camel("ThisIsCamel")
+
+ assert 'camel case' == from_camel('camelCase')
+ assert 'A case' == from_camel('ACase')
+ assert 'MiXedCaSe is not camel case' == from_camel('MiXedCaSe is not camelCase')
+
+ assert "This is camel cased title" == from_camel("ThisIsCamelCasedTitle")
+ assert "This is camel CASED title" == from_camel("ThisIsCamelCASEDTitle")
+
+ assert "These are camel CASED title" == from_camel("TheseAreCamelCASEDTitle")
+
+ assert "Give a camel case string" == from_camel("GiveACamelCaseString")
+
+ assert "Death TO camel case" == from_camel("DeathTOCamelCase")
+ assert "But i like java too:)" == from_camel("ButILikeJavaToo:)")
+
+ assert "Beatdown french DVD rip.mkv" == from_camel("BeatdownFrenchDVDRip.mkv")
+ assert "DO NOTHING ON UPPER CASE" == from_camel("DO NOTHING ON UPPER CASE")
+
+ assert not is_camel("this_is_not_camel")
+ assert is_camel("ThisIsCamel")
+
+ assert "Dark.City.(1998).DC.BDRIP.720p.DTS.X264-CHD.mkv" == \
+ from_camel("Dark.City.(1998).DC.BDRIP.720p.DTS.X264-CHD.mkv")
+ assert not is_camel("Dark.City.(1998).DC.BDRIP.720p.DTS.X264-CHD.mkv")
+
+ assert "A2LiNE" == from_camel("A2LiNE")
+
+ def test_date(self):
+ assert search_year(' in the year 2000... ') == (2000, (13, 17))
+ assert search_year(' they arrived in 1492. ') == (None, None)
+
+ today = date.today()
+ today_year_2 = int(str(today.year)[2:])
+
+ future = today + timedelta(days=1000)
+ future_year_2 = int(str(future.year)[2:])
+
+ past = today - timedelta(days=10000)
+ past_year_2 = int(str(past.year)[2:])
+
+ assert search_date(' Something before 2002-04-22 ') == (date(2002, 4, 22), (18, 28))
+ assert search_date(' 2002-04-22 Something after ') == (date(2002, 4, 22), (1, 11))
+
+ assert search_date(' This happened on 2002-04-22. ') == (date(2002, 4, 22), (18, 28))
+ assert search_date(' This happened on 22-04-2002. ') == (date(2002, 4, 22), (18, 28))
+
+ assert search_date(' This happened on 13-04-%s. ' % (today_year_2,)) == (date(today.year, 4, 13), (18, 26))
+ assert search_date(' This happened on 22-04-%s. ' % (future_year_2,)) == (date(future.year, 4, 22), (18, 26))
+ assert search_date(' This happened on 20-04-%s. ' % past_year_2) == (date(past.year, 4, 20), (18, 26))
+
+ assert search_date(' This happened on 13-06-14. ', year_first=True) == (date(2013, 6, 14), (18, 26))
+ assert search_date(' This happened on 13-05-14. ', year_first=False) == (date(2014, 5, 13), (18, 26))
+
+ assert search_date(' This happened on 04-13-%s. ' % (today_year_2,)) == (date(today.year, 4, 13), (18, 26))
+ assert search_date(' This happened on 04-22-%s. ' % (future_year_2,)) == (date(future.year, 4, 22), (18, 26))
+ assert search_date(' This happened on 04-20-%s. ' % past_year_2) == (date(past.year, 4, 20), (18, 26))
+
+ assert search_date(' This happened on 35-12-%s. ' % (today_year_2,)) == (None, None)
+ assert search_date(' This happened on 37-18-%s. ' % (future_year_2,)) == (None, None)
+ assert search_date(' This happened on 44-42-%s. ' % past_year_2) == (None, None)
+
+ assert search_date(' This happened on %s. ' % (today, )) == (today, (18, 28))
+ assert search_date(' This happened on %s. ' % (future, )) == (future, (18, 28))
+ assert search_date(' This happened on %s. ' % (past, )) == (past, (18, 28))
+
+ assert search_date(' released date: 04-03-1901? ') == (None, None)
+
+ assert search_date(' There\'s no date in here. ') == (None, None)
+
+ assert search_date(' Something 01-02-03 ') == (date(2003, 2, 1), (11, 19))
+ assert search_date(' Something 01-02-03 ', year_first=False, day_first=True) == (date(2003, 2, 1), (11, 19))
+ assert search_date(' Something 01-02-03 ', year_first=True) == (date(2001, 2, 3), (11, 19))
+ assert search_date(' Something 01-02-03 ', day_first=False) == (date(2003, 1, 2), (11, 19))
diff --git a/lib/guessit/textutils.py b/lib/guessit/textutils.py
index a1b7c4bec2b71812dfec47b490e5481a35d5bce3..822139cfaf1ff0df3f603a51f25139bda7218d0e 100644
--- a/lib/guessit/textutils.py
+++ b/lib/guessit/textutils.py
@@ -1,24 +1,25 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
-# Smewt - A smart collection manager
-# Copyright (c) 2008-2012 Nicolas Wack <wackou@gmail.com>
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
-# Smewt is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
-# Smewt is distributed in the hope that it will be useful,
+# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# Lesser GNU General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
+# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
+from __future__ import absolute_import, division, print_function, unicode_literals
+
from guessit import s
from guessit.patterns import sep
import functools
@@ -27,6 +28,7 @@ import re
# string-related functions
+
def normalize_unicode(s):
return unicodedata.normalize('NFC', s)
@@ -36,45 +38,70 @@ def strip_brackets(s):
return s
if ((s[0] == '[' and s[-1] == ']') or
- (s[0] == '(' and s[-1] == ')') or
- (s[0] == '{' and s[-1] == '}')):
+ (s[0] == '(' and s[-1] == ')') or
+ (s[0] == '{' and s[-1] == '}')):
return s[1:-1]
return s
-def clean_string(s):
- for c in sep[:-2]: # do not remove dashes ('-')
- s = s.replace(c, ' ')
- parts = s.split()
+_dotted_rexp = re.compile(r'(?:\W|^)(([A-Za-z]\.){2,}[A-Za-z]\.?)')
+
+
+def clean_default(st):
+ for c in sep:
+ # do not remove certain chars
+ if c in ['-', ',']:
+ continue
+
+ if c == '.':
+ # we should not remove the dots for acronyms and such
+ dotted = _dotted_rexp.search(st)
+ if dotted:
+ s = dotted.group(1)
+ exclude_begin, exclude_end = dotted.span(1)
+
+ st = (st[:exclude_begin].replace(c, ' ') +
+ st[exclude_begin:exclude_end] +
+ st[exclude_end:].replace(c, ' '))
+ continue
+
+ st = st.replace(c, ' ')
+
+ parts = st.split()
result = ' '.join(p for p in parts if p != '')
# now also remove dashes on the outer part of the string
- while result and result[0] in sep:
+ while result and result[0] in '-':
result = result[1:]
- while result and result[-1] in sep:
+ while result and result[-1] in '-':
result = result[:-1]
return result
-
_words_rexp = re.compile('\w+', re.UNICODE)
+
def find_words(s):
return _words_rexp.findall(s.replace('_', ' '))
-def reorder_title(title):
+def iter_words(s):
+ return _words_rexp.finditer(s.replace('_', ' '))
+
+
+def reorder_title(title, articles=('the',), separators=(',', ', ')):
ltitle = title.lower()
- if ltitle[-4:] == ',the':
- return title[-3:] + ' ' + title[:-4]
- if ltitle[-5:] == ', the':
- return title[-3:] + ' ' + title[:-5]
+ for article in articles:
+ for separator in separators:
+ suffix = separator + article
+ if ltitle[-len(suffix):] == suffix:
+ return title[-len(suffix) + len(separator):] + ' ' + title[:-len(suffix)]
return title
def str_replace(string, pos, c):
- return string[:pos] + c + string[pos+1:]
+ return string[:pos] + c + string[pos + 1:]
def str_fill(string, region, c):
@@ -82,7 +109,6 @@ def str_fill(string, region, c):
return string[:start] + c * (end - start) + string[end:]
-
def levenshtein(a, b):
if not a:
return len(b)
@@ -92,25 +118,25 @@ def levenshtein(a, b):
m = len(a)
n = len(b)
d = []
- for i in range(m+1):
- d.append([0] * (n+1))
+ for i in range(m + 1):
+ d.append([0] * (n + 1))
- for i in range(m+1):
+ for i in range(m + 1):
d[i][0] = i
- for j in range(n+1):
+ for j in range(n + 1):
d[0][j] = j
- for i in range(1, m+1):
- for j in range(1, n+1):
- if a[i-1] == b[j-1]:
+ for i in range(1, m + 1):
+ for j in range(1, n + 1):
+ if a[i - 1] == b[j - 1]:
cost = 0
else:
cost = 1
- d[i][j] = min(d[i-1][j] + 1, # deletion
- d[i][j-1] + 1, # insertion
- d[i-1][j-1] + cost # substitution
+ d[i][j] = min(d[i - 1][j] + 1, # deletion
+ d[i][j - 1] + 1, # insertion
+ d[i - 1][j - 1] + cost # substitution
)
return d[m][n]
@@ -137,7 +163,7 @@ def find_first_level_groups_span(string, enclosing):
[(2, 5), (7, 10)]
"""
opening, closing = enclosing
- depth = [] # depth is a stack of indices where we opened a group
+ depth = [] # depth is a stack of indices where we opened a group
result = []
for i, c, in enumerate(string):
if c == opening:
@@ -148,7 +174,7 @@ def find_first_level_groups_span(string, enclosing):
end = i
if not depth:
# we emptied our stack, so we have a 1st level group
- result.append((start, end+1))
+ result.append((start, end + 1))
except IndexError:
# we closed a group which was not opened before
pass
@@ -169,7 +195,7 @@ def split_on_groups(string, groups):
"""
if not groups:
- return [ string ]
+ return [string]
boundaries = sorted(set(functools.reduce(lambda l, x: l + list(x), groups, [])))
if boundaries[0] != 0:
@@ -177,10 +203,10 @@ def split_on_groups(string, groups):
if boundaries[-1] != len(string):
boundaries.append(len(string))
- groups = [ string[start:end] for start, end in zip(boundaries[:-1],
- boundaries[1:]) ]
+ groups = [string[start:end] for start, end in zip(boundaries[:-1],
+ boundaries[1:])]
- return [ g for g in groups if g ] # return only non-empty groups
+ return [g for g in groups if g] # return only non-empty groups
def find_first_level_groups(string, enclosing, blank_sep=None):
@@ -216,6 +242,124 @@ def find_first_level_groups(string, enclosing, blank_sep=None):
if blank_sep:
for start, end in groups:
string = str_replace(string, start, blank_sep)
- string = str_replace(string, end-1, blank_sep)
+ string = str_replace(string, end - 1, blank_sep)
return split_on_groups(string, groups)
+
+
+_camel_word2_set = set(('is', 'to'))
+_camel_word3_set = set(('the'))
+
+
+def _camel_split_and_lower(string, i):
+ """Retrieves a tuple (need_split, need_lower)
+
+ need_split is True if this char is a first letter in a camelCasedString.
+ need_lower is True if this char should be lowercased.
+ """
+
+ def islower(c):
+ return c.isalpha() and not c.isupper()
+
+ previous_char2 = string[i - 2] if i > 1 else None
+ previous_char = string[i - 1] if i > 0 else None
+ char = string[i]
+ next_char = string[i + 1] if i + 1 < len(string) else None
+ next_char2 = string[i + 2] if i + 2 < len(string) else None
+
+ char_upper = char.isupper()
+ char_lower = islower(char)
+
+ # previous_char2_lower = islower(previous_char2) if previous_char2 else False
+ previous_char2_upper = previous_char2.isupper() if previous_char2 else False
+
+ previous_char_lower = islower(previous_char) if previous_char else False
+ previous_char_upper = previous_char.isupper() if previous_char else False
+
+ next_char_upper = next_char.isupper() if next_char else False
+ next_char_lower = islower(next_char) if next_char else False
+
+ next_char2_upper = next_char2.isupper() if next_char2 else False
+ # next_char2_lower = islower(next_char2) if next_char2 else False
+
+ mixedcase_word = (previous_char_upper and char_lower and next_char_upper) or \
+ (previous_char_lower and char_upper and next_char_lower and next_char2_upper) or \
+ (previous_char2_upper and previous_char_lower and char_upper)
+ if mixedcase_word:
+ word2 = (char + next_char).lower() if next_char else None
+ word3 = (char + next_char + next_char2).lower() if next_char and next_char2 else None
+ word2b = (previous_char2 + previous_char).lower() if previous_char2 and previous_char else None
+ if word2 in _camel_word2_set or word2b in _camel_word2_set or word3 in _camel_word3_set:
+ mixedcase_word = False
+
+ uppercase_word = previous_char_upper and char_upper and next_char_upper or (char_upper and next_char_upper and next_char2_upper)
+
+ need_split = char_upper and previous_char_lower and not mixedcase_word
+
+ if not need_split:
+ previous_char_upper = string[i - 1].isupper() if i > 0 else False
+ next_char_lower = (string[i + 1].isalpha() and not string[i + 1].isupper()) if i + 1 < len(string) else False
+ need_split = char_upper and previous_char_upper and next_char_lower
+ uppercase_word = previous_char_upper and not next_char_lower
+
+ need_lower = not uppercase_word and not mixedcase_word and need_split
+
+ return need_split, need_lower
+
+
+def is_camel(string):
+ """
+ >>> is_camel('dogEATDog')
+ True
+ >>> is_camel('DeathToCamelCase')
+ True
+ >>> is_camel('death_to_camel_case')
+ False
+ >>> is_camel('TheBest')
+ True
+ >>> is_camel('The Best')
+ False
+ """
+ for i in range(0, len(string)):
+ need_split, _ = _camel_split_and_lower(string, i)
+ if need_split:
+ return True
+ return False
+
+
+def from_camel(string):
+ """
+ >>> from_camel('dogEATDog') == 'dog EAT dog'
+ True
+ >>> from_camel('DeathToCamelCase') == 'Death to camel case'
+ True
+ >>> from_camel('TheBest') == 'The best'
+ True
+ >>> from_camel('MiXedCaSe is not camelCase') == 'MiXedCaSe is not camel case'
+ True
+ """
+ if not string:
+ return string
+ pieces = []
+
+ for i in range(0, len(string)):
+ char = string[i]
+ need_split, need_lower = _camel_split_and_lower(string, i)
+ if need_split:
+ pieces.append(' ')
+
+ if need_lower:
+ pieces.append(char.lower())
+ else:
+ pieces.append(char)
+ return ''.join(pieces)
+
+
+def common_words(s1, s2):
+ common = []
+ words1 = set(s1.split())
+ for word in s2.split():
+ # strip some chars here, e.g. as in [1]
+ if word in words1:
+ common.append(word)
+ return common
diff --git a/lib/guessit/tlds-alpha-by-domain.txt b/lib/guessit/tlds-alpha-by-domain.txt
new file mode 100644
index 0000000000000000000000000000000000000000..280c794c5471bfad0e77f7ab8b9574b5c736826c
--- /dev/null
+++ b/lib/guessit/tlds-alpha-by-domain.txt
@@ -0,0 +1,341 @@
+# Version 2013112900, Last Updated Fri Nov 29 07:07:01 2013 UTC
+AC
+AD
+AE
+AERO
+AF
+AG
+AI
+AL
+AM
+AN
+AO
+AQ
+AR
+ARPA
+AS
+ASIA
+AT
+AU
+AW
+AX
+AZ
+BA
+BB
+BD
+BE
+BF
+BG
+BH
+BI
+BIKE
+BIZ
+BJ
+BM
+BN
+BO
+BR
+BS
+BT
+BV
+BW
+BY
+BZ
+CA
+CAMERA
+CAT
+CC
+CD
+CF
+CG
+CH
+CI
+CK
+CL
+CLOTHING
+CM
+CN
+CO
+COM
+CONSTRUCTION
+CONTRACTORS
+COOP
+CR
+CU
+CV
+CW
+CX
+CY
+CZ
+DE
+DIAMONDS
+DIRECTORY
+DJ
+DK
+DM
+DO
+DZ
+EC
+EDU
+EE
+EG
+ENTERPRISES
+EQUIPMENT
+ER
+ES
+ESTATE
+ET
+EU
+FI
+FJ
+FK
+FM
+FO
+FR
+GA
+GALLERY
+GB
+GD
+GE
+GF
+GG
+GH
+GI
+GL
+GM
+GN
+GOV
+GP
+GQ
+GR
+GRAPHICS
+GS
+GT
+GU
+GURU
+GW
+GY
+HK
+HM
+HN
+HOLDINGS
+HR
+HT
+HU
+ID
+IE
+IL
+IM
+IN
+INFO
+INT
+IO
+IQ
+IR
+IS
+IT
+JE
+JM
+JO
+JOBS
+JP
+KE
+KG
+KH
+KI
+KITCHEN
+KM
+KN
+KP
+KR
+KW
+KY
+KZ
+LA
+LAND
+LB
+LC
+LI
+LIGHTING
+LK
+LR
+LS
+LT
+LU
+LV
+LY
+MA
+MC
+MD
+ME
+MG
+MH
+MIL
+MK
+ML
+MM
+MN
+MO
+MOBI
+MP
+MQ
+MR
+MS
+MT
+MU
+MUSEUM
+MV
+MW
+MX
+MY
+MZ
+NA
+NAME
+NC
+NE
+NET
+NF
+NG
+NI
+NL
+NO
+NP
+NR
+NU
+NZ
+OM
+ORG
+PA
+PE
+PF
+PG
+PH
+PHOTOGRAPHY
+PK
+PL
+PLUMBING
+PM
+PN
+POST
+PR
+PRO
+PS
+PT
+PW
+PY
+QA
+RE
+RO
+RS
+RU
+RW
+SA
+SB
+SC
+SD
+SE
+SEXY
+SG
+SH
+SI
+SINGLES
+SJ
+SK
+SL
+SM
+SN
+SO
+SR
+ST
+SU
+SV
+SX
+SY
+SZ
+TATTOO
+TC
+TD
+TECHNOLOGY
+TEL
+TF
+TG
+TH
+TIPS
+TJ
+TK
+TL
+TM
+TN
+TO
+TODAY
+TP
+TR
+TRAVEL
+TT
+TV
+TW
+TZ
+UA
+UG
+UK
+US
+UY
+UZ
+VA
+VC
+VE
+VENTURES
+VG
+VI
+VN
+VOYAGE
+VU
+WF
+WS
+XN--3E0B707E
+XN--45BRJ9C
+XN--80AO21A
+XN--80ASEHDB
+XN--80ASWG
+XN--90A3AC
+XN--CLCHC0EA0B2G2A9GCD
+XN--FIQS8S
+XN--FIQZ9S
+XN--FPCRJ9C3D
+XN--FZC2C9E2C
+XN--GECRJ9C
+XN--H2BRJ9C
+XN--J1AMH
+XN--J6W193G
+XN--KPRW13D
+XN--KPRY57D
+XN--L1ACC
+XN--LGBBAT1AD8J
+XN--MGB9AWBF
+XN--MGBA3A4F16A
+XN--MGBAAM7A8H
+XN--MGBAYH7GPA
+XN--MGBBH1A71E
+XN--MGBC0A9AZCG
+XN--MGBERP4A5D4AR
+XN--MGBX4CD0AB
+XN--NGBC5AZD
+XN--O3CW4H
+XN--OGBPF8FL
+XN--P1AI
+XN--PGBS0DH
+XN--Q9JYB4C
+XN--S9BRJ9C
+XN--UNUP4Y
+XN--WGBH1C
+XN--WGBL6A
+XN--XKC2AL3HYE2A
+XN--XKC2DL3A5EE0H
+XN--YFRO4I67O
+XN--YGBI2AMMX
+XXX
+YE
+YT
+ZA
+ZM
+ZW
diff --git a/lib/guessit/transfo/__init__.py b/lib/guessit/transfo/__init__.py
index 45dae5d7dd059532a32e5df221e0d96a545d1dba..cce2dfdaa3012c0aa9de5a0a5643a2da03434d64 100644
--- a/lib/guessit/transfo/__init__.py
+++ b/lib/guessit/transfo/__init__.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,84 +18,13 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import base_text_type, Guess
-from guessit.patterns import canonical_form
-from guessit.textutils import clean_string
-import logging
+from __future__ import absolute_import, division, print_function, unicode_literals
-log = logging.getLogger(__name__)
+class TransformerException(Exception):
+ def __init__(self, transformer, message):
-def found_property(node, name, confidence):
- node.guess = Guess({name: node.clean_value}, confidence=confidence)
- log.debug('Found with confidence %.2f: %s' % (confidence, node.guess))
+ # Call the base class constructor with the parameters it needs
+ Exception.__init__(self, message)
-
-def format_guess(guess):
- """Format all the found values to their natural type.
- For instance, a year would be stored as an int value, etc...
-
- Note that this modifies the dictionary given as input.
- """
- for prop, value in guess.items():
- if prop in ('season', 'episodeNumber', 'year', 'cdNumber',
- 'cdNumberTotal', 'bonusNumber', 'filmNumber'):
- guess[prop] = int(guess[prop])
- elif isinstance(value, base_text_type):
- if prop in ('edition',):
- value = clean_string(value)
- guess[prop] = canonical_form(value).replace('\\', '')
-
- return guess
-
-
-def find_and_split_node(node, strategy, logger):
- string = ' %s ' % node.value # add sentinels
- for matcher, confidence in strategy:
- if getattr(matcher, 'use_node', False):
- result, span = matcher(string, node)
- else:
- result, span = matcher(string)
-
- if result:
- # readjust span to compensate for sentinels
- span = (span[0] - 1, span[1] - 1)
-
- if isinstance(result, Guess):
- if confidence is None:
- confidence = result.confidence(list(result.keys())[0])
- else:
- if confidence is None:
- confidence = 1.0
-
- guess = format_guess(Guess(result, confidence=confidence))
- msg = 'Found with confidence %.2f: %s' % (confidence, guess)
- (logger or log).debug(msg)
-
- node.partition(span)
- absolute_span = (span[0] + node.offset, span[1] + node.offset)
- for child in node.children:
- if child.span == absolute_span:
- child.guess = guess
- else:
- find_and_split_node(child, strategy, logger)
- return
-
-
-class SingleNodeGuesser(object):
- def __init__(self, guess_func, confidence, logger=None):
- self.guess_func = guess_func
- self.confidence = confidence
- self.logger = logger
-
- def process(self, mtree):
- # strategy is a list of pairs (guesser, confidence)
- # - if the guesser returns a guessit.Guess and confidence is specified,
- # it will override it, otherwise it will leave the guess confidence
- # - if the guesser returns a simple dict as a guess and confidence is
- # specified, it will use it, or 1.0 otherwise
- strategy = [ (self.guess_func, self.confidence) ]
-
- for node in mtree.unidentified_leaves():
- find_and_split_node(node, strategy, self.logger)
+ self.transformer = transformer
\ No newline at end of file
diff --git a/lib/guessit/transfo/expected_series.py b/lib/guessit/transfo/expected_series.py
new file mode 100644
index 0000000000000000000000000000000000000000..f2db12d6cdb964e840964976f55e1097f8d685e0
--- /dev/null
+++ b/lib/guessit/transfo/expected_series.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+import re
+
+from guessit.containers import PropertiesContainer
+from guessit.matcher import GuessFinder
+from guessit.plugins.transformers import Transformer
+
+
+class ExpectedSeries(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 230)
+
+ def register_arguments(self, opts, naming_opts, output_opts, information_opts, webservice_opts, other_options):
+ naming_opts.add_argument('-S', '--expected-series', action='append', dest='expected_series',
+ help='Expected series to parse (can be used multiple times)')
+
+ def should_process(self, mtree, options=None):
+ return options and options.get('expected_series')
+
+ @staticmethod
+ def expected_series(string, node=None, options=None):
+ container = PropertiesContainer(enhance=True, canonical_from_pattern=False)
+
+ for expected_serie in options.get('expected_series'):
+ if expected_serie.startswith('re:'):
+ expected_serie = expected_serie[3:]
+ expected_serie = expected_serie.replace(' ', '-')
+ container.register_property('series', expected_serie, enhance=True)
+ else:
+ expected_serie = re.escape(expected_serie)
+ container.register_property('series', expected_serie, enhance=False)
+
+ found = container.find_properties(string, node, options)
+ return container.as_guess(found, string)
+
+ def supported_properties(self):
+ return ['series']
+
+ def process(self, mtree, options=None):
+ GuessFinder(self.expected_series, None, self.log, options).process_nodes(mtree.unidentified_leaves())
diff --git a/lib/guessit/transfo/expected_title.py b/lib/guessit/transfo/expected_title.py
new file mode 100644
index 0000000000000000000000000000000000000000..995779cd6ed4f148b1538587e74fc155b9f9ab6c
--- /dev/null
+++ b/lib/guessit/transfo/expected_title.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import re
+
+from guessit.containers import PropertiesContainer
+from guessit.matcher import GuessFinder
+from guessit.plugins.transformers import Transformer
+
+
+class ExpectedTitle(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 225)
+
+ def register_arguments(self, opts, naming_opts, output_opts, information_opts, webservice_opts, other_options):
+ naming_opts.add_argument('-T', '--expected-title', action='append', dest='expected_title',
+ help='Expected title (can be used multiple times)')
+
+ def should_process(self, mtree, options=None):
+ return options and options.get('expected_title')
+
+ @staticmethod
+ def expected_titles(string, node=None, options=None):
+ container = PropertiesContainer(enhance=True, canonical_from_pattern=False)
+
+ for expected_title in options.get('expected_title'):
+ if expected_title.startswith('re:'):
+ expected_title = expected_title[3:]
+ expected_title = expected_title.replace(' ', '-')
+ container.register_property('title', expected_title, enhance=True)
+ else:
+ expected_title = re.escape(expected_title)
+ container.register_property('title', expected_title, enhance=False)
+
+ found = container.find_properties(string, node, options)
+ return container.as_guess(found, string)
+
+ def supported_properties(self):
+ return ['title']
+
+ def process(self, mtree, options=None):
+ GuessFinder(self.expected_titles, None, self.log, options).process_nodes(mtree.unidentified_leaves())
diff --git a/lib/guessit/transfo/guess_bonus_features.py b/lib/guessit/transfo/guess_bonus_features.py
index e8791b4a6fe07bb6c21ff8e70ac9017ddbf9bcf5..7249b5db4262e6866deb58fa2f4dba75088c6d7c 100644
--- a/lib/guessit/transfo/guess_bonus_features.py
+++ b/lib/guessit/transfo/guess_bonus_features.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,44 +18,52 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit.transfo import found_property
-import logging
+from __future__ import absolute_import, division, print_function, unicode_literals
-log = logging.getLogger(__name__)
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import found_property
-def process(mtree):
- def previous_group(g):
- for leaf in mtree.unidentified_leaves()[::-1]:
- if leaf.node_idx < g.node_idx:
- return leaf
+class GuessBonusFeatures(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, -150)
- def next_group(g):
- for leaf in mtree.unidentified_leaves():
- if leaf.node_idx > g.node_idx:
- return leaf
+ def supported_properties(self):
+ return ['bonusNumber', 'bonusTitle', 'filmNumber', 'filmSeries', 'title', 'series']
- def same_group(g1, g2):
- return g1.node_idx[:2] == g2.node_idx[:2]
+ def process(self, mtree, options=None):
+ def previous_group(g):
+ for leaf in reversed(list(mtree.unidentified_leaves())):
+ if leaf.node_idx < g.node_idx:
+ return leaf
- bonus = [ node for node in mtree.leaves() if 'bonusNumber' in node.guess ]
- if bonus:
- bonusTitle = next_group(bonus[0])
- if same_group(bonusTitle, bonus[0]):
- found_property(bonusTitle, 'bonusTitle', 0.8)
+ def next_group(g):
+ for leaf in mtree.unidentified_leaves():
+ if leaf.node_idx > g.node_idx:
+ return leaf
- filmNumber = [ node for node in mtree.leaves()
- if 'filmNumber' in node.guess ]
- if filmNumber:
- filmSeries = previous_group(filmNumber[0])
- found_property(filmSeries, 'filmSeries', 0.9)
+ def same_group(g1, g2):
+ return g1.node_idx[:2] == g2.node_idx[:2]
- title = next_group(filmNumber[0])
- found_property(title, 'title', 0.9)
+ bonus = [node for node in mtree.leaves() if 'bonusNumber' in node.guess]
+ if bonus:
+ bonus_title = next_group(bonus[0])
+ if bonus_title and same_group(bonus_title, bonus[0]):
+ found_property(bonus_title, 'bonusTitle', confidence=0.8)
- season = [ node for node in mtree.leaves() if 'season' in node.guess ]
- if season and 'bonusNumber' in mtree.info:
- series = previous_group(season[0])
- if same_group(series, season[0]):
- found_property(series, 'series', 0.9)
+ film_number = [node for node in mtree.leaves()
+ if 'filmNumber' in node.guess]
+ if film_number:
+ film_series = previous_group(film_number[0])
+ if film_series:
+ found_property(film_series, 'filmSeries', confidence=0.9)
+
+ title = next_group(film_number[0])
+ if title:
+ found_property(title, 'title', confidence=0.9)
+
+ season = [node for node in mtree.leaves() if 'season' in node.guess]
+ if season and 'bonusNumber' in mtree.info:
+ series = previous_group(season[0])
+ if same_group(series, season[0]):
+ found_property(series, 'series', confidence=0.9)
diff --git a/lib/guessit/transfo/guess_country.py b/lib/guessit/transfo/guess_country.py
index 123a269435051a71e8e82cd19d09d93cd88f3e9f..a9c16d3e8d3fd591df6e3579ad1d68feca987a3f 100644
--- a/lib/guessit/transfo/guess_country.py
+++ b/lib/guessit/transfo/guess_country.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,31 +18,111 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit.country import Country
-from guessit import Guess
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import logging
+from guessit.plugins.transformers import Transformer
+from babelfish import Country
+from guessit import Guess
+from guessit.textutils import iter_words
+from guessit.matcher import GuessFinder, found_guess
+from guessit.language import LNG_COMMON_WORDS
+import babelfish
+
+
log = logging.getLogger(__name__)
-# list of common words which could be interpreted as countries, but which
-# are far too common to be able to say they represent a country
-country_common_words = frozenset([ 'bt', 'bb' ])
-def process(mtree):
- for node in mtree.unidentified_leaves():
- if len(node.node_idx) == 2:
- c = node.value[1:-1].lower()
- if c in country_common_words:
- continue
+class GuessCountry(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, -170)
+ self.replace_language = frozenset(['uk'])
+
+ def register_arguments(self, opts, naming_opts, output_opts, information_opts, webservice_opts, other_options):
+ naming_opts.add_argument('-C', '--allowed-country', action='append', dest='allowed_countries',
+ help='Allowed country (can be used multiple times)')
+
+ def supported_properties(self):
+ return ['country']
+
+ def should_process(self, mtree, options=None):
+ options = options or {}
+ return options.get('country', True)
- # only keep explicit groups (enclosed in parentheses/brackets)
- if node.value[0] + node.value[-1] not in ['()', '[]', '{}']:
+ @staticmethod
+ def _scan_country(country, strict=False):
+ """
+ Find a country if it is at the start or end of country string
+ """
+ words_match = list(iter_words(country.lower()))
+ s = ""
+ start = None
+
+ for word_match in words_match:
+ if not start:
+ start = word_match.start(0)
+ s += word_match.group(0)
+ try:
+ return Country.fromguessit(s), (start, word_match.end(0))
+ except babelfish.Error:
continue
+ words_match.reverse()
+ s = ""
+ end = None
+ for word_match in words_match:
+ if not end:
+ end = word_match.end(0)
+ s = word_match.group(0) + s
try:
- country = Country(c, strict=True)
- except ValueError:
+ return Country.fromguessit(s), (word_match.start(0), end)
+ except babelfish.Error:
continue
- node.guess = Guess(country=country, confidence=1.0)
+ return Country.fromguessit(country), (start, end)
+
+ @staticmethod
+ def is_valid_country(country, options=None):
+ if options and options.get('allowed_countries'):
+ allowed_countries = options.get('allowed_countries')
+ return country.name.lower() in allowed_countries or country.alpha2.lower() in allowed_countries
+ else:
+ return (country.name.lower() not in LNG_COMMON_WORDS and
+ country.alpha2.lower() not in LNG_COMMON_WORDS)
+
+ def guess_country(self, string, node=None, options=None):
+ c = string.strip().lower()
+ if c not in LNG_COMMON_WORDS:
+ try:
+ country, country_span = self._scan_country(c, True)
+ if self.is_valid_country(country, options):
+ guess = Guess(country=country, confidence=1.0, input=node.value, span=(country_span[0] + 1, country_span[1] + 1))
+ return guess
+ except babelfish.Error:
+ pass
+ return None, None
+
+ def process(self, mtree, options=None):
+ GuessFinder(self.guess_country, None, self.log, options).process_nodes(mtree.unidentified_leaves())
+ for node in mtree.leaves_containing('language'):
+ c = node.clean_value.lower()
+ if c in self.replace_language:
+ node.guess.set('language', None)
+ try:
+ country = Country.fromguessit(c)
+ if self.is_valid_country(country, options):
+ guess = Guess(country=country, confidence=0.9, input=node.value, span=node.span)
+ found_guess(node, guess, logger=log)
+ except babelfish.Error:
+ pass
+
+ def post_process(self, mtree, options=None, *args, **kwargs):
+ # if country is in the guessed properties, make it part of the series name
+ series_leaves = list(mtree.leaves_containing('series'))
+ country_leaves = list(mtree.leaves_containing('country'))
+
+ if series_leaves and country_leaves:
+ country_leaf = country_leaves[0]
+ for serie_leaf in series_leaves:
+ serie_leaf.guess['series'] += ' (%s)' % str(country_leaf.guess['country'].guessit)
diff --git a/lib/guessit/transfo/guess_date.py b/lib/guessit/transfo/guess_date.py
index 5108cc58a413541aeb72488f72a0ab846291d91c..0aa7fe16054ad1ccee8751f435c16820ab40e604 100644
--- a/lib/guessit/transfo/guess_date.py
+++ b/lib/guessit/transfo/guess_date.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,21 +18,33 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit.transfo import SingleNodeGuesser
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import GuessFinder
from guessit.date import search_date
-import logging
-log = logging.getLogger(__name__)
+class GuessDate(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 50)
+
+ def register_arguments(self, opts, naming_opts, output_opts, information_opts, webservice_opts, other_options):
+ naming_opts.add_argument('-Y', '--date-year-first', action='store_true', dest='date_year_first', default=None,
+ help='If short date is found, consider the first digits as the year.')
+ naming_opts.add_argument('-D', '--date-day-first', action='store_true', dest='date_day_first', default=None,
+ help='If short date is found, consider the second digits as the day.')
-def guess_date(string):
- date, span = search_date(string)
- if date:
- return { 'date': date }, span
- else:
- return None, None
+ def supported_properties(self):
+ return ['date']
+ @staticmethod
+ def guess_date(string, node=None, options=None):
+ date, span = search_date(string, options.get('date_year_first') if options else False, options.get('date_day_first') if options else False)
+ if date:
+ return {'date': date}, span
+ else:
+ return None, None
-def process(mtree):
- SingleNodeGuesser(guess_date, 1.0, log).process(mtree)
+ def process(self, mtree, options=None):
+ GuessFinder(self.guess_date, 1.0, self.log, options).process_nodes(mtree.unidentified_leaves())
diff --git a/lib/guessit/transfo/guess_episode_details.py b/lib/guessit/transfo/guess_episode_details.py
new file mode 100644
index 0000000000000000000000000000000000000000..cecef2cd60370f306fce6fd188d57b4229b00892
--- /dev/null
+++ b/lib/guessit/transfo/guess_episode_details.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import itertools
+
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import found_guess
+from guessit.containers import PropertiesContainer
+
+
+class GuessEpisodeDetails(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, -205)
+ self.container = PropertiesContainer()
+ self.container.register_property('episodeDetails', 'Special', 'Bonus', 'Omake', 'Ova', 'Oav', 'Pilot', 'Unaired')
+ self.container.register_property('episodeDetails', 'Extras?', canonical_form='Extras')
+
+ def guess_details(self, string, node=None, options=None):
+ properties = self.container.find_properties(string, node, options, 'episodeDetails', multiple=True)
+ guesses = self.container.as_guess(properties, multiple=True)
+ return guesses
+
+ def second_pass_options(self, mtree, options=None):
+ if not mtree.guess.get('type', '').startswith('episode'):
+ for unidentified_leaf in mtree.unidentified_leaves():
+ properties = self.container.find_properties(unidentified_leaf.value, unidentified_leaf, options, 'episodeDetails')
+ guess = self.container.as_guess(properties)
+ if guess:
+ return {'type': 'episode'}
+ return None
+
+ def supported_properties(self):
+ return self.container.get_supported_properties()
+
+ def process(self, mtree, options=None):
+ if (mtree.guess.get('type', '').startswith('episode') and
+ (not mtree.info.get('episodeNumber') or
+ mtree.info.get('season') == 0)):
+
+ for leaf in itertools.chain(mtree.leaves_containing('title'),
+ mtree.unidentified_leaves()):
+ guesses = self.guess_details(leaf.value, leaf, options)
+ for guess in guesses:
+ found_guess(leaf, guess, update_guess=False)
+
+ return None
diff --git a/lib/guessit/transfo/guess_episode_info_from_position.py b/lib/guessit/transfo/guess_episode_info_from_position.py
index 45a0c99b8007c56116b692855ada9b9df77840ab..91c8392976290a2351808773fd4fcac48cc19ba4 100644
--- a/lib/guessit/transfo/guess_episode_info_from_position.py
+++ b/lib/guessit/transfo/guess_episode_info_from_position.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,129 +18,165 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit.transfo import found_property
-from guessit.patterns import non_episode_title, unlikely_series
-import logging
-
-log = logging.getLogger(__name__)
-
-
-def match_from_epnum_position(mtree, node):
- epnum_idx = node.node_idx
-
- # a few helper functions to be able to filter using high-level semantics
- def before_epnum_in_same_pathgroup():
- return [ leaf for leaf in mtree.unidentified_leaves()
- if (leaf.node_idx[0] == epnum_idx[0] and
- leaf.node_idx[1:] < epnum_idx[1:]) ]
-
- def after_epnum_in_same_pathgroup():
- return [ leaf for leaf in mtree.unidentified_leaves()
- if (leaf.node_idx[0] == epnum_idx[0] and
- leaf.node_idx[1:] > epnum_idx[1:]) ]
-
- def after_epnum_in_same_explicitgroup():
- return [ leaf for leaf in mtree.unidentified_leaves()
- if (leaf.node_idx[:2] == epnum_idx[:2] and
- leaf.node_idx[2:] > epnum_idx[2:]) ]
-
- # epnumber is the first group and there are only 2 after it in same
- # path group
- # -> series title - episode title
- title_candidates = [ n for n in after_epnum_in_same_pathgroup()
- if n.clean_value.lower() not in non_episode_title ]
- if ('title' not in mtree.info and # no title
- before_epnum_in_same_pathgroup() == [] and # no groups before
- len(title_candidates) == 2): # only 2 groups after
-
- found_property(title_candidates[0], 'series', confidence=0.4)
- found_property(title_candidates[1], 'title', confidence=0.4)
- return
-
- # if we have at least 1 valid group before the episodeNumber, then it's
- # probably the series name
- series_candidates = before_epnum_in_same_pathgroup()
- if len(series_candidates) >= 1:
- found_property(series_candidates[0], 'series', confidence=0.7)
-
- # only 1 group after (in the same path group) and it's probably the
- # episode title
- title_candidates = [ n for n in after_epnum_in_same_pathgroup()
- if n.clean_value.lower() not in non_episode_title ]
-
- if len(title_candidates) == 1:
- found_property(title_candidates[0], 'title', confidence=0.5)
- return
- else:
- # try in the same explicit group, with lower confidence
- title_candidates = [ n for n in after_epnum_in_same_explicitgroup()
- if n.clean_value.lower() not in non_episode_title
- ]
- if len(title_candidates) == 1:
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from guessit.plugins.transformers import Transformer, get_transformer
+from guessit.textutils import reorder_title
+
+from guessit.matcher import found_property
+
+
+class GuessEpisodeInfoFromPosition(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, -200)
+
+ def supported_properties(self):
+ return ['title', 'series']
+
+ def match_from_epnum_position(self, mtree, node, options):
+ epnum_idx = node.node_idx
+
+ # a few helper functions to be able to filter using high-level semantics
+ def before_epnum_in_same_pathgroup():
+ return [leaf for leaf in mtree.unidentified_leaves(lambda x: len(x.clean_value) > 1)
+ if (leaf.node_idx[0] == epnum_idx[0] and
+ leaf.node_idx[1:] < epnum_idx[1:])]
+
+ def after_epnum_in_same_pathgroup():
+ return [leaf for leaf in mtree.unidentified_leaves(lambda x: len(x.clean_value) > 1)
+ if (leaf.node_idx[0] == epnum_idx[0] and
+ leaf.node_idx[1:] > epnum_idx[1:])]
+
+ def after_epnum_in_same_explicitgroup():
+ return [leaf for leaf in mtree.unidentified_leaves(lambda x: len(x.clean_value) > 1)
+ if (leaf.node_idx[:2] == epnum_idx[:2] and
+ leaf.node_idx[2:] > epnum_idx[2:])]
+
+ # epnumber is the first group and there are only 2 after it in same
+ # path group
+ # -> series title - episode title
+ title_candidates = self._filter_candidates(after_epnum_in_same_pathgroup(), options)
+
+ if ('title' not in mtree.info and # no title
+ 'series' in mtree.info and # series present
+ before_epnum_in_same_pathgroup() == [] and # no groups before
+ len(title_candidates) == 1): # only 1 group after
+
found_property(title_candidates[0], 'title', confidence=0.4)
return
- elif len(title_candidates) > 1:
- found_property(title_candidates[0], 'title', confidence=0.3)
+
+ if ('title' not in mtree.info and # no title
+ before_epnum_in_same_pathgroup() == [] and # no groups before
+ len(title_candidates) == 2): # only 2 groups after
+
+ found_property(title_candidates[0], 'series', confidence=0.4)
+ found_property(title_candidates[1], 'title', confidence=0.4)
return
- # get the one with the longest value
- title_candidates = [ n for n in after_epnum_in_same_pathgroup()
- if n.clean_value.lower() not in non_episode_title ]
- if title_candidates:
- maxidx = -1
- maxv = -1
- for i, c in enumerate(title_candidates):
- if len(c.clean_value) > maxv:
- maxidx = i
- maxv = len(c.clean_value)
- found_property(title_candidates[maxidx], 'title', confidence=0.3)
-
-
-def process(mtree):
- eps = [node for node in mtree.leaves() if 'episodeNumber' in node.guess]
- if eps:
- match_from_epnum_position(mtree, eps[0])
-
- else:
- # if we don't have the episode number, but at least 2 groups in the
- # basename, then it's probably series - eptitle
- basename = mtree.node_at((-2,))
- title_candidates = [ n for n in basename.unidentified_leaves()
- if n.clean_value.lower() not in non_episode_title
- ]
-
- if len(title_candidates) >= 2:
- found_property(title_candidates[0], 'series', 0.4)
- found_property(title_candidates[1], 'title', 0.4)
- elif len(title_candidates) == 1:
- # but if there's only one candidate, it's probably the series name
- found_property(title_candidates[0], 'series', 0.4)
-
- # if we only have 1 remaining valid group in the folder containing the
- # file, then it's likely that it is the series name
- try:
- series_candidates = mtree.node_at((-3,)).unidentified_leaves()
- except ValueError:
- series_candidates = []
-
- if len(series_candidates) == 1:
- found_property(series_candidates[0], 'series', 0.3)
-
- # if there's a path group that only contains the season info, then the
- # previous one is most likely the series title (ie: ../series/season X/..)
- eps = [ node for node in mtree.nodes()
- if 'season' in node.guess and 'episodeNumber' not in node.guess ]
-
- if eps:
- previous = [ node for node in mtree.unidentified_leaves()
- if node.node_idx[0] == eps[0].node_idx[0] - 1 ]
- if len(previous) == 1:
- found_property(previous[0], 'series', 0.5)
-
- # reduce the confidence of unlikely series
- for node in mtree.nodes():
- if 'series' in node.guess:
- if node.guess['series'].lower() in unlikely_series:
- new_confidence = node.guess.confidence('series') * 0.5
- node.guess.set_confidence('series', new_confidence)
+ # if we have at least 1 valid group before the episodeNumber, then it's
+ # probably the series name
+ series_candidates = before_epnum_in_same_pathgroup()
+ if len(series_candidates) >= 1:
+ found_property(series_candidates[0], 'series', confidence=0.7)
+
+ # only 1 group after (in the same path group) and it's probably the
+ # episode title.
+ title_candidates = self._filter_candidates(after_epnum_in_same_pathgroup(), options)
+ if len(title_candidates) == 1:
+ found_property(title_candidates[0], 'title', confidence=0.5)
+ return
+ else:
+ # try in the same explicit group, with lower confidence
+ title_candidates = self._filter_candidates(after_epnum_in_same_explicitgroup(), options)
+ if len(title_candidates) == 1:
+ found_property(title_candidates[0], 'title', confidence=0.4)
+ return
+ elif len(title_candidates) > 1:
+ found_property(title_candidates[0], 'title', confidence=0.3)
+ return
+
+ # get the one with the longest value
+ title_candidates = self._filter_candidates(after_epnum_in_same_pathgroup(), options)
+ if title_candidates:
+ maxidx = -1
+ maxv = -1
+ for i, c in enumerate(title_candidates):
+ if len(c.clean_value) > maxv:
+ maxidx = i
+ maxv = len(c.clean_value)
+ found_property(title_candidates[maxidx], 'title', confidence=0.3)
+
+ def should_process(self, mtree, options=None):
+ options = options or {}
+ return not options.get('skip_title') and mtree.guess.get('type', '').startswith('episode')
+
+ @staticmethod
+ def _filter_candidates(candidates, options):
+ episode_details_transformer = get_transformer('guess_episode_details')
+ if episode_details_transformer:
+ return [n for n in candidates if not episode_details_transformer.container.find_properties(n.value, n, options, re_match=True)]
+ else:
+ return candidates
+
+ def process(self, mtree, options=None):
+ """
+ try to identify the remaining unknown groups by looking at their
+ position relative to other known elements
+ """
+ eps = [node for node in mtree.leaves() if 'episodeNumber' in node.guess]
+
+ if not eps:
+ eps = [node for node in mtree.leaves() if 'date' in node.guess]
+
+ if eps:
+ self.match_from_epnum_position(mtree, eps[0], options)
+
+ else:
+ # if we don't have the episode number, but at least 2 groups in the
+ # basename, then it's probably series - eptitle
+ basename = mtree.node_at((-2,))
+
+ title_candidates = self._filter_candidates(basename.unidentified_leaves(), options)
+
+ if len(title_candidates) >= 2 and 'series' not in mtree.info:
+ found_property(title_candidates[0], 'series', confidence=0.4)
+ found_property(title_candidates[1], 'title', confidence=0.4)
+ elif len(title_candidates) == 1:
+ # but if there's only one candidate, it's probably the series name
+ found_property(title_candidates[0], 'series' if 'series' not in mtree.info else 'title', confidence=0.4)
+
+ # if we only have 1 remaining valid group in the folder containing the
+ # file, then it's likely that it is the series name
+ try:
+ series_candidates = list(mtree.node_at((-3,)).unidentified_leaves())
+ except ValueError:
+ series_candidates = []
+
+ if len(series_candidates) == 1:
+ found_property(series_candidates[0], 'series', confidence=0.3)
+
+ # if there's a path group that only contains the season info, then the
+ # previous one is most likely the series title (ie: ../series/season X/..)
+ eps = [node for node in mtree.nodes()
+ if 'season' in node.guess and 'episodeNumber' not in node.guess]
+
+ if eps:
+ previous = [node for node in mtree.unidentified_leaves()
+ if node.node_idx[0] == eps[0].node_idx[0] - 1]
+ if len(previous) == 1:
+ found_property(previous[0], 'series', confidence=0.5)
+
+ # If we have found title without any serie name, replace it by the serie name.
+ if 'series' not in mtree.info and 'title' in mtree.info:
+ title_leaf = mtree.first_leaf_containing('title')
+ metadata = title_leaf.guess.metadata('title')
+ value = title_leaf.guess['title']
+ del title_leaf.guess['title']
+ title_leaf.guess.set('series', value, metadata=metadata)
+
+ def post_process(self, mtree, options=None):
+ for node in mtree.nodes():
+ if 'series' not in node.guess:
+ continue
+
+ node.guess['series'] = reorder_title(node.guess['series'])
diff --git a/lib/guessit/transfo/guess_episodes_rexps.py b/lib/guessit/transfo/guess_episodes_rexps.py
index 4523441be0c76954cf01c53e955111cd0d1b5391..1fd9b1dd4a870a455522b498892ff4c5a33ddb56 100644
--- a/lib/guessit/transfo/guess_episodes_rexps.py
+++ b/lib/guessit/transfo/guess_episodes_rexps.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,49 +18,178 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import Guess
-from guessit.transfo import SingleNodeGuesser
-from guessit.patterns import episode_rexps
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import re
-import logging
-log = logging.getLogger(__name__)
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import GuessFinder
+from guessit.patterns import sep, build_or_pattern
+from guessit.containers import PropertiesContainer, WeakValidator, NoValidator, ChainedValidator, DefaultValidator, \
+ FormatterValidator
+from guessit.patterns.numeral import numeral, digital_numeral, parse_numeral
+
+
+class GuessEpisodesRexps(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 20)
+
+ range_separators = ['-', 'to', 'a']
+ discrete_separators = ['&', 'and', 'et']
+ of_separators = ['of', 'sur', '/', '\\']
+
+ season_words = ['seasons?', 'saisons?', 'series?']
+ episode_words = ['episodes?']
+
+ season_markers = ['s']
+ episode_markers = ['e', 'ep']
+
+ discrete_sep = sep
+ for range_separator in range_separators:
+ discrete_sep = discrete_sep.replace(range_separator, '')
+ discrete_separators.append(discrete_sep)
+ all_separators = list(range_separators)
+ all_separators.extend(discrete_separators)
+
+ self.container = PropertiesContainer(enhance=False, canonical_from_pattern=False)
+
+ range_separators_re = re.compile(build_or_pattern(range_separators), re.IGNORECASE)
+ discrete_separators_re = re.compile(build_or_pattern(discrete_separators), re.IGNORECASE)
+ all_separators_re = re.compile(build_or_pattern(all_separators), re.IGNORECASE)
+ of_separators_re = re.compile(build_or_pattern(of_separators, escape=True), re.IGNORECASE)
+
+ season_words_re = re.compile(build_or_pattern(season_words), re.IGNORECASE)
+ episode_words_re = re.compile(build_or_pattern(episode_words), re.IGNORECASE)
+
+ season_markers_re = re.compile(build_or_pattern(season_markers), re.IGNORECASE)
+ episode_markers_re = re.compile(build_or_pattern(episode_markers), re.IGNORECASE)
+
+ def list_parser(value, property_list_name, discrete_separators_re=discrete_separators_re, range_separators_re=range_separators_re, allow_discrete=False, fill_gaps=False):
+ discrete_elements = filter(lambda x: x != '', discrete_separators_re.split(value))
+ discrete_elements = [x.strip() for x in discrete_elements]
+
+ proper_discrete_elements = []
+ i = 0
+ while i < len(discrete_elements):
+ if i < len(discrete_elements) - 2 and range_separators_re.match(discrete_elements[i+1]):
+ proper_discrete_elements.append(discrete_elements[i] + discrete_elements[i+1] + discrete_elements[i+2])
+ i += 3
+ else:
+ match = range_separators_re.search(discrete_elements[i])
+ if match and match.start() == 0:
+ proper_discrete_elements[i - 1] += discrete_elements[i]
+ elif match and match.end() == len(discrete_elements[i]):
+ proper_discrete_elements.append(discrete_elements[i] + discrete_elements[i + 1])
+ else:
+ proper_discrete_elements.append(discrete_elements[i])
+ i += 1
+
+ discrete_elements = proper_discrete_elements
+
+ ret = []
+
+ for discrete_element in discrete_elements:
+ range_values = filter(lambda x: x != '', range_separators_re.split(discrete_element))
+ range_values = [x.strip() for x in range_values]
+ if len(range_values) > 1:
+ for x in range(0, len(range_values) - 1):
+ start_range_ep = parse_numeral(range_values[x])
+ end_range_ep = parse_numeral(range_values[x+1])
+ for range_ep in range(start_range_ep, end_range_ep + 1):
+ if range_ep not in ret:
+ ret.append(range_ep)
+ else:
+ discrete_value = parse_numeral(discrete_element)
+ if discrete_value not in ret:
+ ret.append(discrete_value)
+
+ if len(ret) > 1:
+ if not allow_discrete:
+ valid_ret = list()
+ # replace discrete elements by ranges
+ valid_ret.append(ret[0])
+ for i in range(0, len(ret) - 1):
+ previous = valid_ret[len(valid_ret) - 1]
+ if ret[i+1] < previous:
+ pass
+ else:
+ valid_ret.append(ret[i+1])
+ ret = valid_ret
+ if fill_gaps:
+ ret = list(range(min(ret), max(ret) + 1))
+ if len(ret) > 1:
+ return {None: ret[0], property_list_name: ret}
+ if len(ret) > 0:
+ return ret[0]
+ return None
+
+ def episode_parser_x(value):
+ return list_parser(value, 'episodeList', discrete_separators_re=re.compile('x', re.IGNORECASE))
+
+ def episode_parser_e(value):
+ return list_parser(value, 'episodeList', discrete_separators_re=re.compile('e', re.IGNORECASE), fill_gaps=True)
+
+ def episode_parser(value):
+ return list_parser(value, 'episodeList')
+
+ def season_parser(value):
+ return list_parser(value, 'seasonList')
+
+ class ResolutionCollisionValidator(object):
+ @staticmethod
+ def validate(prop, string, node, match, entry_start, entry_end):
+ return len(match.group(2)) < 3 # limit
+
+ self.container.register_property(None, r'(' + season_words_re.pattern + sep + '?(?P<season>' + numeral + ')' + sep + '?' + season_words_re.pattern + '?)', confidence=1.0, formatter=parse_numeral)
+ self.container.register_property(None, r'(' + season_words_re.pattern + sep + '?(?P<season>' + digital_numeral + '(?:' + sep + '?' + all_separators_re.pattern + sep + '?' + digital_numeral + ')*)' + sep + '?' + season_words_re.pattern + '?)' + sep, confidence=1.0, formatter={None: parse_numeral, 'season': season_parser}, validator=ChainedValidator(DefaultValidator(), FormatterValidator('season', lambda x: len(x) > 1 if hasattr(x, '__len__') else False)))
+
+ self.container.register_property(None, r'(' + season_markers_re.pattern + '(?P<season>' + digital_numeral + ')[^0-9]?' + sep + '?(?P<episodeNumber>(?:e' + digital_numeral + '(?:' + sep + '?[e-]' + digital_numeral + ')*)))', confidence=1.0, formatter={None: parse_numeral, 'episodeNumber': episode_parser_e, 'season': season_parser}, validator=NoValidator())
+ # self.container.register_property(None, r'[^0-9]((?P<season>' + digital_numeral + ')[^0-9 .-]?-?(?P<episodeNumber>(?:x' + digital_numeral + '(?:' + sep + '?[x-]' + digital_numeral + ')*)))', confidence=1.0, formatter={None: parse_numeral, 'episodeNumber': episode_parser_x, 'season': season_parser}, validator=ChainedValidator(DefaultValidator(), ResolutionCollisionValidator()))
+ self.container.register_property(None, sep + r'((?P<season>' + digital_numeral + ')' + sep + '' + '(?P<episodeNumber>(?:x' + sep + digital_numeral + '(?:' + sep + '[x-]' + digital_numeral + ')*)))', confidence=1.0, formatter={None: parse_numeral, 'episodeNumber': episode_parser_x, 'season': season_parser}, validator=ChainedValidator(DefaultValidator(), ResolutionCollisionValidator()))
+ self.container.register_property(None, r'((?P<season>' + digital_numeral + ')' + '(?P<episodeNumber>(?:x' + digital_numeral + '(?:[x-]' + digital_numeral + ')*)))', confidence=1.0, formatter={None: parse_numeral, 'episodeNumber': episode_parser_x, 'season': season_parser}, validator=ChainedValidator(DefaultValidator(), ResolutionCollisionValidator()))
+ self.container.register_property(None, r'(' + season_markers_re.pattern + '(?P<season>' + digital_numeral + '(?:' + sep + '?' + all_separators_re.pattern + sep + '?' + digital_numeral + ')*))', confidence=0.6, formatter={None: parse_numeral, 'season': season_parser}, validator=NoValidator())
+
+ self.container.register_property(None, r'((?P<episodeNumber>' + digital_numeral + ')' + sep + '?v(?P<version>\d+))', confidence=0.6, formatter=parse_numeral)
+ self.container.register_property(None, r'(ep' + sep + r'?(?P<episodeNumber>' + digital_numeral + ')' + sep + '?)', confidence=0.7, formatter=parse_numeral)
+ self.container.register_property(None, r'(ep' + sep + r'?(?P<episodeNumber>' + digital_numeral + ')' + sep + '?v(?P<version>\d+))', confidence=0.7, formatter=parse_numeral)
+
+
+ self.container.register_property(None, r'(' + episode_markers_re.pattern + '(?P<episodeNumber>' + digital_numeral + '(?:' + sep + '?' + all_separators_re.pattern + sep + '?' + digital_numeral + ')*))', confidence=0.6, formatter={None: parse_numeral, 'episodeNumber': episode_parser})
+ self.container.register_property(None, r'(' + episode_words_re.pattern + sep + '?(?P<episodeNumber>' + digital_numeral + '(?:' + sep + '?' + all_separators_re.pattern + sep + '?' + digital_numeral + ')*)' + sep + '?' + episode_words_re.pattern + '?)', confidence=0.8, formatter={None: parse_numeral, 'episodeNumber': episode_parser})
-def number_list(s):
- l = [ int(n) for n in re.sub('[^0-9]+', ' ', s).split() ]
+ self.container.register_property(None, r'(' + episode_markers_re.pattern + '(?P<episodeNumber>' + digital_numeral + ')' + sep + '?v(?P<version>\d+))', confidence=0.6, formatter={None: parse_numeral, 'episodeNumber': episode_parser})
+ self.container.register_property(None, r'(' + episode_words_re.pattern + sep + '?(?P<episodeNumber>' + digital_numeral + ')' + sep + '?v(?P<version>\d+))', confidence=0.8, formatter={None: parse_numeral, 'episodeNumber': episode_parser})
- if len(l) == 2:
- # it is an episode interval, return all numbers in between
- return range(l[0], l[1]+1)
- return l
+ self.container.register_property('episodeNumber', r'^ ?(\d{2})' + sep, confidence=0.4, formatter=parse_numeral)
+ self.container.register_property('episodeNumber', r'^ ?(\d{2})' + sep, confidence=0.4, formatter=parse_numeral)
+ self.container.register_property('episodeNumber', r'^ ?0(\d{1,2})' + sep, confidence=0.4, formatter=parse_numeral)
+ self.container.register_property('episodeNumber', sep + r'(\d{2}) ?$', confidence=0.4, formatter=parse_numeral)
+ self.container.register_property('episodeNumber', sep + r'0(\d{1,2}) ?$', confidence=0.4, formatter=parse_numeral)
-def guess_episodes_rexps(string):
- for rexp, confidence, span_adjust in episode_rexps:
- match = re.search(rexp, string, re.IGNORECASE)
- if match:
- guess = Guess(match.groupdict(), confidence=confidence)
- span = (match.start() + span_adjust[0],
- match.end() + span_adjust[1])
+ self.container.register_property(None, r'((?P<episodeNumber>' + numeral + ')' + sep + '?' + of_separators_re.pattern + sep + '?(?P<episodeCount>' + numeral + ')(?:' + sep + '?(?:episodes?|eps?))?)', confidence=0.7, formatter=parse_numeral)
+ self.container.register_property(None, r'((?:episodes?|eps?)' + sep + '?(?P<episodeNumber>' + numeral + ')' + sep + '?' + of_separators_re.pattern + sep + '?(?P<episodeCount>' + numeral + '))', confidence=0.7, formatter=parse_numeral)
+ self.container.register_property(None, r'((?:seasons?|saisons?|s)' + sep + '?(?P<season>' + numeral + ')' + sep + '?' + of_separators_re.pattern + sep + '?(?P<seasonCount>' + numeral + '))', confidence=0.7, formatter=parse_numeral)
+ self.container.register_property(None, r'((?P<season>' + numeral + ')' + sep + '?' + of_separators_re.pattern + sep + '?(?P<seasonCount>' + numeral + ')' + sep + '?(?:seasons?|saisons?|s))', confidence=0.7, formatter=parse_numeral)
- # decide whether we have only a single episode number or an
- # episode list
- if guess.get('episodeNumber'):
- eplist = number_list(guess['episodeNumber'])
- guess.set('episodeNumber', eplist[0], confidence=confidence)
+ self.container.register_canonical_properties('other', 'FiNAL', 'Complete', validator=WeakValidator())
- if len(eplist) > 1:
- guess.set('episodeList', eplist, confidence=confidence)
+ self.container.register_property(None, r'[^0-9]((?P<season>' + digital_numeral + ')[^0-9 .-]?-?(?P<other>xAll))', confidence=1.0, formatter={None: parse_numeral, 'other': lambda x: 'Complete', 'season': season_parser}, validator=ChainedValidator(DefaultValidator(), ResolutionCollisionValidator()))
- if guess.get('bonusNumber'):
- eplist = number_list(guess['bonusNumber'])
- guess.set('bonusNumber', eplist[0], confidence=confidence)
+ def register_arguments(self, opts, naming_opts, output_opts, information_opts, webservice_opts, other_options):
+ naming_opts.add_argument('-E', '--episode-prefer-number', action='store_true', dest='episode_prefer_number', default=False,
+ help='Guess "serie.213.avi" as the episodeNumber 213. Without this option, '
+ 'it will be guessed as season 2, episodeNumber 13')
- return guess, span
+ def supported_properties(self):
+ return ['episodeNumber', 'season', 'episodeList', 'seasonList', 'episodeCount', 'seasonCount', 'version', 'other']
- return None, None
+ def guess_episodes_rexps(self, string, node=None, options=None):
+ found = self.container.find_properties(string, node, options)
+ return self.container.as_guess(found, string)
+ def should_process(self, mtree, options=None):
+ return mtree.guess.get('type', '').startswith('episode')
-def process(mtree):
- SingleNodeGuesser(guess_episodes_rexps, None, log).process(mtree)
+ def process(self, mtree, options=None):
+ GuessFinder(self.guess_episodes_rexps, None, self.log, options).process_nodes(mtree.unidentified_leaves())
diff --git a/lib/guessit/transfo/guess_filetype.py b/lib/guessit/transfo/guess_filetype.py
index 1cfb133f3ddae24836297546f91593eff77eed79..03fb5d5c4f1557eac7424f8875d3edb86756ceb4 100644
--- a/lib/guessit/transfo/guess_filetype.py
+++ b/lib/guessit/transfo/guess_filetype.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,159 +18,220 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import Guess
-from guessit.patterns import (subtitle_exts, video_exts, episode_rexps,
- find_properties, compute_canonical_form)
-from guessit.date import valid_year
-from guessit.textutils import clean_string
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import mimetypes
import os.path
import re
-import mimetypes
-import logging
-
-log = logging.getLogger(__name__)
-
-# List of well known movies and series, hardcoded because they cannot be
-# guessed appropriately otherwise
-MOVIES = [ 'OSS 117' ]
-SERIES = [ 'Band of Brothers' ]
-
-MOVIES = [ m.lower() for m in MOVIES ]
-SERIES = [ s.lower() for s in SERIES ]
-
-def guess_filetype(mtree, filetype):
- # put the filetype inside a dummy container to be able to have the
- # following functions work correctly as closures
- # this is a workaround for python 2 which doesn't have the
- # 'nonlocal' keyword (python 3 does have it)
- filetype_container = [filetype]
- other = {}
- filename = mtree.string
-
- def upgrade_episode():
- if filetype_container[0] == 'video':
- filetype_container[0] = 'episode'
- elif filetype_container[0] == 'subtitle':
- filetype_container[0] = 'episodesubtitle'
-
- def upgrade_movie():
- if filetype_container[0] == 'video':
- filetype_container[0] = 'movie'
- elif filetype_container[0] == 'subtitle':
- filetype_container[0] = 'moviesubtitle'
-
- def upgrade_subtitle():
- if 'movie' in filetype_container[0]:
- filetype_container[0] = 'moviesubtitle'
- elif 'episode' in filetype_container[0]:
- filetype_container[0] = 'episodesubtitle'
- else:
- filetype_container[0] = 'subtitle'
-
- def upgrade(type='unknown'):
- if filetype_container[0] == 'autodetect':
- filetype_container[0] = type
-
-
- # look at the extension first
- fileext = os.path.splitext(filename)[1][1:].lower()
- if fileext in subtitle_exts:
- upgrade_subtitle()
- other = { 'container': fileext }
- elif fileext in video_exts:
- upgrade(type='video')
- other = { 'container': fileext }
- else:
- upgrade(type='unknown')
- other = { 'extension': fileext }
-
-
-
- # check whether we are in a 'Movies', 'Tv Shows', ... folder
- folder_rexps = [ (r'Movies?', upgrade_movie),
- (r'Tv[ _-]?Shows?', upgrade_episode),
- (r'Series', upgrade_episode)
- ]
- for frexp, upgrade_func in folder_rexps:
- frexp = re.compile(frexp, re.IGNORECASE)
- for pathgroup in mtree.children:
- if frexp.match(pathgroup.value):
- upgrade_func()
-
- # check for a few specific cases which will unintentionally make the
- # following heuristics confused (eg: OSS 117 will look like an episode,
- # season 1, epnum 17, when it is in fact a movie)
- fname = clean_string(filename).lower()
- for m in MOVIES:
- if m in fname:
- upgrade_movie()
- for s in SERIES:
- if s in fname:
- upgrade_episode()
- # now look whether there are some specific hints for episode vs movie
- if filetype_container[0] in ('video', 'subtitle'):
- # if we have an episode_rexp (eg: s02e13), it is an episode
- for rexp, _, _ in episode_rexps:
- match = re.search(rexp, filename, re.IGNORECASE)
- if match:
- upgrade_episode()
- break
-
- # if we have a 3-4 digit number that's not a year, maybe an episode
- match = re.search(r'[^0-9]([0-9]{3,4})[^0-9]', filename)
- if match:
- fullnumber = int(match.group()[1:-1])
- #season = fullnumber // 100
- epnumber = fullnumber % 100
- possible = True
-
- # check for validity
- if epnumber > 40:
- possible = False
- if valid_year(fullnumber):
- possible = False
-
- if possible:
- upgrade_episode()
-
- # if we have certain properties characteristic of episodes, it is an ep
- for prop, value, _, _ in find_properties(filename):
- log.debug('prop: %s = %s' % (prop, value))
- if prop == 'episodeFormat':
+from guessit.guess import Guess
+from guessit.patterns.extension import subtitle_exts, info_exts, video_exts
+from guessit.transfo import TransformerException
+from guessit.plugins.transformers import Transformer, get_transformer
+from guessit.matcher import log_found_guess, found_guess
+
+
+class GuessFiletype(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 200)
+
+ # List of well known movies and series, hardcoded because they cannot be
+ # guessed appropriately otherwise
+ MOVIES = ['OSS 117']
+ SERIES = ['Band of Brothers']
+
+ MOVIES = [m.lower() for m in MOVIES]
+ SERIES = [s.lower() for s in SERIES]
+
+ def guess_filetype(self, mtree, options=None):
+ options = options or {}
+
+ # put the filetype inside a dummy container to be able to have the
+ # following functions work correctly as closures
+ # this is a workaround for python 2 which doesn't have the
+ # 'nonlocal' keyword which we could use here in the upgrade_* functions
+ # (python 3 does have it)
+ filetype_container = [mtree.guess.get('type')]
+ other = {}
+ filename = mtree.string
+
+ def upgrade_episode():
+ if filetype_container[0] == 'subtitle':
+ filetype_container[0] = 'episodesubtitle'
+ elif filetype_container[0] == 'info':
+ filetype_container[0] = 'episodeinfo'
+ elif (not filetype_container[0] or
+ filetype_container[0] == 'video'):
+ filetype_container[0] = 'episode'
+
+ def upgrade_movie():
+ if filetype_container[0] == 'subtitle':
+ filetype_container[0] = 'moviesubtitle'
+ elif filetype_container[0] == 'info':
+ filetype_container[0] = 'movieinfo'
+ elif (not filetype_container[0] or
+ filetype_container[0] == 'video'):
+ filetype_container[0] = 'movie'
+
+ def upgrade_subtitle():
+ if filetype_container[0] == 'movie':
+ filetype_container[0] = 'moviesubtitle'
+ elif filetype_container[0] == 'episode':
+ filetype_container[0] = 'episodesubtitle'
+ elif not filetype_container[0]:
+ filetype_container[0] = 'subtitle'
+
+ def upgrade_info():
+ if filetype_container[0] == 'movie':
+ filetype_container[0] = 'movieinfo'
+ elif filetype_container[0] == 'episode':
+ filetype_container[0] = 'episodeinfo'
+ elif not filetype_container[0]:
+ filetype_container[0] = 'info'
+
+ # look at the extension first
+ fileext = os.path.splitext(filename)[1][1:].lower()
+ if fileext in subtitle_exts:
+ upgrade_subtitle()
+ other = {'container': fileext}
+ elif fileext in info_exts:
+ upgrade_info()
+ other = {'container': fileext}
+ elif fileext in video_exts:
+ other = {'container': fileext}
+ else:
+ if fileext and not options.get('name_only'):
+ other = {'extension': fileext}
+ list(mtree.unidentified_leaves())[-1].guess = Guess(other)
+
+ # check whether we are in a 'Movies', 'Tv Shows', ... folder
+ folder_rexps = [(r'Movies?', upgrade_movie),
+ (r'Films?', upgrade_movie),
+ (r'Tv[ _-]?Shows?', upgrade_episode),
+ (r'Series?', upgrade_episode),
+ (r'Episodes?', upgrade_episode)]
+ for frexp, upgrade_func in folder_rexps:
+ frexp = re.compile(frexp, re.IGNORECASE)
+ for pathgroup in mtree.children:
+ if frexp.match(pathgroup.value):
+ upgrade_func()
+ return filetype_container[0], other
+
+ # check for a few specific cases which will unintentionally make the
+ # following heuristics confused (eg: OSS 117 will look like an episode,
+ # season 1, epnum 17, when it is in fact a movie)
+ fname = mtree.clean_string(filename).lower()
+ for m in self.MOVIES:
+ if m in fname:
+ self.log.debug('Found in exception list of movies -> type = movie')
+ upgrade_movie()
+ return filetype_container[0], other
+ for s in self.SERIES:
+ if s in fname:
+ self.log.debug('Found in exception list of series -> type = episode')
upgrade_episode()
- break
+ return filetype_container[0], other
- elif compute_canonical_form('format', value) == 'DVB':
+ # if we have an episode_rexp (eg: s02e13), it is an episode
+ episode_transformer = get_transformer('guess_episodes_rexps')
+ if episode_transformer:
+ filename_parts = list(x.value for x in mtree.unidentified_leaves())
+ filename_parts.append(filename)
+ for filename_part in filename_parts:
+ guess = episode_transformer.guess_episodes_rexps(filename_part)
+ if guess:
+ self.log.debug('Found guess_episodes_rexps: %s -> type = episode', guess)
+ upgrade_episode()
+ return filetype_container[0], other
+
+ properties_transformer = get_transformer('guess_properties')
+ if properties_transformer:
+ # if we have certain properties characteristic of episodes, it is an ep
+ found = properties_transformer.container.find_properties(filename, mtree, options, 'episodeFormat')
+ guess = properties_transformer.container.as_guess(found, filename)
+ if guess:
+ self.log.debug('Found characteristic property of episodes: %s"', guess)
upgrade_episode()
- break
-
- # origin-specific type
- if 'tvu.org.ru' in filename:
- upgrade_episode()
-
- # if no episode info found, assume it's a movie
- upgrade_movie()
-
- filetype = filetype_container[0]
- return filetype, other
-
-
-def process(mtree, filetype='autodetect'):
- filetype, other = guess_filetype(mtree, filetype)
-
- mtree.guess.set('type', filetype, confidence=1.0)
- log.debug('Found with confidence %.2f: %s' % (1.0, mtree.guess))
-
- filetype_info = Guess(other, confidence=1.0)
- # guess the mimetype of the filename
- # TODO: handle other mimetypes not found on the default type_maps
- # mimetypes.types_map['.srt']='text/subtitle'
- mime, _ = mimetypes.guess_type(mtree.string, strict=False)
- if mime is not None:
- filetype_info.update({'mimetype': mime}, confidence=1.0)
+ return filetype_container[0], other
+
+ weak_episode_transformer = get_transformer('guess_weak_episodes_rexps')
+ if weak_episode_transformer:
+ found = properties_transformer.container.find_properties(filename, mtree, options, 'crc32')
+ guess = properties_transformer.container.as_guess(found, filename)
+ if guess:
+ found = weak_episode_transformer.container.find_properties(filename, mtree, options)
+ guess = weak_episode_transformer.container.as_guess(found, filename)
+ if guess:
+ self.log.debug('Found characteristic property of episodes: %s"', guess)
+ upgrade_episode()
+ return filetype_container[0], other
+
+ found = properties_transformer.container.find_properties(filename, mtree, options, 'format')
+ guess = properties_transformer.container.as_guess(found, filename)
+ if guess and guess['format'] in ('HDTV', 'WEBRip', 'WEB-DL', 'DVB'):
+ # Use weak episodes only if TV or WEB source
+ weak_episode_transformer = get_transformer('guess_weak_episodes_rexps')
+ if weak_episode_transformer:
+ guess = weak_episode_transformer.guess_weak_episodes_rexps(filename)
+ if guess:
+ self.log.debug('Found guess_weak_episodes_rexps: %s -> type = episode', guess)
+ upgrade_episode()
+ return filetype_container[0], other
+
+ website_transformer = get_transformer('guess_website')
+ if website_transformer:
+ found = website_transformer.container.find_properties(filename, mtree, options, 'website')
+ guess = website_transformer.container.as_guess(found, filename)
+ if guess:
+ for namepart in ('tv', 'serie', 'episode'):
+ if namepart in guess['website']:
+ # origin-specific type
+ self.log.debug('Found characteristic property of episodes: %s', guess)
+ upgrade_episode()
+ return filetype_container[0], other
+
+ if filetype_container[0] in ('subtitle', 'info') or (not filetype_container[0] and fileext in video_exts):
+ # if no episode info found, assume it's a movie
+ self.log.debug('Nothing characteristic found, assuming type = movie')
+ upgrade_movie()
- node_ext = mtree.node_at((-1,))
- node_ext.guess = filetype_info
- log.debug('Found with confidence %.2f: %s' % (1.0, node_ext.guess))
+ if not filetype_container[0]:
+ self.log.debug('Nothing characteristic found, assuming type = unknown')
+ filetype_container[0] = 'unknown'
+
+ return filetype_container[0], other
+
+ def process(self, mtree, options=None):
+ """guess the file type now (will be useful later)
+ """
+ filetype, other = self.guess_filetype(mtree, options)
+
+ mtree.guess.set('type', filetype, confidence=1.0)
+ log_found_guess(mtree.guess)
+
+ filetype_info = Guess(other, confidence=1.0)
+ # guess the mimetype of the filename
+ # TODO: handle other mimetypes not found on the default type_maps
+ # mimetypes.types_map['.srt']='text/subtitle'
+ mime, _ = mimetypes.guess_type(mtree.string, strict=False)
+ if mime is not None:
+ filetype_info.update({'mimetype': mime}, confidence=1.0)
+
+ node_ext = mtree.node_at((-1,))
+ found_guess(node_ext, filetype_info)
+
+ if mtree.guess.get('type') in [None, 'unknown']:
+ if options.get('name_only'):
+ mtree.guess.set('type', 'movie', confidence=0.6)
+ else:
+ raise TransformerException(__name__, 'Unknown file type')
+
+ def post_process(self, mtree, options=None):
+ # now look whether there are some specific hints for episode vs movie
+ # If we have a date and no year, this is a TV Show.
+ if 'date' in mtree.info and 'year' not in mtree.info and mtree.info.get('type') != 'episode':
+ mtree.guess['type'] = 'episode'
+ for type_leaves in mtree.leaves_containing('type'):
+ type_leaves.guess['type'] = 'episode'
+ for title_leaves in mtree.leaves_containing('title'):
+ title_leaves.guess.rename('title', 'series')
\ No newline at end of file
diff --git a/lib/guessit/transfo/guess_idnumber.py b/lib/guessit/transfo/guess_idnumber.py
index eb3f9306a965d5ab8122dc154cc082f766daf2e0..911b1ae99067174dca5789f7710a8ba6536e2c02 100644
--- a/lib/guessit/transfo/guess_idnumber.py
+++ b/lib/guessit/transfo/guess_idnumber.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
@@ -18,54 +18,64 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit.transfo import SingleNodeGuesser
-from guessit.patterns import find_properties
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import re
-import logging
-log = logging.getLogger(__name__)
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import GuessFinder
+
+
+_DIGIT = 0
+_LETTER = 1
+_OTHER = 2
+
+
+class GuessIdnumber(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 220)
+
+ def supported_properties(self):
+ return ['idNumber']
+
+ _idnum = re.compile(r'(?P<idNumber>[a-zA-Z0-9-]{20,})') # 1.0, (0, 0))
+
+ def guess_idnumber(self, string, node=None, options=None):
+ match = self._idnum.search(string)
+ if match is not None:
+ result = match.groupdict()
+ switch_count = 0
+ switch_letter_count = 0
+ letter_count = 0
+ last_letter = None
+
+ last = _LETTER
+ for c in result['idNumber']:
+ if c in '0123456789':
+ ci = _DIGIT
+ elif c in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ':
+ ci = _LETTER
+ if c != last_letter:
+ switch_letter_count += 1
+ last_letter = c
+ letter_count += 1
+ else:
+ ci = _OTHER
+
+ if ci != last:
+ switch_count += 1
+
+ last = ci
+
+ switch_ratio = float(switch_count) / len(result['idNumber'])
+ letters_ratio = (float(switch_letter_count) / letter_count) if letter_count > 0 else 1
+ # only return the result as probable if we alternate often between
+ # char type (more likely for hash values than for common words)
+ if switch_ratio > 0.4 and letters_ratio > 0.4:
+ return result, match.span()
-def guess_properties(string):
- try:
- prop, value, pos, end = find_properties(string)[0]
- return { prop: value }, (pos, end)
- except IndexError:
return None, None
-_idnum = re.compile(r'(?P<idNumber>[a-zA-Z0-9-]{10,})') # 1.0, (0, 0))
-
-def guess_idnumber(string):
- match = _idnum.search(string)
- if match is not None:
- result = match.groupdict()
- switch_count = 0
- DIGIT = 0
- LETTER = 1
- OTHER = 2
- last = LETTER
- for c in result['idNumber']:
- if c in '0123456789':
- ci = DIGIT
- elif c in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ':
- ci = LETTER
- else:
- ci = OTHER
-
- if ci != last:
- switch_count += 1
-
- last = ci
-
- switch_ratio = float(switch_count) / len(result['idNumber'])
-
- # only return the result as probable if we alternate often between
- # char type (more likely for hash values than for common words)
- if switch_ratio > 0.4:
- return result, match.span()
-
- return None, None
-
-def process(mtree):
- SingleNodeGuesser(guess_idnumber, 0.4, log).process(mtree)
+ def process(self, mtree, options=None):
+ GuessFinder(self.guess_idnumber, 0.4, self.log, options).process_nodes(mtree.unidentified_leaves())
diff --git a/lib/guessit/transfo/guess_language.py b/lib/guessit/transfo/guess_language.py
index f636f2af528a247bfc69d0bf568feb5c0932fa28..42ce72bec4049dfa34f228910e10e7a3bce9fb1c 100644
--- a/lib/guessit/transfo/guess_language.py
+++ b/lib/guessit/transfo/guess_language.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,26 +18,172 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import Guess
-from guessit.transfo import SingleNodeGuesser
-from guessit.language import search_language
-from guessit.textutils import clean_string, find_words
-import logging
+from __future__ import absolute_import, division, print_function, unicode_literals
-log = logging.getLogger(__name__)
+from guessit.language import search_language, subtitle_prefixes, subtitle_suffixes
+from guessit.patterns.extension import subtitle_exts
+from guessit.textutils import find_words
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import GuessFinder
-def guess_language(string):
- language, span, confidence = search_language(string)
- if language:
- return (Guess({'language': language},
- confidence=confidence),
- span)
+class GuessLanguage(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 30)
- return None, None
+ def register_arguments(self, opts, naming_opts, output_opts, information_opts, webservice_opts, other_options):
+ naming_opts.add_argument('-L', '--allowed-languages', action='append', dest='allowed_languages',
+ help='Allowed language (can be used multiple times)')
+ def supported_properties(self):
+ return ['language', 'subtitleLanguage']
-def process(mtree):
- SingleNodeGuesser(guess_language, None, log).process(mtree)
- # Note: 'language' is promoted to 'subtitleLanguage' in the post_process transfo
+ @staticmethod
+ def guess_language(string, node=None, options=None):
+ allowed_languages = None
+ if options and 'allowed_languages' in options:
+ allowed_languages = options.get('allowed_languages')
+ guess = search_language(string, allowed_languages)
+ return guess
+
+ @staticmethod
+ def _skip_language_on_second_pass(mtree, node):
+ """Check if found node is a valid language node, or if it's a false positive.
+
+ :param mtree: Tree detected on first pass.
+ :type mtree: :class:`guessit.matchtree.MatchTree`
+ :param node: Node that contains a language Guess
+ :type node: :class:`guessit.matchtree.MatchTree`
+
+ :return: True if a second pass skipping this node is required
+ :rtype: bool
+ """
+ unidentified_starts = {}
+ unidentified_ends = {}
+
+ property_starts = {}
+ property_ends = {}
+
+ title_starts = {}
+ title_ends = {}
+
+ for unidentified_node in mtree.unidentified_leaves():
+ unidentified_starts[unidentified_node.span[0]] = unidentified_node
+ unidentified_ends[unidentified_node.span[1]] = unidentified_node
+
+ for property_node in mtree.leaves_containing('year'):
+ property_starts[property_node.span[0]] = property_node
+ property_ends[property_node.span[1]] = property_node
+
+ for title_node in mtree.leaves_containing(['title', 'series']):
+ title_starts[title_node.span[0]] = title_node
+ title_ends[title_node.span[1]] = title_node
+
+ return node.span[0] in title_ends.keys() and (node.span[1] in unidentified_starts.keys() or node.span[1] + 1 in property_starts.keys()) or\
+ node.span[1] in title_starts.keys() and (node.span[0] == node.group_node().span[0] or node.span[0] in unidentified_ends.keys() or node.span[0] in property_ends.keys())
+
+ def second_pass_options(self, mtree, options=None):
+ m = mtree.matched()
+ to_skip_language_nodes = []
+
+ for lang_key in ('language', 'subtitleLanguage'):
+ langs = {}
+ lang_nodes = set(mtree.leaves_containing(lang_key))
+
+ for lang_node in lang_nodes:
+ lang = lang_node.guess.get(lang_key, None)
+ if self._skip_language_on_second_pass(mtree, lang_node):
+ # Language probably split the title. Add to skip for 2nd pass.
+
+ # if filetype is subtitle and the language appears last, just before
+ # the extension, then it is likely a subtitle language
+ parts = mtree.clean_string(lang_node.root.value).split()
+ if m.get('type') in ['moviesubtitle', 'episodesubtitle']:
+ if lang_node.value in parts and \
+ (parts.index(lang_node.value) == len(parts) - 2):
+ continue
+ to_skip_language_nodes.append(lang_node)
+ elif lang not in langs:
+ langs[lang] = lang_node
+ else:
+ # The same language was found. Keep the more confident one,
+ # and add others to skip for 2nd pass.
+ existing_lang_node = langs[lang]
+ to_skip = None
+ if (existing_lang_node.guess.confidence('language') >=
+ lang_node.guess.confidence('language')):
+ # lang_node is to remove
+ to_skip = lang_node
+ else:
+ # existing_lang_node is to remove
+ langs[lang] = lang_node
+ to_skip = existing_lang_node
+ to_skip_language_nodes.append(to_skip)
+
+ if to_skip_language_nodes:
+ # Also skip same value nodes
+ skipped_values = [skip_node.value for skip_node in to_skip_language_nodes]
+
+ for lang_key in ('language', 'subtitleLanguage'):
+ lang_nodes = set(mtree.leaves_containing(lang_key))
+
+ for lang_node in lang_nodes:
+ if lang_node not in to_skip_language_nodes and lang_node.value in skipped_values:
+ to_skip_language_nodes.append(lang_node)
+ return {'skip_nodes': to_skip_language_nodes}
+ return None
+
+ def should_process(self, mtree, options=None):
+ options = options or {}
+ return options.get('language', True)
+
+ def process(self, mtree, options=None):
+ GuessFinder(self.guess_language, None, self.log, options).process_nodes(mtree.unidentified_leaves())
+
+ @staticmethod
+ def promote_subtitle(node):
+ if 'language' in node.guess:
+ node.guess.set('subtitleLanguage', node.guess['language'],
+ confidence=node.guess.confidence('language'))
+ del node.guess['language']
+
+ def post_process(self, mtree, options=None):
+ # 1- try to promote language to subtitle language where it makes sense
+ for node in mtree.nodes():
+ if 'language' not in node.guess:
+ continue
+
+ # - if we matched a language in a file with a sub extension and that
+ # the group is the last group of the filename, it is probably the
+ # language of the subtitle
+ # (eg: 'xxx.english.srt')
+ if (mtree.node_at((-1,)).value.lower() in subtitle_exts and
+ node == list(mtree.leaves())[-2]):
+ self.promote_subtitle(node)
+
+ # - if we find in the same explicit group
+ # a subtitle prefix before the language,
+ # or a subtitle suffix after the language,
+ # then upgrade the language
+ explicit_group = mtree.node_at(node.node_idx[:2])
+ group_str = explicit_group.value.lower()
+
+ for sub_prefix in subtitle_prefixes:
+ if (sub_prefix in find_words(group_str) and
+ 0 <= group_str.find(sub_prefix) < (node.span[0] - explicit_group.span[0])):
+ self.promote_subtitle(node)
+
+ for sub_suffix in subtitle_suffixes:
+ if (sub_suffix in find_words(group_str) and
+ (node.span[0] - explicit_group.span[0]) < group_str.find(sub_suffix)):
+ self.promote_subtitle(node)
+
+ # - if a language is in an explicit group just preceded by "st",
+ # it is a subtitle language (eg: '...st[fr-eng]...')
+ try:
+ idx = node.node_idx
+ previous = list(mtree.node_at((idx[0], idx[1] - 1)).leaves())[-1]
+ if previous.value.lower()[-2:] == 'st':
+ self.promote_subtitle(node)
+ except IndexError:
+ pass
diff --git a/lib/guessit/transfo/guess_movie_title_from_position.py b/lib/guessit/transfo/guess_movie_title_from_position.py
index 80817fab231af84ff741ed102ec571b67afc4086..671e4cb5be3ca3f0f81fb5ee3dbaa889575090cd 100644
--- a/lib/guessit/transfo/guess_movie_title_from_position.py
+++ b/lib/guessit/transfo/guess_movie_title_from_position.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,156 +18,156 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import Guess
-import unicodedata
-import logging
-
-log = logging.getLogger(__name__)
-
-
-def process(mtree):
- def found_property(node, name, value, confidence):
- node.guess = Guess({ name: value },
- confidence=confidence)
- log.debug('Found with confidence %.2f: %s' % (confidence, node.guess))
-
- def found_title(node, confidence):
- found_property(node, 'title', node.clean_value, confidence)
-
- basename = mtree.node_at((-2,))
- all_valid = lambda leaf: len(leaf.clean_value) > 0
- basename_leftover = basename.unidentified_leaves(valid=all_valid)
-
- try:
- folder = mtree.node_at((-3,))
- folder_leftover = folder.unidentified_leaves()
- except ValueError:
- folder = None
- folder_leftover = []
-
- log.debug('folder: %s' % folder_leftover)
- log.debug('basename: %s' % basename_leftover)
-
- # specific cases:
- # if we find the same group both in the folder name and the filename,
- # it's a good candidate for title
- if (folder_leftover and basename_leftover and
- folder_leftover[0].clean_value == basename_leftover[0].clean_value):
-
- found_title(folder_leftover[0], confidence=0.8)
- return
-
- # specific cases:
- # if the basename contains a number first followed by an unidentified
- # group, and the folder only contains 1 unidentified one, then we have
- # a series
- # ex: Millenium Trilogy (2009)/(1)The Girl With The Dragon Tattoo(2009).mkv
- try:
- series = folder_leftover[0]
- filmNumber = basename_leftover[0]
- title = basename_leftover[1]
-
- basename_leaves = basename.leaves()
-
- num = int(filmNumber.clean_value)
-
- log.debug('series: %s' % series.clean_value)
- log.debug('title: %s' % title.clean_value)
- if (series.clean_value != title.clean_value and
- series.clean_value != filmNumber.clean_value and
- basename_leaves.index(filmNumber) == 0 and
- basename_leaves.index(title) == 1):
-
- found_title(title, confidence=0.6)
- found_property(series, 'filmSeries',
- series.clean_value, confidence=0.6)
- found_property(filmNumber, 'filmNumber',
- num, confidence=0.6)
- return
- except Exception:
- pass
-
- # specific cases:
- # - movies/tttttt (yyyy)/tttttt.ccc
- try:
- if mtree.node_at((-4, 0)).value.lower() == 'movies':
- folder = mtree.node_at((-3,))
+from __future__ import absolute_import, division, print_function, unicode_literals
- # Note:too generic, might solve all the unittests as they all
- # contain 'movies' in their path
- #
- #if containing_folder.is_leaf() and not containing_folder.guess:
- # containing_folder.guess =
- # Guess({ 'title': clean_string(containing_folder.value) },
- # confidence=0.7)
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import found_property
+from guessit import u
- year_group = folder.first_leaf_containing('year')
- groups_before = folder.previous_unidentified_leaves(year_group)
- found_title(groups_before[0], confidence=0.8)
+class GuessMovieTitleFromPosition(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, -200)
+
+ def supported_properties(self):
+ return ['title']
+
+ def should_process(self, mtree, options=None):
+ options = options or {}
+ return not options.get('skip_title') and not mtree.guess.get('type', '').startswith('episode')
+
+ def process(self, mtree, options=None):
+ """
+ try to identify the remaining unknown groups by looking at their
+ position relative to other known elements
+ """
+ if 'title' in mtree.info:
return
- except Exception:
- pass
-
- # if we have either format or videoCodec in the folder containing the file
- # or one of its parents, then we should probably look for the title in
- # there rather than in the basename
- try:
- props = mtree.previous_leaves_containing(mtree.children[-2],
- [ 'videoCodec', 'format',
- 'language' ])
- except IndexError:
- props = []
-
- if props:
- group_idx = props[0].node_idx[0]
- if all(g.node_idx[0] == group_idx for g in props):
- # if they're all in the same group, take leftover info from there
- leftover = mtree.node_at((group_idx,)).unidentified_leaves()
-
- if leftover:
- found_title(leftover[0], confidence=0.7)
- return
+ basename = mtree.node_at((-2,))
+ all_valid = lambda leaf: len(leaf.clean_value) > 0
+ basename_leftover = list(basename.unidentified_leaves(valid=all_valid))
- # look for title in basename if there are some remaining undidentified
- # groups there
- if basename_leftover:
- title_candidate = basename_leftover[0]
+ try:
+ folder = mtree.node_at((-3,))
+ folder_leftover = list(folder.unidentified_leaves())
+ except ValueError:
+ folder = None
+ folder_leftover = []
+
+ self.log.debug('folder: %s' % u(folder_leftover))
+ self.log.debug('basename: %s' % u(basename_leftover))
+
+ # specific cases:
+ # if we find the same group both in the folder name and the filename,
+ # it's a good candidate for title
+ if folder_leftover and basename_leftover and folder_leftover[0].clean_value == basename_leftover[0].clean_value:
+ found_property(folder_leftover[0], 'title', confidence=0.8)
+ return
+
+ # specific cases:
+ # if the basename contains a number first followed by an unidentified
+ # group, and the folder only contains 1 unidentified one, then we have
+ # a series
+ # ex: Millenium Trilogy (2009)/(1)The Girl With The Dragon Tattoo(2009).mkv
+ if len(folder_leftover) > 0 and len(basename_leftover) > 1:
+ series = folder_leftover[0]
+ film_number = basename_leftover[0]
+ title = basename_leftover[1]
+
+ basename_leaves = list(basename.leaves())
+
+ num = None
+ try:
+ num = int(film_number.clean_value)
+ except ValueError:
+ pass
+
+ if num:
+ self.log.debug('series: %s' % series.clean_value)
+ self.log.debug('title: %s' % title.clean_value)
+ if (series.clean_value != title.clean_value and
+ series.clean_value != film_number.clean_value and
+ basename_leaves.index(film_number) == 0 and
+ basename_leaves.index(title) == 1):
+
+ found_property(title, 'title', confidence=0.6)
+ found_property(series, 'filmSeries', confidence=0.6)
+ found_property(film_number, 'filmNumber', num, confidence=0.6)
+ return
- # if basename is only one word and the containing folder has at least
- # 3 words in it, we should take the title from the folder name
- # ex: Movies/Alice in Wonderland DVDRip.XviD-DiAMOND/dmd-aw.avi
- # ex: Movies/Somewhere.2010.DVDRip.XviD-iLG/i-smwhr.avi <-- TODO: gets caught here?
- if (title_candidate.clean_value.count(' ') == 0 and
- folder_leftover and
- folder_leftover[0].clean_value.count(' ') >= 2):
+ if folder:
+ year_group = folder.first_leaf_containing('year')
+ if year_group:
+ groups_before = folder.previous_unidentified_leaves(year_group)
+ if groups_before:
+ try:
+ node = next(groups_before)
+ found_property(node, 'title', confidence=0.8)
+ return
+ except StopIteration:
+ pass
+
+ # if we have either format or videoCodec in the folder containing the
+ # file or one of its parents, then we should probably look for the title
+ # in there rather than in the basename
+ try:
+ props = list(mtree.previous_leaves_containing(mtree.children[-2],
+ ['videoCodec',
+ 'format',
+ 'language']))
+ except IndexError:
+ props = []
+
+ if props:
+ group_idx = props[0].node_idx[0]
+ if all(g.node_idx[0] == group_idx for g in props):
+ # if they're all in the same group, take leftover info from there
+ leftover = mtree.node_at((group_idx,)).unidentified_leaves()
+ try:
+ found_property(next(leftover), 'title', confidence=0.7)
+ return
+ except StopIteration:
+ pass
+
+ # look for title in basename if there are some remaining unidentified
+ # groups there
+ if basename_leftover:
+ # if basename is only one word and the containing folder has at least
+ # 3 words in it, we should take the title from the folder name
+ # ex: Movies/Alice in Wonderland DVDRip.XviD-DiAMOND/dmd-aw.avi
+ # ex: Movies/Somewhere.2010.DVDRip.XviD-iLG/i-smwhr.avi <-- TODO: gets caught here?
+ if (basename_leftover[0].clean_value.count(' ') == 0 and
+ folder_leftover and folder_leftover[0].clean_value.count(' ') >= 2):
+
+ found_property(folder_leftover[0], 'title', confidence=0.7)
+ return
- found_title(folder_leftover[0], confidence=0.7)
+ # if there are only many unidentified groups, take the first of which is
+ # not inside brackets or parentheses.
+ # ex: Movies/[阿维达].Avida.2006.FRENCH.DVDRiP.XViD-PROD.avi
+ if basename_leftover[0].is_explicit():
+ for basename_leftover_elt in basename_leftover:
+ if not basename_leftover_elt.is_explicit():
+ found_property(basename_leftover_elt, 'title', confidence=0.8)
+ return
+
+ # if all else fails, take the first remaining unidentified group in the
+ # basename as title
+ found_property(basename_leftover[0], 'title', confidence=0.6)
return
- # if there are only 2 unidentified groups, the first of which is inside
- # brackets or parentheses, we take the second one for the title:
- # ex: Movies/[阿维达].Avida.2006.FRENCH.DVDRiP.XViD-PROD.avi
- if len(basename_leftover) == 2 and basename_leftover[0].is_explicit():
- found_title(basename_leftover[1], confidence=0.8)
+ # if there are no leftover groups in the basename, look in the folder name
+ if folder_leftover:
+ found_property(folder_leftover[0], 'title', confidence=0.5)
return
- # if all else fails, take the first remaining unidentified group in the
- # basename as title
- found_title(title_candidate, confidence=0.6)
- return
-
- # if there are no leftover groups in the basename, look in the folder name
- if folder_leftover:
- found_title(folder_leftover[0], confidence=0.5)
- return
-
- # if nothing worked, look if we have a very small group at the beginning
- # of the basename
- basename = mtree.node_at((-2,))
- basename_leftover = basename.unidentified_leaves(valid=lambda leaf: True)
- if basename_leftover:
- found_title(basename_leftover[0], confidence=0.4)
- return
+ # if nothing worked, look if we have a very small group at the beginning
+ # of the basename
+ basename = mtree.node_at((-2,))
+ basename_leftover = basename.unidentified_leaves(valid=lambda leaf: True)
+ try:
+ found_property(next(basename_leftover), 'title', confidence=0.4)
+ return
+ except StopIteration:
+ pass
diff --git a/lib/guessit/transfo/guess_properties.py b/lib/guessit/transfo/guess_properties.py
index dce7db64cee865cbf46db314b9da572757f8bed6..8c08d20df5e5ab6ba23d9aefb6dc7e0e541bbd90 100644
--- a/lib/guessit/transfo/guess_properties.py
+++ b/lib/guessit/transfo/guess_properties.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,21 +18,272 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit.transfo import SingleNodeGuesser
-from guessit.patterns import find_properties
-import logging
+from __future__ import absolute_import, division, print_function, unicode_literals
-log = logging.getLogger(__name__)
+import re
+from guessit.containers import PropertiesContainer, WeakValidator, LeavesValidator, QualitiesContainer, ChainedValidator, DefaultValidator, OnlyOneValidator, LeftValidator, NeighborValidator
+from guessit.patterns import sep, build_or_pattern
+from guessit.patterns.extension import subtitle_exts, video_exts, info_exts
+from guessit.patterns.numeral import numeral, parse_numeral
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import GuessFinder, found_property
-def guess_properties(string):
- try:
- prop, value, pos, end = find_properties(string)[0]
- return { prop: value }, (pos, end)
- except IndexError:
- return None, None
+class GuessProperties(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 35)
-def process(mtree):
- SingleNodeGuesser(guess_properties, 1.0, log).process(mtree)
+ self.container = PropertiesContainer()
+ self.qualities = QualitiesContainer()
+
+ def register_property(propname, props, **kwargs):
+ """props a dict of {value: [patterns]}"""
+ for canonical_form, patterns in props.items():
+ if isinstance(patterns, tuple):
+ patterns2, pattern_kwarg = patterns
+ if kwargs:
+ current_kwarg = dict(kwargs)
+ current_kwarg.update(pattern_kwarg)
+ else:
+ current_kwarg = dict(pattern_kwarg)
+ current_kwarg['canonical_form'] = canonical_form
+ self.container.register_property(propname, *patterns2, **current_kwarg)
+ elif kwargs:
+ current_kwarg = dict(kwargs)
+ current_kwarg['canonical_form'] = canonical_form
+ self.container.register_property(propname, *patterns, **current_kwarg)
+ else:
+ self.container.register_property(propname, *patterns, canonical_form=canonical_form)
+
+ def register_quality(propname, quality_dict):
+ """props a dict of {canonical_form: quality}"""
+ for canonical_form, quality in quality_dict.items():
+ self.qualities.register_quality(propname, canonical_form, quality)
+
+ register_property('container', {'mp4': ['MP4']})
+
+ # http://en.wikipedia.org/wiki/Pirated_movie_release_types
+ register_property('format', {'VHS': ['VHS', 'VHS-Rip'],
+ 'Cam': ['CAM', 'CAMRip', 'HD-CAM'],
+ #'Telesync': ['TELESYNC', 'PDVD'],
+ 'Telesync': (['TS', 'HD-TS'], {'confidence': 0.4}),
+ 'Workprint': ['WORKPRINT', 'WP'],
+ 'Telecine': ['TELECINE', 'TC'],
+ 'PPV': ['PPV', 'PPV-Rip'], # Pay Per View
+ 'TV': ['SD-TV', 'SD-TV-Rip', 'Rip-SD-TV', 'TV-Rip', 'Rip-TV'],
+ 'DVB': ['DVB-Rip', 'DVB', 'PD-TV'],
+ 'DVD': ['DVD', 'DVD-Rip', 'VIDEO-TS', 'DVD-R', 'DVD-9', 'DVD-5'],
+ 'HDTV': ['HD-TV', 'TV-RIP-HD', 'HD-TV-RIP'],
+ 'VOD': ['VOD', 'VOD-Rip'],
+ 'WEBRip': ['WEB-Rip'],
+ 'WEB-DL': ['WEB-DL', 'WEB-HD', 'WEB'],
+ 'HD-DVD': ['HD-(?:DVD)?-Rip', 'HD-DVD'],
+ 'BluRay': ['Blu-ray(?:-Rip)?', 'B[DR]', 'B[DR]-Rip', 'BD[59]', 'BD25', 'BD50']
+ })
+
+ register_quality('format', {'VHS': -100,
+ 'Cam': -90,
+ 'Telesync': -80,
+ 'Workprint': -70,
+ 'Telecine': -60,
+ 'PPV': -50,
+ 'TV': -30,
+ 'DVB': -20,
+ 'DVD': 0,
+ 'HDTV': 20,
+ 'VOD': 40,
+ 'WEBRip': 50,
+ 'WEB-DL': 60,
+ 'HD-DVD': 80,
+ 'BluRay': 100
+ })
+
+ register_property('screenSize', {'360p': ['(?:\d{3,}(?:\\|\/|x|\*))?360(?:i|p?x?)'],
+ '368p': ['(?:\d{3,}(?:\\|\/|x|\*))?368(?:i|p?x?)'],
+ '480p': ['(?:\d{3,}(?:\\|\/|x|\*))?480(?:i|p?x?)'],
+ #'480p': (['hr'], {'confidence': 0.2}), # duplicate dict key
+ '576p': ['(?:\d{3,}(?:\\|\/|x|\*))?576(?:i|p?x?)'],
+ '720p': ['(?:\d{3,}(?:\\|\/|x|\*))?720(?:i|p?x?)'],
+ '900p': ['(?:\d{3,}(?:\\|\/|x|\*))?900(?:i|p?x?)'],
+ '1080i': ['(?:\d{3,}(?:\\|\/|x|\*))?1080i'],
+ '1080p': ['(?:\d{3,}(?:\\|\/|x|\*))?1080p?x?'],
+ '4K': ['(?:\d{3,}(?:\\|\/|x|\*))?2160(?:i|p?x?)']
+ },
+ validator=ChainedValidator(DefaultValidator(), OnlyOneValidator()))
+
+ class ResolutionValidator(object):
+ """Make sure our match is surrounded by separators, or by another entry"""
+ @staticmethod
+ def validate(prop, string, node, match, entry_start, entry_end):
+ """
+ span = _get_span(prop, match)
+ span = _trim_span(span, string[span[0]:span[1]])
+ start, end = span
+
+ sep_start = start <= 0 or string[start - 1] in sep
+ sep_end = end >= len(string) or string[end] in sep
+ start_by_other = start in entry_end
+ end_by_other = end in entry_start
+ if (sep_start or start_by_other) and (sep_end or end_by_other):
+ return True
+ return False
+ """
+ return True
+
+ _digits_re = re.compile('\d+')
+
+ def resolution_formatter(value):
+ digits = _digits_re.findall(value)
+ return 'x'.join(digits)
+
+ self.container.register_property('screenSize', '\d{3,4}-?[x\*]-?\d{3,4}', canonical_from_pattern=False, formatter=resolution_formatter, validator=ChainedValidator(DefaultValidator(), ResolutionValidator()))
+
+ register_quality('screenSize', {'360p': -300,
+ '368p': -200,
+ '480p': -100,
+ '576p': 0,
+ '720p': 100,
+ '900p': 130,
+ '1080i': 180,
+ '1080p': 200,
+ '4K': 400
+ })
+
+ _videoCodecProperty = {'Real': ['Rv\d{2}'], # http://en.wikipedia.org/wiki/RealVideo
+ 'Mpeg2': ['Mpeg2'],
+ 'DivX': ['DVDivX', 'DivX'],
+ 'XviD': ['XviD'],
+ 'h264': ['[hx]-264(?:-AVC)?', 'MPEG-4(?:-AVC)'],
+ 'h265': ['[hx]-265(?:-HEVC)?', 'HEVC']
+ }
+
+ register_property('videoCodec', _videoCodecProperty)
+
+ register_quality('videoCodec', {'Real': -50,
+ 'Mpeg2': -30,
+ 'DivX': -10,
+ 'XviD': 0,
+ 'h264': 100,
+ 'h265': 150
+ })
+
+ # http://blog.mediacoderhq.com/h264-profiles-and-levels/
+ # http://fr.wikipedia.org/wiki/H.264
+ self.container.register_property('videoProfile', 'BP', validator=LeavesValidator(lambdas=[lambda node: 'videoCodec' in node.guess]))
+ self.container.register_property('videoProfile', 'XP', 'EP', canonical_form='XP', validator=LeavesValidator(lambdas=[lambda node: 'videoCodec' in node.guess]))
+ self.container.register_property('videoProfile', 'MP', validator=LeavesValidator(lambdas=[lambda node: 'videoCodec' in node.guess]))
+ self.container.register_property('videoProfile', 'HP', 'HiP', canonical_form='HP', validator=LeavesValidator(lambdas=[lambda node: 'videoCodec' in node.guess]))
+ self.container.register_property('videoProfile', '10.?bit', 'Hi10P', canonical_form='10bit')
+ self.container.register_property('videoProfile', '8.?bit', canonical_form='8bit')
+ self.container.register_property('videoProfile', 'Hi422P', validator=LeavesValidator(lambdas=[lambda node: 'videoCodec' in node.guess]))
+ self.container.register_property('videoProfile', 'Hi444PP', validator=LeavesValidator(lambdas=[lambda node: 'videoCodec' in node.guess]))
+
+ register_quality('videoProfile', {'BP': -20,
+ 'XP': -10,
+ 'MP': 0,
+ 'HP': 10,
+ '10bit': 15,
+ 'Hi422P': 25,
+ 'Hi444PP': 35
+ })
+
+ # has nothing to do here (or on filenames for that matter), but some
+ # releases use it and it helps to identify release groups, so we adapt
+ register_property('videoApi', {'DXVA': ['DXVA']})
+
+ register_property('audioCodec', {'MP3': ['MP3', 'LAME', 'LAME(?:\d)+-(?:\d)+'],
+ 'DolbyDigital': ['DD'],
+ 'AAC': ['AAC'],
+ 'AC3': ['AC3'],
+ 'Flac': ['FLAC'],
+ 'DTS': (['DTS'], {'validator': LeftValidator()}),
+ 'TrueHD': ['True-HD']
+ })
+
+ register_quality('audioCodec', {'MP3': 10,
+ 'DolbyDigital': 30,
+ 'AAC': 35,
+ 'AC3': 40,
+ 'Flac': 45,
+ 'DTS': 60,
+ 'TrueHD': 70
+ })
+
+ self.container.register_property('audioProfile', 'HD', validator=LeavesValidator(lambdas=[lambda node: node.guess.get('audioCodec') == 'DTS']))
+ self.container.register_property('audioProfile', 'HD-MA', canonical_form='HDMA', validator=LeavesValidator(lambdas=[lambda node: node.guess.get('audioCodec') == 'DTS']))
+ self.container.register_property('audioProfile', 'HE', validator=LeavesValidator(lambdas=[lambda node: node.guess.get('audioCodec') == 'AAC']))
+ self.container.register_property('audioProfile', 'LC', validator=LeavesValidator(lambdas=[lambda node: node.guess.get('audioCodec') == 'AAC']))
+ self.container.register_property('audioProfile', 'HQ', validator=LeavesValidator(lambdas=[lambda node: node.guess.get('audioCodec') == 'AC3']))
+
+ register_quality('audioProfile', {'HD': 20,
+ 'HDMA': 50,
+ 'LC': 0,
+ 'HQ': 0,
+ 'HE': 20
+ })
+
+ register_property('audioChannels', {'7.1': ['7[\W_]1', '7ch', '8ch'],
+ '5.1': ['5[\W_]1', '5ch', '6ch'],
+ '2.0': ['2[\W_]0', '2ch', 'stereo'],
+ '1.0': ['1[\W_]0', '1ch', 'mono']
+ })
+
+ register_quality('audioChannels', {'7.1': 200,
+ '5.1': 100,
+ '2.0': 0,
+ '1.0': -100
+ })
+
+ self.container.register_property('episodeFormat', r'Minisodes?', canonical_form='Minisode')
+
+ self.container.register_property('crc32', '(?:[a-fA-F]|[0-9]){8}', enhance=False, canonical_from_pattern=False)
+
+ weak_episode_words = ['pt', 'part']
+ self.container.register_property(None, '(' + build_or_pattern(weak_episode_words) + sep + '?(?P<part>' + numeral + '))[^0-9]', enhance=False, canonical_from_pattern=False, confidence=0.4, formatter=parse_numeral)
+
+ register_property('other', {'AudioFix': ['Audio-Fix', 'Audio-Fixed'],
+ 'SyncFix': ['Sync-Fix', 'Sync-Fixed'],
+ 'DualAudio': ['Dual-Audio'],
+ 'WideScreen': ['ws', 'wide-screen'],
+ 'Netflix': ['Netflix', 'NF']
+ })
+
+ self.container.register_property('other', 'Real', 'Fix', canonical_form='Proper', validator=NeighborValidator())
+ self.container.register_property('other', 'Proper', 'Repack', 'Rerip', canonical_form='Proper')
+ self.container.register_property('other', 'Fansub', canonical_form='Fansub')
+ self.container.register_property('other', 'Fastsub', canonical_form='Fastsub')
+ self.container.register_property('other', '(?:Seasons?' + sep + '?)?Complete', canonical_form='Complete')
+ self.container.register_property('other', 'R5', 'RC', canonical_form='R5')
+ self.container.register_property('other', 'Pre-Air', 'Preair', canonical_form='Preair')
+
+ self.container.register_canonical_properties('other', 'Screener', 'Remux', '3D', 'HD', 'mHD', 'HDLight', 'HQ',
+ 'DDC',
+ 'HR', 'PAL', 'SECAM', 'NTSC')
+ self.container.register_canonical_properties('other', 'Limited', 'Complete', 'Classic', 'Unrated', 'LiNE', 'Bonus', 'Trailer', validator=WeakValidator())
+
+ for prop in self.container.get_properties('format'):
+ self.container.register_property('other', prop.pattern + '(-?Scr(?:eener)?)', canonical_form='Screener')
+
+ for exts in (subtitle_exts, info_exts, video_exts):
+ for container in exts:
+ self.container.register_property('container', container, confidence=0.3)
+
+ def guess_properties(self, string, node=None, options=None):
+ found = self.container.find_properties(string, node, options)
+ return self.container.as_guess(found, string)
+
+ def supported_properties(self):
+ return self.container.get_supported_properties()
+
+ def process(self, mtree, options=None):
+ GuessFinder(self.guess_properties, 1.0, self.log, options).process_nodes(mtree.unidentified_leaves())
+ proper_count = 0
+ for other_leaf in mtree.leaves_containing('other'):
+ if 'other' in other_leaf.info and 'Proper' in other_leaf.info['other']:
+ proper_count += 1
+ if proper_count:
+ found_property(mtree, 'properCount', proper_count)
+
+ def rate_quality(self, guess, *props):
+ return self.qualities.rate_quality(guess, *props)
diff --git a/lib/guessit/transfo/guess_release_group.py b/lib/guessit/transfo/guess_release_group.py
index 76ffe025d7e83e9218a620dd9ed0290a23548ec7..08e8786ddba12ee2bccd3f95de2aed9b8bf06364 100644
--- a/lib/guessit/transfo/guess_release_group.py
+++ b/lib/guessit/transfo/guess_release_group.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,69 +18,202 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit.transfo import SingleNodeGuesser
-from guessit.patterns import prop_multi, compute_canonical_form, _dash, _psep
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import re
-import logging
-
-log = logging.getLogger(__name__)
-
-def get_patterns(property_name):
- return [ p.replace(_dash, _psep) for patterns in prop_multi[property_name].values() for p in patterns ]
-
-CODECS = get_patterns('videoCodec')
-FORMATS = get_patterns('format')
-VAPIS = get_patterns('videoApi')
-
-# RG names following a codec or format, with a potential space or dash inside the name
-GROUP_NAMES = [ r'(?P<videoCodec>' + codec + r')[ \.-](?P<releaseGroup>.+?([- \.].*?)??)[ \.]'
- for codec in CODECS ]
-GROUP_NAMES += [ r'(?P<format>' + fmt + r')[ \.-](?P<releaseGroup>.+?([- \.].*?)??)[ \.]'
- for fmt in FORMATS ]
-GROUP_NAMES += [ r'(?P<videoApi>' + api + r')[ \.-](?P<releaseGroup>.+?([- \.].*?)??)[ \.]'
- for api in VAPIS ]
-
-GROUP_NAMES2 = [ r'\.(?P<videoCodec>' + codec + r')-(?P<releaseGroup>.*?)(-(.*?))?[ \.]'
- for codec in CODECS ]
-GROUP_NAMES2 += [ r'\.(?P<format>' + fmt + r')-(?P<releaseGroup>.*?)(-(.*?))?[ \.]'
- for fmt in FORMATS ]
-GROUP_NAMES2 += [ r'\.(?P<videoApi>' + vapi + r')-(?P<releaseGroup>.*?)(-(.*?))?[ \.]'
- for vapi in VAPIS ]
-
-GROUP_NAMES = [ re.compile(r, re.IGNORECASE) for r in GROUP_NAMES ]
-GROUP_NAMES2 = [ re.compile(r, re.IGNORECASE) for r in GROUP_NAMES2 ]
-
-def adjust_metadata(md):
- return dict((property_name, compute_canonical_form(property_name, value) or value)
- for property_name, value in md.items())
-
-
-def guess_release_group(string):
- # first try to see whether we have both a known codec and a known release group
- for rexp in GROUP_NAMES:
- match = rexp.search(string)
- while match:
- metadata = match.groupdict()
- # make sure this is an actual release group we caught
- release_group = (compute_canonical_form('releaseGroup', metadata['releaseGroup']) or
- compute_canonical_form('weakReleaseGroup', metadata['releaseGroup']))
- if release_group:
- return adjust_metadata(metadata), (match.start(1), match.end(2))
-
- # we didn't find anything conclusive, keep searching
- match = rexp.search(string, match.span()[0]+1)
-
- # pick anything as releaseGroup as long as we have a codec in front
- # this doesn't include a potential dash ('-') ending the release group
- # eg: [...].X264-HiS@SiLUHD-English.[...]
- for rexp in GROUP_NAMES2:
- match = rexp.search(string)
- if match:
- return adjust_metadata(match.groupdict()), (match.start(1), match.end(2))
-
- return None, None
-
-
-def process(mtree):
- SingleNodeGuesser(guess_release_group, 0.8, log).process(mtree)
+
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import GuessFinder, build_guess
+from guessit.containers import PropertiesContainer
+from guessit.patterns import sep
+from guessit.guess import Guess
+from guessit.textutils import strip_brackets
+
+
+class GuessReleaseGroup(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, -190)
+
+ self.container = PropertiesContainer(canonical_from_pattern=False)
+ self._allowed_groupname_pattern = '[\w@#€£$&!\?]'
+ self._forbidden_groupname_lambda = [lambda elt: elt in ['rip', 'by', 'for', 'par', 'pour', 'bonus'],
+ lambda elt: self._is_number(elt)]
+ # If the previous property in this list, the match will be considered as safe
+ # and group name can contain a separator.
+ self.previous_safe_properties = ['videoCodec', 'format', 'videoApi', 'audioCodec', 'audioProfile', 'videoProfile', 'audioChannels', 'screenSize', 'other']
+ self.previous_safe_values = {'other': ['Complete']}
+ self.next_safe_properties = ['extension', 'website']
+ self.next_safe_values = {'format': ['Telesync']}
+ self.next_unsafe_properties = list(self.previous_safe_properties)
+ self.next_unsafe_properties.extend(['episodeNumber', 'season'])
+ self.container.sep_replace_char = '-'
+ self.container.canonical_from_pattern = False
+ self.container.enhance = True
+ self.container.register_property('releaseGroup', self._allowed_groupname_pattern + '+')
+ self.container.register_property('releaseGroup', self._allowed_groupname_pattern + '+-' + self._allowed_groupname_pattern + '+')
+ self.re_sep = re.compile('(' + sep + ')')
+
+ def register_arguments(self, opts, naming_opts, output_opts, information_opts, webservice_opts, other_options):
+ naming_opts.add_argument('-G', '--expected-group', action='append', dest='expected_group',
+ help='Expected release group (can be used multiple times)')
+
+ def supported_properties(self):
+ return self.container.get_supported_properties()
+
+ @staticmethod
+ def _is_number(s):
+ try:
+ int(s)
+ return True
+ except ValueError:
+ return False
+
+ def validate_group_name(self, guess):
+ val = guess['releaseGroup']
+ if len(val) > 1:
+ checked_val = ""
+ forbidden = False
+ for elt in self.re_sep.split(val): # separators are in the list because of capturing group
+ if forbidden:
+ # Previous was forbidden, don't had separator
+ forbidden = False
+ continue
+ for forbidden_lambda in self._forbidden_groupname_lambda:
+ forbidden = forbidden_lambda(elt.lower())
+ if forbidden:
+ if checked_val:
+ # Removing previous separator
+ checked_val = checked_val[0:len(checked_val) - 1]
+ break
+ if not forbidden:
+ checked_val += elt
+
+ val = checked_val
+ if not val:
+ return False
+ if self.re_sep.match(val[-1]):
+ val = val[:len(val)-1]
+ if self.re_sep.match(val[0]):
+ val = val[1:]
+ guess['releaseGroup'] = val
+ forbidden = False
+ for forbidden_lambda in self._forbidden_groupname_lambda:
+ forbidden = forbidden_lambda(val.lower())
+ if forbidden:
+ break
+ if not forbidden:
+ return True
+ return False
+
+ @staticmethod
+ def is_leaf_previous(leaf, node):
+ if leaf.span[1] <= node.span[0]:
+ for idx in range(leaf.span[1], node.span[0]):
+ if leaf.root.value[idx] not in sep:
+ return False
+ return True
+ return False
+
+ def validate_next_leaves(self, node):
+ if 'series' in node.root.info or 'title' in node.root.info:
+ # --expected-series or --expected-title is used.
+ return True
+
+ next_leaf = node.root.next_leaf(node)
+ node_idx = node.node_last_idx
+ while next_leaf and next_leaf.node_last_idx >= node_idx:
+ node_idx = next_leaf.node_last_idx
+ # Check next properties in the same group are not in unsafe properties list
+ for next_unsafe_property in self.next_unsafe_properties:
+ if next_unsafe_property in next_leaf.info:
+ return False
+ next_leaf = next_leaf.root.next_leaf(next_leaf)
+
+ # Make sure to avoid collision with 'series' or 'title' guessed later. Should be more precise.
+ leaves = node.root.unidentified_leaves()
+ return len(list(leaves)) > 1
+
+ def validate_node(self, leaf, node, safe=False):
+ if not self.is_leaf_previous(leaf, node):
+ return False
+ if not self.validate_next_leaves(node):
+ return False
+ if safe:
+ for k, v in leaf.guess.items():
+ if k in self.previous_safe_values and v not in self.previous_safe_values[k]:
+ return False
+ return True
+
+ def guess_release_group(self, string, node=None, options=None):
+ if options and options.get('expected_group'):
+ expected_container = PropertiesContainer(enhance=True, canonical_from_pattern=False)
+ for expected_group in options.get('expected_group'):
+ if expected_group.startswith('re:'):
+ expected_group = expected_group[3:]
+ expected_group = expected_group.replace(' ', '-')
+ expected_container.register_property('releaseGroup', expected_group, enhance=True)
+ else:
+ expected_group = re.escape(expected_group)
+ expected_container.register_property('releaseGroup', expected_group, enhance=False)
+
+ found = expected_container.find_properties(string, node, options, 'releaseGroup')
+ guess = expected_container.as_guess(found, string, self.validate_group_name)
+ if guess:
+ return guess
+
+ found = self.container.find_properties(string, node, options, 'releaseGroup')
+ guess = self.container.as_guess(found, string, self.validate_group_name)
+ validated_guess = None
+ if guess:
+ group_node = node.group_node()
+ if group_node:
+ for leaf in group_node.leaves_containing(self.previous_safe_properties):
+ if self.validate_node(leaf, node, True):
+ if leaf.root.value[leaf.span[1]] == '-':
+ guess.metadata().confidence = 1
+ else:
+ guess.metadata().confidence = 0.7
+ validated_guess = guess
+
+ if not validated_guess:
+ # If previous group last leaf is identified as a safe property,
+ # consider the raw value as a releaseGroup
+ previous_group_node = node.previous_group_node()
+ if previous_group_node:
+ for leaf in previous_group_node.leaves_containing(self.previous_safe_properties):
+ if self.validate_node(leaf, node, False):
+ guess = Guess({'releaseGroup': node.value}, confidence=1, input=node.value, span=(0, len(node.value)))
+ if self.validate_group_name(guess):
+ node.guess = guess
+ validated_guess = guess
+
+ if validated_guess:
+ # If following group nodes have only one unidentified leaf, it belongs to the release group
+ next_group_node = node
+
+ while True:
+ next_group_node = next_group_node.next_group_node()
+ if next_group_node:
+ leaves = list(next_group_node.leaves())
+ if len(leaves) == 1 and not leaves[0].guess:
+ validated_guess['releaseGroup'] = validated_guess['releaseGroup'] + leaves[0].value
+ leaves[0].guess = validated_guess
+ else:
+ break
+ else:
+ break
+
+ if not validated_guess and node.is_explicit() and node.node_last_idx == 0: # first node from group
+ validated_guess = build_guess(node, 'releaseGroup', value=node.value[1:len(node.value)-1])
+ validated_guess.metadata().confidence = 0.4
+ validated_guess.metadata().span = 1, len(node.value)
+ node.guess = validated_guess
+
+ if validated_guess:
+ # Strip brackets
+ validated_guess['releaseGroup'] = strip_brackets(validated_guess['releaseGroup'])
+
+ return validated_guess
+
+ def process(self, mtree, options=None):
+ GuessFinder(self.guess_release_group, None, self.log, options).process_nodes(mtree.unidentified_leaves())
diff --git a/lib/guessit/transfo/guess_video_rexps.py b/lib/guessit/transfo/guess_video_rexps.py
index 6f46b398bb04606c7dea07791961c1307fce9d73..b1dca8ee34143cc2383505737026ae73b1f7f555 100644
--- a/lib/guessit/transfo/guess_video_rexps.py
+++ b/lib/guessit/transfo/guess_video_rexps.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,32 +18,41 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import Guess
-from guessit.transfo import SingleNodeGuesser
-from guessit.patterns import video_rexps, sep
-import re
-import logging
+from __future__ import absolute_import, division, print_function, \
+ unicode_literals
-log = logging.getLogger(__name__)
+from guessit.patterns import _psep
+from guessit.containers import PropertiesContainer
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import GuessFinder
+from guessit.patterns.numeral import parse_numeral
-def guess_video_rexps(string):
- string = '-' + string + '-'
- for rexp, confidence, span_adjust in video_rexps:
- match = re.search(sep + rexp + sep, string, re.IGNORECASE)
- if match:
- metadata = match.groupdict()
- # is this the better place to put it? (maybe, as it is at least
- # the soonest that we can catch it)
- if metadata.get('cdNumberTotal', -1) is None:
- del metadata['cdNumberTotal']
- return (Guess(metadata, confidence=confidence),
- (match.start() + span_adjust[0],
- match.end() + span_adjust[1] - 2))
+class GuessVideoRexps(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 25)
- return None, None
+ self.container = PropertiesContainer(canonical_from_pattern=False)
+ self.container.register_property(None, 'cd' + _psep + '(?P<cdNumber>[0-9])(?:' + _psep + 'of' + _psep + '(?P<cdNumberTotal>[0-9]))?', confidence=1.0, enhance=False, global_span=True, formatter=parse_numeral)
+ self.container.register_property('cdNumberTotal', '([1-9])' + _psep + 'cds?', confidence=0.9, enhance=False, formatter=parse_numeral)
-def process(mtree):
- SingleNodeGuesser(guess_video_rexps, None, log).process(mtree)
+ self.container.register_property('bonusNumber', 'x([0-9]{1,2})', enhance=False, global_span=True, formatter=parse_numeral)
+
+ self.container.register_property('filmNumber', 'f([0-9]{1,2})', enhance=False, global_span=True, formatter=parse_numeral)
+
+ self.container.register_property('edition', 'collector', 'collector-edition', 'edition-collector', canonical_form='Collector Edition')
+ self.container.register_property('edition', 'special-edition', 'edition-special', canonical_form='Special Edition')
+ self.container.register_property('edition', 'criterion', 'criterion-edition', 'edition-criterion', canonical_form='Criterion Edition')
+ self.container.register_property('edition', 'deluxe', 'cdeluxe-edition', 'edition-deluxe', canonical_form='Deluxe Edition')
+ self.container.register_property('edition', 'director\'?s?-cut', 'director\'?s?-cut-edition', 'edition-director\'?s?-cut', canonical_form='Director\'s cut')
+
+ def supported_properties(self):
+ return self.container.get_supported_properties()
+
+ def guess_video_rexps(self, string, node=None, options=None):
+ found = self.container.find_properties(string, node, options)
+ return self.container.as_guess(found, string)
+
+ def process(self, mtree, options=None):
+ GuessFinder(self.guess_video_rexps, None, self.log, options).process_nodes(mtree.unidentified_leaves())
diff --git a/lib/guessit/transfo/guess_weak_episodes_rexps.py b/lib/guessit/transfo/guess_weak_episodes_rexps.py
index 00f553feb15587e594d7a3edbf14d21ff15b1228..71a74cd1d8033c94bbcbd0a8cbb46e249a162a11 100644
--- a/lib/guessit/transfo/guess_weak_episodes_rexps.py
+++ b/lib/guessit/transfo/guess_weak_episodes_rexps.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,45 +18,65 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit import Guess
-from guessit.transfo import SingleNodeGuesser
-from guessit.patterns import weak_episode_rexps
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import re
-import logging
-log = logging.getLogger(__name__)
+from guessit.plugins.transformers import Transformer
+
+from guessit.matcher import GuessFinder
+from guessit.patterns import sep, build_or_pattern
+from guessit.containers import PropertiesContainer
+from guessit.patterns.numeral import numeral, parse_numeral
+from guessit.date import valid_year
+
+
+class GuessWeakEpisodesRexps(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 15)
+
+ of_separators = ['of', 'sur', '/', '\\']
+ of_separators_re = re.compile(build_or_pattern(of_separators, escape=True), re.IGNORECASE)
+
+ self.container = PropertiesContainer(enhance=False, canonical_from_pattern=False)
+ episode_words = ['episodes?']
-def guess_weak_episodes_rexps(string, node):
- if 'episodeNumber' in node.root.info:
- return None, None
+ def _formater(episode_number):
+ epnum = parse_numeral(episode_number)
+ if not valid_year(epnum):
+ if epnum > 100:
+ season, epnum = epnum // 100, epnum % 100
+ # episodes which have a season > 50 are most likely errors
+ # (Simpson is at 25!)
+ if season > 50:
+ return None
+ return {'season': season, 'episodeNumber': epnum}
+ else:
+ return epnum
- for rexp, span_adjust in weak_episode_rexps:
- match = re.search(rexp, string, re.IGNORECASE)
- if match:
- metadata = match.groupdict()
- span = (match.start() + span_adjust[0],
- match.end() + span_adjust[1])
+ self.container.register_property(['episodeNumber', 'season'], '[0-9]{2,4}', confidence=0.6, formatter=_formater, disabler=lambda options: options.get('episode_prefer_number') if options else False)
+ self.container.register_property(['episodeNumber', 'season'], '[0-9]{4}', confidence=0.6, formatter=_formater)
+ self.container.register_property('episodeNumber', '[^0-9](\d{1,3})', confidence=0.6, formatter=parse_numeral, disabler=lambda options: not options.get('episode_prefer_number') if options else True)
+ self.container.register_property(None, '(' + build_or_pattern(episode_words) + sep + '?(?P<episodeNumber>' + numeral + '))[^0-9]', confidence=0.4, formatter=parse_numeral)
+ self.container.register_property(None, r'(?P<episodeNumber>' + numeral + ')' + sep + '?' + of_separators_re.pattern + sep + '?(?P<episodeCount>' + numeral +')', confidence=0.6, formatter=parse_numeral)
+ self.container.register_property('episodeNumber', r'^' + sep + '?(\d{1,3})' + sep, confidence=0.4, formatter=parse_numeral, disabler=lambda options: not options.get('episode_prefer_number') if options else True)
+ self.container.register_property('episodeNumber', sep + r'(\d{1,3})' + sep + '?$', confidence=0.4, formatter=parse_numeral, disabler=lambda options: not options.get('episode_prefer_number') if options else True)
- epnum = int(metadata['episodeNumber'])
- if epnum > 100:
- season, epnum = epnum // 100, epnum % 100
- # episodes which have a season > 25 are most likely errors
- # (Simpsons is at 23!)
- if season > 25:
- continue
- return Guess({ 'season': season,
- 'episodeNumber': epnum },
- confidence=0.6), span
- else:
- return Guess(metadata, confidence=0.3), span
+ def supported_properties(self):
+ return self.container.get_supported_properties()
- return None, None
+ def guess_weak_episodes_rexps(self, string, node=None, options=None):
+ if node and 'episodeNumber' in node.root.info:
+ return None
+ properties = self.container.find_properties(string, node, options)
+ guess = self.container.as_guess(properties, string)
-guess_weak_episodes_rexps.use_node = True
+ return guess
+ def should_process(self, mtree, options=None):
+ return mtree.guess.get('type', '').startswith('episode')
-def process(mtree):
- SingleNodeGuesser(guess_weak_episodes_rexps, 0.6, log).process(mtree)
+ def process(self, mtree, options=None):
+ GuessFinder(self.guess_weak_episodes_rexps, 0.6, self.log, options).process_nodes(mtree.unidentified_leaves())
diff --git a/lib/guessit/transfo/guess_website.py b/lib/guessit/transfo/guess_website.py
index d90e4c2dc8ca3e6098f79012fff09b02c5d96bc2..ead0d65b7cc6c645ad32cdd381f5db5beb538417 100644
--- a/lib/guessit/transfo/guess_website.py
+++ b/lib/guessit/transfo/guess_website.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Rémi Alvergnat <toilal.dev@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,22 +18,42 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit.transfo import SingleNodeGuesser
-from guessit.patterns import websites
-import logging
+from __future__ import absolute_import, division, print_function, unicode_literals
-log = logging.getLogger(__name__)
+from pkg_resources import resource_stream # @UnresolvedImport
+from guessit.patterns import build_or_pattern
+from guessit.containers import PropertiesContainer
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import GuessFinder
-def guess_website(string):
- low = string.lower()
- for site in websites:
- pos = low.find(site.lower())
- if pos != -1:
- return {'website': site}, (pos, pos + len(site))
- return None, None
+TLDS = [l.strip().decode('utf-8')
+ for l in resource_stream('guessit', 'tlds-alpha-by-domain.txt').readlines()
+ if b'--' not in l][1:]
-def process(mtree):
- SingleNodeGuesser(guess_website, 1.0, log).process(mtree)
+
+class GuessWebsite(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 45)
+
+ self.container = PropertiesContainer(enhance=False, canonical_from_pattern=False)
+
+ tlds_pattern = build_or_pattern(TLDS) # All registered domain extension
+ safe_tlds_pattern = build_or_pattern(['com', 'org', 'net']) # For sure a website extension
+ safe_subdomains_pattern = build_or_pattern(['www']) # For sure a website subdomain
+ safe_prefix_tlds_pattern = build_or_pattern(['co', 'com', 'org', 'net']) # Those words before a tlds are sure
+
+ self.container.register_property('website', '(?:' + safe_subdomains_pattern + '\.)+' + r'(?:[a-z-]+\.)+' + r'(?:' + tlds_pattern + r')+')
+ self.container.register_property('website', '(?:' + safe_subdomains_pattern + '\.)*' + r'[a-z-]+\.' + r'(?:' + safe_tlds_pattern + r')+')
+ self.container.register_property('website', '(?:' + safe_subdomains_pattern + '\.)*' + r'[a-z-]+\.' + r'(?:' + safe_prefix_tlds_pattern + r'\.)+' + r'(?:' + tlds_pattern + r')+')
+
+ def supported_properties(self):
+ return self.container.get_supported_properties()
+
+ def guess_website(self, string, node=None, options=None):
+ found = self.container.find_properties(string, node, options, 'website')
+ return self.container.as_guess(found, string)
+
+ def process(self, mtree, options=None):
+ GuessFinder(self.guess_website, 1.0, self.log, options).process_nodes(mtree.unidentified_leaves())
diff --git a/lib/guessit/transfo/guess_year.py b/lib/guessit/transfo/guess_year.py
index 375f31c6a6cfe278dd5c15ffcd9861342cdf829a..e4eb8bb8c4e83658d3f034d3367ef37a32b75e05 100644
--- a/lib/guessit/transfo/guess_year.py
+++ b/lib/guessit/transfo/guess_year.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,33 +18,41 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit.transfo import SingleNodeGuesser
-from guessit.date import search_year
-import logging
+from __future__ import absolute_import, division, print_function, unicode_literals
-log = logging.getLogger(__name__)
+from guessit.plugins.transformers import Transformer
+from guessit.matcher import GuessFinder
+from guessit.date import search_year, valid_year
-def guess_year(string):
- year, span = search_year(string)
- if year:
- return { 'year': year }, span
- else:
- return None, None
+class GuessYear(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, -160)
-def guess_year_skip_first(string):
- year, span = search_year(string)
- if year:
- year2, span2 = guess_year(string[span[1]:])
- if year2:
- return year2, (span2[0]+span[1], span2[1]+span[1])
+ def supported_properties(self):
+ return ['year']
- return None, None
+ @staticmethod
+ def guess_year(string, node=None, options=None):
+ year, span = search_year(string)
+ if year:
+ return {'year': year}, span
+ else:
+ return None, None
+ def second_pass_options(self, mtree, options=None):
+ year_nodes = list(mtree.leaves_containing('year'))
+ if len(year_nodes) > 1:
+ return {'skip_nodes': year_nodes[:len(year_nodes) - 1]}
+ return None
-def process(mtree, skip_first_year=False):
- if skip_first_year:
- SingleNodeGuesser(guess_year_skip_first, 1.0, log).process(mtree)
- else:
- SingleNodeGuesser(guess_year, 1.0, log).process(mtree)
+ def process(self, mtree, options=None):
+ GuessFinder(self.guess_year, 1.0, self.log, options).process_nodes(mtree.unidentified_leaves())
+
+ # if we found a season number that is a valid year, it is usually safe to assume
+ # we can also set the year property to that value
+ for n in mtree.leaves_containing('season'):
+ g = n.guess
+ season = g['season']
+ if valid_year(season):
+ g['year'] = season
diff --git a/lib/guessit/transfo/post_process.py b/lib/guessit/transfo/post_process.py
deleted file mode 100644
index 723ba53eed63497b80a97af5e2730563542721ed..0000000000000000000000000000000000000000
--- a/lib/guessit/transfo/post_process.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
-#
-# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
-#
-# GuessIt is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# GuessIt is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
-#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-
-from __future__ import unicode_literals
-from guessit.patterns import subtitle_exts
-from guessit.textutils import reorder_title, find_words
-import logging
-
-log = logging.getLogger(__name__)
-
-
-def process(mtree):
- # 1- try to promote language to subtitle language where it makes sense
- for node in mtree.nodes():
- if 'language' not in node.guess:
- continue
-
- def promote_subtitle():
- if 'language' in node.guess:
- node.guess.set('subtitleLanguage', node.guess['language'],
- confidence=node.guess.confidence('language'))
- del node.guess['language']
-
- # - if we matched a language in a file with a sub extension and that
- # the group is the last group of the filename, it is probably the
- # language of the subtitle
- # (eg: 'xxx.english.srt')
- if (mtree.node_at((-1,)).value.lower() in subtitle_exts and
- node == mtree.leaves()[-2]):
- promote_subtitle()
-
- # - if we find the word 'sub' before the language, and in the same explicit
- # group, then upgrade the language
- explicit_group = mtree.node_at(node.node_idx[:2])
- group_str = explicit_group.value.lower()
-
- if ('sub' in find_words(group_str) and
- 0 <= group_str.find('sub') < (node.span[0] - explicit_group.span[0])):
- promote_subtitle()
-
- # - if a language is in an explicit group just preceded by "st",
- # it is a subtitle language (eg: '...st[fr-eng]...')
- try:
- idx = node.node_idx
- previous = mtree.node_at((idx[0], idx[1] - 1)).leaves()[-1]
- if previous.value.lower()[-2:] == 'st':
- promote_subtitle()
- except IndexError:
- pass
-
- # 2- ", the" at the end of a series title should be prepended to it
- for node in mtree.nodes():
- if 'series' not in node.guess:
- continue
-
- node.guess['series'] = reorder_title(node.guess['series'])
diff --git a/lib/guessit/transfo/split_explicit_groups.py b/lib/guessit/transfo/split_explicit_groups.py
index 38a80096c28cf356dbef47046448cd015d81fc0e..17041b3a5f2cef97210bafe491a740cd3543e27c 100644
--- a/lib/guessit/transfo/split_explicit_groups.py
+++ b/lib/guessit/transfo/split_explicit_groups.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,27 +18,33 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from functools import reduce
+
+from guessit.plugins.transformers import Transformer
from guessit.textutils import find_first_level_groups
from guessit.patterns import group_delimiters
-import functools
-import logging
-log = logging.getLogger(__name__)
+class SplitExplicitGroups(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 250)
+
+ def process(self, mtree, options=None):
+ """split each of those into explicit groups (separated by parentheses or square brackets)
-def process(mtree):
- """return the string split into explicit groups, that is, those either
- between parenthese, square brackets or curly braces, and those separated
- by a dash."""
- for c in mtree.children:
- groups = find_first_level_groups(c.value, group_delimiters[0])
- for delimiters in group_delimiters:
- flatten = lambda l, x: l + find_first_level_groups(x, delimiters)
- groups = functools.reduce(flatten, groups, [])
+ :return: return the string split into explicit groups, that is, those either
+ between parenthese, square brackets or curly braces, and those separated
+ by a dash."""
+ for c in mtree.children:
+ groups = find_first_level_groups(c.value, group_delimiters[0])
+ for delimiters in group_delimiters:
+ flatten = lambda l, x: l + find_first_level_groups(x, delimiters)
+ groups = reduce(flatten, groups, [])
- # do not do this at this moment, it is not strong enough and can break other
- # patterns, such as dates, etc...
- #groups = functools.reduce(lambda l, x: l + x.split('-'), groups, [])
+ # do not do this at this moment, it is not strong enough and can break other
+ # patterns, such as dates, etc...
+ # groups = functools.reduce(lambda l, x: l + x.split('-'), groups, [])
- c.split_on_components(groups)
+ c.split_on_components(groups)
diff --git a/lib/guessit/transfo/split_on_dash.py b/lib/guessit/transfo/split_on_dash.py
index 9c4213a7c2ac6b960b5de757329b16272e9db2e6..24e888f381f0aabc1f84af6d101ffb1c5e14f62f 100644
--- a/lib/guessit/transfo/split_on_dash.py
+++ b/lib/guessit/transfo/split_on_dash.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,25 +18,31 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
-from guessit.patterns import sep
+from __future__ import absolute_import, division, print_function, unicode_literals
+
import re
-import logging
-log = logging.getLogger(__name__)
+from guessit.plugins.transformers import Transformer
+from guessit.patterns import sep
+
+class SplitOnDash(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 245)
-def process(mtree):
- for node in mtree.unidentified_leaves():
- indices = []
+ def process(self, mtree, options=None):
+ """split into '-' separated subgroups (with required separator chars
+ around the dash)
+ """
+ for node in mtree.unidentified_leaves():
+ indices = []
- didx = 0
- pattern = re.compile(sep + '-' + sep)
- match = pattern.search(node.value)
- while match:
- span = match.span()
- indices.extend([ span[0], span[1] ])
- match = pattern.search(node.value, span[1])
+ pattern = re.compile(sep + '-' + sep)
+ match = pattern.search(node.value)
+ while match:
+ span = match.span()
+ indices.extend([span[0], span[1]])
+ match = pattern.search(node.value, span[1])
- if indices:
- node.partition(indices)
+ if indices:
+ node.partition(indices)
diff --git a/lib/guessit/transfo/split_path_components.py b/lib/guessit/transfo/split_path_components.py
index bb77ec5e0d48c03b6a5b84793b3d3c6f4357bd61..8f2b13760ceab7151cb2aa1b403948e821747046 100644
--- a/lib/guessit/transfo/split_path_components.py
+++ b/lib/guessit/transfo/split_path_components.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2012 Nicolas Wack <wackou@gmail.com>
+# Copyright (c) 2013 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -18,19 +18,29 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-from __future__ import unicode_literals
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from os.path import splitext
+
+from guessit.plugins.transformers import Transformer
from guessit import fileutils
-import os.path
-import logging
-log = logging.getLogger(__name__)
+class SplitPathComponents(Transformer):
+ def __init__(self):
+ Transformer.__init__(self, 255)
+
+ def process(self, mtree, options=None):
+ """first split our path into dirs + basename + ext
-def process(mtree):
- """Returns the filename split into [ dir*, basename, ext ]."""
- components = fileutils.split_path(mtree.value)
- basename = components.pop(-1)
- components += list(os.path.splitext(basename))
- components[-1] = components[-1][1:] # remove the '.' from the extension
+ :return: the filename split into [ dir*, basename, ext ]
+ """
+ if not options.get('name_only'):
+ components = fileutils.split_path(mtree.value)
+ basename = components.pop(-1)
+ components += list(splitext(basename))
+ components[-1] = components[-1][1:] # remove the '.' from the extension
- mtree.split_on_components(components)
+ mtree.split_on_components(components)
+ else:
+ mtree.split_on_components([mtree.value, ''])
diff --git a/lib/pysrt/__init__.py b/lib/pysrt/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..34e967176f0a39dfdafd251f1832f08598166b8f
--- /dev/null
+++ b/lib/pysrt/__init__.py
@@ -0,0 +1,18 @@
+from pysrt.srttime import SubRipTime
+from pysrt.srtitem import SubRipItem
+from pysrt.srtfile import SubRipFile
+from pysrt.srtexc import Error, InvalidItem, InvalidTimeString
+from pysrt.version import VERSION, VERSION_STRING
+
+__all__ = [
+ 'SubRipFile', 'SubRipItem', 'SubRipFile', 'SUPPORT_UTF_32_LE',
+ 'SUPPORT_UTF_32_BE', 'InvalidItem', 'InvalidTimeString'
+]
+
+ERROR_PASS = SubRipFile.ERROR_PASS
+ERROR_LOG = SubRipFile.ERROR_LOG
+ERROR_RAISE = SubRipFile.ERROR_RAISE
+
+open = SubRipFile.open
+stream = SubRipFile.stream
+from_string = SubRipFile.from_string
diff --git a/lib/pysrt/commands.py b/lib/pysrt/commands.py
new file mode 100755
index 0000000000000000000000000000000000000000..05bf78cf57679c2915549c2fef18bda514bde22b
--- /dev/null
+++ b/lib/pysrt/commands.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# pylint: disable-all
+
+import os
+import re
+import sys
+import codecs
+import shutil
+import argparse
+from textwrap import dedent
+
+from chardet import detect
+from pysrt import SubRipFile, SubRipTime, VERSION_STRING
+
+
+def underline(string):
+ return "\033[4m%s\033[0m" % string
+
+
+class TimeAwareArgumentParser(argparse.ArgumentParser):
+
+ RE_TIME_REPRESENTATION = re.compile(r'^\-?(\d+[hms]{0,2}){1,4}$')
+
+ def parse_args(self, args=None, namespace=None):
+ time_index = -1
+ for index, arg in enumerate(args):
+ match = self.RE_TIME_REPRESENTATION.match(arg)
+ if match:
+ time_index = index
+ break
+
+ if time_index >= 0:
+ args.insert(time_index, '--')
+
+ return super(TimeAwareArgumentParser, self).parse_args(args, namespace)
+
+
+class SubRipShifter(object):
+
+ BACKUP_EXTENSION = '.bak'
+ RE_TIME_STRING = re.compile(r'(\d+)([hms]{0,2})')
+ UNIT_RATIOS = {
+ 'ms': 1,
+ '': SubRipTime.SECONDS_RATIO,
+ 's': SubRipTime.SECONDS_RATIO,
+ 'm': SubRipTime.MINUTES_RATIO,
+ 'h': SubRipTime.HOURS_RATIO,
+ }
+ DESCRIPTION = dedent("""\
+ Srt subtitle editor
+
+ It can either shift, split or change the frame rate.
+ """)
+ TIMESTAMP_HELP = "A timestamp in the form: [-][Hh][Mm]S[s][MSms]"
+ SHIFT_EPILOG = dedent("""\
+
+ Examples:
+ 1 minute and 12 seconds foreward (in place):
+ $ srt -i shift 1m12s movie.srt
+
+ half a second foreward:
+ $ srt shift 500ms movie.srt > othername.srt
+
+ 1 second and half backward:
+ $ srt -i shift -1s500ms movie.srt
+
+ 3 seconds backward:
+ $ srt -i shift -3 movie.srt
+ """)
+ RATE_EPILOG = dedent("""\
+
+ Examples:
+ Convert 23.9fps subtitles to 25fps:
+ $ srt -i rate 23.9 25 movie.srt
+ """)
+ LIMITS_HELP = "Each parts duration in the form: [Hh][Mm]S[s][MSms]"
+ SPLIT_EPILOG = dedent("""\
+
+ Examples:
+ For a movie in 2 parts with the first part 48 minutes and 18 seconds long:
+ $ srt split 48m18s movie.srt
+ => creates movie.1.srt and movie.2.srt
+
+ For a movie in 3 parts of 20 minutes each:
+ $ srt split 20m 20m movie.srt
+ => creates movie.1.srt, movie.2.srt and movie.3.srt
+ """)
+ FRAME_RATE_HELP = "A frame rate in fps (commonly 23.9 or 25)"
+ ENCODING_HELP = dedent("""\
+ Change file encoding. Useful for players accepting only latin1 subtitles.
+ List of supported encodings: http://docs.python.org/library/codecs.html#standard-encodings
+ """)
+ BREAK_EPILOG = dedent("""\
+ Break lines longer than defined length
+ """)
+ LENGTH_HELP = "Maximum number of characters per line"
+
+ def __init__(self):
+ self.output_file_path = None
+
+ def build_parser(self):
+ parser = TimeAwareArgumentParser(description=self.DESCRIPTION, formatter_class=argparse.RawTextHelpFormatter)
+ parser.add_argument('-i', '--in-place', action='store_true', dest='in_place',
+ help="Edit file in-place, saving a backup as file.bak (do not works for the split command)")
+ parser.add_argument('-e', '--output-encoding', metavar=underline('encoding'), action='store', dest='output_encoding',
+ type=self.parse_encoding, help=self.ENCODING_HELP)
+ parser.add_argument('-v', '--version', action='version', version='%%(prog)s %s' % VERSION_STRING)
+ subparsers = parser.add_subparsers(title='commands')
+
+ shift_parser = subparsers.add_parser('shift', help="Shift subtitles by specified time offset", epilog=self.SHIFT_EPILOG, formatter_class=argparse.RawTextHelpFormatter)
+ shift_parser.add_argument('time_offset', action='store', metavar=underline('offset'),
+ type=self.parse_time, help=self.TIMESTAMP_HELP)
+ shift_parser.set_defaults(action=self.shift)
+
+ rate_parser = subparsers.add_parser('rate', help="Convert subtitles from a frame rate to another", epilog=self.RATE_EPILOG, formatter_class=argparse.RawTextHelpFormatter)
+ rate_parser.add_argument('initial', action='store', type=float, help=self.FRAME_RATE_HELP)
+ rate_parser.add_argument('final', action='store', type=float, help=self.FRAME_RATE_HELP)
+ rate_parser.set_defaults(action=self.rate)
+
+ split_parser = subparsers.add_parser('split', help="Split a file in multiple parts", epilog=self.SPLIT_EPILOG, formatter_class=argparse.RawTextHelpFormatter)
+ split_parser.add_argument('limits', action='store', nargs='+', type=self.parse_time, help=self.LIMITS_HELP)
+ split_parser.set_defaults(action=self.split)
+
+ break_parser = subparsers.add_parser('break', help="Break long lines", epilog=self.BREAK_EPILOG, formatter_class=argparse.RawTextHelpFormatter)
+ break_parser.add_argument('length', action='store', type=int, help=self.LENGTH_HELP)
+ break_parser.set_defaults(action=self.break_lines)
+
+ parser.add_argument('file', action='store')
+
+ return parser
+
+ def run(self, args):
+ self.arguments = self.build_parser().parse_args(args)
+ if self.arguments.in_place:
+ self.create_backup()
+ self.arguments.action()
+
+ def parse_time(self, time_string):
+ negative = time_string.startswith('-')
+ if negative:
+ time_string = time_string[1:]
+ ordinal = sum(int(value) * self.UNIT_RATIOS[unit] for value, unit
+ in self.RE_TIME_STRING.findall(time_string))
+ return -ordinal if negative else ordinal
+
+ def parse_encoding(self, encoding_name):
+ try:
+ codecs.lookup(encoding_name)
+ except LookupError as error:
+ raise argparse.ArgumentTypeError(error.message)
+ return encoding_name
+
+ def shift(self):
+ self.input_file.shift(milliseconds=self.arguments.time_offset)
+ self.input_file.write_into(self.output_file)
+
+ def rate(self):
+ ratio = self.arguments.final / self.arguments.initial
+ self.input_file.shift(ratio=ratio)
+ self.input_file.write_into(self.output_file)
+
+ def split(self):
+ limits = [0] + self.arguments.limits + [self.input_file[-1].end.ordinal + 1]
+ base_name, extension = os.path.splitext(self.arguments.file)
+ for index, (start, end) in enumerate(zip(limits[:-1], limits[1:])):
+ file_name = '%s.%s%s' % (base_name, index + 1, extension)
+ part_file = self.input_file.slice(ends_after=start, starts_before=end)
+ part_file.shift(milliseconds=-start)
+ part_file.clean_indexes()
+ part_file.save(path=file_name, encoding=self.output_encoding)
+
+ def create_backup(self):
+ backup_file = self.arguments.file + self.BACKUP_EXTENSION
+ if not os.path.exists(backup_file):
+ shutil.copy2(self.arguments.file, backup_file)
+ self.output_file_path = self.arguments.file
+ self.arguments.file = backup_file
+
+ def break_lines(self):
+ split_re = re.compile(r'(.{,%i})(?:\s+|$)' % self.arguments.length)
+ for item in self.input_file:
+ item.text = '\n'.join(split_re.split(item.text)[1::2])
+ self.input_file.write_into(self.output_file)
+
+ @property
+ def output_encoding(self):
+ return self.arguments.output_encoding or self.input_file.encoding
+
+ @property
+ def input_file(self):
+ if not hasattr(self, '_source_file'):
+ with open(self.arguments.file, 'rb') as f:
+ content = f.read()
+ encoding = detect(content).get('encoding')
+ encoding = self.normalize_encoding(encoding)
+
+ self._source_file = SubRipFile.open(self.arguments.file,
+ encoding=encoding, error_handling=SubRipFile.ERROR_LOG)
+ return self._source_file
+
+ @property
+ def output_file(self):
+ if not hasattr(self, '_output_file'):
+ if self.output_file_path:
+ self._output_file = codecs.open(self.output_file_path, 'w+', encoding=self.output_encoding)
+ else:
+ self._output_file = sys.stdout
+ return self._output_file
+
+ def normalize_encoding(self, encoding):
+ return encoding.lower().replace('-', '_')
+
+
+def main():
+ SubRipShifter().run(sys.argv[1:])
+
+if __name__ == '__main__':
+ main()
diff --git a/lib/pysrt/comparablemixin.py b/lib/pysrt/comparablemixin.py
new file mode 100644
index 0000000000000000000000000000000000000000..3ae70b075b517c293d8f0c4bb84809fa84963c6c
--- /dev/null
+++ b/lib/pysrt/comparablemixin.py
@@ -0,0 +1,26 @@
+class ComparableMixin(object):
+ def _compare(self, other, method):
+ try:
+ return method(self._cmpkey(), other._cmpkey())
+ except (AttributeError, TypeError):
+ # _cmpkey not implemented, or return different type,
+ # so I can't compare with "other".
+ return NotImplemented
+
+ def __lt__(self, other):
+ return self._compare(other, lambda s, o: s < o)
+
+ def __le__(self, other):
+ return self._compare(other, lambda s, o: s <= o)
+
+ def __eq__(self, other):
+ return self._compare(other, lambda s, o: s == o)
+
+ def __ge__(self, other):
+ return self._compare(other, lambda s, o: s >= o)
+
+ def __gt__(self, other):
+ return self._compare(other, lambda s, o: s > o)
+
+ def __ne__(self, other):
+ return self._compare(other, lambda s, o: s != o)
diff --git a/lib/pysrt/compat.py b/lib/pysrt/compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..d34d199b9d9e869fdb0d0f8f80bf7fe3155cad2c
--- /dev/null
+++ b/lib/pysrt/compat.py
@@ -0,0 +1,22 @@
+
+import sys
+
+# Syntax sugar.
+_ver = sys.version_info
+
+#: Python 2.x?
+is_py2 = (_ver[0] == 2)
+
+#: Python 3.x?
+is_py3 = (_ver[0] == 3)
+
+from io import open as io_open
+
+if is_py2:
+ basestring = basestring
+ str = unicode
+ open = io_open
+elif is_py3:
+ basestring = (str, bytes)
+ str = str
+ open = open
diff --git a/lib/pysrt/srtexc.py b/lib/pysrt/srtexc.py
new file mode 100644
index 0000000000000000000000000000000000000000..971b47098084941e2cc7e761cee04899eb7b048c
--- /dev/null
+++ b/lib/pysrt/srtexc.py
@@ -0,0 +1,31 @@
+"""
+Exception classes
+"""
+
+
+class Error(Exception):
+ """
+ Pysrt's base exception
+ """
+ pass
+
+
+class InvalidTimeString(Error):
+ """
+ Raised when parser fail on bad formated time strings
+ """
+ pass
+
+
+class InvalidItem(Error):
+ """
+ Raised when parser fail to parse a sub title item
+ """
+ pass
+
+
+class InvalidIndex(InvalidItem):
+ """
+ Raised when parser fail to parse a sub title index
+ """
+ pass
diff --git a/lib/pysrt/srtfile.py b/lib/pysrt/srtfile.py
new file mode 100644
index 0000000000000000000000000000000000000000..c03595af5016ab67388ea3497c77f23a590fe8c7
--- /dev/null
+++ b/lib/pysrt/srtfile.py
@@ -0,0 +1,312 @@
+# -*- coding: utf-8 -*-
+import os
+import sys
+import codecs
+
+try:
+ from collections import UserList
+except ImportError:
+ from UserList import UserList
+
+from itertools import chain
+from copy import copy
+
+from pysrt.srtexc import Error
+from pysrt.srtitem import SubRipItem
+from pysrt.compat import str
+
+BOMS = ((codecs.BOM_UTF32_LE, 'utf_32_le'),
+ (codecs.BOM_UTF32_BE, 'utf_32_be'),
+ (codecs.BOM_UTF16_LE, 'utf_16_le'),
+ (codecs.BOM_UTF16_BE, 'utf_16_be'),
+ (codecs.BOM_UTF8, 'utf_8'))
+CODECS_BOMS = dict((codec, str(bom, codec)) for bom, codec in BOMS)
+BIGGER_BOM = max(len(bom) for bom, encoding in BOMS)
+
+
+class SubRipFile(UserList, object):
+ """
+ SubRip file descriptor.
+
+ Provide a pure Python mapping on all metadata.
+
+ SubRipFile(items, eol, path, encoding)
+
+ items -> list of SubRipItem. Default to [].
+ eol -> str: end of line character. Default to linesep used in opened file
+ if any else to os.linesep.
+ path -> str: path where file will be saved. To open an existant file see
+ SubRipFile.open.
+ encoding -> str: encoding used at file save. Default to utf-8.
+ """
+ ERROR_PASS = 0
+ ERROR_LOG = 1
+ ERROR_RAISE = 2
+
+ DEFAULT_ENCODING = 'utf_8'
+
+ def __init__(self, items=None, eol=None, path=None, encoding='utf-8'):
+ UserList.__init__(self, items or [])
+ self._eol = eol
+ self.path = path
+ self.encoding = encoding
+
+ def _get_eol(self):
+ return self._eol or os.linesep
+
+ def _set_eol(self, eol):
+ self._eol = self._eol or eol
+
+ eol = property(_get_eol, _set_eol)
+
+ def slice(self, starts_before=None, starts_after=None, ends_before=None,
+ ends_after=None):
+ """
+ slice([starts_before][, starts_after][, ends_before][, ends_after]) \
+-> SubRipFile clone
+
+ All arguments are optional, and should be coercible to SubRipTime
+ object.
+
+ It reduce the set of subtitles to those that match match given time
+ constraints.
+
+ The returned set is a clone, but still contains references to original
+ subtitles. So if you shift this returned set, subs contained in the
+ original SubRipFile instance will be altered too.
+
+ Example:
+ >>> subs.slice(ends_after={'seconds': 20}).shift(seconds=2)
+ """
+ clone = copy(self)
+
+ if starts_before:
+ clone.data = (i for i in clone.data if i.start < starts_before)
+ if starts_after:
+ clone.data = (i for i in clone.data if i.start > starts_after)
+ if ends_before:
+ clone.data = (i for i in clone.data if i.end < ends_before)
+ if ends_after:
+ clone.data = (i for i in clone.data if i.end > ends_after)
+
+ clone.data = list(clone.data)
+ return clone
+
+ def at(self, timestamp=None, **kwargs):
+ """
+ at(timestamp) -> SubRipFile clone
+
+ timestamp argument should be coercible to SubRipFile object.
+
+ A specialization of slice. Return all subtiles visible at the
+ timestamp mark.
+
+ Example:
+ >>> subs.at((0, 0, 20, 0)).shift(seconds=2)
+ >>> subs.at(seconds=20).shift(seconds=2)
+ """
+ time = timestamp or kwargs
+ return self.slice(starts_before=time, ends_after=time)
+
+ def shift(self, *args, **kwargs):
+ """shift(hours, minutes, seconds, milliseconds, ratio)
+
+ Shift `start` and `end` attributes of each items of file either by
+ applying a ratio or by adding an offset.
+
+ `ratio` should be either an int or a float.
+ Example to convert subtitles from 23.9 fps to 25 fps:
+ >>> subs.shift(ratio=25/23.9)
+
+ All "time" arguments are optional and have a default value of 0.
+ Example to delay all subs from 2 seconds and half
+ >>> subs.shift(seconds=2, milliseconds=500)
+ """
+ for item in self:
+ item.shift(*args, **kwargs)
+
+ def clean_indexes(self):
+ """
+ clean_indexes()
+
+ Sort subs and reset their index attribute. Should be called after
+ destructive operations like split or such.
+ """
+ self.sort()
+ for index, item in enumerate(self):
+ item.index = index + 1
+
+ @property
+ def text(self):
+ return '\n'.join(i.text for i in self)
+
+ @classmethod
+ def open(cls, path='', encoding=None, error_handling=ERROR_PASS):
+ """
+ open([path, [encoding]])
+
+ If you do not provide any encoding, it can be detected if the file
+ contain a bit order mark, unless it is set to utf-8 as default.
+ """
+ source_file, encoding = cls._open_unicode_file(path, claimed_encoding=encoding)
+ new_file = cls(path=path, encoding=encoding)
+ new_file.read(source_file, error_handling=error_handling)
+ source_file.close()
+ return new_file
+
+ @classmethod
+ def from_string(cls, source, **kwargs):
+ """
+ from_string(source, **kwargs) -> SubRipFile
+
+ `source` -> a unicode instance or at least a str instance encoded with
+ `sys.getdefaultencoding()`
+ """
+ error_handling = kwargs.pop('error_handling', None)
+ new_file = cls(**kwargs)
+ new_file.read(source.splitlines(True), error_handling=error_handling)
+ return new_file
+
+ def read(self, source_file, error_handling=ERROR_PASS):
+ """
+ read(source_file, [error_handling])
+
+ This method parse subtitles contained in `source_file` and append them
+ to the current instance.
+
+ `source_file` -> Any iterable that yield unicode strings, like a file
+ opened with `codecs.open()` or an array of unicode.
+ """
+ self.eol = self._guess_eol(source_file)
+ self.extend(self.stream(source_file, error_handling=error_handling))
+ return self
+
+ @classmethod
+ def stream(cls, source_file, error_handling=ERROR_PASS):
+ """
+ stream(source_file, [error_handling])
+
+ This method yield SubRipItem instances a soon as they have been parsed
+ without storing them. It is a kind of SAX parser for .srt files.
+
+ `source_file` -> Any iterable that yield unicode strings, like a file
+ opened with `codecs.open()` or an array of unicode.
+
+ Example:
+ >>> import pysrt
+ >>> import codecs
+ >>> file = codecs.open('movie.srt', encoding='utf-8')
+ >>> for sub in pysrt.stream(file):
+ ... sub.text += "\nHello !"
+ ... print unicode(sub)
+ """
+ string_buffer = []
+ for index, line in enumerate(chain(source_file, '\n')):
+ if line.strip():
+ string_buffer.append(line)
+ else:
+ source = string_buffer
+ string_buffer = []
+ if source and all(source):
+ try:
+ yield SubRipItem.from_lines(source)
+ except Error as error:
+ error.args += (''.join(source), )
+ cls._handle_error(error, error_handling, index)
+
+ def save(self, path=None, encoding=None, eol=None):
+ """
+ save([path][, encoding][, eol])
+
+ Use initial path if no other provided.
+ Use initial encoding if no other provided.
+ Use initial eol if no other provided.
+ """
+ path = path or self.path
+ encoding = encoding or self.encoding
+
+ save_file = codecs.open(path, 'w+', encoding=encoding)
+ self.write_into(save_file, eol=eol)
+ save_file.close()
+
+ def write_into(self, output_file, eol=None):
+ """
+ write_into(output_file [, eol])
+
+ Serialize current state into `output_file`.
+
+ `output_file` -> Any instance that respond to `write()`, typically a
+ file object
+ """
+ output_eol = eol or self.eol
+
+ for item in self:
+ string_repr = str(item)
+ if output_eol != '\n':
+ string_repr = string_repr.replace('\n', output_eol)
+ output_file.write(string_repr)
+ # Only add trailing eol if it's not already present.
+ # It was kept in the SubRipItem's text before but it really
+ # belongs here. Existing applications might give us subtitles
+ # which already contain a trailing eol though.
+ if not string_repr.endswith(2 * output_eol):
+ output_file.write(output_eol)
+
+ @classmethod
+ def _guess_eol(cls, string_iterable):
+ first_line = cls._get_first_line(string_iterable)
+ for eol in ('\r\n', '\r', '\n'):
+ if first_line.endswith(eol):
+ return eol
+ return os.linesep
+
+ @classmethod
+ def _get_first_line(cls, string_iterable):
+ if hasattr(string_iterable, 'tell'):
+ previous_position = string_iterable.tell()
+
+ try:
+ first_line = next(iter(string_iterable))
+ except StopIteration:
+ return ''
+ if hasattr(string_iterable, 'seek'):
+ string_iterable.seek(previous_position)
+
+ return first_line
+
+ @classmethod
+ def _detect_encoding(cls, path):
+ file_descriptor = open(path, 'rb')
+ first_chars = file_descriptor.read(BIGGER_BOM)
+ file_descriptor.close()
+
+ for bom, encoding in BOMS:
+ if first_chars.startswith(bom):
+ return encoding
+
+ # TODO: maybe a chardet integration
+ return cls.DEFAULT_ENCODING
+
+ @classmethod
+ def _open_unicode_file(cls, path, claimed_encoding=None):
+ encoding = claimed_encoding or cls._detect_encoding(path)
+ source_file = codecs.open(path, 'rU', encoding=encoding)
+
+ # get rid of BOM if any
+ possible_bom = CODECS_BOMS.get(encoding, None)
+ if possible_bom:
+ file_bom = source_file.read(len(possible_bom))
+ if not file_bom == possible_bom:
+ source_file.seek(0) # if not rewind
+ return source_file, encoding
+
+ @classmethod
+ def _handle_error(cls, error, error_handling, index):
+ if error_handling == cls.ERROR_RAISE:
+ error.args = (index, ) + error.args
+ raise error
+ if error_handling == cls.ERROR_LOG:
+ name = type(error).__name__
+ sys.stderr.write('PySRT-%s(line %s): \n' % (name, index))
+ sys.stderr.write(error.args[0].encode('ascii', 'replace'))
+ sys.stderr.write('\n')
diff --git a/lib/pysrt/srtitem.py b/lib/pysrt/srtitem.py
new file mode 100644
index 0000000000000000000000000000000000000000..55979059a608f316f2783cb361809fc06904481c
--- /dev/null
+++ b/lib/pysrt/srtitem.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+SubRip's subtitle parser
+"""
+
+from pysrt.srtexc import InvalidItem, InvalidIndex
+from pysrt.srttime import SubRipTime
+from pysrt.comparablemixin import ComparableMixin
+from pysrt.compat import str, is_py2
+import re
+
+
+class SubRipItem(ComparableMixin):
+ """
+ SubRipItem(index, start, end, text, position)
+
+ index -> int: index of item in file. 0 by default.
+ start, end -> SubRipTime or coercible.
+ text -> unicode: text content for item.
+ position -> unicode: raw srt/vtt "display coordinates" string
+ """
+ ITEM_PATTERN = str('%s\n%s --> %s%s\n%s\n')
+ TIMESTAMP_SEPARATOR = '-->'
+
+ def __init__(self, index=0, start=None, end=None, text='', position=''):
+ try:
+ self.index = int(index)
+ except (TypeError, ValueError): # try to cast as int, but it's not mandatory
+ self.index = index
+
+ self.start = SubRipTime.coerce(start or 0)
+ self.end = SubRipTime.coerce(end or 0)
+ self.position = str(position)
+ self.text = str(text)
+
+ @property
+ def duration(self):
+ return self.end - self.start
+
+ @property
+ def text_without_tags(self):
+ RE_TAG = re.compile(r'<[^>]*?>')
+ return RE_TAG.sub('', self.text)
+
+ @property
+ def characters_per_second(self):
+ characters_count = len(self.text_without_tags.replace('\n', ''))
+ try:
+ return characters_count / (self.duration.ordinal / 1000.0)
+ except ZeroDivisionError:
+ return 0.0
+
+ def __str__(self):
+ position = ' %s' % self.position if self.position.strip() else ''
+ return self.ITEM_PATTERN % (self.index, self.start, self.end,
+ position, self.text)
+ if is_py2:
+ __unicode__ = __str__
+
+ def __str__(self):
+ raise NotImplementedError('Use unicode() instead!')
+
+ def _cmpkey(self):
+ return (self.start, self.end)
+
+ def shift(self, *args, **kwargs):
+ """
+ shift(hours, minutes, seconds, milliseconds, ratio)
+
+ Add given values to start and end attributes.
+ All arguments are optional and have a default value of 0.
+ """
+ self.start.shift(*args, **kwargs)
+ self.end.shift(*args, **kwargs)
+
+ @classmethod
+ def from_string(cls, source):
+ return cls.from_lines(source.splitlines(True))
+
+ @classmethod
+ def from_lines(cls, lines):
+ if len(lines) < 2:
+ raise InvalidItem()
+ lines = [l.rstrip() for l in lines]
+ index = None
+ if cls.TIMESTAMP_SEPARATOR not in lines[0]:
+ index = lines.pop(0)
+ start, end, position = cls.split_timestamps(lines[0])
+ body = '\n'.join(lines[1:])
+ return cls(index, start, end, body, position)
+
+ @classmethod
+ def split_timestamps(cls, line):
+ timestamps = line.split(cls.TIMESTAMP_SEPARATOR)
+ if len(timestamps) != 2:
+ raise InvalidItem()
+ start, end_and_position = timestamps
+ end_and_position = end_and_position.lstrip().split(' ', 1)
+ end = end_and_position[0]
+ position = end_and_position[1] if len(end_and_position) > 1 else ''
+ return (s.strip() for s in (start, end, position))
diff --git a/lib/pysrt/srttime.py b/lib/pysrt/srttime.py
new file mode 100644
index 0000000000000000000000000000000000000000..0481b6870d5431a7aa9d45454937372c21402cf9
--- /dev/null
+++ b/lib/pysrt/srttime.py
@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+"""
+SubRip's time format parser: HH:MM:SS,mmm
+"""
+import re
+from datetime import time
+
+from pysrt.srtexc import InvalidTimeString
+from pysrt.comparablemixin import ComparableMixin
+from pysrt.compat import str, basestring
+
+
+class TimeItemDescriptor(object):
+ # pylint: disable-msg=R0903
+ def __init__(self, ratio, super_ratio=0):
+ self.ratio = int(ratio)
+ self.super_ratio = int(super_ratio)
+
+ def _get_ordinal(self, instance):
+ if self.super_ratio:
+ return instance.ordinal % self.super_ratio
+ return instance.ordinal
+
+ def __get__(self, instance, klass):
+ if instance is None:
+ raise AttributeError
+ return self._get_ordinal(instance) // self.ratio
+
+ def __set__(self, instance, value):
+ part = self._get_ordinal(instance) - instance.ordinal % self.ratio
+ instance.ordinal += value * self.ratio - part
+
+
+class SubRipTime(ComparableMixin):
+ TIME_PATTERN = '%02d:%02d:%02d,%03d'
+ TIME_REPR = 'SubRipTime(%d, %d, %d, %d)'
+ RE_TIME_SEP = re.compile(r'\:|\.|\,')
+ RE_INTEGER = re.compile(r'^(\d+)')
+ SECONDS_RATIO = 1000
+ MINUTES_RATIO = SECONDS_RATIO * 60
+ HOURS_RATIO = MINUTES_RATIO * 60
+
+ hours = TimeItemDescriptor(HOURS_RATIO)
+ minutes = TimeItemDescriptor(MINUTES_RATIO, HOURS_RATIO)
+ seconds = TimeItemDescriptor(SECONDS_RATIO, MINUTES_RATIO)
+ milliseconds = TimeItemDescriptor(1, SECONDS_RATIO)
+
+ def __init__(self, hours=0, minutes=0, seconds=0, milliseconds=0):
+ """
+ SubRipTime(hours, minutes, seconds, milliseconds)
+
+ All arguments are optional and have a default value of 0.
+ """
+ super(SubRipTime, self).__init__()
+ self.ordinal = hours * self.HOURS_RATIO \
+ + minutes * self.MINUTES_RATIO \
+ + seconds * self.SECONDS_RATIO \
+ + milliseconds
+
+ def __repr__(self):
+ return self.TIME_REPR % tuple(self)
+
+ def __str__(self):
+ if self.ordinal < 0:
+ # Represent negative times as zero
+ return str(SubRipTime.from_ordinal(0))
+ return self.TIME_PATTERN % tuple(self)
+
+ def _compare(self, other, method):
+ return super(SubRipTime, self)._compare(self.coerce(other), method)
+
+ def _cmpkey(self):
+ return self.ordinal
+
+ def __add__(self, other):
+ return self.from_ordinal(self.ordinal + self.coerce(other).ordinal)
+
+ def __iadd__(self, other):
+ self.ordinal += self.coerce(other).ordinal
+ return self
+
+ def __sub__(self, other):
+ return self.from_ordinal(self.ordinal - self.coerce(other).ordinal)
+
+ def __isub__(self, other):
+ self.ordinal -= self.coerce(other).ordinal
+ return self
+
+ def __mul__(self, ratio):
+ return self.from_ordinal(int(round(self.ordinal * ratio)))
+
+ def __imul__(self, ratio):
+ self.ordinal = int(round(self.ordinal * ratio))
+ return self
+
+ @classmethod
+ def coerce(cls, other):
+ """
+ Coerce many types to SubRipTime instance.
+ Supported types:
+ - str/unicode
+ - int/long
+ - datetime.time
+ - any iterable
+ - dict
+ """
+ if isinstance(other, SubRipTime):
+ return other
+ if isinstance(other, basestring):
+ return cls.from_string(other)
+ if isinstance(other, int):
+ return cls.from_ordinal(other)
+ if isinstance(other, time):
+ return cls.from_time(other)
+ try:
+ return cls(**other)
+ except TypeError:
+ return cls(*other)
+
+ def __iter__(self):
+ yield self.hours
+ yield self.minutes
+ yield self.seconds
+ yield self.milliseconds
+
+ def shift(self, *args, **kwargs):
+ """
+ shift(hours, minutes, seconds, milliseconds)
+
+ All arguments are optional and have a default value of 0.
+ """
+ if 'ratio' in kwargs:
+ self *= kwargs.pop('ratio')
+ self += self.__class__(*args, **kwargs)
+
+ @classmethod
+ def from_ordinal(cls, ordinal):
+ """
+ int -> SubRipTime corresponding to a total count of milliseconds
+ """
+ return cls(milliseconds=int(ordinal))
+
+ @classmethod
+ def from_string(cls, source):
+ """
+ str/unicode(HH:MM:SS,mmm) -> SubRipTime corresponding to serial
+ raise InvalidTimeString
+ """
+ items = cls.RE_TIME_SEP.split(source)
+ if len(items) != 4:
+ raise InvalidTimeString
+ return cls(*(cls.parse_int(i) for i in items))
+
+ @classmethod
+ def parse_int(cls, digits):
+ try:
+ return int(digits)
+ except ValueError:
+ match = cls.RE_INTEGER.match(digits)
+ if match:
+ return int(match.group())
+ return 0
+
+ @classmethod
+ def from_time(cls, source):
+ """
+ datetime.time -> SubRipTime corresponding to time object
+ """
+ return cls(hours=source.hour, minutes=source.minute,
+ seconds=source.second, milliseconds=source.microsecond // 1000)
+
+ def to_time(self):
+ """
+ Convert SubRipTime instance into a pure datetime.time object
+ """
+ return time(self.hours, self.minutes, self.seconds,
+ self.milliseconds * 1000)
diff --git a/lib/pysrt/version.py b/lib/pysrt/version.py
new file mode 100644
index 0000000000000000000000000000000000000000..f04e34e8569db57b54892d7a7f7f8e9849c7b4fc
--- /dev/null
+++ b/lib/pysrt/version.py
@@ -0,0 +1,2 @@
+VERSION = (1, 0, 1)
+VERSION_STRING = '.'.join(str(i) for i in VERSION)
diff --git a/lib/rtorrent/lib/xmlrpc/requests_transport.py b/lib/rtorrent/lib/xmlrpc/requests_transport.py
index ffa1ea97aa73dc809af652104fa410a4a30a89e7..9a4556773cff66b14e2f99e1af4f0d49157f1bb4 100644
--- a/lib/rtorrent/lib/xmlrpc/requests_transport.py
+++ b/lib/rtorrent/lib/xmlrpc/requests_transport.py
@@ -27,11 +27,11 @@ except ImportError:
import traceback
-from lib import requests
-from lib.requests.exceptions import RequestException
-from lib.requests.auth import HTTPBasicAuth
-from lib.requests.auth import HTTPDigestAuth
-from lib.requests.packages.urllib3 import disable_warnings # @UnresolvedImport
+import requests
+from requests.exceptions import RequestException
+from requests.auth import HTTPBasicAuth
+from requests.auth import HTTPDigestAuth
+from requests.packages.urllib3 import disable_warnings # @UnresolvedImport
class RequestsTransport(xmlrpc_client.Transport):
diff --git a/lib/stevedore/__init__.py b/lib/stevedore/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..93a56b2e54c845dd5a84b4b29d0bcb22ac0e6394
--- /dev/null
+++ b/lib/stevedore/__init__.py
@@ -0,0 +1,36 @@
+# flake8: noqa
+
+__all__ = [
+ 'ExtensionManager',
+ 'EnabledExtensionManager',
+ 'NamedExtensionManager',
+ 'HookManager',
+ 'DriverManager',
+]
+
+from .extension import ExtensionManager
+from .enabled import EnabledExtensionManager
+from .named import NamedExtensionManager
+from .hook import HookManager
+from .driver import DriverManager
+
+import logging
+
+# Configure a NullHandler for our log messages in case
+# the app we're used from does not set up logging.
+LOG = logging.getLogger('stevedore')
+
+if hasattr(logging, 'NullHandler'):
+ LOG.addHandler(logging.NullHandler())
+else:
+ class NullHandler(logging.Handler):
+ def handle(self, record):
+ pass
+
+ def emit(self, record):
+ pass
+
+ def createLock(self):
+ self.lock = None
+
+ LOG.addHandler(NullHandler())
diff --git a/lib/stevedore/dispatch.py b/lib/stevedore/dispatch.py
new file mode 100644
index 0000000000000000000000000000000000000000..226d3ae251925a91b799e47e8aa96a63f2f767f7
--- /dev/null
+++ b/lib/stevedore/dispatch.py
@@ -0,0 +1,216 @@
+import logging
+
+from .enabled import EnabledExtensionManager
+
+LOG = logging.getLogger(__name__)
+
+
+class DispatchExtensionManager(EnabledExtensionManager):
+ """Loads all plugins and filters on execution.
+
+ This is useful for long-running processes that need to pass
+ different inputs to different extensions.
+
+ :param namespace: The namespace for the entry points.
+ :type namespace: str
+ :param check_func: Function to determine which extensions to load.
+ :type check_func: callable
+ :param invoke_on_load: Boolean controlling whether to invoke the
+ object returned by the entry point after the driver is loaded.
+ :type invoke_on_load: bool
+ :param invoke_args: Positional arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_args: tuple
+ :param invoke_kwds: Named arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_kwds: dict
+ :param propagate_map_exceptions: Boolean controlling whether exceptions
+ are propagated up through the map call or whether they are logged and
+ then ignored
+ :type invoke_on_load: bool
+ """
+
+ def map(self, filter_func, func, *args, **kwds):
+ """Iterate over the extensions invoking func() for any where
+ filter_func() returns True.
+
+ The signature of filter_func() should be::
+
+ def filter_func(ext, *args, **kwds):
+ pass
+
+ The first argument to filter_func(), 'ext', is the
+ :class:`~stevedore.extension.Extension`
+ instance. filter_func() should return True if the extension
+ should be invoked for the input arguments.
+
+ The signature for func() should be::
+
+ def func(ext, *args, **kwds):
+ pass
+
+ The first argument to func(), 'ext', is the
+ :class:`~stevedore.extension.Extension` instance.
+
+ Exceptions raised from within func() are propagated up and
+ processing stopped if self.propagate_map_exceptions is True,
+ otherwise they are logged and ignored.
+
+ :param filter_func: Callable to test each extension.
+ :param func: Callable to invoke for each extension.
+ :param args: Variable arguments to pass to func()
+ :param kwds: Keyword arguments to pass to func()
+ :returns: List of values returned from func()
+ """
+ if not self.extensions:
+ # FIXME: Use a more specific exception class here.
+ raise RuntimeError('No %s extensions found' % self.namespace)
+ response = []
+ for e in self.extensions:
+ if filter_func(e, *args, **kwds):
+ self._invoke_one_plugin(response.append, func, e, args, kwds)
+ return response
+
+ def map_method(self, filter_func, method_name, *args, **kwds):
+ """Iterate over the extensions invoking each one's object method called
+ `method_name` for any where filter_func() returns True.
+
+ This is equivalent of using :meth:`map` with func set to
+ `lambda x: x.obj.method_name()`
+ while being more convenient.
+
+ Exceptions raised from within the called method are propagated up
+ and processing stopped if self.propagate_map_exceptions is True,
+ otherwise they are logged and ignored.
+
+ .. versionadded:: 0.12
+
+ :param filter_func: Callable to test each extension.
+ :param method_name: The extension method name to call
+ for each extension.
+ :param args: Variable arguments to pass to method
+ :param kwds: Keyword arguments to pass to method
+ :returns: List of values returned from methods
+ """
+ return self.map(filter_func, self._call_extension_method,
+ method_name, *args, **kwds)
+
+
+class NameDispatchExtensionManager(DispatchExtensionManager):
+ """Loads all plugins and filters on execution.
+
+ This is useful for long-running processes that need to pass
+ different inputs to different extensions and can predict the name
+ of the extensions before calling them.
+
+ The check_func argument should return a boolean, with ``True``
+ indicating that the extension should be loaded and made available
+ and ``False`` indicating that the extension should be ignored.
+
+ :param namespace: The namespace for the entry points.
+ :type namespace: str
+ :param check_func: Function to determine which extensions to load.
+ :type check_func: callable
+ :param invoke_on_load: Boolean controlling whether to invoke the
+ object returned by the entry point after the driver is loaded.
+ :type invoke_on_load: bool
+ :param invoke_args: Positional arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_args: tuple
+ :param invoke_kwds: Named arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_kwds: dict
+ :param propagate_map_exceptions: Boolean controlling whether exceptions
+ are propagated up through the map call or whether they are logged and
+ then ignored
+ :type invoke_on_load: bool
+ :param on_load_failure_callback: Callback function that will be called when
+ a entrypoint can not be loaded. The arguments that will be provided
+ when this is called (when an entrypoint fails to load) are
+ (manager, entrypoint, exception)
+ :type on_load_failure_callback: function
+ :param verify_requirements: Use setuptools to enforce the
+ dependencies of the plugin(s) being loaded. Defaults to False.
+ :type verify_requirements: bool
+
+ """
+
+ def __init__(self, namespace, check_func, invoke_on_load=False,
+ invoke_args=(), invoke_kwds={},
+ propagate_map_exceptions=False,
+ on_load_failure_callback=None,
+ verify_requirements=False):
+ super(NameDispatchExtensionManager, self).__init__(
+ namespace=namespace,
+ check_func=check_func,
+ invoke_on_load=invoke_on_load,
+ invoke_args=invoke_args,
+ invoke_kwds=invoke_kwds,
+ propagate_map_exceptions=propagate_map_exceptions,
+ on_load_failure_callback=on_load_failure_callback,
+ verify_requirements=verify_requirements,
+ )
+
+ def _init_plugins(self, extensions):
+ super(NameDispatchExtensionManager, self)._init_plugins(extensions)
+ self.by_name = dict((e.name, e) for e in self.extensions)
+
+ def map(self, names, func, *args, **kwds):
+ """Iterate over the extensions invoking func() for any where
+ the name is in the given list of names.
+
+ The signature for func() should be::
+
+ def func(ext, *args, **kwds):
+ pass
+
+ The first argument to func(), 'ext', is the
+ :class:`~stevedore.extension.Extension` instance.
+
+ Exceptions raised from within func() are propagated up and
+ processing stopped if self.propagate_map_exceptions is True,
+ otherwise they are logged and ignored.
+
+ :param names: List or set of name(s) of extension(s) to invoke.
+ :param func: Callable to invoke for each extension.
+ :param args: Variable arguments to pass to func()
+ :param kwds: Keyword arguments to pass to func()
+ :returns: List of values returned from func()
+ """
+ response = []
+ for name in names:
+ try:
+ e = self.by_name[name]
+ except KeyError:
+ LOG.debug('Missing extension %r being ignored', name)
+ else:
+ self._invoke_one_plugin(response.append, func, e, args, kwds)
+ return response
+
+ def map_method(self, names, method_name, *args, **kwds):
+ """Iterate over the extensions invoking each one's object method called
+ `method_name` for any where the name is in the given list of names.
+
+ This is equivalent of using :meth:`map` with func set to
+ `lambda x: x.obj.method_name()`
+ while being more convenient.
+
+ Exceptions raised from within the called method are propagated up
+ and processing stopped if self.propagate_map_exceptions is True,
+ otherwise they are logged and ignored.
+
+ .. versionadded:: 0.12
+
+ :param names: List or set of name(s) of extension(s) to invoke.
+ :param method_name: The extension method name
+ to call for each extension.
+ :param args: Variable arguments to pass to method
+ :param kwds: Keyword arguments to pass to method
+ :returns: List of values returned from methods
+ """
+ return self.map(names, self._call_extension_method,
+ method_name, *args, **kwds)
diff --git a/lib/stevedore/driver.py b/lib/stevedore/driver.py
new file mode 100644
index 0000000000000000000000000000000000000000..fedc359bd5dbb7fc84b8e5cd43ad190c13ea4038
--- /dev/null
+++ b/lib/stevedore/driver.py
@@ -0,0 +1,132 @@
+from .named import NamedExtensionManager
+
+
+class DriverManager(NamedExtensionManager):
+ """Load a single plugin with a given name from the namespace.
+
+ :param namespace: The namespace for the entry points.
+ :type namespace: str
+ :param name: The name of the driver to load.
+ :type name: str
+ :param invoke_on_load: Boolean controlling whether to invoke the
+ object returned by the entry point after the driver is loaded.
+ :type invoke_on_load: bool
+ :param invoke_args: Positional arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_args: tuple
+ :param invoke_kwds: Named arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_kwds: dict
+ :param on_load_failure_callback: Callback function that will be called when
+ a entrypoint can not be loaded. The arguments that will be provided
+ when this is called (when an entrypoint fails to load) are
+ (manager, entrypoint, exception)
+ :type on_load_failure_callback: function
+ :param verify_requirements: Use setuptools to enforce the
+ dependencies of the plugin(s) being loaded. Defaults to False.
+ :type verify_requirements: bool
+ """
+
+ def __init__(self, namespace, name,
+ invoke_on_load=False, invoke_args=(), invoke_kwds={},
+ on_load_failure_callback=None,
+ verify_requirements=False):
+ on_load_failure_callback = on_load_failure_callback \
+ or self._default_on_load_failure
+ super(DriverManager, self).__init__(
+ namespace=namespace,
+ names=[name],
+ invoke_on_load=invoke_on_load,
+ invoke_args=invoke_args,
+ invoke_kwds=invoke_kwds,
+ on_load_failure_callback=on_load_failure_callback,
+ verify_requirements=verify_requirements,
+ )
+
+ @staticmethod
+ def _default_on_load_failure(drivermanager, ep, err):
+ raise
+
+ @classmethod
+ def make_test_instance(cls, extension, namespace='TESTING',
+ propagate_map_exceptions=False,
+ on_load_failure_callback=None,
+ verify_requirements=False):
+ """Construct a test DriverManager
+
+ Test instances are passed a list of extensions to work from rather
+ than loading them from entry points.
+
+ :param extension: Pre-configured Extension instance
+ :type extension: :class:`~stevedore.extension.Extension`
+ :param namespace: The namespace for the manager; used only for
+ identification since the extensions are passed in.
+ :type namespace: str
+ :param propagate_map_exceptions: Boolean controlling whether exceptions
+ are propagated up through the map call or whether they are logged
+ and then ignored
+ :type propagate_map_exceptions: bool
+ :param on_load_failure_callback: Callback function that will
+ be called when a entrypoint can not be loaded. The
+ arguments that will be provided when this is called (when
+ an entrypoint fails to load) are (manager, entrypoint,
+ exception)
+ :type on_load_failure_callback: function
+ :param verify_requirements: Use setuptools to enforce the
+ dependencies of the plugin(s) being loaded. Defaults to False.
+ :type verify_requirements: bool
+ :return: The manager instance, initialized for testing
+
+ """
+
+ o = super(DriverManager, cls).make_test_instance(
+ [extension], namespace=namespace,
+ propagate_map_exceptions=propagate_map_exceptions,
+ on_load_failure_callback=on_load_failure_callback,
+ verify_requirements=verify_requirements)
+ return o
+
+ def _init_plugins(self, extensions):
+ super(DriverManager, self)._init_plugins(extensions)
+
+ if not self.extensions:
+ name = self._names[0]
+ raise RuntimeError('No %r driver found, looking for %r' %
+ (self.namespace, name))
+ if len(self.extensions) > 1:
+ discovered_drivers = ','.join(e.entry_point_target
+ for e in self.extensions)
+
+ raise RuntimeError('Multiple %r drivers found: %s' %
+ (self.namespace, discovered_drivers))
+
+ def __call__(self, func, *args, **kwds):
+ """Invokes func() for the single loaded extension.
+
+ The signature for func() should be::
+
+ def func(ext, *args, **kwds):
+ pass
+
+ The first argument to func(), 'ext', is the
+ :class:`~stevedore.extension.Extension` instance.
+
+ Exceptions raised from within func() are logged and ignored.
+
+ :param func: Callable to invoke for each extension.
+ :param args: Variable arguments to pass to func()
+ :param kwds: Keyword arguments to pass to func()
+ :returns: List of values returned from func()
+ """
+ results = self.map(func, *args, **kwds)
+ if results:
+ return results[0]
+
+ @property
+ def driver(self):
+ """Returns the driver being used by this manager.
+ """
+ ext = self.extensions[0]
+ return ext.obj if ext.obj else ext.plugin
diff --git a/lib/stevedore/enabled.py b/lib/stevedore/enabled.py
new file mode 100644
index 0000000000000000000000000000000000000000..d02c74826d8d20d1f89b99ea08fc9b90a3eade46
--- /dev/null
+++ b/lib/stevedore/enabled.py
@@ -0,0 +1,72 @@
+import logging
+
+from .extension import ExtensionManager
+
+
+LOG = logging.getLogger(__name__)
+
+
+class EnabledExtensionManager(ExtensionManager):
+ """Loads only plugins that pass a check function.
+
+ The check_func argument should return a boolean, with ``True``
+ indicating that the extension should be loaded and made available
+ and ``False`` indicating that the extension should be ignored.
+
+ :param namespace: The namespace for the entry points.
+ :type namespace: str
+ :param check_func: Function to determine which extensions to load.
+ :type check_func: callable, taking an :class:`Extension`
+ instance as argument
+ :param invoke_on_load: Boolean controlling whether to invoke the
+ object returned by the entry point after the driver is loaded.
+ :type invoke_on_load: bool
+ :param invoke_args: Positional arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_args: tuple
+ :param invoke_kwds: Named arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_kwds: dict
+ :param propagate_map_exceptions: Boolean controlling whether exceptions
+ are propagated up through the map call or whether they are logged and
+ then ignored
+ :type propagate_map_exceptions: bool
+ :param on_load_failure_callback: Callback function that will be called when
+ a entrypoint can not be loaded. The arguments that will be provided
+ when this is called (when an entrypoint fails to load) are
+ (manager, entrypoint, exception)
+ :type on_load_failure_callback: function
+ :param verify_requirements: Use setuptools to enforce the
+ dependencies of the plugin(s) being loaded. Defaults to False.
+ :type verify_requirements: bool
+
+ """
+
+ def __init__(self, namespace, check_func, invoke_on_load=False,
+ invoke_args=(), invoke_kwds={},
+ propagate_map_exceptions=False,
+ on_load_failure_callback=None,
+ verify_requirements=False,):
+ self.check_func = check_func
+ super(EnabledExtensionManager, self).__init__(
+ namespace,
+ invoke_on_load=invoke_on_load,
+ invoke_args=invoke_args,
+ invoke_kwds=invoke_kwds,
+ propagate_map_exceptions=propagate_map_exceptions,
+ on_load_failure_callback=on_load_failure_callback,
+ verify_requirements=verify_requirements,
+ )
+
+ def _load_one_plugin(self, ep, invoke_on_load, invoke_args, invoke_kwds,
+ verify_requirements):
+ ext = super(EnabledExtensionManager, self)._load_one_plugin(
+ ep, invoke_on_load, invoke_args, invoke_kwds,
+ verify_requirements,
+ )
+ if ext and not self.check_func(ext):
+ LOG.debug('ignoring extension %r', ep.name)
+ return None
+ return ext
diff --git a/lib/stevedore/example/__init__.py b/lib/stevedore/example/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lib/stevedore/example/base.py b/lib/stevedore/example/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c8ca4ca361a7bb19504feaf68089ff7228029fc
--- /dev/null
+++ b/lib/stevedore/example/base.py
@@ -0,0 +1,22 @@
+import abc
+
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class FormatterBase(object):
+ """Base class for example plugin used in the tutoral.
+ """
+
+ def __init__(self, max_width=60):
+ self.max_width = max_width
+
+ @abc.abstractmethod
+ def format(self, data):
+ """Format the data and return unicode text.
+
+ :param data: A dictionary with string keys and simple types as
+ values.
+ :type data: dict(str:?)
+ :returns: Iterable producing the formatted text.
+ """
diff --git a/lib/stevedore/example/fields.py b/lib/stevedore/example/fields.py
new file mode 100644
index 0000000000000000000000000000000000000000..f5c8e194a46bc87cc89ac54c0797d54c404cb4c3
--- /dev/null
+++ b/lib/stevedore/example/fields.py
@@ -0,0 +1,36 @@
+import textwrap
+
+from stevedore.example import base
+
+
+class FieldList(base.FormatterBase):
+ """Format values as a reStructuredText field list.
+
+ For example::
+
+ : name1 : value
+ : name2 : value
+ : name3 : a long value
+ will be wrapped with
+ a hanging indent
+ """
+
+ def format(self, data):
+ """Format the data and return unicode text.
+
+ :param data: A dictionary with string keys and simple types as
+ values.
+ :type data: dict(str:?)
+ """
+ for name, value in sorted(data.items()):
+ full_text = ': {name} : {value}'.format(
+ name=name,
+ value=value,
+ )
+ wrapped_text = textwrap.fill(
+ full_text,
+ initial_indent='',
+ subsequent_indent=' ',
+ width=self.max_width,
+ )
+ yield wrapped_text + '\n'
diff --git a/lib/stevedore/example/load_as_driver.py b/lib/stevedore/example/load_as_driver.py
new file mode 100644
index 0000000000000000000000000000000000000000..d8c47f5f61ddb79111143af715871d9ac2ddfa1c
--- /dev/null
+++ b/lib/stevedore/example/load_as_driver.py
@@ -0,0 +1,37 @@
+from __future__ import print_function
+
+import argparse
+
+from stevedore import driver
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ 'format',
+ nargs='?',
+ default='simple',
+ help='the output format',
+ )
+ parser.add_argument(
+ '--width',
+ default=60,
+ type=int,
+ help='maximum output width for text',
+ )
+ parsed_args = parser.parse_args()
+
+ data = {
+ 'a': 'A',
+ 'b': 'B',
+ 'long': 'word ' * 80,
+ }
+
+ mgr = driver.DriverManager(
+ namespace='stevedore.example.formatter',
+ name=parsed_args.format,
+ invoke_on_load=True,
+ invoke_args=(parsed_args.width,),
+ )
+ for chunk in mgr.driver.format(data):
+ print(chunk, end='')
diff --git a/lib/stevedore/example/load_as_extension.py b/lib/stevedore/example/load_as_extension.py
new file mode 100644
index 0000000000000000000000000000000000000000..436206a365dfd6bde7b4b6f284ffa937826582e2
--- /dev/null
+++ b/lib/stevedore/example/load_as_extension.py
@@ -0,0 +1,39 @@
+from __future__ import print_function
+
+import argparse
+
+from stevedore import extension
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '--width',
+ default=60,
+ type=int,
+ help='maximum output width for text',
+ )
+ parsed_args = parser.parse_args()
+
+ data = {
+ 'a': 'A',
+ 'b': 'B',
+ 'long': 'word ' * 80,
+ }
+
+ mgr = extension.ExtensionManager(
+ namespace='stevedore.example.formatter',
+ invoke_on_load=True,
+ invoke_args=(parsed_args.width,),
+ )
+
+ def format_data(ext, data):
+ return (ext.name, ext.obj.format(data))
+
+ results = mgr.map(format_data, data)
+
+ for name, result in results:
+ print('Formatter: {0}'.format(name))
+ for chunk in result:
+ print(chunk, end='')
+ print('')
diff --git a/lib/stevedore/example/setup.py b/lib/stevedore/example/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..afc7789c6e4005a83ad8640a1fc5d4feff85eca4
--- /dev/null
+++ b/lib/stevedore/example/setup.py
@@ -0,0 +1,46 @@
+from setuptools import setup, find_packages
+
+setup(
+ name='stevedore-examples',
+ version='1.0',
+
+ description='Demonstration package for stevedore',
+
+ author='Doug Hellmann',
+ author_email='doug.hellmann@dreamhost.com',
+
+ url='https://github.com/dreamhost/stevedore',
+ download_url='https://github.com/dreamhost/stevedore/tarball/master',
+
+ classifiers=['Development Status :: 3 - Alpha',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.6',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.4',
+ 'Intended Audience :: Developers',
+ 'Environment :: Console',
+ ],
+
+ platforms=['Any'],
+
+ scripts=[],
+
+ provides=['stevedore.examples',
+ ],
+
+ packages=find_packages(),
+ include_package_data=True,
+
+ entry_points={
+ 'stevedore.example.formatter': [
+ 'simple = stevedore.example.simple:Simple',
+ 'field = stevedore.example.fields:FieldList',
+ 'plain = stevedore.example.simple:Simple',
+ ],
+ },
+
+ zip_safe=False,
+)
diff --git a/lib/stevedore/example/simple.py b/lib/stevedore/example/simple.py
new file mode 100644
index 0000000000000000000000000000000000000000..1cad96af3ef6f5fe87c9a36cfc5a37633cc433fa
--- /dev/null
+++ b/lib/stevedore/example/simple.py
@@ -0,0 +1,20 @@
+from stevedore.example import base
+
+
+class Simple(base.FormatterBase):
+ """A very basic formatter.
+ """
+
+ def format(self, data):
+ """Format the data and return unicode text.
+
+ :param data: A dictionary with string keys and simple types as
+ values.
+ :type data: dict(str:?)
+ """
+ for name, value in sorted(data.items()):
+ line = '{name} = {value}\n'.format(
+ name=name,
+ value=value,
+ )
+ yield line
diff --git a/lib/stevedore/extension.py b/lib/stevedore/extension.py
new file mode 100644
index 0000000000000000000000000000000000000000..5da97c5d93efe7522fbaa5a28cbfbe9896bf3cfc
--- /dev/null
+++ b/lib/stevedore/extension.py
@@ -0,0 +1,288 @@
+"""ExtensionManager
+"""
+
+import pkg_resources
+
+import logging
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Extension(object):
+ """Book-keeping object for tracking extensions.
+
+ The arguments passed to the constructor are saved as attributes of
+ the instance using the same names, and can be accessed by the
+ callables passed to :meth:`map` or when iterating over an
+ :class:`ExtensionManager` directly.
+
+ :param name: The entry point name.
+ :type name: str
+ :param entry_point: The EntryPoint instance returned by
+ :mod:`pkg_resources`.
+ :type entry_point: EntryPoint
+ :param plugin: The value returned by entry_point.load()
+ :param obj: The object returned by ``plugin(*args, **kwds)`` if the
+ manager invoked the extension on load.
+
+ """
+
+ def __init__(self, name, entry_point, plugin, obj):
+ self.name = name
+ self.entry_point = entry_point
+ self.plugin = plugin
+ self.obj = obj
+
+ @property
+ def entry_point_target(self):
+ """The module and attribute referenced by this extension's entry_point.
+
+ :return: A string representation of the target of the entry point in
+ 'dotted.module:object' format.
+ """
+ return '%s:%s' % (self.entry_point.module_name,
+ self.entry_point.attrs[0])
+
+
+class ExtensionManager(object):
+ """Base class for all of the other managers.
+
+ :param namespace: The namespace for the entry points.
+ :type namespace: str
+ :param invoke_on_load: Boolean controlling whether to invoke the
+ object returned by the entry point after the driver is loaded.
+ :type invoke_on_load: bool
+ :param invoke_args: Positional arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_args: tuple
+ :param invoke_kwds: Named arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_kwds: dict
+ :param propagate_map_exceptions: Boolean controlling whether exceptions
+ are propagated up through the map call or whether they are logged and
+ then ignored
+ :type propagate_map_exceptions: bool
+ :param on_load_failure_callback: Callback function that will be called when
+ a entrypoint can not be loaded. The arguments that will be provided
+ when this is called (when an entrypoint fails to load) are
+ (manager, entrypoint, exception)
+ :type on_load_failure_callback: function
+ :param verify_requirements: Use setuptools to enforce the
+ dependencies of the plugin(s) being loaded. Defaults to False.
+ :type verify_requirements: bool
+ """
+
+ def __init__(self, namespace,
+ invoke_on_load=False,
+ invoke_args=(),
+ invoke_kwds={},
+ propagate_map_exceptions=False,
+ on_load_failure_callback=None,
+ verify_requirements=False):
+ self._init_attributes(
+ namespace,
+ propagate_map_exceptions=propagate_map_exceptions,
+ on_load_failure_callback=on_load_failure_callback)
+ extensions = self._load_plugins(invoke_on_load,
+ invoke_args,
+ invoke_kwds,
+ verify_requirements)
+ self._init_plugins(extensions)
+
+ @classmethod
+ def make_test_instance(cls, extensions, namespace='TESTING',
+ propagate_map_exceptions=False,
+ on_load_failure_callback=None,
+ verify_requirements=False):
+ """Construct a test ExtensionManager
+
+ Test instances are passed a list of extensions to work from rather
+ than loading them from entry points.
+
+ :param extensions: Pre-configured Extension instances to use
+ :type extensions: list of :class:`~stevedore.extension.Extension`
+ :param namespace: The namespace for the manager; used only for
+ identification since the extensions are passed in.
+ :type namespace: str
+ :param propagate_map_exceptions: When calling map, controls whether
+ exceptions are propagated up through the map call or whether they
+ are logged and then ignored
+ :type propagate_map_exceptions: bool
+ :param on_load_failure_callback: Callback function that will
+ be called when a entrypoint can not be loaded. The
+ arguments that will be provided when this is called (when
+ an entrypoint fails to load) are (manager, entrypoint,
+ exception)
+ :type on_load_failure_callback: function
+ :param verify_requirements: Use setuptools to enforce the
+ dependencies of the plugin(s) being loaded. Defaults to False.
+ :type verify_requirements: bool
+ :return: The manager instance, initialized for testing
+
+ """
+
+ o = cls.__new__(cls)
+ o._init_attributes(namespace,
+ propagate_map_exceptions=propagate_map_exceptions,
+ on_load_failure_callback=on_load_failure_callback)
+ o._init_plugins(extensions)
+ return o
+
+ def _init_attributes(self, namespace, propagate_map_exceptions=False,
+ on_load_failure_callback=None):
+ self.namespace = namespace
+ self.propagate_map_exceptions = propagate_map_exceptions
+ self._on_load_failure_callback = on_load_failure_callback
+
+ def _init_plugins(self, extensions):
+ self.extensions = extensions
+ self._extensions_by_name = None
+
+ ENTRY_POINT_CACHE = {}
+
+ def _find_entry_points(self, namespace):
+ if namespace not in self.ENTRY_POINT_CACHE:
+ eps = list(pkg_resources.iter_entry_points(namespace))
+ self.ENTRY_POINT_CACHE[namespace] = eps
+ return self.ENTRY_POINT_CACHE[namespace]
+
+ def _load_plugins(self, invoke_on_load, invoke_args, invoke_kwds,
+ verify_requirements):
+ extensions = []
+ for ep in self._find_entry_points(self.namespace):
+ LOG.debug('found extension %r', ep)
+ try:
+ ext = self._load_one_plugin(ep,
+ invoke_on_load,
+ invoke_args,
+ invoke_kwds,
+ verify_requirements,
+ )
+ if ext:
+ extensions.append(ext)
+ except (KeyboardInterrupt, AssertionError):
+ raise
+ except Exception as err:
+ if self._on_load_failure_callback is not None:
+ self._on_load_failure_callback(self, ep, err)
+ else:
+ LOG.error('Could not load %r: %s', ep.name, err)
+ LOG.exception(err)
+ return extensions
+
+ def _load_one_plugin(self, ep, invoke_on_load, invoke_args, invoke_kwds,
+ verify_requirements):
+ # NOTE(dhellmann): Using require=False is deprecated in
+ # setuptools 11.3.
+ if hasattr(ep, 'resolve') and hasattr(ep, 'require'):
+ if verify_requirements:
+ ep.require()
+ plugin = ep.resolve()
+ else:
+ plugin = ep.load(require=verify_requirements)
+ if invoke_on_load:
+ obj = plugin(*invoke_args, **invoke_kwds)
+ else:
+ obj = None
+ return Extension(ep.name, ep, plugin, obj)
+
+ def names(self):
+ "Returns the names of the discovered extensions"
+ # We want to return the names of the extensions in the order
+ # they would be used by map(), since some subclasses change
+ # that order.
+ return [e.name for e in self.extensions]
+
+ def map(self, func, *args, **kwds):
+ """Iterate over the extensions invoking func() for each.
+
+ The signature for func() should be::
+
+ def func(ext, *args, **kwds):
+ pass
+
+ The first argument to func(), 'ext', is the
+ :class:`~stevedore.extension.Extension` instance.
+
+ Exceptions raised from within func() are propagated up and
+ processing stopped if self.propagate_map_exceptions is True,
+ otherwise they are logged and ignored.
+
+ :param func: Callable to invoke for each extension.
+ :param args: Variable arguments to pass to func()
+ :param kwds: Keyword arguments to pass to func()
+ :returns: List of values returned from func()
+ """
+ if not self.extensions:
+ # FIXME: Use a more specific exception class here.
+ raise RuntimeError('No %s extensions found' % self.namespace)
+ response = []
+ for e in self.extensions:
+ self._invoke_one_plugin(response.append, func, e, args, kwds)
+ return response
+
+ @staticmethod
+ def _call_extension_method(extension, method_name, *args, **kwds):
+ return getattr(extension.obj, method_name)(*args, **kwds)
+
+ def map_method(self, method_name, *args, **kwds):
+ """Iterate over the extensions invoking a method by name.
+
+ This is equivalent of using :meth:`map` with func set to
+ `lambda x: x.obj.method_name()`
+ while being more convenient.
+
+ Exceptions raised from within the called method are propagated up
+ and processing stopped if self.propagate_map_exceptions is True,
+ otherwise they are logged and ignored.
+
+ .. versionadded:: 0.12
+
+ :param method_name: The extension method name
+ to call for each extension.
+ :param args: Variable arguments to pass to method
+ :param kwds: Keyword arguments to pass to method
+ :returns: List of values returned from methods
+ """
+ return self.map(self._call_extension_method,
+ method_name, *args, **kwds)
+
+ def _invoke_one_plugin(self, response_callback, func, e, args, kwds):
+ try:
+ response_callback(func(e, *args, **kwds))
+ except Exception as err:
+ if self.propagate_map_exceptions:
+ raise
+ else:
+ LOG.error('error calling %r: %s', e.name, err)
+ LOG.exception(err)
+
+ def __iter__(self):
+ """Produce iterator for the manager.
+
+ Iterating over an ExtensionManager produces the :class:`Extension`
+ instances in the order they would be invoked.
+ """
+ return iter(self.extensions)
+
+ def __getitem__(self, name):
+ """Return the named extension.
+
+ Accessing an ExtensionManager as a dictionary (``em['name']``)
+ produces the :class:`Extension` instance with the
+ specified name.
+ """
+ if self._extensions_by_name is None:
+ d = {}
+ for e in self.extensions:
+ d[e.name] = e
+ self._extensions_by_name = d
+ return self._extensions_by_name[name]
+
+ def __contains__(self, name):
+ """Return true if name is in list of enabled extensions.
+ """
+ return any(extension.name == name for extension in self.extensions)
diff --git a/lib/stevedore/hook.py b/lib/stevedore/hook.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2570db0f31391ef33322dde3be2d95efebb09d0
--- /dev/null
+++ b/lib/stevedore/hook.py
@@ -0,0 +1,64 @@
+from .named import NamedExtensionManager
+
+
+class HookManager(NamedExtensionManager):
+ """Coordinate execution of multiple extensions using a common name.
+
+ :param namespace: The namespace for the entry points.
+ :type namespace: str
+ :param name: The name of the hooks to load.
+ :type name: str
+ :param invoke_on_load: Boolean controlling whether to invoke the
+ object returned by the entry point after the driver is loaded.
+ :type invoke_on_load: bool
+ :param invoke_args: Positional arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_args: tuple
+ :param invoke_kwds: Named arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_kwds: dict
+ :param on_load_failure_callback: Callback function that will be called when
+ a entrypoint can not be loaded. The arguments that will be provided
+ when this is called (when an entrypoint fails to load) are
+ (manager, entrypoint, exception)
+ :type on_load_failure_callback: function
+ :param verify_requirements: Use setuptools to enforce the
+ dependencies of the plugin(s) being loaded. Defaults to False.
+ :type verify_requirements: bool
+ """
+
+ def __init__(self, namespace, name,
+ invoke_on_load=False, invoke_args=(), invoke_kwds={},
+ on_load_failure_callback=None,
+ verify_requirements=False):
+ super(HookManager, self).__init__(
+ namespace,
+ [name],
+ invoke_on_load=invoke_on_load,
+ invoke_args=invoke_args,
+ invoke_kwds=invoke_kwds,
+ on_load_failure_callback=on_load_failure_callback,
+ verify_requirements=verify_requirements,
+ )
+
+ def _init_attributes(self, namespace, names, name_order=False,
+ propagate_map_exceptions=False,
+ on_load_failure_callback=None):
+ super(HookManager, self)._init_attributes(
+ namespace, names,
+ propagate_map_exceptions=propagate_map_exceptions,
+ on_load_failure_callback=on_load_failure_callback)
+ self._name = names[0]
+
+ def __getitem__(self, name):
+ """Return the named extensions.
+
+ Accessing a HookManager as a dictionary (``em['name']``)
+ produces a list of the :class:`Extension` instance(s) with the
+ specified name, in the order they would be invoked by map().
+ """
+ if name != self._name:
+ raise KeyError(name)
+ return self.extensions
diff --git a/lib/stevedore/named.py b/lib/stevedore/named.py
new file mode 100644
index 0000000000000000000000000000000000000000..18fe235b2c7dec5ae6ba2e5fb1aec8fa02cbee14
--- /dev/null
+++ b/lib/stevedore/named.py
@@ -0,0 +1,124 @@
+from .extension import ExtensionManager
+
+
+class NamedExtensionManager(ExtensionManager):
+ """Loads only the named extensions.
+
+ This is useful for explicitly enabling extensions in a
+ configuration file, for example.
+
+ :param namespace: The namespace for the entry points.
+ :type namespace: str
+ :param names: The names of the extensions to load.
+ :type names: list(str)
+ :param invoke_on_load: Boolean controlling whether to invoke the
+ object returned by the entry point after the driver is loaded.
+ :type invoke_on_load: bool
+ :param invoke_args: Positional arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_args: tuple
+ :param invoke_kwds: Named arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_kwds: dict
+ :param name_order: If true, sort the loaded extensions to match the
+ order used in ``names``.
+ :type name_order: bool
+ :param propagate_map_exceptions: Boolean controlling whether exceptions
+ are propagated up through the map call or whether they are logged and
+ then ignored
+ :type propagate_map_exceptions: bool
+ :param on_load_failure_callback: Callback function that will be called when
+ a entrypoint can not be loaded. The arguments that will be provided
+ when this is called (when an entrypoint fails to load) are
+ (manager, entrypoint, exception)
+ :type on_load_failure_callback: function
+ :param verify_requirements: Use setuptools to enforce the
+ dependencies of the plugin(s) being loaded. Defaults to False.
+ :type verify_requirements: bool
+
+ """
+
+ def __init__(self, namespace, names,
+ invoke_on_load=False, invoke_args=(), invoke_kwds={},
+ name_order=False, propagate_map_exceptions=False,
+ on_load_failure_callback=None,
+ verify_requirements=False):
+ self._init_attributes(
+ namespace, names, name_order=name_order,
+ propagate_map_exceptions=propagate_map_exceptions,
+ on_load_failure_callback=on_load_failure_callback)
+ extensions = self._load_plugins(invoke_on_load,
+ invoke_args,
+ invoke_kwds,
+ verify_requirements)
+ self._init_plugins(extensions)
+
+ @classmethod
+ def make_test_instance(cls, extensions, namespace='TESTING',
+ propagate_map_exceptions=False,
+ on_load_failure_callback=None,
+ verify_requirements=False):
+ """Construct a test NamedExtensionManager
+
+ Test instances are passed a list of extensions to use rather than
+ loading them from entry points.
+
+ :param extensions: Pre-configured Extension instances
+ :type extensions: list of :class:`~stevedore.extension.Extension`
+ :param namespace: The namespace for the manager; used only for
+ identification since the extensions are passed in.
+ :type namespace: str
+ :param propagate_map_exceptions: Boolean controlling whether exceptions
+ are propagated up through the map call or whether they are logged
+ and then ignored
+ :type propagate_map_exceptions: bool
+ :param on_load_failure_callback: Callback function that will
+ be called when a entrypoint can not be loaded. The
+ arguments that will be provided when this is called (when
+ an entrypoint fails to load) are (manager, entrypoint,
+ exception)
+ :type on_load_failure_callback: function
+ :param verify_requirements: Use setuptools to enforce the
+ dependencies of the plugin(s) being loaded. Defaults to False.
+ :type verify_requirements: bool
+ :return: The manager instance, initialized for testing
+
+ """
+
+ o = cls.__new__(cls)
+ names = [e.name for e in extensions]
+ o._init_attributes(namespace, names,
+ propagate_map_exceptions=propagate_map_exceptions,
+ on_load_failure_callback=on_load_failure_callback)
+ o._init_plugins(extensions)
+ return o
+
+ def _init_attributes(self, namespace, names, name_order=False,
+ propagate_map_exceptions=False,
+ on_load_failure_callback=None):
+ super(NamedExtensionManager, self)._init_attributes(
+ namespace, propagate_map_exceptions=propagate_map_exceptions,
+ on_load_failure_callback=on_load_failure_callback)
+
+ self._names = names
+ self._name_order = name_order
+
+ def _init_plugins(self, extensions):
+ super(NamedExtensionManager, self)._init_plugins(extensions)
+
+ if self._name_order:
+ self.extensions = [self[n] for n in self._names]
+
+ def _load_one_plugin(self, ep, invoke_on_load, invoke_args, invoke_kwds,
+ verify_requirements):
+ # Check the name before going any further to prevent
+ # undesirable code from being loaded at all if we are not
+ # going to use it.
+ if ep.name not in self._names:
+ return None
+ return super(NamedExtensionManager, self)._load_one_plugin(
+ ep, invoke_on_load, invoke_args, invoke_kwds,
+ verify_requirements,
+ )
diff --git a/lib/stevedore/sphinxext.py b/lib/stevedore/sphinxext.py
new file mode 100644
index 0000000000000000000000000000000000000000..524f9c93c8d17cec963e829f8f92c7c302e0bf7d
--- /dev/null
+++ b/lib/stevedore/sphinxext.py
@@ -0,0 +1,108 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import unicode_literals
+
+import inspect
+
+from docutils import nodes
+from docutils.parsers import rst
+from docutils.parsers.rst import directives
+from docutils.statemachine import ViewList
+from sphinx.util.nodes import nested_parse_with_titles
+
+from stevedore import extension
+
+
+def _get_docstring(plugin):
+ return inspect.getdoc(plugin) or ''
+
+
+def _simple_list(mgr):
+ for name in sorted(mgr.names()):
+ ext = mgr[name]
+ doc = _get_docstring(ext.plugin) or '\n'
+ summary = doc.splitlines()[0].strip()
+ yield('* %s -- %s' % (ext.name, summary),
+ ext.entry_point.module_name)
+
+
+def _detailed_list(mgr, over='', under='-'):
+ for name in sorted(mgr.names()):
+ ext = mgr[name]
+ if over:
+ yield (over * len(ext.name), ext.entry_point.module_name)
+ yield (ext.name, ext.entry_point.module_name)
+ if under:
+ yield (under * len(ext.name), ext.entry_point.module_name)
+ yield ('\n', ext.entry_point.module_name)
+ doc = _get_docstring(ext.plugin)
+ if doc:
+ yield (doc, ext.entry_point.module_name)
+ else:
+ yield ('.. warning:: No documentation found in %s'
+ % ext.entry_point,
+ ext.entry_point.module_name)
+ yield ('\n', ext.entry_point.module_name)
+
+
+class ListPluginsDirective(rst.Directive):
+ """Present a simple list of the plugins in a namespace."""
+
+ option_spec = {
+ 'class': directives.class_option,
+ 'detailed': directives.flag,
+ 'overline-style': directives.single_char_or_unicode,
+ 'underline-style': directives.single_char_or_unicode,
+ }
+
+ has_content = True
+
+ def run(self):
+ env = self.state.document.settings.env
+ app = env.app
+
+ namespace = ' '.join(self.content).strip()
+ app.info('documenting plugins from %r' % namespace)
+ overline_style = self.options.get('overline-style', '')
+ underline_style = self.options.get('underline-style', '=')
+
+ def report_load_failure(mgr, ep, err):
+ app.warn(u'Failed to load %s: %s' % (ep.module_name, err))
+
+ mgr = extension.ExtensionManager(
+ namespace,
+ on_load_failure_callback=report_load_failure,
+ )
+
+ result = ViewList()
+
+ if 'detailed' in self.options:
+ data = _detailed_list(
+ mgr, over=overline_style, under=underline_style)
+ else:
+ data = _simple_list(mgr)
+ for text, source in data:
+ for line in text.splitlines():
+ result.append(line, source)
+
+ # Parse what we have into a new section.
+ node = nodes.section()
+ node.document = self.state.document
+ nested_parse_with_titles(self.state, result, node)
+
+ return node.children
+
+
+def setup(app):
+ app.info('loading stevedore.sphinxext')
+ app.add_directive('list-plugins', ListPluginsDirective)
diff --git a/lib/stevedore/tests/__init__.py b/lib/stevedore/tests/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lib/stevedore/tests/extension_unimportable.py b/lib/stevedore/tests/extension_unimportable.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lib/stevedore/tests/manager.py b/lib/stevedore/tests/manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..28c37321762dfb549872083f64d120f46d080e9c
--- /dev/null
+++ b/lib/stevedore/tests/manager.py
@@ -0,0 +1,59 @@
+"""TestExtensionManager
+
+Extension manager used only for testing.
+"""
+
+import logging
+import warnings
+
+from stevedore import extension
+
+
+LOG = logging.getLogger(__name__)
+
+
+class TestExtensionManager(extension.ExtensionManager):
+ """ExtensionManager that is explicitly initialized for tests.
+
+ .. deprecated:: 0.13
+
+ Use the :func:`make_test_instance` class method of the class
+ being replaced by the test instance instead of using this class
+ directly.
+
+ :param extensions: Pre-configured Extension instances to use
+ instead of loading them from entry points.
+ :type extensions: list of :class:`~stevedore.extension.Extension`
+ :param namespace: The namespace for the entry points.
+ :type namespace: str
+ :param invoke_on_load: Boolean controlling whether to invoke the
+ object returned by the entry point after the driver is loaded.
+ :type invoke_on_load: bool
+ :param invoke_args: Positional arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_args: tuple
+ :param invoke_kwds: Named arguments to pass when invoking
+ the object returned by the entry point. Only used if invoke_on_load
+ is True.
+ :type invoke_kwds: dict
+
+ """
+
+ def __init__(self, extensions,
+ namespace='test',
+ invoke_on_load=False,
+ invoke_args=(),
+ invoke_kwds={}):
+ super(TestExtensionManager, self).__init__(namespace,
+ invoke_on_load,
+ invoke_args,
+ invoke_kwds,
+ )
+ self.extensions = extensions
+ warnings.warn(
+ 'TestExtesionManager has been replaced by make_test_instance()',
+ DeprecationWarning)
+
+ def _load_plugins(self, *args, **kwds):
+ return []
diff --git a/lib/stevedore/tests/test_callback.py b/lib/stevedore/tests/test_callback.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e9f5b18ac5ea96ea0b3074b51efda4bf930e5a7
--- /dev/null
+++ b/lib/stevedore/tests/test_callback.py
@@ -0,0 +1,25 @@
+"""Tests for failure loading callback
+"""
+from testtools.matchers import GreaterThan
+
+from stevedore import extension
+from stevedore.tests import utils
+
+
+class TestCallback(utils.TestCase):
+ def test_extension_failure_custom_callback(self):
+ errors = []
+
+ def failure_callback(manager, entrypoint, error):
+ errors.append((manager, entrypoint, error))
+
+ em = extension.ExtensionManager('stevedore.test.extension',
+ invoke_on_load=True,
+ on_load_failure_callback=
+ failure_callback)
+ extensions = list(em.extensions)
+ self.assertThat(len(extensions), GreaterThan(0))
+ self.assertEqual(len(errors), 2)
+ for manager, entrypoint, error in errors:
+ self.assertIs(manager, em)
+ self.assertIsInstance(error, (IOError, ImportError))
diff --git a/lib/stevedore/tests/test_dispatch.py b/lib/stevedore/tests/test_dispatch.py
new file mode 100644
index 0000000000000000000000000000000000000000..1cd3bd43cd9bda3a9d2d583f1f36d97be18edbb1
--- /dev/null
+++ b/lib/stevedore/tests/test_dispatch.py
@@ -0,0 +1,91 @@
+from stevedore.tests import utils
+from stevedore import dispatch
+
+
+def check_dispatch(ep, *args, **kwds):
+ return ep.name == 't2'
+
+
+class TestDispatch(utils.TestCase):
+ def check_dispatch(ep, *args, **kwds):
+ return ep.name == 't2'
+
+ def test_dispatch(self):
+
+ def invoke(ep, *args, **kwds):
+ return (ep.name, args, kwds)
+
+ em = dispatch.DispatchExtensionManager('stevedore.test.extension',
+ lambda *args, **kwds: True,
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+ self.assertEqual(len(em.extensions), 2)
+ self.assertEqual(set(em.names()), set(['t1', 't2']))
+
+ results = em.map(check_dispatch,
+ invoke,
+ 'first',
+ named='named value',
+ )
+ expected = [('t2', ('first',), {'named': 'named value'})]
+ self.assertEqual(results, expected)
+
+ def test_dispatch_map_method(self):
+ em = dispatch.DispatchExtensionManager('stevedore.test.extension',
+ lambda *args, **kwds: True,
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+
+ results = em.map_method(check_dispatch, 'get_args_and_data', 'first')
+ self.assertEqual(results, [(('a',), {'b': 'B'}, 'first')])
+
+ def test_name_dispatch(self):
+
+ def invoke(ep, *args, **kwds):
+ return (ep.name, args, kwds)
+
+ em = dispatch.NameDispatchExtensionManager('stevedore.test.extension',
+ lambda *args, **kwds: True,
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+ self.assertEqual(len(em.extensions), 2)
+ self.assertEqual(set(em.names()), set(['t1', 't2']))
+
+ results = em.map(['t2'], invoke, 'first', named='named value',)
+ expected = [('t2', ('first',), {'named': 'named value'})]
+ self.assertEqual(results, expected)
+
+ def test_name_dispatch_ignore_missing(self):
+
+ def invoke(ep, *args, **kwds):
+ return (ep.name, args, kwds)
+
+ em = dispatch.NameDispatchExtensionManager(
+ 'stevedore.test.extension',
+ lambda *args, **kwds: True,
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+
+ results = em.map(['t3', 't1'], invoke, 'first', named='named value',)
+ expected = [('t1', ('first',), {'named': 'named value'})]
+ self.assertEqual(results, expected)
+
+ def test_name_dispatch_map_method(self):
+ em = dispatch.NameDispatchExtensionManager(
+ 'stevedore.test.extension',
+ lambda *args, **kwds: True,
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+
+ results = em.map_method(['t3', 't1'], 'get_args_and_data', 'first')
+ self.assertEqual(results, [(('a',), {'b': 'B'}, 'first')])
diff --git a/lib/stevedore/tests/test_driver.py b/lib/stevedore/tests/test_driver.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff9c3eac05744ca7acd72e72c3e4b055cdf30190
--- /dev/null
+++ b/lib/stevedore/tests/test_driver.py
@@ -0,0 +1,76 @@
+"""Tests for stevedore.extension
+"""
+
+import pkg_resources
+
+from stevedore import driver
+from stevedore import extension
+from stevedore.tests import test_extension
+from stevedore.tests import utils
+
+
+class TestCallback(utils.TestCase):
+ def test_detect_plugins(self):
+ em = driver.DriverManager('stevedore.test.extension', 't1')
+ names = sorted(em.names())
+ self.assertEqual(names, ['t1'])
+
+ def test_call(self):
+ def invoke(ext, *args, **kwds):
+ return (ext.name, args, kwds)
+ em = driver.DriverManager('stevedore.test.extension', 't1')
+ result = em(invoke, 'a', b='C')
+ self.assertEqual(result, ('t1', ('a',), {'b': 'C'}))
+
+ def test_driver_property_not_invoked_on_load(self):
+ em = driver.DriverManager('stevedore.test.extension', 't1',
+ invoke_on_load=False)
+ d = em.driver
+ self.assertIs(d, test_extension.FauxExtension)
+
+ def test_driver_property_invoked_on_load(self):
+ em = driver.DriverManager('stevedore.test.extension', 't1',
+ invoke_on_load=True)
+ d = em.driver
+ self.assertIsInstance(d, test_extension.FauxExtension)
+
+ def test_no_drivers(self):
+ try:
+ driver.DriverManager('stevedore.test.extension.none', 't1')
+ except RuntimeError as err:
+ self.assertIn("No 'stevedore.test.extension.none' driver found",
+ str(err))
+
+ def test_bad_driver(self):
+ try:
+ driver.DriverManager('stevedore.test.extension', 'e2')
+ except ImportError:
+ pass
+ else:
+ self.assertEquals(False, "No error raised")
+
+ def test_multiple_drivers(self):
+ # The idea for this test was contributed by clayg:
+ # https://gist.github.com/clayg/6311348
+ extensions = [
+ extension.Extension(
+ 'backend',
+ pkg_resources.EntryPoint.parse('backend = pkg1:driver'),
+ 'pkg backend',
+ None,
+ ),
+ extension.Extension(
+ 'backend',
+ pkg_resources.EntryPoint.parse('backend = pkg2:driver'),
+ 'pkg backend',
+ None,
+ ),
+ ]
+ try:
+ dm = driver.DriverManager.make_test_instance(extensions[0])
+ # Call the initialization code that verifies the extension
+ dm._init_plugins(extensions)
+ except RuntimeError as err:
+ self.assertIn("Multiple", str(err))
+ else:
+ self.fail('Should have had an error')
diff --git a/lib/stevedore/tests/test_enabled.py b/lib/stevedore/tests/test_enabled.py
new file mode 100644
index 0000000000000000000000000000000000000000..7040d032efa27a286f9b8503362d31525462ea13
--- /dev/null
+++ b/lib/stevedore/tests/test_enabled.py
@@ -0,0 +1,30 @@
+from stevedore import enabled
+from stevedore.tests import utils
+
+
+class TestEnabled(utils.TestCase):
+ def test_enabled(self):
+ def check_enabled(ep):
+ return ep.name == 't2'
+ em = enabled.EnabledExtensionManager(
+ 'stevedore.test.extension',
+ check_enabled,
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+ self.assertEqual(len(em.extensions), 1)
+ self.assertEqual(em.names(), ['t2'])
+
+ def test_enabled_after_load(self):
+ def check_enabled(ext):
+ return ext.obj and ext.name == 't2'
+ em = enabled.EnabledExtensionManager(
+ 'stevedore.test.extension',
+ check_enabled,
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+ self.assertEqual(len(em.extensions), 1)
+ self.assertEqual(em.names(), ['t2'])
diff --git a/lib/stevedore/tests/test_example_fields.py b/lib/stevedore/tests/test_example_fields.py
new file mode 100644
index 0000000000000000000000000000000000000000..a3eb39775ea02c0ffb273ecd6cfa6ebfee08593a
--- /dev/null
+++ b/lib/stevedore/tests/test_example_fields.py
@@ -0,0 +1,29 @@
+"""Tests for stevedore.exmaple.fields
+"""
+
+from stevedore.example import fields
+from stevedore.tests import utils
+
+
+class TestExampleFields(utils.TestCase):
+ def test_simple_items(self):
+ f = fields.FieldList(100)
+ text = ''.join(f.format({'a': 'A', 'b': 'B'}))
+ expected = '\n'.join([
+ ': a : A',
+ ': b : B',
+ '',
+ ])
+ self.assertEqual(text, expected)
+
+ def test_long_item(self):
+ f = fields.FieldList(25)
+ text = ''.join(f.format({'name':
+ 'a value longer than the allowed width'}))
+ expected = '\n'.join([
+ ': name : a value longer',
+ ' than the allowed',
+ ' width',
+ '',
+ ])
+ self.assertEqual(text, expected)
diff --git a/lib/stevedore/tests/test_example_simple.py b/lib/stevedore/tests/test_example_simple.py
new file mode 100644
index 0000000000000000000000000000000000000000..b8ef43119f539f6a9a023c16b378d4a060016c56
--- /dev/null
+++ b/lib/stevedore/tests/test_example_simple.py
@@ -0,0 +1,17 @@
+"""Tests for stevedore.exmaple.simple
+"""
+
+from stevedore.example import simple
+from stevedore.tests import utils
+
+
+class TestExampleSimple(utils.TestCase):
+ def test_simple_items(self):
+ f = simple.Simple(100)
+ text = ''.join(f.format({'a': 'A', 'b': 'B'}))
+ expected = '\n'.join([
+ 'a = A',
+ 'b = B',
+ '',
+ ])
+ self.assertEqual(text, expected)
diff --git a/lib/stevedore/tests/test_extension.py b/lib/stevedore/tests/test_extension.py
new file mode 100644
index 0000000000000000000000000000000000000000..b05b377682756efbd8115349d35e6d52a7ad952a
--- /dev/null
+++ b/lib/stevedore/tests/test_extension.py
@@ -0,0 +1,211 @@
+"""Tests for stevedore.extension
+"""
+
+import mock
+
+from stevedore import extension
+from stevedore.tests import utils
+
+
+ALL_NAMES = ['e1', 't1', 't2']
+WORKING_NAMES = ['t1', 't2']
+
+
+class FauxExtension(object):
+ def __init__(self, *args, **kwds):
+ self.args = args
+ self.kwds = kwds
+
+ def get_args_and_data(self, data):
+ return self.args, self.kwds, data
+
+
+class BrokenExtension(object):
+ def __init__(self, *args, **kwds):
+ raise IOError("Did not create")
+
+
+class TestCallback(utils.TestCase):
+ def test_detect_plugins(self):
+ em = extension.ExtensionManager('stevedore.test.extension')
+ names = sorted(em.names())
+ self.assertEqual(names, ALL_NAMES)
+
+ def test_get_by_name(self):
+ em = extension.ExtensionManager('stevedore.test.extension')
+ e = em['t1']
+ self.assertEqual(e.name, 't1')
+
+ def test_contains_by_name(self):
+ em = extension.ExtensionManager('stevedore.test.extension')
+ self.assertEqual('t1' in em, True)
+
+ def test_get_by_name_missing(self):
+ em = extension.ExtensionManager('stevedore.test.extension')
+ try:
+ em['t3']
+ except KeyError:
+ pass
+ else:
+ assert False, 'Failed to raise KeyError'
+
+ def test_load_multiple_times_entry_points(self):
+ # We expect to get the same EntryPoint object because we save them
+ # in the cache.
+ em1 = extension.ExtensionManager('stevedore.test.extension')
+ eps1 = [ext.entry_point for ext in em1]
+ em2 = extension.ExtensionManager('stevedore.test.extension')
+ eps2 = [ext.entry_point for ext in em2]
+ self.assertIs(eps1[0], eps2[0])
+
+ def test_load_multiple_times_plugins(self):
+ # We expect to get the same plugin object (module or class)
+ # because the underlying import machinery will cache the values.
+ em1 = extension.ExtensionManager('stevedore.test.extension')
+ plugins1 = [ext.plugin for ext in em1]
+ em2 = extension.ExtensionManager('stevedore.test.extension')
+ plugins2 = [ext.plugin for ext in em2]
+ self.assertIs(plugins1[0], plugins2[0])
+
+ def test_use_cache(self):
+ # If we insert something into the cache of entry points,
+ # the manager should not have to call into pkg_resources
+ # to find the plugins.
+ cache = extension.ExtensionManager.ENTRY_POINT_CACHE
+ cache['stevedore.test.faux'] = []
+ with mock.patch('pkg_resources.iter_entry_points',
+ side_effect=
+ AssertionError('called iter_entry_points')):
+ em = extension.ExtensionManager('stevedore.test.faux')
+ names = em.names()
+ self.assertEqual(names, [])
+
+ def test_iterable(self):
+ em = extension.ExtensionManager('stevedore.test.extension')
+ names = sorted(e.name for e in em)
+ self.assertEqual(names, ALL_NAMES)
+
+ def test_invoke_on_load(self):
+ em = extension.ExtensionManager('stevedore.test.extension',
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+ self.assertEqual(len(em.extensions), 2)
+ for e in em.extensions:
+ self.assertEqual(e.obj.args, ('a',))
+ self.assertEqual(e.obj.kwds, {'b': 'B'})
+
+ def test_map_return_values(self):
+ def mapped(ext, *args, **kwds):
+ return ext.name
+
+ em = extension.ExtensionManager('stevedore.test.extension',
+ invoke_on_load=True,
+ )
+ results = em.map(mapped)
+ self.assertEqual(sorted(results), WORKING_NAMES)
+
+ def test_map_arguments(self):
+ objs = []
+
+ def mapped(ext, *args, **kwds):
+ objs.append((ext, args, kwds))
+
+ em = extension.ExtensionManager('stevedore.test.extension',
+ invoke_on_load=True,
+ )
+ em.map(mapped, 1, 2, a='A', b='B')
+ self.assertEqual(len(objs), 2)
+ names = sorted([o[0].name for o in objs])
+ self.assertEqual(names, WORKING_NAMES)
+ for o in objs:
+ self.assertEqual(o[1], (1, 2))
+ self.assertEqual(o[2], {'a': 'A', 'b': 'B'})
+
+ def test_map_eats_errors(self):
+ def mapped(ext, *args, **kwds):
+ raise RuntimeError('hard coded error')
+
+ em = extension.ExtensionManager('stevedore.test.extension',
+ invoke_on_load=True,
+ )
+ results = em.map(mapped, 1, 2, a='A', b='B')
+ self.assertEqual(results, [])
+
+ def test_map_propagate_exceptions(self):
+ def mapped(ext, *args, **kwds):
+ raise RuntimeError('hard coded error')
+
+ em = extension.ExtensionManager('stevedore.test.extension',
+ invoke_on_load=True,
+ propagate_map_exceptions=True
+ )
+
+ try:
+ em.map(mapped, 1, 2, a='A', b='B')
+ assert False
+ except RuntimeError:
+ pass
+
+ def test_map_errors_when_no_plugins(self):
+ expected_str = 'No stevedore.test.extension.none extensions found'
+
+ def mapped(ext, *args, **kwds):
+ pass
+
+ em = extension.ExtensionManager('stevedore.test.extension.none',
+ invoke_on_load=True,
+ )
+ try:
+ em.map(mapped, 1, 2, a='A', b='B')
+ except RuntimeError as err:
+ self.assertEqual(expected_str, str(err))
+
+ def test_map_method(self):
+ em = extension.ExtensionManager('stevedore.test.extension',
+ invoke_on_load=True,
+ )
+
+ result = em.map_method('get_args_and_data', 42)
+ self.assertEqual(set(r[2] for r in result), set([42]))
+
+
+class TestLoadRequirementsNewSetuptools(utils.TestCase):
+ # setuptools 11.3 and later
+
+ def setUp(self):
+ super(TestLoadRequirementsNewSetuptools, self).setUp()
+ self.mock_ep = mock.Mock(spec=['require', 'resolve', 'load', 'name'])
+ self.em = extension.ExtensionManager.make_test_instance([])
+
+ def test_verify_requirements(self):
+ self.em._load_one_plugin(self.mock_ep, False, (), {},
+ verify_requirements=True)
+ self.mock_ep.require.assert_called_once_with()
+ self.mock_ep.resolve.assert_called_once_with()
+
+ def test_no_verify_requirements(self):
+ self.em._load_one_plugin(self.mock_ep, False, (), {},
+ verify_requirements=False)
+ self.assertEqual(0, self.mock_ep.require.call_count)
+ self.mock_ep.resolve.assert_called_once_with()
+
+
+class TestLoadRequirementsOldSetuptools(utils.TestCase):
+ # Before setuptools 11.3
+
+ def setUp(self):
+ super(TestLoadRequirementsOldSetuptools, self).setUp()
+ self.mock_ep = mock.Mock(spec=['load', 'name'])
+ self.em = extension.ExtensionManager.make_test_instance([])
+
+ def test_verify_requirements(self):
+ self.em._load_one_plugin(self.mock_ep, False, (), {},
+ verify_requirements=True)
+ self.mock_ep.load.assert_called_once_with(require=True)
+
+ def test_no_verify_requirements(self):
+ self.em._load_one_plugin(self.mock_ep, False, (), {},
+ verify_requirements=False)
+ self.mock_ep.load.assert_called_once_with(require=False)
diff --git a/lib/stevedore/tests/test_hook.py b/lib/stevedore/tests/test_hook.py
new file mode 100644
index 0000000000000000000000000000000000000000..b95f4b84880d2ba72430978a0888fef3df7598d3
--- /dev/null
+++ b/lib/stevedore/tests/test_hook.py
@@ -0,0 +1,43 @@
+from stevedore import hook
+from stevedore.tests import utils
+
+
+class TestHook(utils.TestCase):
+ def test_hook(self):
+ em = hook.HookManager(
+ 'stevedore.test.extension',
+ 't1',
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+ self.assertEqual(len(em.extensions), 1)
+ self.assertEqual(em.names(), ['t1'])
+
+ def test_get_by_name(self):
+ em = hook.HookManager(
+ 'stevedore.test.extension',
+ 't1',
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+ e_list = em['t1']
+ self.assertEqual(len(e_list), 1)
+ e = e_list[0]
+ self.assertEqual(e.name, 't1')
+
+ def test_get_by_name_missing(self):
+ em = hook.HookManager(
+ 'stevedore.test.extension',
+ 't1',
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+ try:
+ em['t2']
+ except KeyError:
+ pass
+ else:
+ assert False, 'Failed to raise KeyError'
diff --git a/lib/stevedore/tests/test_named.py b/lib/stevedore/tests/test_named.py
new file mode 100644
index 0000000000000000000000000000000000000000..bbac13709d33ddfe7bc09f263b4bd6d81c069b23
--- /dev/null
+++ b/lib/stevedore/tests/test_named.py
@@ -0,0 +1,58 @@
+from stevedore import named
+from stevedore.tests import utils
+
+import mock
+
+
+class TestNamed(utils.TestCase):
+ def test_named(self):
+ em = named.NamedExtensionManager(
+ 'stevedore.test.extension',
+ names=['t1'],
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+ actual = em.names()
+ self.assertEqual(actual, ['t1'])
+
+ def test_enabled_before_load(self):
+ # Set up the constructor for the FauxExtension to cause an
+ # AssertionError so the test fails if the class is instantiated,
+ # which should only happen if it is loaded before the name of the
+ # extension is compared against the names that should be loaded by
+ # the manager.
+ init_name = 'stevedore.tests.test_extension.FauxExtension.__init__'
+ with mock.patch(init_name) as m:
+ m.side_effect = AssertionError
+ em = named.NamedExtensionManager(
+ 'stevedore.test.extension',
+ # Look for an extension that does not exist so the
+ # __init__ we mocked should never be invoked.
+ names=['no-such-extension'],
+ invoke_on_load=True,
+ invoke_args=('a',),
+ invoke_kwds={'b': 'B'},
+ )
+ actual = em.names()
+ self.assertEqual(actual, [])
+
+ def test_extensions_listed_in_name_order(self):
+ # Since we don't know the "natural" order of the extensions, run
+ # the test both ways: if the sorting is broken, one of them will
+ # fail
+ em = named.NamedExtensionManager(
+ 'stevedore.test.extension',
+ names=['t1', 't2'],
+ name_order=True
+ )
+ actual = em.names()
+ self.assertEqual(actual, ['t1', 't2'])
+
+ em = named.NamedExtensionManager(
+ 'stevedore.test.extension',
+ names=['t2', 't1'],
+ name_order=True
+ )
+ actual = em.names()
+ self.assertEqual(actual, ['t2', 't1'])
diff --git a/lib/stevedore/tests/test_sphinxext.py b/lib/stevedore/tests/test_sphinxext.py
new file mode 100644
index 0000000000000000000000000000000000000000..60b47944f7a85539dd916947a175b90a45a8558a
--- /dev/null
+++ b/lib/stevedore/tests/test_sphinxext.py
@@ -0,0 +1,120 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Tests for the sphinx extension
+"""
+
+from __future__ import unicode_literals
+
+from stevedore import extension
+from stevedore import sphinxext
+from stevedore.tests import utils
+
+import mock
+import pkg_resources
+
+
+def _make_ext(name, docstring):
+ def inner():
+ pass
+
+ inner.__doc__ = docstring
+ m1 = mock.Mock(spec=pkg_resources.EntryPoint)
+ m1.module_name = '%s_module' % name
+ s = mock.Mock(return_value='ENTRY_POINT(%s)' % name)
+ m1.__str__ = s
+ return extension.Extension(name, m1, inner, None)
+
+
+class TestSphinxExt(utils.TestCase):
+
+ def setUp(self):
+ super(TestSphinxExt, self).setUp()
+ self.exts = [
+ _make_ext('test1', 'One-line docstring'),
+ _make_ext('test2', 'Multi-line docstring\n\nAnother para'),
+ ]
+ self.em = extension.ExtensionManager.make_test_instance(self.exts)
+
+ def test_simple_list(self):
+ results = list(sphinxext._simple_list(self.em))
+ self.assertEqual(
+ [
+ ('* test1 -- One-line docstring', 'test1_module'),
+ ('* test2 -- Multi-line docstring', 'test2_module'),
+ ],
+ results,
+ )
+
+ def test_simple_list_no_docstring(self):
+ ext = [_make_ext('nodoc', None)]
+ em = extension.ExtensionManager.make_test_instance(ext)
+ results = list(sphinxext._simple_list(em))
+ self.assertEqual(
+ [
+ ('* nodoc -- ', 'nodoc_module'),
+ ],
+ results,
+ )
+
+ def test_detailed_list(self):
+ results = list(sphinxext._detailed_list(self.em))
+ self.assertEqual(
+ [
+ ('test1', 'test1_module'),
+ ('-----', 'test1_module'),
+ ('\n', 'test1_module'),
+ ('One-line docstring', 'test1_module'),
+ ('\n', 'test1_module'),
+ ('test2', 'test2_module'),
+ ('-----', 'test2_module'),
+ ('\n', 'test2_module'),
+ ('Multi-line docstring\n\nAnother para', 'test2_module'),
+ ('\n', 'test2_module'),
+ ],
+ results,
+ )
+
+ def test_detailed_list_format(self):
+ results = list(sphinxext._detailed_list(self.em, over='+', under='+'))
+ self.assertEqual(
+ [
+ ('+++++', 'test1_module'),
+ ('test1', 'test1_module'),
+ ('+++++', 'test1_module'),
+ ('\n', 'test1_module'),
+ ('One-line docstring', 'test1_module'),
+ ('\n', 'test1_module'),
+ ('+++++', 'test2_module'),
+ ('test2', 'test2_module'),
+ ('+++++', 'test2_module'),
+ ('\n', 'test2_module'),
+ ('Multi-line docstring\n\nAnother para', 'test2_module'),
+ ('\n', 'test2_module'),
+ ],
+ results,
+ )
+
+ def test_detailed_list_no_docstring(self):
+ ext = [_make_ext('nodoc', None)]
+ em = extension.ExtensionManager.make_test_instance(ext)
+ results = list(sphinxext._detailed_list(em))
+ self.assertEqual(
+ [
+ ('nodoc', 'nodoc_module'),
+ ('-----', 'nodoc_module'),
+ ('\n', 'nodoc_module'),
+ ('.. warning:: No documentation found in ENTRY_POINT(nodoc)',
+ 'nodoc_module'),
+ ('\n', 'nodoc_module'),
+ ],
+ results,
+ )
diff --git a/lib/stevedore/tests/test_test_manager.py b/lib/stevedore/tests/test_test_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..70aa1003b30ce3740893b40396b6eb0aaf0dd99d
--- /dev/null
+++ b/lib/stevedore/tests/test_test_manager.py
@@ -0,0 +1,204 @@
+from mock import Mock, sentinel
+from stevedore import (ExtensionManager, NamedExtensionManager, HookManager,
+ DriverManager, EnabledExtensionManager)
+from stevedore.dispatch import (DispatchExtensionManager,
+ NameDispatchExtensionManager)
+from stevedore.extension import Extension
+from stevedore.tests import utils
+
+
+test_extension = Extension('test_extension', None, None, None)
+test_extension2 = Extension('another_one', None, None, None)
+
+mock_entry_point = Mock(module_name='test.extension', attrs=['obj'])
+a_driver = Extension('test_driver', mock_entry_point, sentinel.driver_plugin,
+ sentinel.driver_obj)
+
+
+# base ExtensionManager
+class TestTestManager(utils.TestCase):
+ def test_instance_should_use_supplied_extensions(self):
+ extensions = [test_extension, test_extension2]
+ em = ExtensionManager.make_test_instance(extensions)
+ self.assertEqual(extensions, em.extensions)
+
+ def test_instance_should_have_default_namespace(self):
+ em = ExtensionManager.make_test_instance([])
+ self.assertEqual(em.namespace, 'TESTING')
+
+ def test_instance_should_use_supplied_namespace(self):
+ namespace = 'testing.1.2.3'
+ em = ExtensionManager.make_test_instance([], namespace=namespace)
+ self.assertEqual(namespace, em.namespace)
+
+ def test_extension_name_should_be_listed(self):
+ em = ExtensionManager.make_test_instance([test_extension])
+ self.assertIn(test_extension.name, em.names())
+
+ def test_iterator_should_yield_extension(self):
+ em = ExtensionManager.make_test_instance([test_extension])
+ self.assertEqual(test_extension, next(iter(em)))
+
+ def test_manager_should_allow_name_access(self):
+ em = ExtensionManager.make_test_instance([test_extension])
+ self.assertEqual(test_extension, em[test_extension.name])
+
+ def test_manager_should_call(self):
+ em = ExtensionManager.make_test_instance([test_extension])
+ func = Mock()
+ em.map(func)
+ func.assert_called_once_with(test_extension)
+
+ def test_manager_should_call_all(self):
+ em = ExtensionManager.make_test_instance([test_extension2,
+ test_extension])
+ func = Mock()
+ em.map(func)
+ func.assert_any_call(test_extension2)
+ func.assert_any_call(test_extension)
+
+ def test_manager_return_values(self):
+ def mapped(ext, *args, **kwds):
+ return ext.name
+
+ em = ExtensionManager.make_test_instance([test_extension2,
+ test_extension])
+ results = em.map(mapped)
+ self.assertEqual(sorted(results), ['another_one', 'test_extension'])
+
+ def test_manager_should_eat_exceptions(self):
+ em = ExtensionManager.make_test_instance([test_extension])
+
+ func = Mock(side_effect=RuntimeError('hard coded error'))
+
+ results = em.map(func, 1, 2, a='A', b='B')
+ self.assertEqual(results, [])
+
+ def test_manager_should_propagate_exceptions(self):
+ em = ExtensionManager.make_test_instance([test_extension],
+ propagate_map_exceptions=True)
+ self.skipTest('Skipping temporarily')
+ func = Mock(side_effect=RuntimeError('hard coded error'))
+ em.map(func, 1, 2, a='A', b='B')
+
+ # NamedExtensionManager
+ def test_named_manager_should_use_supplied_extensions(self):
+ extensions = [test_extension, test_extension2]
+ em = NamedExtensionManager.make_test_instance(extensions)
+ self.assertEqual(extensions, em.extensions)
+
+ def test_named_manager_should_have_default_namespace(self):
+ em = NamedExtensionManager.make_test_instance([])
+ self.assertEqual(em.namespace, 'TESTING')
+
+ def test_named_manager_should_use_supplied_namespace(self):
+ namespace = 'testing.1.2.3'
+ em = NamedExtensionManager.make_test_instance([], namespace=namespace)
+ self.assertEqual(namespace, em.namespace)
+
+ def test_named_manager_should_populate_names(self):
+ extensions = [test_extension, test_extension2]
+ em = NamedExtensionManager.make_test_instance(extensions)
+ self.assertEqual(em.names(), ['test_extension', 'another_one'])
+
+ # HookManager
+ def test_hook_manager_should_use_supplied_extensions(self):
+ extensions = [test_extension, test_extension2]
+ em = HookManager.make_test_instance(extensions)
+ self.assertEqual(extensions, em.extensions)
+
+ def test_hook_manager_should_be_first_extension_name(self):
+ extensions = [test_extension, test_extension2]
+ em = HookManager.make_test_instance(extensions)
+ # This will raise KeyError if the names don't match
+ assert(em[test_extension.name])
+
+ def test_hook_manager_should_have_default_namespace(self):
+ em = HookManager.make_test_instance([test_extension])
+ self.assertEqual(em.namespace, 'TESTING')
+
+ def test_hook_manager_should_use_supplied_namespace(self):
+ namespace = 'testing.1.2.3'
+ em = HookManager.make_test_instance([test_extension],
+ namespace=namespace)
+ self.assertEqual(namespace, em.namespace)
+
+ def test_hook_manager_should_return_named_extensions(self):
+ hook1 = Extension('captain', None, None, None)
+ hook2 = Extension('captain', None, None, None)
+ em = HookManager.make_test_instance([hook1, hook2])
+ self.assertEqual([hook1, hook2], em['captain'])
+
+ # DriverManager
+ def test_driver_manager_should_use_supplied_extension(self):
+ em = DriverManager.make_test_instance(a_driver)
+ self.assertEqual([a_driver], em.extensions)
+
+ def test_driver_manager_should_have_default_namespace(self):
+ em = DriverManager.make_test_instance(a_driver)
+ self.assertEqual(em.namespace, 'TESTING')
+
+ def test_driver_manager_should_use_supplied_namespace(self):
+ namespace = 'testing.1.2.3'
+ em = DriverManager.make_test_instance(a_driver, namespace=namespace)
+ self.assertEqual(namespace, em.namespace)
+
+ def test_instance_should_use_driver_name(self):
+ em = DriverManager.make_test_instance(a_driver)
+ self.assertEqual(['test_driver'], em.names())
+
+ def test_instance_call(self):
+ def invoke(ext, *args, **kwds):
+ return ext.name, args, kwds
+
+ em = DriverManager.make_test_instance(a_driver)
+ result = em(invoke, 'a', b='C')
+ self.assertEqual(result, ('test_driver', ('a',), {'b': 'C'}))
+
+ def test_instance_driver_property(self):
+ em = DriverManager.make_test_instance(a_driver)
+ self.assertEqual(sentinel.driver_obj, em.driver)
+
+ # EnabledExtensionManager
+ def test_enabled_instance_should_use_supplied_extensions(self):
+ extensions = [test_extension, test_extension2]
+ em = EnabledExtensionManager.make_test_instance(extensions)
+ self.assertEqual(extensions, em.extensions)
+
+ # DispatchExtensionManager
+ def test_dispatch_instance_should_use_supplied_extensions(self):
+ extensions = [test_extension, test_extension2]
+ em = DispatchExtensionManager.make_test_instance(extensions)
+ self.assertEqual(extensions, em.extensions)
+
+ def test_dispatch_map_should_invoke_filter_for_extensions(self):
+ em = DispatchExtensionManager.make_test_instance([test_extension,
+ test_extension2])
+ filter_func = Mock(return_value=False)
+ args = ('A',)
+ kw = {'big': 'Cheese'}
+ em.map(filter_func, None, *args, **kw)
+ filter_func.assert_any_call(test_extension, *args, **kw)
+ filter_func.assert_any_call(test_extension2, *args, **kw)
+
+ # NameDispatchExtensionManager
+ def test_name_dispatch_instance_should_use_supplied_extensions(self):
+ extensions = [test_extension, test_extension2]
+ em = NameDispatchExtensionManager.make_test_instance(extensions)
+
+ self.assertEqual(extensions, em.extensions)
+
+ def test_name_dispatch_instance_should_build_extension_name_map(self):
+ extensions = [test_extension, test_extension2]
+ em = NameDispatchExtensionManager.make_test_instance(extensions)
+ self.assertEqual(test_extension, em.by_name[test_extension.name])
+ self.assertEqual(test_extension2, em.by_name[test_extension2.name])
+
+ def test_named_dispatch_map_should_invoke_filter_for_extensions(self):
+ em = NameDispatchExtensionManager.make_test_instance([test_extension,
+ test_extension2])
+ func = Mock()
+ args = ('A',)
+ kw = {'BIGGER': 'Cheese'}
+ em.map(['test_extension'], func, *args, **kw)
+ func.assert_called_once_with(test_extension, *args, **kw)
diff --git a/lib/stevedore/tests/utils.py b/lib/stevedore/tests/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..01e2a4645fc49dcafbc0f643e1895eccf781024a
--- /dev/null
+++ b/lib/stevedore/tests/utils.py
@@ -0,0 +1,5 @@
+from oslotest import base as test_base
+
+
+class TestCase(test_base.BaseTestCase):
+ pass
diff --git a/lib/subliminal/__init__.py b/lib/subliminal/__init__.py
index cb4904a59d58a53b9863c9206b6e1ff5d8de80eb..943bb1d13c60d9d8e8a0ebb5355308e352a96fd2 100644
--- a/lib/subliminal/__init__.py
+++ b/lib/subliminal/__init__.py
@@ -1,34 +1,20 @@
# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from .api import list_subtitles, download_subtitles
-from .async import Pool
-from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE,
- MATCHING_CONFIDENCE)
-from .infos import __version__
+__title__ = 'subliminal'
+__version__ = '0.8.0-dev'
+__author__ = 'Antoine Bertin'
+__license__ = 'MIT'
+__copyright__ = 'Copyright 2013 Antoine Bertin'
+
import logging
-try:
- from logging import NullHandler
-except ImportError:
- class NullHandler(logging.Handler):
- def emit(self, record):
- pass
+from .api import list_subtitles, download_subtitles, download_best_subtitles, save_subtitles
+from .cache import MutexLock, region as cache_region
+from .exceptions import Error, ProviderError
+from .providers import Provider, ProviderPool, provider_manager
+from .subtitle import Subtitle
+from .video import VIDEO_EXTENSIONS, SUBTITLE_EXTENSIONS, Video, Episode, Movie, scan_videos, scan_video
+class NullHandler(logging.Handler):
+ def emit(self, record):
+ pass
-__all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE',
- 'MATCHING_CONFIDENCE', 'list_subtitles', 'download_subtitles', 'Pool', 'language']
-logging.getLogger("subliminal").addHandler(NullHandler())
+logging.getLogger(__name__).addHandler(NullHandler())
diff --git a/lib/subliminal/api.py b/lib/subliminal/api.py
index 867fafea456ae442dab4919f9dc8d116729c1afd..47d6a2cb93cc7ce638c873f8bca1f1af81ce5d19 100644
--- a/lib/subliminal/api.py
+++ b/lib/subliminal/api.py
@@ -1,113 +1,140 @@
# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE,
- MATCHING_CONFIDENCE, create_list_tasks, consume_task, create_download_tasks,
- group_by_video, key_subtitles)
-from .language import language_set, language_list, LANGUAGES
+from __future__ import unicode_literals
+import collections
+import io
import logging
-import sys
+import operator
+import os.path
+import babelfish
+from .providers import ProviderPool
+from .subtitle import get_subtitle_path
-__all__ = ['list_subtitles', 'download_subtitles']
-logger = logging.getLogger("subliminal")
+logger = logging.getLogger(__name__)
-def list_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None):
- """List subtitles in given paths according to the criteria
+def list_subtitles(videos, languages, providers=None, provider_configs=None):
+ """List subtitles for `videos` with the given `languages` using the specified `providers`
- :param paths: path(s) to video file or folder
- :type paths: string or list
- :param languages: languages to search for, in preferred order
- :type languages: list of :class:`~subliminal.language.Language` or string
- :param list services: services to use for the search, in preferred order
- :param bool force: force searching for subtitles even if some are detected
- :param bool multi: search multiple languages for the same video
- :param string cache_dir: path to the cache directory to use
- :param int max_depth: maximum depth for scanning entries
- :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
+ :param videos: videos to list subtitles for
+ :type videos: set of :class:`~subliminal.video.Video`
+ :param languages: languages of subtitles to search for
+ :type languages: set of :class:`babelfish.Language`
+ :param providers: providers to use, if not all
+ :type providers: list of string or None
+ :param provider_configs: configuration for providers
+ :type provider_configs: dict of provider name => provider constructor kwargs or None
:return: found subtitles
- :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
+ :rtype: dict of :class:`~subliminal.video.Video` => [:class:`~subliminal.subtitle.Subtitle`]
"""
- services = services or SERVICES
- languages = language_set(languages) if languages is not None else language_set(LANGUAGES)
- if isinstance(paths, basestring):
- paths = [paths]
- if any([not isinstance(p, unicode) for p in paths]):
- logger.warning(u'Not all entries are unicode') if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- results = []
- service_instances = {}
- tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
- for task in tasks:
- try:
- result = consume_task(task, service_instances)
- results.append((task.video, result))
- except:
- logger.error(u'Error consuming task %r' % task, exc_info=True) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- for service_instance in service_instances.itervalues():
- service_instance.terminate()
- return group_by_video(results)
-
-
-def download_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None, order=None):
- """Download subtitles in given paths according to the criteria
-
- :param paths: path(s) to video file or folder
- :type paths: string or list
- :param languages: languages to search for, in preferred order
- :type languages: list of :class:`~subliminal.language.Language` or string
- :param list services: services to use for the search, in preferred order
- :param bool force: force searching for subtitles even if some are detected
- :param bool multi: search multiple languages for the same video
- :param string cache_dir: path to the cache directory to use
- :param int max_depth: maximum depth for scanning entries
- :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
- :param order: preferred order for subtitles sorting
- :type list: list of :data:`~subliminal.core.LANGUAGE_INDEX`, :data:`~subliminal.core.SERVICE_INDEX`, :data:`~subliminal.core.SERVICE_CONFIDENCE`, :data:`~subliminal.core.MATCHING_CONFIDENCE`
- :return: downloaded subtitles
- :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
-
- .. note::
-
- If you use ``multi=True``, :data:`~subliminal.core.LANGUAGE_INDEX` has to be the first item of the ``order`` list
- or you might get unexpected results.
+ subtitles = collections.defaultdict(list)
+ with ProviderPool(providers, provider_configs) as pp:
+ for video in videos:
+ logger.info('Listing subtitles for %r', video)
+ video_subtitles = pp.list_subtitles(video, languages)
+ logger.info('Found %d subtitles total', len(video_subtitles))
+ subtitles[video].extend(video_subtitles)
+ return subtitles
+
+
+def download_subtitles(subtitles, provider_configs=None):
+ """Download subtitles
+
+ :param subtitles: subtitles to download
+ :type subtitles: list of :class:`~subliminal.subtitle.Subtitle`
+ :param provider_configs: configuration for providers
+ :type provider_configs: dict of provider name => provider constructor kwargs or None
+
+ """
+ with ProviderPool(provider_configs=provider_configs) as pp:
+ for subtitle in subtitles:
+ logger.info('Downloading subtitle %r', subtitle)
+ pp.download_subtitle(subtitle)
+
+
+def download_best_subtitles(videos, languages, providers=None, provider_configs=None, min_score=0,
+ hearing_impaired=False, single=False):
+ """Download the best subtitles for `videos` with the given `languages` using the specified `providers`
+
+ :param videos: videos to download subtitles for
+ :type videos: set of :class:`~subliminal.video.Video`
+ :param languages: languages of subtitles to download
+ :type languages: set of :class:`babelfish.Language`
+ :param providers: providers to use for the search, if not all
+ :type providers: list of string or None
+ :param provider_configs: configuration for providers
+ :type provider_configs: dict of provider name => provider constructor kwargs or None
+ :param int min_score: minimum score for subtitles to download
+ :param bool hearing_impaired: download hearing impaired subtitles
+ :param bool single: do not download for videos with an undetermined subtitle language detected
+
+ """
+ downloaded_subtitles = collections.defaultdict(list)
+ with ProviderPool(providers, provider_configs) as pp:
+ for video in videos:
+ # filter
+ if single and babelfish.Language('und') in video.subtitle_languages:
+ logger.debug('Skipping video %r: undetermined language found')
+ continue
+
+ # list
+ logger.info('Listing subtitles for %r', video)
+ video_subtitles = pp.list_subtitles(video, languages)
+ logger.info('Found %d subtitles total', len(video_subtitles))
+
+ # download
+ downloaded_languages = set()
+ for subtitle, score in sorted([(s, s.compute_score(video)) for s in video_subtitles],
+ key=operator.itemgetter(1), reverse=True):
+ if score < min_score:
+ logger.info('No subtitle with score >= %d', min_score)
+ break
+ if subtitle.hearing_impaired != hearing_impaired:
+ logger.debug('Skipping subtitle: hearing impaired != %r', hearing_impaired)
+ continue
+ if subtitle.language in downloaded_languages:
+ logger.debug('Skipping subtitle: %r already downloaded', subtitle.language)
+ continue
+ logger.info('Downloading subtitle %r with score %d', subtitle, score)
+ if pp.download_subtitle(subtitle):
+ downloaded_languages.add(subtitle.language)
+ downloaded_subtitles[video].append(subtitle)
+ if single or downloaded_languages == languages:
+ logger.debug('All languages downloaded')
+ break
+ return downloaded_subtitles
+
+
+def save_subtitles(subtitles, single=False, directory=None, encoding=None):
+ """Save subtitles on disk next to the video or in a specific folder if `folder_path` is specified
+
+ :param bool single: download with .srt extension if ``True``, add language identifier otherwise
+ :param directory: path to directory where to save the subtitles, if any
+ :type directory: string or None
+ :param encoding: encoding for the subtitles or ``None`` to use the original encoding
+ :type encoding: string or None
"""
- services = services or SERVICES
- languages = language_list(languages) if languages is not None else language_list(LANGUAGES)
- if isinstance(paths, basestring):
- paths = [paths]
- order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
- subtitles_by_video = list_subtitles(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
- for video, subtitles in subtitles_by_video.iteritems():
- try:
- subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
- except StopIteration:
- break
- results = []
- service_instances = {}
- tasks = create_download_tasks(subtitles_by_video, languages, multi)
- for task in tasks:
- try:
- result = consume_task(task, service_instances)
- results.append((task.video, result))
- except:
- logger.error(u'Error consuming task %r' % task, exc_info=True) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- for service_instance in service_instances.itervalues():
- service_instance.terminate()
- return group_by_video(results)
+ for video, video_subtitles in subtitles.items():
+ saved_languages = set()
+ for video_subtitle in video_subtitles:
+ if video_subtitle.content is None:
+ logger.debug('Skipping subtitle %r: no content', video_subtitle)
+ continue
+ if video_subtitle.language in saved_languages:
+ logger.debug('Skipping subtitle %r: language already saved', video_subtitle)
+ continue
+ subtitle_path = get_subtitle_path(video.name, None if single else video_subtitle.language)
+ if directory is not None:
+ subtitle_path = os.path.join(directory, os.path.split(subtitle_path)[1])
+ logger.info('Saving %r to %r', video_subtitle, subtitle_path)
+ if encoding is None:
+ with io.open(subtitle_path, 'wb') as f:
+ f.write(video_subtitle.content)
+ else:
+ with io.open(subtitle_path, 'w', encoding=encoding) as f:
+ f.write(video_subtitle.text)
+ saved_languages.add(video_subtitle.language)
+ if single:
+ break
diff --git a/lib/subliminal/async.py b/lib/subliminal/async.py
deleted file mode 100644
index 6e5af16fe84ac853f9856eab4b3b65c6a498461c..0000000000000000000000000000000000000000
--- a/lib/subliminal/async.py
+++ /dev/null
@@ -1,143 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from .core import (consume_task, LANGUAGE_INDEX, SERVICE_INDEX,
- SERVICE_CONFIDENCE, MATCHING_CONFIDENCE, SERVICES, create_list_tasks,
- create_download_tasks, group_by_video, key_subtitles)
-from .language import language_list, language_set, LANGUAGES
-from .tasks import StopTask
-import Queue
-import logging
-import threading
-import sys
-
-
-__all__ = ['Worker', 'Pool']
-logger = logging.getLogger("subliminal")
-
-
-class Worker(threading.Thread):
- """Consume tasks and put the result in the queue"""
- def __init__(self, tasks, results):
- super(Worker, self).__init__()
- self.tasks = tasks
- self.results = results
- self.services = {}
-
- def run(self):
- while 1:
- result = []
- try:
- task = self.tasks.get(block=True)
- if isinstance(task, StopTask):
- break
- result = consume_task(task, self.services)
- self.results.put((task.video, result))
- except:
- logger.error(u'Exception raised in worker %s' % self.name, exc_info=True) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- finally:
- self.tasks.task_done()
- self.terminate()
- logger.debug(u'Thread %s terminated' % self.name) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
-
- def terminate(self):
- """Terminate instantiated services"""
- for service_name, service in self.services.iteritems():
- try:
- service.terminate()
- except:
- logger.error(u'Exception raised when terminating service %s' % service_name, exc_info=True) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
-
-
-class Pool(object):
- """Pool of workers"""
- def __init__(self, size):
- self.tasks = Queue.Queue()
- self.results = Queue.Queue()
- self.workers = []
- for _ in range(size):
- self.workers.append(Worker(self.tasks, self.results))
-
- def __enter__(self):
- self.start()
- return self
-
- def __exit__(self, *args):
- self.stop()
- self.join()
-
- def start(self):
- """Start workers"""
- for worker in self.workers:
- worker.start()
-
- def stop(self):
- """Stop workers"""
- for _ in self.workers:
- self.tasks.put(StopTask())
-
- def join(self):
- """Join the task queue"""
- self.tasks.join()
-
- def collect(self):
- """Collect available results
-
- :return: results of tasks
- :rtype: list of :class:`~subliminal.tasks.Task`
-
- """
- results = []
- while 1:
- try:
- result = self.results.get(block=False)
- results.append(result)
- except Queue.Empty:
- break
- return results
-
- def list_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None):
- """See :meth:`subliminal.list_subtitles`"""
- services = services or SERVICES
- languages = language_set(languages) if languages is not None else language_set(LANGUAGES)
- if isinstance(paths, basestring):
- paths = [paths]
- if any([not isinstance(p, unicode) for p in paths]):
- logger.warning(u'Not all entries are unicode') if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
- for task in tasks:
- self.tasks.put(task)
- self.join()
- results = self.collect()
- return group_by_video(results)
-
- def download_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None, order=None):
- """See :meth:`subliminal.download_subtitles`"""
- services = services or SERVICES
- languages = language_list(languages) if languages is not None else language_list(LANGUAGES)
- if isinstance(paths, basestring):
- paths = [paths]
- order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
- subtitles_by_video = self.list_subtitles(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
- for video, subtitles in subtitles_by_video.iteritems():
- subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
- tasks = create_download_tasks(subtitles_by_video, languages, multi)
- for task in tasks:
- self.tasks.put(task)
- self.join()
- results = self.collect()
- return group_by_video(results)
diff --git a/lib/subliminal/cache.py b/lib/subliminal/cache.py
index e8e08c3bda2279c9118b878752ecf746fa19434d..1cc1fe182b6e588b2204139d3f73f3d124735252 100644
--- a/lib/subliminal/cache.py
+++ b/lib/subliminal/cache.py
@@ -1,135 +1,62 @@
# -*- coding: utf-8 -*-
-# Copyright 2012 Nicolas Wack <wackou@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from collections import defaultdict
-from functools import wraps
-import logging
-import os.path
-import threading
-try:
- import cPickle as pickle
-except ImportError:
- import pickle
-import sys
-
-
-__all__ = ['Cache', 'cachedmethod']
-logger = logging.getLogger("subliminal")
-
-
-class Cache(object):
- """A Cache object contains cached values for methods. It can have
- separate internal caches, one for each service
-
- """
- def __init__(self, cache_dir):
- self.cache_dir = cache_dir
- self.cache = defaultdict(dict)
- self.lock = threading.RLock()
-
- def __del__(self):
- for service_name in self.cache:
- self.save(service_name)
-
- def cache_location(self, service_name):
- return os.path.join(self.cache_dir, 'subliminal_%s.cache' % service_name)
-
- def load(self, service_name):
- with self.lock:
- if service_name in self.cache:
- # already loaded
- return
-
- self.cache[service_name] = defaultdict(dict)
- filename = self.cache_location(service_name)
- logger.debug(u'Cache: loading cache from %s' % filename) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- try:
- self.cache[service_name] = pickle.load(open(filename, 'rb'))
- except IOError:
- logger.info('Cache: Cache file "%s" doesn\'t exist, creating it' % filename) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- except EOFError:
- logger.error('Cache: cache file "%s" is corrupted... Removing it.' % filename) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- os.remove(filename)
-
- def save(self, service_name):
- filename = self.cache_location(service_name)
- logger.debug(u'Cache: saving cache to %s' % filename) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- with self.lock:
- pickle.dump(self.cache[service_name], open(filename, 'wb'))
-
- def clear(self, service_name):
- try:
- os.remove(self.cache_location(service_name))
- except OSError:
- pass
- self.cache[service_name] = defaultdict(dict)
-
- def cached_func_key(self, func, cls=None):
- try:
- cls = func.im_class
- except:
- pass
- return ('%s.%s' % (cls.__module__, cls.__name__), func.__name__)
-
- def function_cache(self, service_name, func):
- func_key = self.cached_func_key(func)
- return self.cache[service_name][func_key]
-
- def cache_for(self, service_name, func, args, result):
- # no need to lock here, dict ops are atomic
- self.function_cache(service_name, func)[args] = result
-
- def cached_value(self, service_name, func, args):
- """Raises KeyError if not found"""
- # no need to lock here, dict ops are atomic
- return self.function_cache(service_name, func)[args]
-
-
-def cachedmethod(function):
- """Decorator to make a method use the cache.
-
- .. note::
-
- This can NOT be used with static functions, it has to be used on
- methods of some class
-
- """
- @wraps(function)
- def cached(*args):
- c = args[0].config.cache
- service_name = args[0].__class__.__name__
- func_key = c.cached_func_key(function, cls=args[0].__class__)
- func_cache = c.cache[service_name][func_key]
-
- # we need to remove the first element of args for the key, as it is the
- # instance pointer and we don't want the cache to know which instance
- # called it, it is shared among all instances of the same class
- key = args[1:]
-
- if key in func_cache:
- result = func_cache[key]
- logger.debug(u'Using cached value for %s(%s), returns: %s' % (func_key, key, result)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return result
-
- result = function(*args)
-
- # note: another thread could have already cached a value in the
- # meantime, but that's ok as we prefer to keep the latest value in
- # the cache
- func_cache[key] = result
- return result
- return cached
+import datetime
+import inspect
+from dogpile.cache import make_region # @UnresolvedImport
+from dogpile.cache.backends.file import AbstractFileLock # @UnresolvedImport
+from dogpile.cache.compat import string_type # @UnresolvedImport
+from dogpile.core.readwrite_lock import ReadWriteMutex # @UnresolvedImport
+
+
+#: Subliminal's cache version
+CACHE_VERSION = 1
+
+EXPIRE_SECONDS = datetime.timedelta(weeks=3)
+
+#: Expiration time for show caching
+SHOW_EXPIRATION_TIME = EXPIRE_SECONDS.days * 1440 + EXPIRE_SECONDS.seconds
+
+#: Expiration time for episode caching
+EPISODE_EXPIRATION_TIME = EXPIRE_SECONDS.days * 1440 + EXPIRE_SECONDS.seconds
+
+
+def subliminal_key_generator(namespace, fn, to_str=string_type):
+ """Add a :data:`CACHE_VERSION` to dogpile.cache's default function_key_generator"""
+ if namespace is None:
+ namespace = '%d:%s:%s' % (CACHE_VERSION, fn.__module__, fn.__name__)
+ else:
+ namespace = '%d:%s:%s|%s' % (CACHE_VERSION, fn.__module__, fn.__name__, namespace)
+
+ args = inspect.getargspec(fn)
+ has_self = args[0] and args[0][0] in ('self', 'cls')
+
+ def generate_key(*args, **kw):
+ if kw:
+ raise ValueError('Keyword arguments not supported')
+ if has_self:
+ args = args[1:]
+ return namespace + '|' + ' '.join(map(to_str, args))
+ return generate_key
+
+
+class MutexLock(AbstractFileLock):
+ """:class:`MutexLock` is a thread-based rw lock based on :class:`dogpile.core.ReadWriteMutex`"""
+ def __init__(self, filename):
+ self.mutex = ReadWriteMutex()
+
+ def acquire_read_lock(self, wait):
+ ret = self.mutex.acquire_read_lock(wait)
+ return wait or ret
+
+ def acquire_write_lock(self, wait):
+ ret = self.mutex.acquire_write_lock(wait)
+ return wait or ret
+
+ def release_read_lock(self):
+ return self.mutex.release_read_lock()
+
+ def release_write_lock(self):
+ return self.mutex.release_write_lock()
+
+
+#: The dogpile.cache region
+region = make_region(function_key_generator=subliminal_key_generator)
diff --git a/lib/subliminal/cli.py b/lib/subliminal/cli.py
new file mode 100644
index 0000000000000000000000000000000000000000..cabcdfc8bd872a81498ddc9d44572f5964136fd4
--- /dev/null
+++ b/lib/subliminal/cli.py
@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals, print_function
+import argparse
+import datetime
+import logging
+import os
+import re
+import sys
+import babelfish
+import xdg.BaseDirectory
+from subliminal import (__version__, cache_region, MutexLock, provider_manager, Video, Episode, Movie, scan_videos,
+ download_best_subtitles, save_subtitles)
+try:
+ import colorlog
+except ImportError:
+ colorlog = None
+
+
+DEFAULT_CACHE_FILE = os.path.join(xdg.BaseDirectory.save_cache_path('subliminal'), 'cli.dbm')
+
+
+def subliminal():
+ parser = argparse.ArgumentParser(prog='subliminal', description='Subtitles, faster than your thoughts',
+ epilog='Suggestions and bug reports are greatly appreciated: '
+ 'https://github.com/Diaoul/subliminal/issues', add_help=False)
+
+ # required arguments
+ required_arguments_group = parser.add_argument_group('required arguments')
+ required_arguments_group.add_argument('paths', nargs='+', metavar='PATH', help='path to video file or folder')
+ required_arguments_group.add_argument('-l', '--languages', nargs='+', required=True, metavar='LANGUAGE',
+ help='wanted languages as IETF codes e.g. fr, pt-BR, sr-Cyrl ')
+
+ # configuration
+ configuration_group = parser.add_argument_group('configuration')
+ configuration_group.add_argument('-s', '--single', action='store_true',
+ help='download without language code in subtitle\'s filename i.e. .srt only')
+ configuration_group.add_argument('-c', '--cache-file', default=DEFAULT_CACHE_FILE,
+ help='cache file (default: %(default)s)')
+
+ # filtering
+ filtering_group = parser.add_argument_group('filtering')
+ filtering_group.add_argument('-p', '--providers', nargs='+', metavar='PROVIDER',
+ help='providers to use (%s)' % ', '.join(provider_manager.available_providers))
+ filtering_group.add_argument('-m', '--min-score', type=int, default=0,
+ help='minimum score for subtitles (0-%d for episodes, 0-%d for movies)'
+ % (Episode.scores['hash'], Movie.scores['hash']))
+ filtering_group.add_argument('-a', '--age', help='download subtitles for videos newer than AGE e.g. 12h, 1w2d')
+ filtering_group.add_argument('-h', '--hearing-impaired', action='store_true',
+ help='download hearing impaired subtitles')
+ filtering_group.add_argument('-f', '--force', action='store_true',
+ help='force subtitle download for videos with existing subtitles')
+
+ # addic7ed
+ addic7ed_group = parser.add_argument_group('addic7ed')
+ addic7ed_group.add_argument('--addic7ed-username', metavar='USERNAME', help='username for addic7ed provider')
+ addic7ed_group.add_argument('--addic7ed-password', metavar='PASSWORD', help='password for addic7ed provider')
+
+ # output
+ output_group = parser.add_argument_group('output')
+ output_group.add_argument('-d', '--directory',
+ help='save subtitles in the given directory rather than next to the video')
+ output_group.add_argument('-e', '--encoding', default=None,
+ help='encoding to convert the subtitle to (default: no conversion)')
+ output_exclusive_group = output_group.add_mutually_exclusive_group()
+ output_exclusive_group.add_argument('-q', '--quiet', action='store_true', help='disable output')
+ output_exclusive_group.add_argument('-v', '--verbose', action='store_true', help='verbose output')
+ output_group.add_argument('--log-file', help='log into a file instead of stdout')
+ output_group.add_argument('--color', action='store_true', help='add color to console output (requires colorlog)')
+
+ # troubleshooting
+ troubleshooting_group = parser.add_argument_group('troubleshooting')
+ troubleshooting_group.add_argument('--debug', action='store_true', help='debug output')
+ troubleshooting_group.add_argument('--version', action='version', version=__version__)
+ troubleshooting_group.add_argument('--help', action='help', help='show this help message and exit')
+
+ # parse args
+ args = parser.parse_args()
+
+ # parse paths
+ try:
+ args.paths = [os.path.abspath(os.path.expanduser(p.decode('utf-8') if isinstance(p, bytes) else p))
+ for p in args.paths]
+ except UnicodeDecodeError:
+ parser.error('argument paths: encodings is not utf-8: %r' % args.paths)
+
+ # parse languages
+ try:
+ args.languages = {babelfish.Language.fromietf(l) for l in args.languages}
+ except babelfish.Error:
+ parser.error('argument -l/--languages: codes are not IETF: %r' % args.languages)
+
+ # parse age
+ if args.age is not None:
+ match = re.match(r'^(?:(?P<weeks>\d+?)w)?(?:(?P<days>\d+?)d)?(?:(?P<hours>\d+?)h)?$', args.age)
+ if not match:
+ parser.error('argument -a/--age: invalid age: %r' % args.age)
+ args.age = datetime.timedelta(**{k: int(v) for k, v in match.groupdict(0).items()})
+
+ # parse cache-file
+ args.cache_file = os.path.abspath(os.path.expanduser(args.cache_file))
+ if not os.path.exists(os.path.split(args.cache_file)[0]):
+ parser.error('argument -c/--cache-file: directory %r for cache file does not exist'
+ % os.path.split(args.cache_file)[0])
+
+ # parse provider configs
+ provider_configs = {}
+ if (args.addic7ed_username is not None and args.addic7ed_password is None
+ or args.addic7ed_username is None and args.addic7ed_password is not None):
+ parser.error('argument --addic7ed-username/--addic7ed-password: both arguments are required or none')
+ if args.addic7ed_username is not None and args.addic7ed_password is not None:
+ provider_configs['addic7ed'] = {'username': args.addic7ed_username, 'password': args.addic7ed_password}
+
+ # parse color
+ if args.color and colorlog is None:
+ parser.error('argument --color: colorlog required')
+
+ # setup output
+ if args.log_file is None:
+ handler = logging.StreamHandler()
+ else:
+ handler = logging.FileHandler(args.log_file, encoding='utf-8')
+ if args.debug:
+ if args.color:
+ if args.log_file is None:
+ log_format = '%(log_color)s%(levelname)-8s%(reset)s [%(blue)s%(name)s-%(funcName)s:%(lineno)d%(reset)s] %(message)s'
+ else:
+ log_format = '%(purple)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s [%(blue)s%(name)s-%(funcName)s:%(lineno)d%(reset)s] %(message)s'
+ handler.setFormatter(colorlog.ColoredFormatter(log_format,
+ log_colors=dict(colorlog.default_log_colors.items() + [('DEBUG', 'cyan')])))
+ else:
+ if args.log_file is None:
+ log_format = '%(levelname)-8s [%(name)s-%(funcName)s:%(lineno)d] %(message)s'
+ else:
+ log_format = '%(asctime)s %(levelname)-8s [%(name)s-%(funcName)s:%(lineno)d] %(message)s'
+ handler.setFormatter(logging.Formatter(log_format))
+ logging.getLogger().addHandler(handler)
+ logging.getLogger().setLevel(logging.DEBUG)
+ elif args.verbose:
+ if args.color:
+ if args.log_file is None:
+ log_format = '%(log_color)s%(levelname)-8s%(reset)s [%(blue)s%(name)s%(reset)s] %(message)s'
+ else:
+ log_format = '%(purple)s%(asctime)s%(reset)s %(log_color)s%(levelname)-8s%(reset)s [%(blue)s%(name)s%(reset)s] %(message)s'
+ handler.setFormatter(colorlog.ColoredFormatter(log_format))
+ else:
+ log_format = '%(levelname)-8s [%(name)s] %(message)s'
+ if args.log_file is not None:
+ log_format = '%(asctime)s ' + log_format
+ handler.setFormatter(logging.Formatter(log_format))
+ logging.getLogger('subliminal').addHandler(handler)
+ logging.getLogger('subliminal').setLevel(logging.INFO)
+ elif not args.quiet:
+ if args.color:
+ if args.log_file is None:
+ log_format = '[%(log_color)s%(levelname)s%(reset)s] %(message)s'
+ else:
+ log_format = '%(purple)s%(asctime)s%(reset)s [%(log_color)s%(levelname)s%(reset)s] %(message)s'
+ handler.setFormatter(colorlog.ColoredFormatter(log_format))
+ else:
+ if args.log_file is None:
+ log_format = '%(levelname)s: %(message)s'
+ else:
+ log_format = '%(asctime)s %(levelname)s: %(message)s'
+ handler.setFormatter(logging.Formatter(log_format))
+ logging.getLogger('subliminal.api').addHandler(handler)
+ logging.getLogger('subliminal.api').setLevel(logging.INFO)
+
+ # configure cache
+ cache_region.configure('dogpile.cache.dbm', expiration_time=datetime.timedelta(days=30), # @UndefinedVariable
+ arguments={'filename': args.cache_file, 'lock_factory': MutexLock})
+
+ # scan videos
+ videos = scan_videos([p for p in args.paths if os.path.exists(p)], subtitles=not args.force,
+ embedded_subtitles=not args.force, age=args.age)
+
+ # guess videos
+ videos.extend([Video.fromname(p) for p in args.paths if not os.path.exists(p)])
+
+ # download best subtitles
+ subtitles = download_best_subtitles(videos, args.languages, providers=args.providers,
+ provider_configs=provider_configs, min_score=args.min_score,
+ hearing_impaired=args.hearing_impaired, single=args.single)
+
+ # save subtitles
+ save_subtitles(subtitles, single=args.single, directory=args.directory, encoding=args.encoding)
+
+ # result output
+ if not subtitles:
+ if not args.quiet:
+ print('No subtitles downloaded', file=sys.stderr)
+ exit(1)
+ if not args.quiet:
+ subtitles_count = sum([len(s) for s in subtitles.values()])
+ if subtitles_count == 1:
+ print('%d subtitle downloaded' % subtitles_count)
+ else:
+ print('%d subtitles downloaded' % subtitles_count)
diff --git a/lib/subliminal/compat.py b/lib/subliminal/compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..28bd3e849cbc14cb89ae9f6d0c285159221f5d78
--- /dev/null
+++ b/lib/subliminal/compat.py
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+import sys
+import socket
+
+
+if sys.version_info[0] == 2:
+ from xmlrpclib import ServerProxy, Transport
+ from httplib import HTTPConnection
+elif sys.version_info[0] == 3:
+ from xmlrpc.client import ServerProxy, Transport
+ from http.client import HTTPConnection
+
+
+class TimeoutTransport(Transport, object):
+ def __init__(self, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, *args, **kwargs):
+ super(TimeoutTransport, self).__init__(*args, **kwargs)
+ self.timeout = timeout
+
+ def make_connection(self, host):
+ h = HTTPConnection(host, timeout=self.timeout)
+ return h
diff --git a/lib/subliminal/converters/__init__.py b/lib/subliminal/converters/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lib/subliminal/converters/addic7ed.py b/lib/subliminal/converters/addic7ed.py
new file mode 100644
index 0000000000000000000000000000000000000000..0e862931ddd760dc633d0b59fa0e8f7f0451da72
--- /dev/null
+++ b/lib/subliminal/converters/addic7ed.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from babelfish import LanguageReverseConverter, language_converters
+
+
+class Addic7edConverter(LanguageReverseConverter):
+ def __init__(self):
+ self.name_converter = language_converters['name']
+ self.from_addic7ed = {'Català': ('cat',), 'Chinese (Simplified)': ('zho',), 'Chinese (Traditional)': ('zho',),
+ 'Euskera': ('eus',), 'Galego': ('glg',), 'Greek': ('ell',), 'Malay': ('msa',),
+ 'Portuguese (Brazilian)': ('por', 'BR'), 'Serbian (Cyrillic)': ('srp', None, 'Cyrl'),
+ 'Serbian (Latin)': ('srp',), 'Spanish (Latin America)': ('spa',),
+ 'Spanish (Spain)': ('spa',)}
+ self.to_addic7ed = {('cat',): 'Català', ('zho',): 'Chinese (Simplified)', ('eus',): 'Euskera',
+ ('glg',): 'Galego', ('ell',): 'Greek', ('msa',): 'Malay',
+ ('por', 'BR'): 'Portuguese (Brazilian)', ('srp', None, 'Cyrl'): 'Serbian (Cyrillic)'}
+ self.codes = self.name_converter.codes | set(self.from_addic7ed.keys())
+
+ def convert(self, alpha3, country=None, script=None):
+ if (alpha3, country, script) in self.to_addic7ed:
+ return self.to_addic7ed[(alpha3, country, script)]
+ if (alpha3, country) in self.to_addic7ed:
+ return self.to_addic7ed[(alpha3, country)]
+ if (alpha3,) in self.to_addic7ed:
+ return self.to_addic7ed[(alpha3,)]
+ return self.name_converter.convert(alpha3, country, script)
+
+ def reverse(self, addic7ed):
+ if addic7ed in self.from_addic7ed:
+ return self.from_addic7ed[addic7ed]
+ return self.name_converter.reverse(addic7ed)
diff --git a/lib/subliminal/converters/podnapisi.py b/lib/subliminal/converters/podnapisi.py
new file mode 100644
index 0000000000000000000000000000000000000000..d73cb1c1fb978596ca5fff073c68195409754a1a
--- /dev/null
+++ b/lib/subliminal/converters/podnapisi.py
@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from babelfish import LanguageReverseConverter, LanguageConvertError, LanguageReverseError
+
+
+class PodnapisiConverter(LanguageReverseConverter):
+ def __init__(self):
+ self.from_podnapisi = {2: ('eng',), 28: ('spa',), 26: ('pol',), 36: ('srp',), 1: ('slv',), 38: ('hrv',),
+ 9: ('ita',), 8: ('fra',), 48: ('por', 'BR'), 23: ('nld',), 12: ('ara',), 13: ('ron',),
+ 33: ('bul',), 32: ('por',), 16: ('ell',), 15: ('hun',), 31: ('fin',), 30: ('tur',),
+ 7: ('ces',), 25: ('swe',), 27: ('rus',), 24: ('dan',), 22: ('heb',), 51: ('vie',),
+ 52: ('fas',), 5: ('deu',), 14: ('spa', 'AR'), 54: ('ind',), 47: ('srp', None, 'Cyrl'),
+ 3: ('nor',), 20: ('est',), 10: ('bos',), 17: ('zho',), 37: ('slk',), 35: ('mkd',),
+ 11: ('jpn',), 4: ('kor',), 29: ('sqi',), 6: ('isl',), 19: ('lit',), 46: ('ukr',),
+ 44: ('tha',), 53: ('cat',), 56: ('sin',), 21: ('lav',), 40: ('cmn',), 55: ('msa',),
+ 42: ('hin',), 50: ('bel',)}
+ self.to_podnapisi = {v: k for k, v in self.from_podnapisi.items()}
+ self.codes = set(self.from_podnapisi.keys())
+
+ def convert(self, alpha3, country=None, script=None):
+ if (alpha3,) in self.to_podnapisi:
+ return self.to_podnapisi[(alpha3,)]
+ if (alpha3, country) in self.to_podnapisi:
+ return self.to_podnapisi[(alpha3, country)]
+ if (alpha3, country, script) in self.to_podnapisi:
+ return self.to_podnapisi[(alpha3, country, script)]
+ raise LanguageConvertError(alpha3, country, script)
+
+ def reverse(self, podnapisi):
+ if podnapisi not in self.from_podnapisi:
+ raise LanguageReverseError(podnapisi)
+ return self.from_podnapisi[podnapisi]
diff --git a/lib/subliminal/converters/tvsubtitles.py b/lib/subliminal/converters/tvsubtitles.py
new file mode 100644
index 0000000000000000000000000000000000000000..e9b7e74fd8af4e0f772e4cd47539be1cc321b268
--- /dev/null
+++ b/lib/subliminal/converters/tvsubtitles.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from babelfish import LanguageReverseConverter, language_converters
+
+
+class TVsubtitlesConverter(LanguageReverseConverter):
+ def __init__(self):
+ self.alpha2_converter = language_converters['alpha2']
+ self.from_tvsubtitles = {'br': ('por', 'BR'), 'ua': ('ukr',), 'gr': ('ell',), 'cn': ('zho',), 'jp': ('jpn',),
+ 'cz': ('ces',)}
+ self.to_tvsubtitles = {v: k for k, v in self.from_tvsubtitles}
+ self.codes = self.alpha2_converter.codes | set(self.from_tvsubtitles.keys())
+
+ def convert(self, alpha3, country=None, script=None):
+ if (alpha3, country) in self.to_tvsubtitles:
+ return self.to_tvsubtitles[(alpha3, country)]
+ if (alpha3,) in self.to_tvsubtitles:
+ return self.to_tvsubtitles[(alpha3,)]
+ return self.alpha2_converter.convert(alpha3, country, script)
+
+ def reverse(self, tvsubtitles):
+ if tvsubtitles in self.from_tvsubtitles:
+ return self.from_tvsubtitles[tvsubtitles]
+ return self.alpha2_converter.reverse(tvsubtitles)
diff --git a/lib/subliminal/core.py b/lib/subliminal/core.py
deleted file mode 100644
index 27c45cabce3e6b4e826bbe1c4d0a8aaef5d433c9..0000000000000000000000000000000000000000
--- a/lib/subliminal/core.py
+++ /dev/null
@@ -1,277 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from .exceptions import DownloadFailedError
-from .services import ServiceConfig
-from .tasks import DownloadTask, ListTask
-from .utils import get_keywords
-from .videos import Episode, Movie, scan
-from .language import Language
-from collections import defaultdict
-from itertools import groupby
-import bs4
-import guessit
-import logging
-import sys
-
-
-__all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE', 'MATCHING_CONFIDENCE',
- 'create_list_tasks', 'create_download_tasks', 'consume_task', 'matching_confidence',
- 'key_subtitles', 'group_by_video']
-logger = logging.getLogger("subliminal")
-SERVICES = ['opensubtitles', 'subswiki', 'subtitulos', 'thesubdb', 'addic7ed', 'tvsubtitles', 'itasa',
- 'usub', 'subscenter']
-LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE = range(4)
-
-
-def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter):
- """Create a list of :class:`~subliminal.tasks.ListTask` from one or more paths using the given criteria
-
- :param paths: path(s) to video file or folder
- :type paths: string or list
- :param set languages: languages to search for
- :param list services: services to use for the search
- :param bool force: force searching for subtitles even if some are detected
- :param bool multi: search multiple languages for the same video
- :param string cache_dir: path to the cache directory to use
- :param int max_depth: maximum depth for scanning entries
- :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
- :return: the created tasks
- :rtype: list of :class:`~subliminal.tasks.ListTask`
-
- """
- scan_result = []
- for p in paths:
- scan_result.extend(scan(p, max_depth, scan_filter))
- logger.debug(u'Found %d videos in %r with maximum depth %d' % (len(scan_result), paths, max_depth)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- tasks = []
- config = ServiceConfig(multi, cache_dir)
- services = filter_services(services)
- for video, detected_subtitles in scan_result:
- detected_languages = set(s.language for s in detected_subtitles)
- wanted_languages = languages.copy()
- if not force and multi:
- wanted_languages -= detected_languages
- if not wanted_languages:
- logger.debug(u'No need to list multi subtitles %r for %r because %r detected' % (languages, video, detected_languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- if not force and not multi and Language('Undetermined') in detected_languages:
- logger.debug(u'No need to list single subtitles %r for %r because one detected' % (languages, video)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- logger.debug(u'Listing subtitles %r for %r with services %r' % (wanted_languages, video, services)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- for service_name in services:
- mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
- service = mod.Service
- if not service.check_validity(video, wanted_languages):
- continue
- task = ListTask(video, wanted_languages & service.languages, service_name, config)
- logger.debug(u'Created task %r' % task) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- tasks.append(task)
- return tasks
-
-
-def create_download_tasks(subtitles_by_video, languages, multi):
- """Create a list of :class:`~subliminal.tasks.DownloadTask` from a list results grouped by video
-
- :param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results with ordered subtitles
- :type subtitles_by_video: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`]
- :param languages: languages in preferred order
- :type languages: :class:`~subliminal.language.language_list`
- :param bool multi: download multiple languages for the same video
- :return: the created tasks
- :rtype: list of :class:`~subliminal.tasks.DownloadTask`
-
- """
- tasks = []
- for video, subtitles in subtitles_by_video.iteritems():
- if not subtitles:
- continue
- if not multi:
- task = DownloadTask(video, list(subtitles))
- logger.debug(u'Created task %r' % task) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- tasks.append(task)
- continue
- for _, by_language in groupby(subtitles, lambda s: languages.index(s.language)):
- task = DownloadTask(video, list(by_language))
- logger.debug(u'Created task %r' % task) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- tasks.append(task)
- return tasks
-
-
-def consume_task(task, services=None):
- """Consume a task. If the ``services`` parameter is given, the function will attempt
- to get the service from it. In case the service is not in ``services``, it will be initialized
- and put in ``services``
-
- :param task: task to consume
- :type task: :class:`~subliminal.tasks.ListTask` or :class:`~subliminal.tasks.DownloadTask`
- :param dict services: mapping between the service name and an instance of this service
- :return: the result of the task
- :rtype: list of :class:`~subliminal.subtitles.ResultSubtitle`
-
- """
- if services is None:
- services = {}
- logger.info(u'Consuming %r' % task) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- result = None
- if isinstance(task, ListTask):
- service = get_service(services, task.service, config=task.config)
- result = service.list(task.video, task.languages)
- elif isinstance(task, DownloadTask):
- for subtitle in task.subtitles:
- service = get_service(services, subtitle.service)
- try:
- service.download(subtitle)
- result = [subtitle]
- break
- except DownloadFailedError:
- logger.warning(u'Could not download subtitle %r, trying next' % subtitle) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- if result is None:
- logger.error(u'No subtitles could be downloaded for video %r' % task.video) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return result
-
-
-def matching_confidence(video, subtitle):
- """Compute the probability (confidence) that the subtitle matches the video
-
- :param video: video to match
- :type video: :class:`~subliminal.videos.Video`
- :param subtitle: subtitle to match
- :type subtitle: :class:`~subliminal.subtitles.Subtitle`
- :return: the matching probability
- :rtype: float
-
- """
- guess = guessit.guess_file_info(subtitle.release, 'autodetect')
- video_keywords = get_keywords(video.guess)
- subtitle_keywords = get_keywords(guess) | subtitle.keywords
- logger.debug(u'Video keywords %r - Subtitle keywords %r' % (video_keywords, subtitle_keywords)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- replacement = {'keywords': len(video_keywords & subtitle_keywords)}
- if isinstance(video, Episode):
- replacement.update({'series': 0, 'season': 0, 'episode': 0})
- matching_format = '{series:b}{season:b}{episode:b}{keywords:03b}'
- best = matching_format.format(series=1, season=1, episode=1, keywords=len(video_keywords))
- if guess['type'] in ['episode', 'episodesubtitle']:
- if 'series' in guess and guess['series'].lower() == video.series.lower():
- replacement['series'] = 1
- if 'season' in guess and guess['season'] == video.season:
- replacement['season'] = 1
- if 'episodeNumber' in guess and guess['episodeNumber'] == video.episode:
- replacement['episode'] = 1
- elif isinstance(video, Movie):
- replacement.update({'title': 0, 'year': 0})
- matching_format = '{title:b}{year:b}{keywords:03b}'
- best = matching_format.format(title=1, year=1, keywords=len(video_keywords))
- if guess['type'] in ['movie', 'moviesubtitle']:
- if 'title' in guess and guess['title'].lower() == video.title.lower():
- replacement['title'] = 1
- if 'year' in guess and guess['year'] == video.year:
- replacement['year'] = 1
- else:
- logger.debug(u'Not able to compute confidence for %r' % video) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return 0.0
- logger.debug(u'Found %r' % replacement) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- confidence = float(int(matching_format.format(**replacement), 2)) / float(int(best, 2))
- logger.info(u'Computed confidence %.4f for %r and %r' % (confidence, video, subtitle)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return confidence
-
-
-def get_service(services, service_name, config=None):
- """Get a service from its name in the service dict with the specified config.
- If the service does not exist in the service dict, it is created and added to the dict.
-
- :param dict services: dict where to get existing services or put created ones
- :param string service_name: name of the service to get
- :param config: config to use for the service
- :type config: :class:`~subliminal.services.ServiceConfig` or None
- :return: the corresponding service
- :rtype: :class:`~subliminal.services.ServiceBase`
-
- """
- if service_name not in services:
- mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
- services[service_name] = mod.Service()
- services[service_name].init()
- services[service_name].config = config
- return services[service_name]
-
-
-def key_subtitles(subtitle, video, languages, services, order):
- """Create a key to sort subtitle using the given order
-
- :param subtitle: subtitle to sort
- :type subtitle: :class:`~subliminal.subtitles.ResultSubtitle`
- :param video: video to match
- :type video: :class:`~subliminal.videos.Video`
- :param list languages: languages in preferred order
- :param list services: services in preferred order
- :param order: preferred order for subtitles sorting
- :type list: list of :data:`LANGUAGE_INDEX`, :data:`SERVICE_INDEX`, :data:`SERVICE_CONFIDENCE`, :data:`MATCHING_CONFIDENCE`
- :return: a key ready to use for subtitles sorting
- :rtype: int
-
- """
- key = ''
- for sort_item in order:
- if sort_item == LANGUAGE_INDEX:
- key += '{0:03d}'.format(len(languages) - languages.index(subtitle.language) - 1)
- key += '{0:01d}'.format(subtitle.language == languages[languages.index(subtitle.language)])
- elif sort_item == SERVICE_INDEX:
- key += '{0:02d}'.format(len(services) - services.index(subtitle.service) - 1)
- elif sort_item == SERVICE_CONFIDENCE:
- key += '{0:04d}'.format(int(subtitle.confidence * 1000))
- elif sort_item == MATCHING_CONFIDENCE:
- confidence = 0
- if subtitle.release:
- confidence = matching_confidence(video, subtitle)
- key += '{0:04d}'.format(int(confidence * 1000))
- return int(key)
-
-
-def group_by_video(list_results):
- """Group the results of :class:`ListTasks <subliminal.tasks.ListTask>` into a
- dictionary of :class:`~subliminal.videos.Video` => :class:`~subliminal.subtitles.Subtitle`
-
- :param list_results:
- :type list_results: list of result of :class:`~subliminal.tasks.ListTask`
- :return: subtitles grouped by videos
- :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`]
-
- """
- result = defaultdict(list)
- for video, subtitles in list_results:
- result[video] += subtitles or []
- return result
-
-
-def filter_services(services):
- """Filter out services that are not available because of a missing feature
-
- :param list services: service names to filter
- :return: a copy of the initial list of service names without unavailable ones
- :rtype: list
-
- """
- filtered_services = services[:]
- for service_name in services:
- mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
- service = mod.Service
- if service.required_features is not None and bs4.builder_registry.lookup(*service.required_features) is None:
- logger.warning(u'Service %s not available: none of available features could be used. One of %r required' % (service_name, service.required_features)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- filtered_services.remove(service_name)
- return filtered_services
diff --git a/lib/subliminal/exceptions.py b/lib/subliminal/exceptions.py
index 66e3dd51c2adce7f4e6f4302311911ffe76554c4..be954800a055c7e7bb1c6348dfdd0be7d5899b21 100644
--- a/lib/subliminal/exceptions.py
+++ b/lib/subliminal/exceptions.py
@@ -1,32 +1,22 @@
# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
+from __future__ import unicode_literals
class Error(Exception):
"""Base class for exceptions in subliminal"""
- pass
-class ServiceError(Error):
- """"Exception raised by services"""
- pass
+class ProviderError(Error):
+ """Exception raised by providers"""
-class DownloadFailedError(Error):
- """"Exception raised when a download task has failed in service"""
- pass
+class ConfigurationError(ProviderError):
+ """Exception raised by providers when badly configured"""
+
+
+class AuthenticationError(ProviderError):
+ """Exception raised by providers when authentication failed"""
+
+
+class DownloadLimitExceeded(ProviderError):
+ """Exception raised by providers when download limit is exceeded"""
diff --git a/lib/subliminal/infos.py b/lib/subliminal/infos.py
deleted file mode 100644
index 5ab2084ac5f7f17bcd7578a2b93efa4a0c38bb7f..0000000000000000000000000000000000000000
--- a/lib/subliminal/infos.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-__version__ = '0.6.3'
diff --git a/lib/subliminal/language.py b/lib/subliminal/language.py
deleted file mode 100644
index baf724c363d66b98132ac5445790ed789c1d1e09..0000000000000000000000000000000000000000
--- a/lib/subliminal/language.py
+++ /dev/null
@@ -1,1051 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from .utils import to_unicode
-import re
-import logging
-import sys
-
-
-logger = logging.getLogger("subliminal")
-
-
-COUNTRIES = [('AF', 'AFG', '004', u'Afghanistan'),
- ('AX', 'ALA', '248', u'Åland Islands'),
- ('AL', 'ALB', '008', u'Albania'),
- ('DZ', 'DZA', '012', u'Algeria'),
- ('AS', 'ASM', '016', u'American Samoa'),
- ('AD', 'AND', '020', u'Andorra'),
- ('AO', 'AGO', '024', u'Angola'),
- ('AI', 'AIA', '660', u'Anguilla'),
- ('AQ', 'ATA', '010', u'Antarctica'),
- ('AG', 'ATG', '028', u'Antigua and Barbuda'),
- ('AR', 'ARG', '032', u'Argentina'),
- ('AM', 'ARM', '051', u'Armenia'),
- ('AW', 'ABW', '533', u'Aruba'),
- ('AU', 'AUS', '036', u'Australia'),
- ('AT', 'AUT', '040', u'Austria'),
- ('AZ', 'AZE', '031', u'Azerbaijan'),
- ('BS', 'BHS', '044', u'Bahamas'),
- ('BH', 'BHR', '048', u'Bahrain'),
- ('BD', 'BGD', '050', u'Bangladesh'),
- ('BB', 'BRB', '052', u'Barbados'),
- ('BY', 'BLR', '112', u'Belarus'),
- ('BE', 'BEL', '056', u'Belgium'),
- ('BZ', 'BLZ', '084', u'Belize'),
- ('BJ', 'BEN', '204', u'Benin'),
- ('BM', 'BMU', '060', u'Bermuda'),
- ('BT', 'BTN', '064', u'Bhutan'),
- ('BO', 'BOL', '068', u'Bolivia, Plurinational State of'),
- ('BQ', 'BES', '535', u'Bonaire, Sint Eustatius and Saba'),
- ('BA', 'BIH', '070', u'Bosnia and Herzegovina'),
- ('BW', 'BWA', '072', u'Botswana'),
- ('BV', 'BVT', '074', u'Bouvet Island'),
- ('BR', 'BRA', '076', u'Brazil'),
- ('IO', 'IOT', '086', u'British Indian Ocean Territory'),
- ('BN', 'BRN', '096', u'Brunei Darussalam'),
- ('BG', 'BGR', '100', u'Bulgaria'),
- ('BF', 'BFA', '854', u'Burkina Faso'),
- ('BI', 'BDI', '108', u'Burundi'),
- ('KH', 'KHM', '116', u'Cambodia'),
- ('CM', 'CMR', '120', u'Cameroon'),
- ('CA', 'CAN', '124', u'Canada'),
- ('CV', 'CPV', '132', u'Cape Verde'),
- ('KY', 'CYM', '136', u'Cayman Islands'),
- ('CF', 'CAF', '140', u'Central African Republic'),
- ('TD', 'TCD', '148', u'Chad'),
- ('CL', 'CHL', '152', u'Chile'),
- ('CN', 'CHN', '156', u'China'),
- ('CX', 'CXR', '162', u'Christmas Island'),
- ('CC', 'CCK', '166', u'Cocos (Keeling) Islands'),
- ('CO', 'COL', '170', u'Colombia'),
- ('KM', 'COM', '174', u'Comoros'),
- ('CG', 'COG', '178', u'Congo'),
- ('CD', 'COD', '180', u'Congo, The Democratic Republic of the'),
- ('CK', 'COK', '184', u'Cook Islands'),
- ('CR', 'CRI', '188', u'Costa Rica'),
- ('CI', 'CIV', '384', u'Côte d\'Ivoire'),
- ('HR', 'HRV', '191', u'Croatia'),
- ('CU', 'CUB', '192', u'Cuba'),
- ('CW', 'CUW', '531', u'Curaçao'),
- ('CY', 'CYP', '196', u'Cyprus'),
- ('CZ', 'CZE', '203', u'Czech Republic'),
- ('DK', 'DNK', '208', u'Denmark'),
- ('DJ', 'DJI', '262', u'Djibouti'),
- ('DM', 'DMA', '212', u'Dominica'),
- ('DO', 'DOM', '214', u'Dominican Republic'),
- ('EC', 'ECU', '218', u'Ecuador'),
- ('EG', 'EGY', '818', u'Egypt'),
- ('SV', 'SLV', '222', u'El Salvador'),
- ('GQ', 'GNQ', '226', u'Equatorial Guinea'),
- ('ER', 'ERI', '232', u'Eritrea'),
- ('EE', 'EST', '233', u'Estonia'),
- ('ET', 'ETH', '231', u'Ethiopia'),
- ('FK', 'FLK', '238', u'Falkland Islands (Malvinas)'),
- ('FO', 'FRO', '234', u'Faroe Islands'),
- ('FJ', 'FJI', '242', u'Fiji'),
- ('FI', 'FIN', '246', u'Finland'),
- ('FR', 'FRA', '250', u'France'),
- ('GF', 'GUF', '254', u'French Guiana'),
- ('PF', 'PYF', '258', u'French Polynesia'),
- ('TF', 'ATF', '260', u'French Southern Territories'),
- ('GA', 'GAB', '266', u'Gabon'),
- ('GM', 'GMB', '270', u'Gambia'),
- ('GE', 'GEO', '268', u'Georgia'),
- ('DE', 'DEU', '276', u'Germany'),
- ('GH', 'GHA', '288', u'Ghana'),
- ('GI', 'GIB', '292', u'Gibraltar'),
- ('GR', 'GRC', '300', u'Greece'),
- ('GL', 'GRL', '304', u'Greenland'),
- ('GD', 'GRD', '308', u'Grenada'),
- ('GP', 'GLP', '312', u'Guadeloupe'),
- ('GU', 'GUM', '316', u'Guam'),
- ('GT', 'GTM', '320', u'Guatemala'),
- ('GG', 'GGY', '831', u'Guernsey'),
- ('GN', 'GIN', '324', u'Guinea'),
- ('GW', 'GNB', '624', u'Guinea-Bissau'),
- ('GY', 'GUY', '328', u'Guyana'),
- ('HT', 'HTI', '332', u'Haiti'),
- ('HM', 'HMD', '334', u'Heard Island and McDonald Islands'),
- ('VA', 'VAT', '336', u'Holy See (Vatican City State)'),
- ('HN', 'HND', '340', u'Honduras'),
- ('HK', 'HKG', '344', u'Hong Kong'),
- ('HU', 'HUN', '348', u'Hungary'),
- ('IS', 'ISL', '352', u'Iceland'),
- ('IN', 'IND', '356', u'India'),
- ('ID', 'IDN', '360', u'Indonesia'),
- ('IR', 'IRN', '364', u'Iran, Islamic Republic of'),
- ('IQ', 'IRQ', '368', u'Iraq'),
- ('IE', 'IRL', '372', u'Ireland'),
- ('IM', 'IMN', '833', u'Isle of Man'),
- ('IL', 'ISR', '376', u'Israel'),
- ('IT', 'ITA', '380', u'Italy'),
- ('JM', 'JAM', '388', u'Jamaica'),
- ('JP', 'JPN', '392', u'Japan'),
- ('JE', 'JEY', '832', u'Jersey'),
- ('JO', 'JOR', '400', u'Jordan'),
- ('KZ', 'KAZ', '398', u'Kazakhstan'),
- ('KE', 'KEN', '404', u'Kenya'),
- ('KI', 'KIR', '296', u'Kiribati'),
- ('KP', 'PRK', '408', u'Korea, Democratic People\'s Republic of'),
- ('KR', 'KOR', '410', u'Korea, Republic of'),
- ('KW', 'KWT', '414', u'Kuwait'),
- ('KG', 'KGZ', '417', u'Kyrgyzstan'),
- ('LA', 'LAO', '418', u'Lao People\'s Democratic Republic'),
- ('LV', 'LVA', '428', u'Latvia'),
- ('LB', 'LBN', '422', u'Lebanon'),
- ('LS', 'LSO', '426', u'Lesotho'),
- ('LR', 'LBR', '430', u'Liberia'),
- ('LY', 'LBY', '434', u'Libya'),
- ('LI', 'LIE', '438', u'Liechtenstein'),
- ('LT', 'LTU', '440', u'Lithuania'),
- ('LU', 'LUX', '442', u'Luxembourg'),
- ('MO', 'MAC', '446', u'Macao'),
- ('MK', 'MKD', '807', u'Macedonia, Republic of'),
- ('MG', 'MDG', '450', u'Madagascar'),
- ('MW', 'MWI', '454', u'Malawi'),
- ('MY', 'MYS', '458', u'Malaysia'),
- ('MV', 'MDV', '462', u'Maldives'),
- ('ML', 'MLI', '466', u'Mali'),
- ('MT', 'MLT', '470', u'Malta'),
- ('MH', 'MHL', '584', u'Marshall Islands'),
- ('MQ', 'MTQ', '474', u'Martinique'),
- ('MR', 'MRT', '478', u'Mauritania'),
- ('MU', 'MUS', '480', u'Mauritius'),
- ('YT', 'MYT', '175', u'Mayotte'),
- ('MX', 'MEX', '484', u'Mexico'),
- ('FM', 'FSM', '583', u'Micronesia, Federated States of'),
- ('MD', 'MDA', '498', u'Moldova, Republic of'),
- ('MC', 'MCO', '492', u'Monaco'),
- ('MN', 'MNG', '496', u'Mongolia'),
- ('ME', 'MNE', '499', u'Montenegro'),
- ('MS', 'MSR', '500', u'Montserrat'),
- ('MA', 'MAR', '504', u'Morocco'),
- ('MZ', 'MOZ', '508', u'Mozambique'),
- ('MM', 'MMR', '104', u'Myanmar'),
- ('NA', 'NAM', '516', u'Namibia'),
- ('NR', 'NRU', '520', u'Nauru'),
- ('NP', 'NPL', '524', u'Nepal'),
- ('NL', 'NLD', '528', u'Netherlands'),
- ('NC', 'NCL', '540', u'New Caledonia'),
- ('NZ', 'NZL', '554', u'New Zealand'),
- ('NI', 'NIC', '558', u'Nicaragua'),
- ('NE', 'NER', '562', u'Niger'),
- ('NG', 'NGA', '566', u'Nigeria'),
- ('NU', 'NIU', '570', u'Niue'),
- ('NF', 'NFK', '574', u'Norfolk Island'),
- ('MP', 'MNP', '580', u'Northern Mariana Islands'),
- ('NO', 'NOR', '578', u'Norway'),
- ('OM', 'OMN', '512', u'Oman'),
- ('PK', 'PAK', '586', u'Pakistan'),
- ('PW', 'PLW', '585', u'Palau'),
- ('PS', 'PSE', '275', u'Palestinian Territory, Occupied'),
- ('PA', 'PAN', '591', u'Panama'),
- ('PG', 'PNG', '598', u'Papua New Guinea'),
- ('PY', 'PRY', '600', u'Paraguay'),
- ('PE', 'PER', '604', u'Peru'),
- ('PH', 'PHL', '608', u'Philippines'),
- ('PN', 'PCN', '612', u'Pitcairn'),
- ('PL', 'POL', '616', u'Poland'),
- ('PT', 'PRT', '620', u'Portugal'),
- ('PR', 'PRI', '630', u'Puerto Rico'),
- ('QA', 'QAT', '634', u'Qatar'),
- ('RE', 'REU', '638', u'Réunion'),
- ('RO', 'ROU', '642', u'Romania'),
- ('RU', 'RUS', '643', u'Russian Federation'),
- ('RW', 'RWA', '646', u'Rwanda'),
- ('BL', 'BLM', '652', u'Saint Barthélemy'),
- ('SH', 'SHN', '654', u'Saint Helena, Ascension and Tristan da Cunha'),
- ('KN', 'KNA', '659', u'Saint Kitts and Nevis'),
- ('LC', 'LCA', '662', u'Saint Lucia'),
- ('MF', 'MAF', '663', u'Saint Martin (French part)'),
- ('PM', 'SPM', '666', u'Saint Pierre and Miquelon'),
- ('VC', 'VCT', '670', u'Saint Vincent and the Grenadines'),
- ('WS', 'WSM', '882', u'Samoa'),
- ('SM', 'SMR', '674', u'San Marino'),
- ('ST', 'STP', '678', u'Sao Tome and Principe'),
- ('SA', 'SAU', '682', u'Saudi Arabia'),
- ('SN', 'SEN', '686', u'Senegal'),
- ('RS', 'SRB', '688', u'Serbia'),
- ('SC', 'SYC', '690', u'Seychelles'),
- ('SL', 'SLE', '694', u'Sierra Leone'),
- ('SG', 'SGP', '702', u'Singapore'),
- ('SX', 'SXM', '534', u'Sint Maarten (Dutch part)'),
- ('SK', 'SVK', '703', u'Slovakia'),
- ('SI', 'SVN', '705', u'Slovenia'),
- ('SB', 'SLB', '090', u'Solomon Islands'),
- ('SO', 'SOM', '706', u'Somalia'),
- ('ZA', 'ZAF', '710', u'South Africa'),
- ('GS', 'SGS', '239', u'South Georgia and the South Sandwich Islands'),
- ('ES', 'ESP', '724', u'Spain'),
- ('LK', 'LKA', '144', u'Sri Lanka'),
- ('SD', 'SDN', '729', u'Sudan'),
- ('SR', 'SUR', '740', u'Suriname'),
- ('SS', 'SSD', '728', u'South Sudan'),
- ('SJ', 'SJM', '744', u'Svalbard and Jan Mayen'),
- ('SZ', 'SWZ', '748', u'Swaziland'),
- ('SE', 'SWE', '752', u'Sweden'),
- ('CH', 'CHE', '756', u'Switzerland'),
- ('SY', 'SYR', '760', u'Syrian Arab Republic'),
- ('TW', 'TWN', '158', u'Taiwan, Province of China'),
- ('TJ', 'TJK', '762', u'Tajikistan'),
- ('TZ', 'TZA', '834', u'Tanzania, United Republic of'),
- ('TH', 'THA', '764', u'Thailand'),
- ('TL', 'TLS', '626', u'Timor-Leste'),
- ('TG', 'TGO', '768', u'Togo'),
- ('TK', 'TKL', '772', u'Tokelau'),
- ('TO', 'TON', '776', u'Tonga'),
- ('TT', 'TTO', '780', u'Trinidad and Tobago'),
- ('TN', 'TUN', '788', u'Tunisia'),
- ('TR', 'TUR', '792', u'Turkey'),
- ('TM', 'TKM', '795', u'Turkmenistan'),
- ('TC', 'TCA', '796', u'Turks and Caicos Islands'),
- ('TV', 'TUV', '798', u'Tuvalu'),
- ('UG', 'UGA', '800', u'Uganda'),
- ('UA', 'UKR', '804', u'Ukraine'),
- ('AE', 'ARE', '784', u'United Arab Emirates'),
- ('GB', 'GBR', '826', u'United Kingdom'),
- ('US', 'USA', '840', u'United States'),
- ('UM', 'UMI', '581', u'United States Minor Outlying Islands'),
- ('UY', 'URY', '858', u'Uruguay'),
- ('UZ', 'UZB', '860', u'Uzbekistan'),
- ('VU', 'VUT', '548', u'Vanuatu'),
- ('VE', 'VEN', '862', u'Venezuela, Bolivarian Republic of'),
- ('VN', 'VNM', '704', u'Viet Nam'),
- ('VG', 'VGB', '092', u'Virgin Islands, British'),
- ('VI', 'VIR', '850', u'Virgin Islands, U.S.'),
- ('WF', 'WLF', '876', u'Wallis and Futuna'),
- ('EH', 'ESH', '732', u'Western Sahara'),
- ('YE', 'YEM', '887', u'Yemen'),
- ('ZM', 'ZMB', '894', u'Zambia'),
- ('ZW', 'ZWE', '716', u'Zimbabwe')]
-
-
-LANGUAGES = [('aar', '', 'aa', u'Afar', u'afar'),
- ('abk', '', 'ab', u'Abkhazian', u'abkhaze'),
- ('ace', '', '', u'Achinese', u'aceh'),
- ('ach', '', '', u'Acoli', u'acoli'),
- ('ada', '', '', u'Adangme', u'adangme'),
- ('ady', '', '', u'Adyghe; Adygei', u'adyghé'),
- ('afa', '', '', u'Afro-Asiatic languages', u'afro-asiatiques, langues'),
- ('afh', '', '', u'Afrihili', u'afrihili'),
- ('afr', '', 'af', u'Afrikaans', u'afrikaans'),
- ('ain', '', '', u'Ainu', u'aïnou'),
- ('aka', '', 'ak', u'Akan', u'akan'),
- ('akk', '', '', u'Akkadian', u'akkadien'),
- ('alb', 'sqi', 'sq', u'Albanian', u'albanais'),
- ('ale', '', '', u'Aleut', u'aléoute'),
- ('alg', '', '', u'Algonquian languages', u'algonquines, langues'),
- ('alt', '', '', u'Southern Altai', u'altai du Sud'),
- ('amh', '', 'am', u'Amharic', u'amharique'),
- ('ang', '', '', u'English, Old (ca.450-1100)', u'anglo-saxon (ca.450-1100)'),
- ('anp', '', '', u'Angika', u'angika'),
- ('apa', '', '', u'Apache languages', u'apaches, langues'),
- ('ara', '', 'ar', u'Arabic', u'arabe'),
- ('arc', '', '', u'Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)', u'araméen d\'empire (700-300 BCE)'),
- ('arg', '', 'an', u'Aragonese', u'aragonais'),
- ('arm', 'hye', 'hy', u'Armenian', u'arménien'),
- ('arn', '', '', u'Mapudungun; Mapuche', u'mapudungun; mapuche; mapuce'),
- ('arp', '', '', u'Arapaho', u'arapaho'),
- ('art', '', '', u'Artificial languages', u'artificielles, langues'),
- ('arw', '', '', u'Arawak', u'arawak'),
- ('asm', '', 'as', u'Assamese', u'assamais'),
- ('ast', '', '', u'Asturian; Bable; Leonese; Asturleonese', u'asturien; bable; léonais; asturoléonais'),
- ('ath', '', '', u'Athapascan languages', u'athapascanes, langues'),
- ('aus', '', '', u'Australian languages', u'australiennes, langues'),
- ('ava', '', 'av', u'Avaric', u'avar'),
- ('ave', '', 'ae', u'Avestan', u'avestique'),
- ('awa', '', '', u'Awadhi', u'awadhi'),
- ('aym', '', 'ay', u'Aymara', u'aymara'),
- ('aze', '', 'az', u'Azerbaijani', u'azéri'),
- ('bad', '', '', u'Banda languages', u'banda, langues'),
- ('bai', '', '', u'Bamileke languages', u'bamiléké, langues'),
- ('bak', '', 'ba', u'Bashkir', u'bachkir'),
- ('bal', '', '', u'Baluchi', u'baloutchi'),
- ('bam', '', 'bm', u'Bambara', u'bambara'),
- ('ban', '', '', u'Balinese', u'balinais'),
- ('baq', 'eus', 'eu', u'Basque', u'basque'),
- ('bas', '', '', u'Basa', u'basa'),
- ('bat', '', '', u'Baltic languages', u'baltes, langues'),
- ('bej', '', '', u'Beja; Bedawiyet', u'bedja'),
- ('bel', '', 'be', u'Belarusian', u'biélorusse'),
- ('bem', '', '', u'Bemba', u'bemba'),
- ('ben', '', 'bn', u'Bengali', u'bengali'),
- ('ber', '', '', u'Berber languages', u'berbères, langues'),
- ('bho', '', '', u'Bhojpuri', u'bhojpuri'),
- ('bih', '', 'bh', u'Bihari languages', u'langues biharis'),
- ('bik', '', '', u'Bikol', u'bikol'),
- ('bin', '', '', u'Bini; Edo', u'bini; edo'),
- ('bis', '', 'bi', u'Bislama', u'bichlamar'),
- ('bla', '', '', u'Siksika', u'blackfoot'),
- ('bnt', '', '', u'Bantu (Other)', u'bantoues, autres langues'),
- ('bos', '', 'bs', u'Bosnian', u'bosniaque'),
- ('bra', '', '', u'Braj', u'braj'),
- ('bre', '', 'br', u'Breton', u'breton'),
- ('btk', '', '', u'Batak languages', u'batak, langues'),
- ('bua', '', '', u'Buriat', u'bouriate'),
- ('bug', '', '', u'Buginese', u'bugi'),
- ('bul', '', 'bg', u'Bulgarian', u'bulgare'),
- ('bur', 'mya', 'my', u'Burmese', u'birman'),
- ('byn', '', '', u'Blin; Bilin', u'blin; bilen'),
- ('cad', '', '', u'Caddo', u'caddo'),
- ('cai', '', '', u'Central American Indian languages', u'amérindiennes de L\'Amérique centrale, langues'),
- ('car', '', '', u'Galibi Carib', u'karib; galibi; carib'),
- ('cat', '', 'ca', u'Catalan; Valencian', u'catalan; valencien'),
- ('cau', '', '', u'Caucasian languages', u'caucasiennes, langues'),
- ('ceb', '', '', u'Cebuano', u'cebuano'),
- ('cel', '', '', u'Celtic languages', u'celtiques, langues; celtes, langues'),
- ('cha', '', 'ch', u'Chamorro', u'chamorro'),
- ('chb', '', '', u'Chibcha', u'chibcha'),
- ('che', '', 'ce', u'Chechen', u'tchétchène'),
- ('chg', '', '', u'Chagatai', u'djaghataï'),
- ('chi', 'zho', 'zh', u'Chinese', u'chinois'),
- ('chk', '', '', u'Chuukese', u'chuuk'),
- ('chm', '', '', u'Mari', u'mari'),
- ('chn', '', '', u'Chinook jargon', u'chinook, jargon'),
- ('cho', '', '', u'Choctaw', u'choctaw'),
- ('chp', '', '', u'Chipewyan; Dene Suline', u'chipewyan'),
- ('chr', '', '', u'Cherokee', u'cherokee'),
- ('chu', '', 'cu', u'Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic', u'slavon d\'église; vieux slave; slavon liturgique; vieux bulgare'),
- ('chv', '', 'cv', u'Chuvash', u'tchouvache'),
- ('chy', '', '', u'Cheyenne', u'cheyenne'),
- ('cmc', '', '', u'Chamic languages', u'chames, langues'),
- ('cop', '', '', u'Coptic', u'copte'),
- ('cor', '', 'kw', u'Cornish', u'cornique'),
- ('cos', '', 'co', u'Corsican', u'corse'),
- ('cpe', '', '', u'Creoles and pidgins, English based', u'créoles et pidgins basés sur l\'anglais'),
- ('cpf', '', '', u'Creoles and pidgins, French-based ', u'créoles et pidgins basés sur le français'),
- ('cpp', '', '', u'Creoles and pidgins, Portuguese-based ', u'créoles et pidgins basés sur le portugais'),
- ('cre', '', 'cr', u'Cree', u'cree'),
- ('crh', '', '', u'Crimean Tatar; Crimean Turkish', u'tatar de Crimé'),
- ('crp', '', '', u'Creoles and pidgins ', u'créoles et pidgins'),
- ('csb', '', '', u'Kashubian', u'kachoube'),
- ('cus', '', '', u'Cushitic languages', u'couchitiques, langues'),
- ('cze', 'ces', 'cs', u'Czech', u'tchèque'),
- ('dak', '', '', u'Dakota', u'dakota'),
- ('dan', '', 'da', u'Danish', u'danois'),
- ('dar', '', '', u'Dargwa', u'dargwa'),
- ('day', '', '', u'Land Dayak languages', u'dayak, langues'),
- ('del', '', '', u'Delaware', u'delaware'),
- ('den', '', '', u'Slave (Athapascan)', u'esclave (athapascan)'),
- ('dgr', '', '', u'Dogrib', u'dogrib'),
- ('din', '', '', u'Dinka', u'dinka'),
- ('div', '', 'dv', u'Divehi; Dhivehi; Maldivian', u'maldivien'),
- ('doi', '', '', u'Dogri', u'dogri'),
- ('dra', '', '', u'Dravidian languages', u'dravidiennes, langues'),
- ('dsb', '', '', u'Lower Sorbian', u'bas-sorabe'),
- ('dua', '', '', u'Duala', u'douala'),
- ('dum', '', '', u'Dutch, Middle (ca.1050-1350)', u'néerlandais moyen (ca. 1050-1350)'),
- ('dut', 'nld', 'nl', u'Dutch; Flemish', u'néerlandais; flamand'),
- ('dyu', '', '', u'Dyula', u'dioula'),
- ('dzo', '', 'dz', u'Dzongkha', u'dzongkha'),
- ('efi', '', '', u'Efik', u'efik'),
- ('egy', '', '', u'Egyptian (Ancient)', u'égyptien'),
- ('eka', '', '', u'Ekajuk', u'ekajuk'),
- ('elx', '', '', u'Elamite', u'élamite'),
- ('eng', '', 'en', u'English', u'anglais'),
- ('enm', '', '', u'English, Middle (1100-1500)', u'anglais moyen (1100-1500)'),
- ('epo', '', 'eo', u'Esperanto', u'espéranto'),
- ('est', '', 'et', u'Estonian', u'estonien'),
- ('ewe', '', 'ee', u'Ewe', u'éwé'),
- ('ewo', '', '', u'Ewondo', u'éwondo'),
- ('fan', '', '', u'Fang', u'fang'),
- ('fao', '', 'fo', u'Faroese', u'féroïen'),
- ('fat', '', '', u'Fanti', u'fanti'),
- ('fij', '', 'fj', u'Fijian', u'fidjien'),
- ('fil', '', '', u'Filipino; Pilipino', u'filipino; pilipino'),
- ('fin', '', 'fi', u'Finnish', u'finnois'),
- ('fiu', '', '', u'Finno-Ugrian languages', u'finno-ougriennes, langues'),
- ('fon', '', '', u'Fon', u'fon'),
- ('fre', 'fra', 'fr', u'French', u'français'),
- ('frm', '', '', u'French, Middle (ca.1400-1600)', u'français moyen (1400-1600)'),
- ('fro', '', '', u'French, Old (842-ca.1400)', u'français ancien (842-ca.1400)'),
- ('frr', '', '', u'Northern Frisian', u'frison septentrional'),
- ('frs', '', '', u'Eastern Frisian', u'frison oriental'),
- ('fry', '', 'fy', u'Western Frisian', u'frison occidental'),
- ('ful', '', 'ff', u'Fulah', u'peul'),
- ('fur', '', '', u'Friulian', u'frioulan'),
- ('gaa', '', '', u'Ga', u'ga'),
- ('gay', '', '', u'Gayo', u'gayo'),
- ('gba', '', '', u'Gbaya', u'gbaya'),
- ('gem', '', '', u'Germanic languages', u'germaniques, langues'),
- ('geo', 'kat', 'ka', u'Georgian', u'géorgien'),
- ('ger', 'deu', 'de', u'German', u'allemand'),
- ('gez', '', '', u'Geez', u'guèze'),
- ('gil', '', '', u'Gilbertese', u'kiribati'),
- ('gla', '', 'gd', u'Gaelic; Scottish Gaelic', u'gaélique; gaélique écossais'),
- ('gle', '', 'ga', u'Irish', u'irlandais'),
- ('glg', '', 'gl', u'Galician', u'galicien'),
- ('glv', '', 'gv', u'Manx', u'manx; mannois'),
- ('gmh', '', '', u'German, Middle High (ca.1050-1500)', u'allemand, moyen haut (ca. 1050-1500)'),
- ('goh', '', '', u'German, Old High (ca.750-1050)', u'allemand, vieux haut (ca. 750-1050)'),
- ('gon', '', '', u'Gondi', u'gond'),
- ('gor', '', '', u'Gorontalo', u'gorontalo'),
- ('got', '', '', u'Gothic', u'gothique'),
- ('grb', '', '', u'Grebo', u'grebo'),
- ('grc', '', '', u'Greek, Ancient (to 1453)', u'grec ancien (jusqu\'à 1453)'),
- ('gre', 'ell', 'el', u'Greek, Modern (1453-)', u'grec moderne (après 1453)'),
- ('grn', '', 'gn', u'Guarani', u'guarani'),
- ('gsw', '', '', u'Swiss German; Alemannic; Alsatian', u'suisse alémanique; alémanique; alsacien'),
- ('guj', '', 'gu', u'Gujarati', u'goudjrati'),
- ('gwi', '', '', u'Gwich\'in', u'gwich\'in'),
- ('hai', '', '', u'Haida', u'haida'),
- ('hat', '', 'ht', u'Haitian; Haitian Creole', u'haïtien; créole haïtien'),
- ('hau', '', 'ha', u'Hausa', u'haoussa'),
- ('haw', '', '', u'Hawaiian', u'hawaïen'),
- ('heb', '', 'he', u'Hebrew', u'hébreu'),
- ('her', '', 'hz', u'Herero', u'herero'),
- ('hil', '', '', u'Hiligaynon', u'hiligaynon'),
- ('him', '', '', u'Himachali languages; Western Pahari languages', u'langues himachalis; langues paharis occidentales'),
- ('hin', '', 'hi', u'Hindi', u'hindi'),
- ('hit', '', '', u'Hittite', u'hittite'),
- ('hmn', '', '', u'Hmong; Mong', u'hmong'),
- ('hmo', '', 'ho', u'Hiri Motu', u'hiri motu'),
- ('hrv', '', 'hr', u'Croatian', u'croate'),
- ('hsb', '', '', u'Upper Sorbian', u'haut-sorabe'),
- ('hun', '', 'hu', u'Hungarian', u'hongrois'),
- ('hup', '', '', u'Hupa', u'hupa'),
- ('iba', '', '', u'Iban', u'iban'),
- ('ibo', '', 'ig', u'Igbo', u'igbo'),
- ('ice', 'isl', 'is', u'Icelandic', u'islandais'),
- ('ido', '', 'io', u'Ido', u'ido'),
- ('iii', '', 'ii', u'Sichuan Yi; Nuosu', u'yi de Sichuan'),
- ('ijo', '', '', u'Ijo languages', u'ijo, langues'),
- ('iku', '', 'iu', u'Inuktitut', u'inuktitut'),
- ('ile', '', 'ie', u'Interlingue; Occidental', u'interlingue'),
- ('ilo', '', '', u'Iloko', u'ilocano'),
- ('ina', '', 'ia', u'Interlingua (International Auxiliary Language Association)', u'interlingua (langue auxiliaire internationale)'),
- ('inc', '', '', u'Indic languages', u'indo-aryennes, langues'),
- ('ind', '', 'id', u'Indonesian', u'indonésien'),
- ('ine', '', '', u'Indo-European languages', u'indo-européennes, langues'),
- ('inh', '', '', u'Ingush', u'ingouche'),
- ('ipk', '', 'ik', u'Inupiaq', u'inupiaq'),
- ('ira', '', '', u'Iranian languages', u'iraniennes, langues'),
- ('iro', '', '', u'Iroquoian languages', u'iroquoises, langues'),
- ('ita', '', 'it', u'Italian', u'italien'),
- ('jav', '', 'jv', u'Javanese', u'javanais'),
- ('jbo', '', '', u'Lojban', u'lojban'),
- ('jpn', '', 'ja', u'Japanese', u'japonais'),
- ('jpr', '', '', u'Judeo-Persian', u'judéo-persan'),
- ('jrb', '', '', u'Judeo-Arabic', u'judéo-arabe'),
- ('kaa', '', '', u'Kara-Kalpak', u'karakalpak'),
- ('kab', '', '', u'Kabyle', u'kabyle'),
- ('kac', '', '', u'Kachin; Jingpho', u'kachin; jingpho'),
- ('kal', '', 'kl', u'Kalaallisut; Greenlandic', u'groenlandais'),
- ('kam', '', '', u'Kamba', u'kamba'),
- ('kan', '', 'kn', u'Kannada', u'kannada'),
- ('kar', '', '', u'Karen languages', u'karen, langues'),
- ('kas', '', 'ks', u'Kashmiri', u'kashmiri'),
- ('kau', '', 'kr', u'Kanuri', u'kanouri'),
- ('kaw', '', '', u'Kawi', u'kawi'),
- ('kaz', '', 'kk', u'Kazakh', u'kazakh'),
- ('kbd', '', '', u'Kabardian', u'kabardien'),
- ('kha', '', '', u'Khasi', u'khasi'),
- ('khi', '', '', u'Khoisan languages', u'khoïsan, langues'),
- ('khm', '', 'km', u'Central Khmer', u'khmer central'),
- ('kho', '', '', u'Khotanese; Sakan', u'khotanais; sakan'),
- ('kik', '', 'ki', u'Kikuyu; Gikuyu', u'kikuyu'),
- ('kin', '', 'rw', u'Kinyarwanda', u'rwanda'),
- ('kir', '', 'ky', u'Kirghiz; Kyrgyz', u'kirghiz'),
- ('kmb', '', '', u'Kimbundu', u'kimbundu'),
- ('kok', '', '', u'Konkani', u'konkani'),
- ('kom', '', 'kv', u'Komi', u'kom'),
- ('kon', '', 'kg', u'Kongo', u'kongo'),
- ('kor', '', 'ko', u'Korean', u'coréen'),
- ('kos', '', '', u'Kosraean', u'kosrae'),
- ('kpe', '', '', u'Kpelle', u'kpellé'),
- ('krc', '', '', u'Karachay-Balkar', u'karatchai balkar'),
- ('krl', '', '', u'Karelian', u'carélien'),
- ('kro', '', '', u'Kru languages', u'krou, langues'),
- ('kru', '', '', u'Kurukh', u'kurukh'),
- ('kua', '', 'kj', u'Kuanyama; Kwanyama', u'kuanyama; kwanyama'),
- ('kum', '', '', u'Kumyk', u'koumyk'),
- ('kur', '', 'ku', u'Kurdish', u'kurde'),
- ('kut', '', '', u'Kutenai', u'kutenai'),
- ('lad', '', '', u'Ladino', u'judéo-espagnol'),
- ('lah', '', '', u'Lahnda', u'lahnda'),
- ('lam', '', '', u'Lamba', u'lamba'),
- ('lao', '', 'lo', u'Lao', u'lao'),
- ('lat', '', 'la', u'Latin', u'latin'),
- ('lav', '', 'lv', u'Latvian', u'letton'),
- ('lez', '', '', u'Lezghian', u'lezghien'),
- ('lim', '', 'li', u'Limburgan; Limburger; Limburgish', u'limbourgeois'),
- ('lin', '', 'ln', u'Lingala', u'lingala'),
- ('lit', '', 'lt', u'Lithuanian', u'lituanien'),
- ('lol', '', '', u'Mongo', u'mongo'),
- ('loz', '', '', u'Lozi', u'lozi'),
- ('ltz', '', 'lb', u'Luxembourgish; Letzeburgesch', u'luxembourgeois'),
- ('lua', '', '', u'Luba-Lulua', u'luba-lulua'),
- ('lub', '', 'lu', u'Luba-Katanga', u'luba-katanga'),
- ('lug', '', 'lg', u'Ganda', u'ganda'),
- ('lui', '', '', u'Luiseno', u'luiseno'),
- ('lun', '', '', u'Lunda', u'lunda'),
- ('luo', '', '', u'Luo (Kenya and Tanzania)', u'luo (Kenya et Tanzanie)'),
- ('lus', '', '', u'Lushai', u'lushai'),
- ('mac', 'mkd', 'mk', u'Macedonian', u'macédonien'),
- ('mad', '', '', u'Madurese', u'madourais'),
- ('mag', '', '', u'Magahi', u'magahi'),
- ('mah', '', 'mh', u'Marshallese', u'marshall'),
- ('mai', '', '', u'Maithili', u'maithili'),
- ('mak', '', '', u'Makasar', u'makassar'),
- ('mal', '', 'ml', u'Malayalam', u'malayalam'),
- ('man', '', '', u'Mandingo', u'mandingue'),
- ('mao', 'mri', 'mi', u'Maori', u'maori'),
- ('map', '', '', u'Austronesian languages', u'austronésiennes, langues'),
- ('mar', '', 'mr', u'Marathi', u'marathe'),
- ('mas', '', '', u'Masai', u'massaï'),
- ('may', 'msa', 'ms', u'Malay', u'malais'),
- ('mdf', '', '', u'Moksha', u'moksa'),
- ('mdr', '', '', u'Mandar', u'mandar'),
- ('men', '', '', u'Mende', u'mendé'),
- ('mga', '', '', u'Irish, Middle (900-1200)', u'irlandais moyen (900-1200)'),
- ('mic', '', '', u'Mi\'kmaq; Micmac', u'mi\'kmaq; micmac'),
- ('min', '', '', u'Minangkabau', u'minangkabau'),
- ('mkh', '', '', u'Mon-Khmer languages', u'môn-khmer, langues'),
- ('mlg', '', 'mg', u'Malagasy', u'malgache'),
- ('mlt', '', 'mt', u'Maltese', u'maltais'),
- ('mnc', '', '', u'Manchu', u'mandchou'),
- ('mni', '', '', u'Manipuri', u'manipuri'),
- ('mno', '', '', u'Manobo languages', u'manobo, langues'),
- ('moh', '', '', u'Mohawk', u'mohawk'),
- ('mon', '', 'mn', u'Mongolian', u'mongol'),
- ('mos', '', '', u'Mossi', u'moré'),
- ('mun', '', '', u'Munda languages', u'mounda, langues'),
- ('mus', '', '', u'Creek', u'muskogee'),
- ('mwl', '', '', u'Mirandese', u'mirandais'),
- ('mwr', '', '', u'Marwari', u'marvari'),
- ('myn', '', '', u'Mayan languages', u'maya, langues'),
- ('myv', '', '', u'Erzya', u'erza'),
- ('nah', '', '', u'Nahuatl languages', u'nahuatl, langues'),
- ('nai', '', '', u'North American Indian languages', u'nord-amérindiennes, langues'),
- ('nap', '', '', u'Neapolitan', u'napolitain'),
- ('nau', '', 'na', u'Nauru', u'nauruan'),
- ('nav', '', 'nv', u'Navajo; Navaho', u'navaho'),
- ('nbl', '', 'nr', u'Ndebele, South; South Ndebele', u'ndébélé du Sud'),
- ('nde', '', 'nd', u'Ndebele, North; North Ndebele', u'ndébélé du Nord'),
- ('ndo', '', 'ng', u'Ndonga', u'ndonga'),
- ('nds', '', '', u'Low German; Low Saxon; German, Low; Saxon, Low', u'bas allemand; bas saxon; allemand, bas; saxon, bas'),
- ('nep', '', 'ne', u'Nepali', u'népalais'),
- ('new', '', '', u'Nepal Bhasa; Newari', u'nepal bhasa; newari'),
- ('nia', '', '', u'Nias', u'nias'),
- ('nic', '', '', u'Niger-Kordofanian languages', u'nigéro-kordofaniennes, langues'),
- ('niu', '', '', u'Niuean', u'niué'),
- ('nno', '', 'nn', u'Norwegian Nynorsk; Nynorsk, Norwegian', u'norvégien nynorsk; nynorsk, norvégien'),
- ('nob', '', 'nb', u'Bokmål, Norwegian; Norwegian Bokmål', u'norvégien bokmål'),
- ('nog', '', '', u'Nogai', u'nogaï; nogay'),
- ('non', '', '', u'Norse, Old', u'norrois, vieux'),
- ('nor', '', 'no', u'Norwegian', u'norvégien'),
- ('nqo', '', '', u'N\'Ko', u'n\'ko'),
- ('nso', '', '', u'Pedi; Sepedi; Northern Sotho', u'pedi; sepedi; sotho du Nord'),
- ('nub', '', '', u'Nubian languages', u'nubiennes, langues'),
- ('nwc', '', '', u'Classical Newari; Old Newari; Classical Nepal Bhasa', u'newari classique'),
- ('nya', '', 'ny', u'Chichewa; Chewa; Nyanja', u'chichewa; chewa; nyanja'),
- ('nym', '', '', u'Nyamwezi', u'nyamwezi'),
- ('nyn', '', '', u'Nyankole', u'nyankolé'),
- ('nyo', '', '', u'Nyoro', u'nyoro'),
- ('nzi', '', '', u'Nzima', u'nzema'),
- ('oci', '', 'oc', u'Occitan (post 1500); Provençal', u'occitan (après 1500); provençal'),
- ('oji', '', 'oj', u'Ojibwa', u'ojibwa'),
- ('ori', '', 'or', u'Oriya', u'oriya'),
- ('orm', '', 'om', u'Oromo', u'galla'),
- ('osa', '', '', u'Osage', u'osage'),
- ('oss', '', 'os', u'Ossetian; Ossetic', u'ossète'),
- ('ota', '', '', u'Turkish, Ottoman (1500-1928)', u'turc ottoman (1500-1928)'),
- ('oto', '', '', u'Otomian languages', u'otomi, langues'),
- ('paa', '', '', u'Papuan languages', u'papoues, langues'),
- ('pag', '', '', u'Pangasinan', u'pangasinan'),
- ('pal', '', '', u'Pahlavi', u'pahlavi'),
- ('pam', '', '', u'Pampanga; Kapampangan', u'pampangan'),
- ('pan', '', 'pa', u'Panjabi; Punjabi', u'pendjabi'),
- ('pap', '', '', u'Papiamento', u'papiamento'),
- ('pau', '', '', u'Palauan', u'palau'),
- ('peo', '', '', u'Persian, Old (ca.600-400 B.C.)', u'perse, vieux (ca. 600-400 av. J.-C.)'),
- ('per', 'fas', 'fa', u'Persian', u'persan'),
- ('phi', '', '', u'Philippine languages', u'philippines, langues'),
- ('phn', '', '', u'Phoenician', u'phénicien'),
- ('pli', '', 'pi', u'Pali', u'pali'),
- ('pol', '', 'pl', u'Polish', u'polonais'),
- ('pon', '', '', u'Pohnpeian', u'pohnpei'),
- ('pob', '', 'pb', u'Brazilian Portuguese', u'brazilian portuguese'),
- ('por', '', 'pt', u'Portuguese', u'portugais'),
- ('pra', '', '', u'Prakrit languages', u'prâkrit, langues'),
- ('pro', '', '', u'Provençal, Old (to 1500)', u'provençal ancien (jusqu\'à 1500)'),
- ('pus', '', 'ps', u'Pushto; Pashto', u'pachto'),
- ('que', '', 'qu', u'Quechua', u'quechua'),
- ('raj', '', '', u'Rajasthani', u'rajasthani'),
- ('rap', '', '', u'Rapanui', u'rapanui'),
- ('rar', '', '', u'Rarotongan; Cook Islands Maori', u'rarotonga; maori des îles Cook'),
- ('roa', '', '', u'Romance languages', u'romanes, langues'),
- ('roh', '', 'rm', u'Romansh', u'romanche'),
- ('rom', '', '', u'Romany', u'tsigane'),
- ('rum', 'ron', 'ro', u'Romanian; Moldavian; Moldovan', u'roumain; moldave'),
- ('run', '', 'rn', u'Rundi', u'rundi'),
- ('rup', '', '', u'Aromanian; Arumanian; Macedo-Romanian', u'aroumain; macédo-roumain'),
- ('rus', '', 'ru', u'Russian', u'russe'),
- ('sad', '', '', u'Sandawe', u'sandawe'),
- ('sag', '', 'sg', u'Sango', u'sango'),
- ('sah', '', '', u'Yakut', u'iakoute'),
- ('sai', '', '', u'South American Indian (Other)', u'indiennes d\'Amérique du Sud, autres langues'),
- ('sal', '', '', u'Salishan languages', u'salishennes, langues'),
- ('sam', '', '', u'Samaritan Aramaic', u'samaritain'),
- ('san', '', 'sa', u'Sanskrit', u'sanskrit'),
- ('sas', '', '', u'Sasak', u'sasak'),
- ('sat', '', '', u'Santali', u'santal'),
- ('scn', '', '', u'Sicilian', u'sicilien'),
- ('sco', '', '', u'Scots', u'écossais'),
- ('sel', '', '', u'Selkup', u'selkoupe'),
- ('sem', '', '', u'Semitic languages', u'sémitiques, langues'),
- ('sga', '', '', u'Irish, Old (to 900)', u'irlandais ancien (jusqu\'à 900)'),
- ('sgn', '', '', u'Sign Languages', u'langues des signes'),
- ('shn', '', '', u'Shan', u'chan'),
- ('sid', '', '', u'Sidamo', u'sidamo'),
- ('sin', '', 'si', u'Sinhala; Sinhalese', u'singhalais'),
- ('sio', '', '', u'Siouan languages', u'sioux, langues'),
- ('sit', '', '', u'Sino-Tibetan languages', u'sino-tibétaines, langues'),
- ('sla', '', '', u'Slavic languages', u'slaves, langues'),
- ('slo', 'slk', 'sk', u'Slovak', u'slovaque'),
- ('slv', '', 'sl', u'Slovenian', u'slovène'),
- ('sma', '', '', u'Southern Sami', u'sami du Sud'),
- ('sme', '', 'se', u'Northern Sami', u'sami du Nord'),
- ('smi', '', '', u'Sami languages', u'sames, langues'),
- ('smj', '', '', u'Lule Sami', u'sami de Lule'),
- ('smn', '', '', u'Inari Sami', u'sami d\'Inari'),
- ('smo', '', 'sm', u'Samoan', u'samoan'),
- ('sms', '', '', u'Skolt Sami', u'sami skolt'),
- ('sna', '', 'sn', u'Shona', u'shona'),
- ('snd', '', 'sd', u'Sindhi', u'sindhi'),
- ('snk', '', '', u'Soninke', u'soninké'),
- ('sog', '', '', u'Sogdian', u'sogdien'),
- ('som', '', 'so', u'Somali', u'somali'),
- ('son', '', '', u'Songhai languages', u'songhai, langues'),
- ('sot', '', 'st', u'Sotho, Southern', u'sotho du Sud'),
- ('spa', '', 'es', u'Spanish; Castilian', u'espagnol; castillan'),
- ('srd', '', 'sc', u'Sardinian', u'sarde'),
- ('srn', '', '', u'Sranan Tongo', u'sranan tongo'),
- ('srp', '', 'sr', u'Serbian', u'serbe'),
- ('srr', '', '', u'Serer', u'sérère'),
- ('ssa', '', '', u'Nilo-Saharan languages', u'nilo-sahariennes, langues'),
- ('ssw', '', 'ss', u'Swati', u'swati'),
- ('suk', '', '', u'Sukuma', u'sukuma'),
- ('sun', '', 'su', u'Sundanese', u'soundanais'),
- ('sus', '', '', u'Susu', u'soussou'),
- ('sux', '', '', u'Sumerian', u'sumérien'),
- ('swa', '', 'sw', u'Swahili', u'swahili'),
- ('swe', '', 'sv', u'Swedish', u'suédois'),
- ('syc', '', '', u'Classical Syriac', u'syriaque classique'),
- ('syr', '', '', u'Syriac', u'syriaque'),
- ('tah', '', 'ty', u'Tahitian', u'tahitien'),
- ('tai', '', '', u'Tai languages', u'tai, langues'),
- ('tam', '', 'ta', u'Tamil', u'tamoul'),
- ('tat', '', 'tt', u'Tatar', u'tatar'),
- ('tel', '', 'te', u'Telugu', u'télougou'),
- ('tem', '', '', u'Timne', u'temne'),
- ('ter', '', '', u'Tereno', u'tereno'),
- ('tet', '', '', u'Tetum', u'tetum'),
- ('tgk', '', 'tg', u'Tajik', u'tadjik'),
- ('tgl', '', 'tl', u'Tagalog', u'tagalog'),
- ('tha', '', 'th', u'Thai', u'thaï'),
- ('tib', 'bod', 'bo', u'Tibetan', u'tibétain'),
- ('tig', '', '', u'Tigre', u'tigré'),
- ('tir', '', 'ti', u'Tigrinya', u'tigrigna'),
- ('tiv', '', '', u'Tiv', u'tiv'),
- ('tkl', '', '', u'Tokelau', u'tokelau'),
- ('tlh', '', '', u'Klingon; tlhIngan-Hol', u'klingon'),
- ('tli', '', '', u'Tlingit', u'tlingit'),
- ('tmh', '', '', u'Tamashek', u'tamacheq'),
- ('tog', '', '', u'Tonga (Nyasa)', u'tonga (Nyasa)'),
- ('ton', '', 'to', u'Tonga (Tonga Islands)', u'tongan (Îles Tonga)'),
- ('tpi', '', '', u'Tok Pisin', u'tok pisin'),
- ('tsi', '', '', u'Tsimshian', u'tsimshian'),
- ('tsn', '', 'tn', u'Tswana', u'tswana'),
- ('tso', '', 'ts', u'Tsonga', u'tsonga'),
- ('tuk', '', 'tk', u'Turkmen', u'turkmène'),
- ('tum', '', '', u'Tumbuka', u'tumbuka'),
- ('tup', '', '', u'Tupi languages', u'tupi, langues'),
- ('tur', '', 'tr', u'Turkish', u'turc'),
- ('tut', '', '', u'Altaic languages', u'altaïques, langues'),
- ('tvl', '', '', u'Tuvalu', u'tuvalu'),
- ('twi', '', 'tw', u'Twi', u'twi'),
- ('tyv', '', '', u'Tuvinian', u'touva'),
- ('udm', '', '', u'Udmurt', u'oudmourte'),
- ('uga', '', '', u'Ugaritic', u'ougaritique'),
- ('uig', '', 'ug', u'Uighur; Uyghur', u'ouïgour'),
- ('ukr', '', 'uk', u'Ukrainian', u'ukrainien'),
- ('umb', '', '', u'Umbundu', u'umbundu'),
- ('und', '', '', u'Undetermined', u'indéterminée'),
- ('urd', '', 'ur', u'Urdu', u'ourdou'),
- ('uzb', '', 'uz', u'Uzbek', u'ouszbek'),
- ('vai', '', '', u'Vai', u'vaï'),
- ('ven', '', 've', u'Venda', u'venda'),
- ('vie', '', 'vi', u'Vietnamese', u'vietnamien'),
- ('vol', '', 'vo', u'Volapük', u'volapük'),
- ('vot', '', '', u'Votic', u'vote'),
- ('wak', '', '', u'Wakashan languages', u'wakashanes, langues'),
- ('wal', '', '', u'Walamo', u'walamo'),
- ('war', '', '', u'Waray', u'waray'),
- ('was', '', '', u'Washo', u'washo'),
- ('wel', 'cym', 'cy', u'Welsh', u'gallois'),
- ('wen', '', '', u'Sorbian languages', u'sorabes, langues'),
- ('wln', '', 'wa', u'Walloon', u'wallon'),
- ('wol', '', 'wo', u'Wolof', u'wolof'),
- ('xal', '', '', u'Kalmyk; Oirat', u'kalmouk; oïrat'),
- ('xho', '', 'xh', u'Xhosa', u'xhosa'),
- ('yao', '', '', u'Yao', u'yao'),
- ('yap', '', '', u'Yapese', u'yapois'),
- ('yid', '', 'yi', u'Yiddish', u'yiddish'),
- ('yor', '', 'yo', u'Yoruba', u'yoruba'),
- ('ypk', '', '', u'Yupik languages', u'yupik, langues'),
- ('zap', '', '', u'Zapotec', u'zapotèque'),
- ('zbl', '', '', u'Blissymbols; Blissymbolics; Bliss', u'symboles Bliss; Bliss'),
- ('zen', '', '', u'Zenaga', u'zenaga'),
- ('zha', '', 'za', u'Zhuang; Chuang', u'zhuang; chuang'),
- ('znd', '', '', u'Zande languages', u'zandé, langues'),
- ('zul', '', 'zu', u'Zulu', u'zoulou'),
- ('zun', '', '', u'Zuni', u'zuni'),
- ('zza', '', '', u'Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki', u'zaza; dimili; dimli; kirdki; kirmanjki; zazaki')]
-
-
-class Country(object):
- """Country according to ISO-3166
-
- :param string country: country name, alpha2 code, alpha3 code or numeric code
- :param list countries: all countries
- :type countries: see :data:`~subliminal.language.COUNTRIES`
-
- """
- def __init__(self, country, countries=None):
- countries = countries or COUNTRIES
- country = to_unicode(country.strip().lower())
- country_tuple = None
-
- # Try to find the country
- if len(country) == 2:
- country_tuple = dict((c[0].lower(), c) for c in countries).get(country)
- elif len(country) == 3 and not country.isdigit():
- country_tuple = dict((c[1].lower(), c) for c in countries).get(country)
- elif len(country) == 3 and country.isdigit():
- country_tuple = dict((c[2].lower(), c) for c in countries).get(country)
- if country_tuple is None:
- country_tuple = dict((c[3].lower(), c) for c in countries).get(country)
-
- # Raise ValueError if nothing is found
- if country_tuple is None:
- raise ValueError('Country %s does not exist' % country)
-
- # Set default attrs
- self.alpha2 = country_tuple[0]
- self.alpha3 = country_tuple[1]
- self.numeric = country_tuple[2]
- self.name = country_tuple[3]
-
- def __hash__(self):
- return hash(self.alpha3)
-
- def __eq__(self, other):
- if isinstance(other, Country):
- return self.alpha3 == other.alpha3
- return False
-
- def __ne__(self, other):
- return not self == other
-
- def __unicode__(self):
- return self.name
-
- def __str__(self):
- return unicode(self).encode('utf-8')
-
- def __repr__(self):
- return 'Country(%s)' % self
-
-
-class Language(object):
- """Language according to ISO-639
-
- :param string language: language name (english or french), alpha2 code, alpha3 code, terminologic code or numeric code, eventually with a country
- :param country: country of the language
- :type country: :class:`Country` or string
- :param languages: all languages
- :type languages: see :data:`~subliminal.language.LANGUAGES`
- :param countries: all countries
- :type countries: see :data:`~subliminal.language.COUNTRIES`
- :param bool strict: whether to raise a ValueError on unknown language or not
-
- :class:`Language` implements the inclusion test, with the ``in`` keyword::
-
- >>> Language('pt-BR') in Language('pt') # Portuguese (Brazil) is included in Portuguese
- True
- >>> Language('pt') in Language('pt-BR') # Portuguese is not included in Portuguese (Brazil)
- False
-
- """
- with_country_regexps = [re.compile('(.*)\((.*)\)'), re.compile('(.*)[-_](.*)')]
-
- def __init__(self, language, country=None, languages=None, countries=None, strict=True):
- languages = languages or LANGUAGES
- countries = countries or COUNTRIES
-
- # Get the country
- self.country = None
- if isinstance(country, Country):
- self.country = country
- elif isinstance(country, basestring):
- try:
- self.country = Country(country, countries)
- except ValueError:
- logger.warning(u'Country %s could not be identified' % country) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- if strict:
- raise
-
- # Language + Country format
- #TODO: Improve this part
- if language is None:
- language = 'und'
- if country is None:
- for regexp in [r.match(language) for r in self.with_country_regexps]:
- if regexp:
- language = regexp.group(1)
- try:
- self.country = Country(regexp.group(2), countries)
- except ValueError:
- logger.warning(u'Country %s could not be identified' % country) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- if strict:
- raise
- break
-
- # Try to find the language
- language = to_unicode(language.strip().lower())
- language_tuple = None
- if len(language) == 2:
- language_tuple = dict((l[2].lower(), l) for l in languages).get(language)
- elif len(language) == 3:
- language_tuple = dict((l[0].lower(), l) for l in languages).get(language)
- if language_tuple is None:
- language_tuple = dict((l[1].lower(), l) for l in languages).get(language)
- if language_tuple is None:
- language_tuple = dict((l[3].split('; ')[0].lower(), l) for l in languages).get(language)
- if language_tuple is None:
- language_tuple = dict((l[4].split('; ')[0].lower(), l) for l in languages).get(language)
-
- # Raise ValueError if strict or continue with Undetermined
- if language_tuple is None:
- if strict:
- raise ValueError('Language %s does not exist' % language)
- language_tuple = dict((l[0].lower(), l) for l in languages).get('und')
-
- # Set attributes
- self.alpha2 = language_tuple[2]
- self.alpha3 = language_tuple[0]
- self.terminologic = language_tuple[1]
- self.name = language_tuple[3]
- self.french_name = language_tuple[4]
-
- def __hash__(self):
- if self.country is None:
- return hash(self.alpha3)
- return hash(self.alpha3 + self.country.alpha3)
-
- def __eq__(self, other):
- if isinstance(other, Language):
- return self.alpha3 == other.alpha3 and self.country == other.country
- return False
-
- def __contains__(self, item):
- if isinstance(item, Language):
- if self == item:
- return True
- if self.country is None:
- return self.alpha3 == item.alpha3
- return False
-
- def __ne__(self, other):
- return not self == other
-
- def __nonzero__(self):
- return self.alpha3 != 'und'
-
- def __unicode__(self):
- if self.country is None:
- return self.name
- return '%s (%s)' % (self.name, self.country)
-
- def __str__(self):
- return unicode(self).encode('utf-8')
-
- def __repr__(self):
- if self.country is None:
- return 'Language(%s)' % self.name.encode('utf-8')
- return 'Language(%s, country=%s)' % (self.name.encode('utf-8'), self.country)
-
-
-class language_set(set):
- """Set of :class:`Language` with some specificities.
-
- :param iterable: where to take elements from
- :type iterable: iterable of :class:`Languages <Language>` or string
- :param languages: all languages
- :type languages: see :data:`~subliminal.language.LANGUAGES`
- :param bool strict: whether to raise a ValueError on invalid language or not
-
- The following redefinitions are meant to reflect the inclusion logic in :class:`Language`
-
- * Inclusion test, with the ``in`` keyword
- * Intersection
- * Substraction
-
- Here is an illustration of the previous points::
-
- >>> Language('en') in language_set(['en-US', 'en-CA'])
- False
- >>> Language('en-US') in language_set(['en', 'fr'])
- True
- >>> language_set(['en']) & language_set(['en-US', 'en-CA'])
- language_set([Language(English, country=Canada), Language(English, country=United States)])
- >>> language_set(['en-US', 'en-CA', 'fr']) - language_set(['en'])
- language_set([Language(French)])
-
- """
- def __init__(self, iterable=None, languages=None, strict=True):
- iterable = iterable or []
- languages = languages or LANGUAGES
- items = []
- for i in iterable:
- if isinstance(i, Language):
- items.append(i)
- continue
- if isinstance(i, tuple):
- items.append(Language(i[0], languages=languages, strict=strict))
- continue
- items.append(Language(i, languages=languages, strict=strict))
- super(language_set, self).__init__(items)
-
- def __contains__(self, item):
- for i in self:
- if item in i:
- return True
- return super(language_set, self).__contains__(item)
-
- def __and__(self, other):
- results = language_set()
- for i in self:
- for j in other:
- if i in j:
- results.add(i)
- for i in other:
- for j in self:
- if i in j:
- results.add(i)
- return results
-
- def __sub__(self, other):
- results = language_set()
- for i in self:
- if i not in other:
- results.add(i)
- return results
-
-
-class language_list(list):
- """List of :class:`Language` with some specificities.
-
- :param iterable: where to take elements from
- :type iterable: iterable of :class:`Languages <Language>` or string
- :param languages: all languages
- :type languages: see :data:`~subliminal.language.LANGUAGES`
- :param bool strict: whether to raise a ValueError on invalid language or not
-
- The following redefinitions are meant to reflect the inclusion logic in :class:`Language`
-
- * Inclusion test, with the ``in`` keyword
- * Index
-
- Here is an illustration of the previous points::
-
- >>> Language('en') in language_list(['en-US', 'en-CA'])
- False
- >>> Language('en-US') in language_list(['en', 'fr-BE'])
- True
- >>> language_list(['en', 'fr-BE']).index(Language('en-US'))
- 0
-
- """
- def __init__(self, iterable=None, languages=None, strict=True):
- iterable = iterable or []
- languages = languages or LANGUAGES
- items = []
- for i in iterable:
- if isinstance(i, Language):
- items.append(i)
- continue
- if isinstance(i, tuple):
- items.append(Language(i[0], languages=languages, strict=strict))
- continue
- items.append(Language(i, languages=languages, strict=strict))
- super(language_list, self).__init__(items)
-
- def __contains__(self, item):
- for i in self:
- if item in i:
- return True
- return super(language_list, self).__contains__(item)
-
- def index(self, x, strict=False):
- if not strict:
- for i in range(len(self)):
- if x in self[i]:
- return i
- return super(language_list, self).index(x)
diff --git a/lib/subliminal/providers/__init__.py b/lib/subliminal/providers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..fc0cb9556171513efe5c2d53597cea5f5117303c
--- /dev/null
+++ b/lib/subliminal/providers/__init__.py
@@ -0,0 +1,338 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import contextlib
+import logging
+import socket
+import babelfish
+from pkg_resources import iter_entry_points, EntryPoint
+import requests
+from ..video import Episode, Movie
+
+
+logger = logging.getLogger(__name__)
+
+
+class Provider(object):
+ """Base class for providers
+
+ If any configuration is possible for the provider, like credentials, it must take place during instantiation
+
+ :param \*\*kwargs: configuration
+ :raise: :class:`~subliminal.exceptions.ProviderConfigurationError` if there is a configuration error
+
+ """
+ #: Supported BabelFish languages
+ languages = set()
+
+ #: Supported video types
+ video_types = (Episode, Movie)
+
+ #: Required hash, if any
+ required_hash = None
+
+ def __init__(self, **kwargs):
+ pass
+
+ def __enter__(self):
+ self.initialize()
+ return self
+
+ def __exit__(self, type, value, traceback): # @ReservedAssignment
+ self.terminate()
+
+ def initialize(self):
+ """Initialize the provider
+
+ Must be called when starting to work with the provider. This is the place for network initialization
+ or login operations.
+
+ .. note:
+ This is called automatically if you use the :keyword:`with` statement
+
+
+ :raise: :class:`~subliminal.exceptions.ProviderNotAvailable` if the provider is unavailable
+
+ """
+ pass
+
+ def terminate(self):
+ """Terminate the provider
+
+ Must be called when done with the provider. This is the place for network shutdown or logout operations.
+
+ .. note:
+ This is called automatically if you use the :keyword:`with` statement
+
+ :raise: :class:`~subliminal.exceptions.ProviderNotAvailable` if the provider is unavailable
+ """
+ pass
+
+ @classmethod
+ def check(cls, video):
+ """Check if the `video` can be processed
+
+ The video is considered invalid if not an instance of :attr:`video_types` or if the :attr:`required_hash` is
+ not present in :attr:`~subliminal.video.Video`'s `hashes` attribute.
+
+ :param video: the video to check
+ :type video: :class:`~subliminal.video.Video`
+ :return: `True` if the `video` and `languages` are valid, `False` otherwise
+ :rtype: bool
+
+ """
+ if not isinstance(video, cls.video_types):
+ return False
+ if cls.required_hash is not None and cls.required_hash not in video.hashes:
+ return False
+ return True
+
+ def query(self, languages, *args, **kwargs):
+ """Query the provider for subtitles
+
+ This method arguments match as much as possible the actual parameters for querying the provider
+
+ :param languages: languages to search for
+ :type languages: set of :class:`babelfish.Language`
+ :param \*args: other required arguments
+ :param \*\*kwargs: other optional arguments
+ :return: the subtitles
+ :rtype: list of :class:`~subliminal.subtitle.Subtitle`
+ :raise: :class:`~subliminal.exceptions.ProviderNotAvailable` if the provider is unavailable
+ :raise: :class:`~subliminal.exceptions.ProviderError` if something unexpected occured
+
+ """
+ raise NotImplementedError
+
+ def list_subtitles(self, video, languages):
+ """List subtitles for the `video` with the given `languages`
+
+ This is a proxy for the :meth:`query` method. The parameters passed to the :meth:`query` method may
+ vary depending on the amount of information available in the `video`
+
+ :param video: video to list subtitles for
+ :type video: :class:`~subliminal.video.Video`
+ :param languages: languages to search for
+ :type languages: set of :class:`babelfish.Language`
+ :return: the subtitles
+ :rtype: list of :class:`~subliminal.subtitle.Subtitle`
+ :raise: :class:`~subliminal.exceptions.ProviderNotAvailable` if the provider is unavailable
+ :raise: :class:`~subliminal.exceptions.ProviderError` if something unexpected occured
+
+ """
+ raise NotImplementedError
+
+ def download_subtitle(self, subtitle):
+ """Download the `subtitle` an fill its :attr:`~subliminal.subtitle.Subtitle.content` attribute with
+ subtitle's text
+
+ :param subtitle: subtitle to download
+ :type subtitle: :class:`~subliminal.subtitle.Subtitle`
+ :raise: :class:`~subliminal.exceptions.ProviderNotAvailable` if the provider is unavailable
+ :raise: :class:`~subliminal.exceptions.ProviderError` if something unexpected occured
+
+ """
+ raise NotImplementedError
+
+ def __repr__(self):
+ return '<%s [%r]>' % (self.__class__.__name__, self.video_types)
+
+
+class ProviderManager(object):
+ """Manager for providers behaving like a dict with lazy loading
+
+ Loading is done in this order:
+
+ * Entry point providers
+ * Registered providers
+
+ .. attribute:: entry_point
+
+ The entry point where to look for providers
+
+ """
+ entry_point = 'subliminal.providers'
+
+ def __init__(self):
+ #: Registered providers with entry point syntax
+ self.registered_providers = ['addic7ed = subliminal.providers.addic7ed:Addic7edProvider',
+ 'opensubtitles = subliminal.providers.opensubtitles:OpenSubtitlesProvider',
+ 'podnapisi = subliminal.providers.podnapisi:PodnapisiProvider',
+ 'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider',
+ 'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider']
+
+ #: Loaded providers
+ self.providers = {}
+
+ @property
+ def available_providers(self):
+ """Available providers"""
+ available_providers = set(self.providers.keys())
+ available_providers.update([ep.name for ep in iter_entry_points(self.entry_point)])
+ available_providers.update([EntryPoint.parse(c).name for c in self.registered_providers])
+ return available_providers
+
+ def __getitem__(self, name):
+ """Get a provider, lazy loading it if necessary"""
+ if name in self.providers:
+ return self.providers[name]
+ for ep in iter_entry_points(self.entry_point):
+ if ep.name == name:
+ self.providers[ep.name] = ep.load()
+ return self.providers[ep.name]
+ for ep in (EntryPoint.parse(c) for c in self.registered_providers):
+ if ep.name == name:
+ self.providers[ep.name] = ep.load(require=False)
+ return self.providers[ep.name]
+ raise KeyError(name)
+
+ def __setitem__(self, name, provider):
+ """Load a provider"""
+ self.providers[name] = provider
+
+ def __delitem__(self, name):
+ """Unload a provider"""
+ del self.providers[name]
+
+ def __iter__(self):
+ """Iterator over loaded providers"""
+ return iter(self.providers)
+
+ def register(self, entry_point):
+ """Register a provider
+
+ :param string entry_point: provider to register (entry point syntax)
+ :raise: ValueError if already registered
+
+ """
+ if entry_point in self.registered_providers:
+ raise ValueError('Entry point \'%s\' already registered' % entry_point)
+ entry_point_name = EntryPoint.parse(entry_point).name
+ if entry_point_name in self.available_providers:
+ raise ValueError('An entry point with name \'%s\' already registered' % entry_point_name)
+ self.registered_providers.insert(0, entry_point)
+
+ def unregister(self, entry_point):
+ """Unregister a provider
+
+ :param string entry_point: provider to unregister (entry point syntax)
+
+ """
+ self.registered_providers.remove(entry_point)
+
+ def __contains__(self, name):
+ return name in self.providers
+
+provider_manager = ProviderManager()
+
+
+class ProviderPool(object):
+ """A pool of providers with the same API as a single :class:`Provider`
+
+ The :class:`ProviderPool` supports the ``with`` statement to :meth:`terminate` the providers
+
+ :param providers: providers to use, if not all
+ :type providers: list of string or None
+ :param provider_configs: configuration for providers
+ :type provider_configs: dict of provider name => provider constructor kwargs or None
+
+ """
+ def __init__(self, providers=None, provider_configs=None):
+ self.provider_configs = provider_configs or {}
+ self.providers = dict([(p, provider_manager[p]) for p in (providers or provider_manager.available_providers)])
+ self.initialized_providers = {}
+ self.discarded_providers = set()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, type, value, traceback): # @ReservedAssignment
+ self.terminate()
+
+ def get_initialized_provider(self, name):
+ """Get a :class:`Provider` by name, initializing it if necessary
+
+ :param string name: name of the provider
+ :return: the initialized provider
+ :rtype: :class:`Provider`
+
+ """
+ if name in self.initialized_providers:
+ return self.initialized_providers[name]
+ provider = self.providers[name](**self.provider_configs.get(name, {}))
+ provider.initialize()
+ self.initialized_providers[name] = provider
+ return provider
+
+ def list_subtitles(self, video, languages):
+ """List subtitles for `video` with the given `languages`
+
+ :param video: video to list subtitles for
+ :type video: :class:`~subliminal.video.Video`
+ :param languages: languages of subtitles to search for
+ :type languages: set of :class:`babelfish.Language`
+ :return: found subtitles
+ :rtype: list of :class:`~subliminal.subtitle.Subtitle`
+
+ """
+ subtitles = []
+ for provider_name, provider_class in self.providers.items():
+ if not provider_class.check(video):
+ logger.info('Skipping provider %r: not a valid video', provider_name)
+ continue
+ provider_languages = provider_class.languages & languages - video.subtitle_languages
+ if not provider_languages:
+ logger.info('Skipping provider %r: no language to search for', provider_name)
+ continue
+ if provider_name in self.discarded_providers:
+ logger.debug('Skipping discarded provider %r', provider_name)
+ continue
+ try:
+ provider = self.get_initialized_provider(provider_name)
+ logger.info('Listing subtitles with provider %r and languages %r', provider_name, provider_languages)
+ provider_subtitles = provider.list_subtitles(video, provider_languages)
+ logger.info('Found %d subtitles', len(provider_subtitles))
+ subtitles.extend(provider_subtitles)
+ except (requests.exceptions.Timeout, socket.timeout):
+ logger.warning('Provider %r timed out, discarding it', provider_name)
+ self.discarded_providers.add(provider_name)
+ except:
+ logger.exception('Unexpected error in provider %r, discarding it', provider_name)
+ self.discarded_providers.add(provider_name)
+ return subtitles
+
+ def download_subtitle(self, subtitle):
+ """Download a subtitle
+
+ :param subtitle: subtitle to download
+ :type subtitle: :class:`~subliminal.subtitle.Subtitle`
+ :return: ``True`` if the subtitle has been successfully downloaded, ``False`` otherwise
+ :rtype: bool
+
+ """
+ if subtitle.provider_name in self.discarded_providers:
+ logger.debug('Discarded provider %r', subtitle.provider_name)
+ return False
+ try:
+ provider = self.get_initialized_provider(subtitle.provider_name)
+ provider.download_subtitle(subtitle)
+ if not subtitle.is_valid:
+ logger.warning('Invalid subtitle')
+ return False
+ return True
+ except (requests.exceptions.Timeout, socket.timeout):
+ logger.warning('Provider %r timed out, discarding it', subtitle.provider_name)
+ self.discarded_providers.add(subtitle.provider_name)
+ except:
+ logger.exception('Unexpected error in provider %r, discarding it', subtitle.provider_name)
+ self.discarded_providers.add(subtitle.provider_name)
+ return False
+
+ def terminate(self):
+ """Terminate all the initialized providers"""
+ for (provider_name, provider) in self.initialized_providers.items():
+ try:
+ provider.terminate()
+ except (requests.exceptions.Timeout, socket.timeout):
+ logger.warning('Provider %r timed out, unable to terminate', provider_name)
+ except:
+ logger.exception('Unexpected error in provider %r', provider_name)
diff --git a/lib/subliminal/providers/addic7ed.py b/lib/subliminal/providers/addic7ed.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a7f4bd53a71f2342ba7ace0517c2c8d485bc9cd
--- /dev/null
+++ b/lib/subliminal/providers/addic7ed.py
@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import logging
+import babelfish
+import bs4
+import requests
+from . import Provider
+from .. import __version__
+from ..cache import region, SHOW_EXPIRATION_TIME
+from ..exceptions import ConfigurationError, AuthenticationError, DownloadLimitExceeded, ProviderError
+from ..subtitle import Subtitle, fix_line_endings, compute_guess_properties_matches
+from ..video import Episode
+
+
+logger = logging.getLogger(__name__)
+babelfish.language_converters.register('addic7ed = subliminal.converters.addic7ed:Addic7edConverter')
+
+
+class Addic7edSubtitle(Subtitle):
+ provider_name = 'addic7ed'
+
+ def __init__(self, language, series, season, episode, title, year, version, hearing_impaired, download_link,
+ page_link):
+ super(Addic7edSubtitle, self).__init__(language, hearing_impaired, page_link)
+ self.series = series
+ self.season = season
+ self.episode = episode
+ self.title = title
+ self.year = year
+ self.version = version
+ self.download_link = download_link
+
+ def compute_matches(self, video):
+ matches = set()
+ # series
+ if video.series and self.series == video.series:
+ matches.add('series')
+ # season
+ if video.season and self.season == video.season:
+ matches.add('season')
+ # episode
+ if video.episode and self.episode == video.episode:
+ matches.add('episode')
+ # title
+ if video.title and self.title.lower() == video.title.lower():
+ matches.add('title')
+ # year
+ if self.year == video.year:
+ matches.add('year')
+ # release_group
+ if video.release_group and self.version and video.release_group.lower() in self.version.lower():
+ matches.add('release_group')
+ """
+ # resolution
+ if video.resolution and self.version and video.resolution in self.version.lower():
+ matches.add('resolution')
+ # format
+ if video.format and self.version and video.format in self.version.lower:
+ matches.add('format')
+ """
+ # we don't have the complete filename, so we need to guess the matches separately
+ # guess resolution (screenSize in guessit)
+ matches |= compute_guess_properties_matches(video, self.version, 'screenSize')
+ # guess format
+ matches |= compute_guess_properties_matches(video, self.version, 'format')
+ # guess video codec
+ matches |= compute_guess_properties_matches(video, self.version, 'videoCodec')
+ return matches
+
+
+class Addic7edProvider(Provider):
+ languages = {babelfish.Language('por', 'BR')} | {babelfish.Language(l)
+ for l in ['ara', 'aze', 'ben', 'bos', 'bul', 'cat', 'ces', 'dan', 'deu', 'ell', 'eng', 'eus', 'fas',
+ 'fin', 'fra', 'glg', 'heb', 'hrv', 'hun', 'hye', 'ind', 'ita', 'jpn', 'kor', 'mkd', 'msa',
+ 'nld', 'nor', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'spa', 'sqi', 'srp', 'swe', 'tha',
+ 'tur', 'ukr', 'vie', 'zho']}
+ video_types = (Episode,)
+ server = 'http://www.addic7ed.com'
+
+ def __init__(self, username=None, password=None):
+ if username is not None and password is None or username is None and password is not None:
+ raise ConfigurationError('Username and password must be specified')
+ self.username = username
+ self.password = password
+ self.logged_in = False
+
+ def initialize(self):
+ self.session = requests.Session()
+ self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__.split('-')[0]}
+ # login
+ if self.username is not None and self.password is not None:
+ logger.debug('Logging in')
+ data = {'username': self.username, 'password': self.password, 'Submit': 'Log in'}
+ r = self.session.post(self.server + '/dologin.php', data, timeout=10, allow_redirects=False)
+ if r.status_code == 302:
+ logger.info('Logged in')
+ self.logged_in = True
+ else:
+ raise AuthenticationError(self.username)
+
+ def terminate(self):
+ # logout
+ if self.logged_in:
+ r = self.session.get(self.server + '/logout.php', timeout=10)
+ logger.info('Logged out')
+ if r.status_code != 200:
+ raise ProviderError('Request failed with status code %d' % r.status_code)
+ self.session.close()
+
+ def get(self, url, params=None):
+ """Make a GET request on `url` with the given parameters
+
+ :param string url: part of the URL to reach with the leading slash
+ :param params: params of the request
+ :return: the response
+ :rtype: :class:`bs4.BeautifulSoup`
+
+ """
+ r = self.session.get(self.server + url, params=params, timeout=10)
+ if r.status_code != 200:
+ raise ProviderError('Request failed with status code %d' % r.status_code)
+ return bs4.BeautifulSoup(r.content, ['permissive'])
+
+ @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
+ def get_show_ids(self):
+ """Load the shows page with default series to show ids mapping
+
+ :return: series to show ids
+ :rtype: dict
+
+ """
+ soup = self.get('/shows.php')
+ show_ids = {}
+ for html_show in soup.select('td.version > h3 > a[href^="/show/"]'):
+ show_ids[html_show.string.lower()] = int(html_show['href'][6:])
+ return show_ids
+
+ @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
+ def find_show_id(self, series, year=None):
+ """Find the show id from the `series` with optional `year`
+
+ Use this only if the show id cannot be found with :meth:`get_show_ids`
+
+ :param string series: series of the episode in lowercase
+ :param year: year of the series, if any
+ :type year: int or None
+ :return: the show id, if any
+ :rtype: int or None
+
+ """
+ series_year = series
+ if year is not None:
+ series_year += ' (%d)' % year
+ params = {'search': series_year, 'Submit': 'Search'}
+ logger.debug('Searching series %r', params)
+ suggested_shows = self.get('/search.php', params).select('span.titulo > a[href^="/show/"]')
+ if not suggested_shows:
+ logger.info('Series %r not found', series_year)
+ return None
+ return int(suggested_shows[0]['href'][6:])
+
+ def query(self, series, season, year=None):
+ show_ids = self.get_show_ids()
+ show_id = None
+ if year is not None: # search with the year
+ series_year = '%s (%d)' % (series.lower(), year)
+ if series_year in show_ids:
+ show_id = show_ids[series_year]
+ else:
+ show_id = self.find_show_id(series.lower(), year)
+ if show_id is None: # search without the year
+ year = None
+ if series.lower() in show_ids:
+ show_id = show_ids[series.lower()]
+ else:
+ show_id = self.find_show_id(series.lower())
+ if show_id is None:
+ return []
+ params = {'show_id': show_id, 'season': season}
+ logger.debug('Searching subtitles %r', params)
+ link = '/show/{show_id}&season={season}'.format(**params)
+ soup = self.get(link)
+ subtitles = []
+ for row in soup('tr', class_='epeven completed'):
+ cells = row('td')
+ if cells[5].string != 'Completed':
+ continue
+ if not cells[3].string:
+ continue
+ subtitles.append(Addic7edSubtitle(babelfish.Language.fromaddic7ed(cells[3].string), series, season,
+ int(cells[1].string), cells[2].string, year, cells[4].string,
+ bool(cells[6].string), cells[9].a['href'],
+ self.server + cells[2].a['href']))
+ return subtitles
+
+ def list_subtitles(self, video, languages):
+ return [s for s in self.query(video.series, video.season, video.year)
+ if s.language in languages and s.episode == video.episode]
+
+ def download_subtitle(self, subtitle):
+ r = self.session.get(self.server + subtitle.download_link, timeout=10, headers={'Referer': subtitle.page_link})
+ if r.status_code != 200:
+ raise ProviderError('Request failed with status code %d' % r.status_code)
+ if r.headers['Content-Type'] == 'text/html':
+ raise DownloadLimitExceeded
+ subtitle.content = fix_line_endings(r.content)
diff --git a/lib/subliminal/providers/opensubtitles.py b/lib/subliminal/providers/opensubtitles.py
new file mode 100644
index 0000000000000000000000000000000000000000..795799d2301d521a859322225ab813ba4fdd1bf7
--- /dev/null
+++ b/lib/subliminal/providers/opensubtitles.py
@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import base64
+import logging
+import os
+import re
+import zlib
+import babelfish
+import guessit
+from . import Provider
+from .. import __version__
+from ..compat import ServerProxy, TimeoutTransport
+from ..exceptions import ProviderError, AuthenticationError, DownloadLimitExceeded
+from ..subtitle import Subtitle, fix_line_endings, compute_guess_matches
+from ..video import Episode, Movie
+
+
+logger = logging.getLogger(__name__)
+
+
+class OpenSubtitlesSubtitle(Subtitle):
+ provider_name = 'opensubtitles'
+ series_re = re.compile('^"(?P<series_name>.*)" (?P<series_title>.*)$')
+
+ def __init__(self, language, hearing_impaired, id, matched_by, movie_kind, hash, movie_name, movie_release_name, # @ReservedAssignment
+ movie_year, movie_imdb_id, series_season, series_episode, page_link):
+ super(OpenSubtitlesSubtitle, self).__init__(language, hearing_impaired, page_link)
+ self.id = id
+ self.matched_by = matched_by
+ self.movie_kind = movie_kind
+ self.hash = hash
+ self.movie_name = movie_name
+ self.movie_release_name = movie_release_name
+ self.movie_year = movie_year
+ self.movie_imdb_id = movie_imdb_id
+ self.series_season = series_season
+ self.series_episode = series_episode
+
+ @property
+ def series_name(self):
+ return self.series_re.match(self.movie_name).group('series_name')
+
+ @property
+ def series_title(self):
+ return self.series_re.match(self.movie_name).group('series_title')
+
+ def compute_matches(self, video):
+ matches = set()
+ # episode
+ if isinstance(video, Episode) and self.movie_kind == 'episode':
+ # series
+ if video.series and self.series_name.lower() == video.series.lower():
+ matches.add('series')
+ # season
+ if video.season and self.series_season == video.season:
+ matches.add('season')
+ # episode
+ if video.episode and self.series_episode == video.episode:
+ matches.add('episode')
+ # guess
+ matches |= compute_guess_matches(video, guessit.guess_episode_info(self.movie_release_name + '.mkv'))
+ # movie
+ elif isinstance(video, Movie) and self.movie_kind == 'movie':
+ # year
+ if video.year and self.movie_year == video.year:
+ matches.add('year')
+ # guess
+ matches |= compute_guess_matches(video, guessit.guess_movie_info(self.movie_release_name + '.mkv'))
+ else:
+ logger.info('%r is not a valid movie_kind for %r', self.movie_kind, video)
+ return matches
+ # hash
+ if 'opensubtitles' in video.hashes and self.hash == video.hashes['opensubtitles']:
+ matches.add('hash')
+ # imdb_id
+ if video.imdb_id and self.movie_imdb_id == video.imdb_id:
+ matches.add('imdb_id')
+ # title
+ if video.title and self.movie_name.lower() == video.title.lower():
+ matches.add('title')
+ return matches
+
+
+class OpenSubtitlesProvider(Provider):
+ languages = {babelfish.Language.fromopensubtitles(l) for l in babelfish.language_converters['opensubtitles'].codes}
+
+ def __init__(self):
+ self.server = ServerProxy('http://api.opensubtitles.org/xml-rpc', transport=TimeoutTransport(10))
+ self.token = None
+
+ def initialize(self):
+ response = checked(self.server.LogIn('', '', 'eng', 'subliminal v%s' % __version__.split('-')[0]))
+ self.token = response['token']
+
+ def terminate(self):
+ checked(self.server.LogOut(self.token))
+ self.server.close()
+
+ def no_operation(self):
+ checked(self.server.NoOperation(self.token))
+
+ def query(self, languages, hash=None, size=None, imdb_id=None, query=None, season=None, episode=None): # @ReservedAssignment
+ searches = []
+ if hash and size:
+ searches.append({'moviehash': hash, 'moviebytesize': str(size)})
+ if imdb_id:
+ searches.append({'imdbid': imdb_id})
+ if query and season and episode:
+ searches.append({'query': query, 'season': season, 'episode': episode})
+ elif query:
+ searches.append({'query': query})
+ if not searches:
+ raise ValueError('One or more parameter missing')
+ for search in searches:
+ search['sublanguageid'] = ','.join(l.opensubtitles for l in languages)
+ logger.debug('Searching subtitles %r', searches)
+ response = checked(self.server.SearchSubtitles(self.token, searches))
+ if not response['data']:
+ logger.debug('No subtitle found')
+ return []
+ return [OpenSubtitlesSubtitle(babelfish.Language.fromopensubtitles(r['SubLanguageID']),
+ bool(int(r['SubHearingImpaired'])), r['IDSubtitleFile'], r['MatchedBy'],
+ r['MovieKind'], r['MovieHash'], r['MovieName'], r['MovieReleaseName'],
+ int(r['MovieYear']) if r['MovieYear'] else None, int(r['IDMovieImdb']),
+ int(r['SeriesSeason']) if r['SeriesSeason'] else None,
+ int(r['SeriesEpisode']) if r['SeriesEpisode'] else None, r['SubtitlesLink'])
+ for r in response['data']]
+
+ def list_subtitles(self, video, languages):
+ query = None
+ season = None
+ episode = None
+ if ('opensubtitles' not in video.hashes or not video.size) and not video.imdb_id:
+ query = video.name.split(os.sep)[-1]
+ if isinstance(video, Episode):
+ query = video.series
+ season = video.season
+ episode = video.episode
+ return self.query(languages, hash=video.hashes.get('opensubtitles'), size=video.size, imdb_id=video.imdb_id,
+ query=query, season=season, episode=episode)
+
+ def download_subtitle(self, subtitle):
+ response = checked(self.server.DownloadSubtitles(self.token, [subtitle.id]))
+ if not response['data']:
+ raise ProviderError('Nothing to download')
+ subtitle.content = fix_line_endings(zlib.decompress(base64.b64decode(response['data'][0]['data']), 47))
+
+
+class OpenSubtitlesError(ProviderError):
+ """Base class for non-generic :class:`OpenSubtitlesProvider` exceptions"""
+
+
+class Unauthorized(OpenSubtitlesError, AuthenticationError):
+ """Exception raised when status is '401 Unauthorized'"""
+
+
+class NoSession(OpenSubtitlesError, AuthenticationError):
+ """Exception raised when status is '406 No session'"""
+
+
+class DownloadLimitReached(OpenSubtitlesError, DownloadLimitExceeded):
+ """Exception raised when status is '407 Download limit reached'"""
+
+
+class InvalidImdbid(OpenSubtitlesError):
+ """Exception raised when status is '413 Invalid ImdbID'"""
+
+
+class UnknownUserAgent(OpenSubtitlesError, AuthenticationError):
+ """Exception raised when status is '414 Unknown User Agent'"""
+
+
+class DisabledUserAgent(OpenSubtitlesError, AuthenticationError):
+ """Exception raised when status is '415 Disabled user agent'"""
+
+
+class ServiceUnavailable(OpenSubtitlesError):
+ """Exception raised when status is '503 Service Unavailable'"""
+
+
+def checked(response):
+ """Check a response status before returning it
+
+ :param response: a response from a XMLRPC call to OpenSubtitles
+ :return: the response
+ :raise: :class:`OpenSubtitlesError`
+
+ """
+ status_code = int(response['status'][:3])
+ if status_code == 401:
+ raise Unauthorized
+ if status_code == 406:
+ raise NoSession
+ if status_code == 407:
+ raise DownloadLimitReached
+ if status_code == 413:
+ raise InvalidImdbid
+ if status_code == 414:
+ raise UnknownUserAgent
+ if status_code == 415:
+ raise DisabledUserAgent
+ if status_code == 503:
+ raise ServiceUnavailable
+ if status_code != 200:
+ raise OpenSubtitlesError(response['status'])
+ return response
diff --git a/lib/subliminal/providers/podnapisi.py b/lib/subliminal/providers/podnapisi.py
new file mode 100644
index 0000000000000000000000000000000000000000..328cee0731b53ae77fa6c340268016a13bf708e5
--- /dev/null
+++ b/lib/subliminal/providers/podnapisi.py
@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import io
+import logging
+import re
+import xml.etree.ElementTree
+import zipfile
+import babelfish
+import bs4
+import guessit
+import requests
+from . import Provider
+from .. import __version__
+from ..exceptions import ProviderError
+from ..subtitle import Subtitle, fix_line_endings, compute_guess_matches
+from ..video import Episode, Movie
+
+
+logger = logging.getLogger(__name__)
+babelfish.language_converters.register('podnapisi = subliminal.converters.podnapisi:PodnapisiConverter')
+
+
+class PodnapisiSubtitle(Subtitle):
+ provider_name = 'podnapisi'
+
+ def __init__(self, language, id, releases, hearing_impaired, page_link, series=None, season=None, episode=None, # @ReservedAssignment
+ title=None, year=None):
+ super(PodnapisiSubtitle, self).__init__(language, hearing_impaired, page_link)
+ self.id = id
+ self.releases = releases
+ self.hearing_impaired = hearing_impaired
+ self.series = series
+ self.season = season
+ self.episode = episode
+ self.title = title
+ self.year = year
+
+ def compute_matches(self, video):
+ matches = set()
+ # episode
+ if isinstance(video, Episode):
+ # series
+ if video.series and self.series.lower() == video.series.lower():
+ matches.add('series')
+ # season
+ if video.season and self.season == video.season:
+ matches.add('season')
+ # episode
+ if video.episode and self.episode == video.episode:
+ matches.add('episode')
+ # guess
+ for release in self.releases:
+ matches |= compute_guess_matches(video, guessit.guess_episode_info(release + '.mkv'))
+ # movie
+ elif isinstance(video, Movie):
+ # title
+ if video.title and self.title.lower() == video.title.lower():
+ matches.add('title')
+ # guess
+ for release in self.releases:
+ matches |= compute_guess_matches(video, guessit.guess_movie_info(release + '.mkv'))
+ # year
+ if self.year == video.year:
+ matches.add('year')
+ return matches
+
+
+class PodnapisiProvider(Provider):
+ languages = {babelfish.Language.frompodnapisi(l) for l in babelfish.language_converters['podnapisi'].codes}
+ video_types = (Episode, Movie)
+ search_url = 'http://simple.podnapisi.net/ppodnapisi/search'
+ download_url_suffix = '/download'
+
+ def initialize(self):
+ self.session = requests.Session()
+ self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__.split('-')[0]}
+
+ def terminate(self):
+ self.session.close()
+
+ def get(self, url, params=None, is_xml=True):
+ """Make a GET request on `url` with the given parameters
+
+ :param string url: the URL to reach
+ :param dict params: params of the request
+ :param bool is_xml: whether the response content is XML or not
+ :return: the response
+ :rtype: :class:`xml.etree.ElementTree.Element` or :class:`bs4.BeautifulSoup`
+
+ """
+ r = self.session.get(url, params=params, timeout=10)
+ if r.status_code != 200:
+ raise ProviderError('Request failed with status code %d' % r.status_code)
+ if is_xml:
+ return xml.etree.ElementTree.fromstring(r.content)
+ else:
+ return bs4.BeautifulSoup(r.content, ['permissive'])
+
+ def query(self, language, series=None, season=None, episode=None, title=None, year=None):
+ params = {'sXML': 1, 'sJ': language.podnapisi}
+ if series and season and episode:
+ params['sK'] = series
+ params['sTS'] = season
+ params['sTE'] = episode
+ elif title:
+ params['sK'] = title
+ else:
+ raise ValueError('Missing parameters series and season and episode or title')
+ if year:
+ params['sY'] = year
+ logger.debug('Searching episode %r', params)
+ subtitles = []
+ while True:
+ root = self.get(self.search_url, params)
+ if not int(root.find('pagination/results').text):
+ logger.debug('No subtitle found')
+ break
+ if series and season and episode:
+ subtitles.extend([PodnapisiSubtitle(language, int(s.find('id').text),
+ s.find('release').text.split() if s.find('release').text else [],
+ 'n' in (s.find('flags').text or ''), s.find('url').text,
+ series=series, season=season, episode=episode,
+ year=s.find('year').text)
+ for s in root.findall('subtitle')])
+ elif title:
+ subtitles.extend([PodnapisiSubtitle(language, int(s.find('id').text),
+ s.find('release').text.split() if s.find('release').text else [],
+ 'n' in (s.find('flags').text or ''), s.find('url').text,
+ title=title, year=s.find('year').text)
+ for s in root.findall('subtitle')])
+ if int(root.find('pagination/current').text) >= int(root.find('pagination/count').text):
+ break
+ params['page'] = int(root.find('pagination/current').text) + 1
+ return subtitles
+
+ def list_subtitles(self, video, languages):
+ if isinstance(video, Episode):
+ return [s for l in languages for s in self.query(l, series=video.series, season=video.season,
+ episode=video.episode, year=video.year)]
+ elif isinstance(video, Movie):
+ return [s for l in languages for s in self.query(l, title=video.title, year=video.year)]
+
+ def download_subtitle(self, subtitle):
+ download_url = subtitle.page_link + self.download_url_suffix
+ r = self.session.get(download_url, timeout=10)
+ if r.status_code != 200:
+ raise ProviderError('Request failed with status code %d' % r.status_code)
+ with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
+ if len(zf.namelist()) > 1:
+ raise ProviderError('More than one file to unzip')
+ subtitle.content = fix_line_endings(zf.read(zf.namelist()[0]))
diff --git a/lib/subliminal/providers/thesubdb.py b/lib/subliminal/providers/thesubdb.py
new file mode 100644
index 0000000000000000000000000000000000000000..446231736e87fb1895639142aab54591f3d8f0c8
--- /dev/null
+++ b/lib/subliminal/providers/thesubdb.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import logging
+import babelfish
+import requests
+from . import Provider
+from .. import __version__
+from ..exceptions import ProviderError
+from ..subtitle import Subtitle, fix_line_endings
+
+
+logger = logging.getLogger(__name__)
+
+
+class TheSubDBSubtitle(Subtitle):
+ provider_name = 'thesubdb'
+
+ def __init__(self, language, hash): # @ReservedAssignment
+ super(TheSubDBSubtitle, self).__init__(language)
+ self.hash = hash
+
+ def compute_matches(self, video):
+ matches = set()
+ # hash
+ if 'thesubdb' in video.hashes and video.hashes['thesubdb'] == self.hash:
+ matches.add('hash')
+ return matches
+
+
+class TheSubDBProvider(Provider):
+ languages = {babelfish.Language.fromalpha2(l) for l in ['en', 'es', 'fr', 'it', 'nl', 'pl', 'pt', 'ro', 'sv', 'tr']}
+ required_hash = 'thesubdb'
+
+ def initialize(self):
+ self.session = requests.Session()
+ self.session.headers = {'User-Agent': 'SubDB/1.0 (subliminal/%s; https://github.com/Diaoul/subliminal)' %
+ __version__.split('-')[0]}
+
+ def terminate(self):
+ self.session.close()
+
+ def get(self, params):
+ """Make a GET request on the server with the given parameters
+
+ :param params: params of the request
+ :return: the response
+ :rtype: :class:`requests.Response`
+
+ """
+ return self.session.get('http://api.thesubdb.com', params=params, timeout=10)
+
+ def query(self, hash): # @ReservedAssignment
+ params = {'action': 'search', 'hash': hash}
+ logger.debug('Searching subtitles %r', params)
+ r = self.get(params)
+ if r.status_code == 404:
+ logger.debug('No subtitle found')
+ return []
+ elif r.status_code != 200:
+ raise ProviderError('Request failed with status code %d' % r.status_code)
+ return [TheSubDBSubtitle(language, hash) for language in
+ {babelfish.Language.fromalpha2(l) for l in r.content.decode('utf-8').split(',')}]
+
+ def list_subtitles(self, video, languages):
+ return [s for s in self.query(video.hashes['thesubdb']) if s.language in languages]
+
+ def download_subtitle(self, subtitle):
+ params = {'action': 'download', 'hash': subtitle.hash, 'language': subtitle.language.alpha2}
+ r = self.get(params)
+ if r.status_code != 200:
+ raise ProviderError('Request failed with status code %d' % r.status_code)
+ subtitle.content = fix_line_endings(r.content)
diff --git a/lib/subliminal/providers/tvsubtitles.py b/lib/subliminal/providers/tvsubtitles.py
new file mode 100644
index 0000000000000000000000000000000000000000..3f21928b0d94eb37d7b5860b6998418c1eb2878e
--- /dev/null
+++ b/lib/subliminal/providers/tvsubtitles.py
@@ -0,0 +1,191 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import io
+import logging
+import re
+import zipfile
+import babelfish
+import bs4
+import requests
+from . import Provider
+from .. import __version__
+from ..cache import region, SHOW_EXPIRATION_TIME, EPISODE_EXPIRATION_TIME
+from ..exceptions import ProviderError
+from ..subtitle import Subtitle, fix_line_endings, compute_guess_properties_matches
+from ..video import Episode
+
+
+logger = logging.getLogger(__name__)
+babelfish.language_converters.register('tvsubtitles = subliminal.converters.tvsubtitles:TVsubtitlesConverter')
+
+
+class TVsubtitlesSubtitle(Subtitle):
+ provider_name = 'tvsubtitles'
+
+ def __init__(self, language, series, season, episode, year, id, rip, release, page_link): # @ReservedAssignment
+ super(TVsubtitlesSubtitle, self).__init__(language, page_link=page_link)
+ self.series = series
+ self.season = season
+ self.episode = episode
+ self.year = year
+ self.id = id
+ self.rip = rip
+ self.release = release
+
+ def compute_matches(self, video):
+ matches = set()
+ # series
+ if video.series and self.series == video.series:
+ matches.add('series')
+ # season
+ if video.season and self.season == video.season:
+ matches.add('season')
+ # episode
+ if video.episode and self.episode == video.episode:
+ matches.add('episode')
+ # year
+ if self.year == video.year:
+ matches.add('year')
+ # release_group
+ if video.release_group and self.release and video.release_group.lower() in self.release.lower():
+ matches.add('release_group')
+ """
+ # video_codec
+ if video.video_codec and self.release and (video.video_codec in self.release.lower()
+ or video.video_codec == 'h264' and 'x264' in self.release.lower()):
+ matches.add('video_codec')
+ # resolution
+ if video.resolution and self.rip and video.resolution in self.rip.lower():
+ matches.add('resolution')
+ # format
+ if video.format and self.rip and video.format in self.rip.lower():
+ matches.add('format')
+ """
+ # we don't have the complete filename, so we need to guess the matches separately
+ # guess video_codec (videoCodec in guessit)
+ matches |= compute_guess_properties_matches(video, self.release, 'videoCodec')
+ # guess resolution (screenSize in guessit)
+ matches |= compute_guess_properties_matches(video, self.rip, 'screenSize')
+ # guess format
+ matches |= compute_guess_properties_matches(video, self.rip, 'format')
+ return matches
+
+
+class TVsubtitlesProvider(Provider):
+ languages = {babelfish.Language('por', 'BR')} | {babelfish.Language(l)
+ for l in ['ara', 'bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'fin', 'fra', 'hun', 'ita', 'jpn', 'kor',
+ 'nld', 'pol', 'por', 'ron', 'rus', 'spa', 'swe', 'tur', 'ukr', 'zho']}
+ video_types = (Episode,)
+ server = 'http://www.tvsubtitles.net'
+ episode_id_re = re.compile('^episode-\d+\.html$')
+ subtitle_re = re.compile('^\/subtitle-\d+\.html$')
+ link_re = re.compile('^(?P<series>[A-Za-z0-9 \'.]+).*\((?P<first_year>\d{4})-\d{4}\)$')
+
+ def initialize(self):
+ self.session = requests.Session()
+ self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__.split('-')[0]}
+
+ def terminate(self):
+ self.session.close()
+
+ def request(self, url, params=None, data=None, method='GET'):
+ """Make a `method` request on `url` with the given parameters
+
+ :param string url: part of the URL to reach with the leading slash
+ :param dict params: params of the request
+ :param dict data: data of the request
+ :param string method: method of the request
+ :return: the response
+ :rtype: :class:`bs4.BeautifulSoup`
+
+ """
+ r = self.session.request(method, self.server + url, params=params, data=data, timeout=10)
+ if r.status_code != 200:
+ raise ProviderError('Request failed with status code %d' % r.status_code)
+ return bs4.BeautifulSoup(r.content, ['permissive'])
+
+ @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
+ def find_show_id(self, series, year=None):
+ """Find the show id from the `series` with optional `year`
+
+ :param string series: series of the episode in lowercase
+ :param year: year of the series, if any
+ :type year: int or None
+ :return: the show id, if any
+ :rtype: int or None
+
+ """
+ data = {'q': series}
+ logger.debug('Searching series %r', data)
+ soup = self.request('/search.php', data=data, method='POST')
+ links = soup.select('div.left li div a[href^="/tvshow-"]')
+ if not links:
+ logger.info('Series %r not found', series)
+ return None
+ matched_links = [link for link in links if self.link_re.match(link.string)]
+ for link in matched_links: # first pass with exact match on series
+ match = self.link_re.match(link.string)
+ if match.group('series').lower().replace('.', ' ').strip() == series:
+ if year is not None and int(match.group('first_year')) != year:
+ continue
+ return int(link['href'][8:-5])
+ for link in matched_links: # less selective second pass
+ match = self.link_re.match(link.string)
+ if match.group('series').lower().replace('.', ' ').strip().startswith(series):
+ if year is not None and int(match.group('first_year')) != year:
+ continue
+ return int(link['href'][8:-5])
+ return None
+
+ @region.cache_on_arguments(expiration_time=EPISODE_EXPIRATION_TIME)
+ def find_episode_ids(self, show_id, season):
+ """Find episode ids from the show id and the season
+
+ :param int show_id: show id
+ :param int season: season of the episode
+ :return: episode ids per episode number
+ :rtype: dict
+
+ """
+ params = {'show_id': show_id, 'season': season}
+ logger.debug('Searching episodes %r', params)
+ soup = self.request('/tvshow-{show_id}-{season}.html'.format(**params))
+ episode_ids = {}
+ for row in soup.select('table#table5 tr'):
+ if not row('a', href=self.episode_id_re):
+ continue
+ cells = row('td')
+ episode_ids[int(cells[0].string.split('x')[1])] = int(cells[1].a['href'][8:-5])
+ return episode_ids
+
+ def query(self, series, season, episode, year=None):
+ show_id = self.find_show_id(series.lower(), year)
+ if show_id is None:
+ return []
+ episode_ids = self.find_episode_ids(show_id, season)
+ if episode not in episode_ids:
+ logger.info('Episode %d not found', episode)
+ return []
+ params = {'episode_id': episode_ids[episode]}
+ logger.debug('Searching episode %r', params)
+ link = '/episode-{episode_id}.html'.format(**params)
+ soup = self.request(link)
+ return [TVsubtitlesSubtitle(babelfish.Language.fromtvsubtitles(row.h5.img['src'][13:-4]), series, season,
+ episode, year if year and show_id != self.find_show_id(series.lower()) else None,
+ int(row['href'][10:-5]), row.find('p', title='rip').text.strip() or None,
+ row.find('p', title='release').text.strip() or None,
+ self.server + '/subtitle-%d.html' % int(row['href'][10:-5]))
+ for row in soup('a', href=self.subtitle_re)]
+
+ def list_subtitles(self, video, languages):
+ return [s for s in self.query(video.series, video.season, video.episode, video.year) if s.language in languages]
+
+ def download_subtitle(self, subtitle):
+ r = self.session.get(self.server + '/download-{subtitle_id}.html'.format(subtitle_id=subtitle.id),
+ timeout=10)
+ if r.status_code != 200:
+ raise ProviderError('Request failed with status code %d' % r.status_code)
+ with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
+ if len(zf.namelist()) > 1:
+ raise ProviderError('More than one file to unzip')
+ subtitle.content = fix_line_endings(zf.read(zf.namelist()[0]))
diff --git a/lib/subliminal/score.py b/lib/subliminal/score.py
new file mode 100755
index 0000000000000000000000000000000000000000..f9dcaedee5880fa8f4f7713c6505c0fc47fc3e50
--- /dev/null
+++ b/lib/subliminal/score.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import print_function, unicode_literals
+from sympy import Eq, symbols, solve
+
+
+# Symbols
+release_group, resolution, format, video_codec, audio_codec = symbols('release_group resolution format video_codec audio_codec')
+imdb_id, hash, title, series, tvdb_id, season, episode = symbols('imdb_id hash title series tvdb_id season episode') # @ReservedAssignment
+year = symbols('year')
+
+
+def get_episode_equations():
+ """Get the score equations for a :class:`~subliminal.video.Episode`
+
+ The equations are the following:
+
+ 1. hash = resolution + format + video_codec + audio_codec + series + season + episode + year + release_group
+ 2. series = resolution + video_codec + audio_codec + season + episode + release_group + 1
+ 3. year = series
+ 4. tvdb_id = series + year
+ 5. season = resolution + video_codec + audio_codec + 1
+ 6. imdb_id = series + season + episode + year
+ 7. format = video_codec + audio_codec
+ 8. resolution = video_codec
+ 9. video_codec = 2 * audio_codec
+ 10. title = season + episode
+ 11. season = episode
+ 12. release_group = season
+ 13. audio_codec = 1
+
+ :return: the score equations for an episode
+ :rtype: list of :class:`sympy.Eq`
+
+ """
+ equations = []
+ equations.append(Eq(hash, resolution + format + video_codec + audio_codec + series + season + episode + year + release_group))
+ equations.append(Eq(series, resolution + video_codec + audio_codec + season + episode + release_group + 1))
+ equations.append(Eq(series, year))
+ equations.append(Eq(tvdb_id, series + year))
+ equations.append(Eq(season, resolution + video_codec + audio_codec + 1))
+ equations.append(Eq(imdb_id, series + season + episode + year))
+ equations.append(Eq(format, video_codec + audio_codec))
+ equations.append(Eq(resolution, video_codec))
+ equations.append(Eq(video_codec, 2 * audio_codec))
+ equations.append(Eq(title, season + episode))
+ equations.append(Eq(season, episode))
+ equations.append(Eq(release_group, season))
+ equations.append(Eq(audio_codec, 1))
+ return equations
+
+
+def get_movie_equations():
+ """Get the score equations for a :class:`~subliminal.video.Movie`
+
+ The equations are the following:
+
+ 1. hash = resolution + format + video_codec + audio_codec + title + year + release_group
+ 2. imdb_id = hash
+ 3. resolution = video_codec
+ 4. video_codec = 2 * audio_codec
+ 5. format = video_codec + audio_codec
+ 6. title = resolution + video_codec + audio_codec + year + 1
+ 7. release_group = resolution + video_codec + audio_codec + 1
+ 8. year = release_group + 1
+ 9. audio_codec = 1
+
+ :return: the score equations for a movie
+ :rtype: list of :class:`sympy.Eq`
+
+ """
+ equations = []
+ equations.append(Eq(hash, resolution + format + video_codec + audio_codec + title + year + release_group))
+ equations.append(Eq(imdb_id, hash))
+ equations.append(Eq(resolution, video_codec))
+ equations.append(Eq(video_codec, 2 * audio_codec))
+ equations.append(Eq(format, video_codec + audio_codec))
+ equations.append(Eq(title, resolution + video_codec + audio_codec + year + 1))
+ equations.append(Eq(video_codec, 2 * audio_codec))
+ equations.append(Eq(release_group, resolution + video_codec + audio_codec + 1))
+ equations.append(Eq(year, release_group + 1))
+ equations.append(Eq(audio_codec, 1))
+ return equations
+
+
+if __name__ == '__main__':
+ print(solve(get_episode_equations(), [release_group, resolution, format, video_codec, audio_codec, imdb_id,
+ hash, series, tvdb_id, season, episode, title, year]))
+ print(solve(get_movie_equations(), [release_group, resolution, format, video_codec, audio_codec, imdb_id,
+ hash, title, year]))
diff --git a/lib/subliminal/services/__init__.py b/lib/subliminal/services/__init__.py
deleted file mode 100644
index 044afc98b81158708cf2e0f5dca56cf0bf3ae391..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/__init__.py
+++ /dev/null
@@ -1,260 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from ..cache import Cache
-from ..exceptions import DownloadFailedError, ServiceError
-from ..language import language_set, Language
-from ..subtitles import EXTENSIONS
-import logging
-import os
-from lib import requests
-import threading
-import zipfile
-import sys
-
-
-__all__ = ['ServiceBase', 'ServiceConfig']
-logger = logging.getLogger("subliminal")
-
-
-class ServiceBase(object):
- """Service base class
-
- :param config: service configuration
- :type config: :class:`ServiceConfig`
-
- """
- #: URL to the service server
- server_url = ''
-
- #: User Agent for any HTTP-based requests
- user_agent = 'subliminal v0.6'
-
- #: Whether based on an API or not
- api_based = False
-
- #: Timeout for web requests
- timeout = 5
-
- #: :class:`~subliminal.language.language_set` of available languages
- languages = language_set()
-
- #: Map between language objects and language codes used in the service
- language_map = {}
-
- #: Default attribute of a :class:`~subliminal.language.Language` to get with :meth:`get_code`
- language_code = 'alpha2'
-
- #: Accepted video classes (:class:`~subliminal.videos.Episode`, :class:`~subliminal.videos.Movie`, :class:`~subliminal.videos.UnknownVideo`)
- videos = []
-
- #: Whether the video has to exist or not
- require_video = False
-
- #: List of required features for BeautifulSoup
- required_features = None
-
- def __init__(self, config=None):
- self.config = config or ServiceConfig()
- self.session = None
-
- def __enter__(self):
- self.init()
- return self
-
- def __exit__(self, *args):
- self.terminate()
-
- def init(self):
- """Initialize connection"""
- logger.debug(u'Initializing %s' % self.__class__.__name__) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- self.session = requests.session()
- self.session.headers.update({'User-Agent': self.user_agent})
-
- def init_cache(self):
- """Initialize cache, make sure it is loaded from disk"""
- if not self.config or not self.config.cache:
- raise ServiceError('Cache directory is required')
- self.config.cache.load(self.__class__.__name__)
-
- def save_cache(self):
- self.config.cache.save(self.__class__.__name__)
-
- def clear_cache(self):
- self.config.cache.clear(self.__class__.__name__)
-
- def cache_for(self, func, args, result):
- return self.config.cache.cache_for(self.__class__.__name__, func, args, result)
-
- def cached_value(self, func, args):
- return self.config.cache.cached_value(self.__class__.__name__, func, args)
-
- def terminate(self):
- """Terminate connection"""
- logger.debug(u'Terminating %s' % self.__class__.__name__) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
-
- def get_code(self, language):
- """Get the service code for a :class:`~subliminal.language.Language`
-
- It uses the :data:`language_map` and if there's no match, falls back
- on the :data:`language_code` attribute of the given :class:`~subliminal.language.Language`
-
- """
- if language in self.language_map:
- return self.language_map[language]
- if self.language_code is None:
- raise ValueError('%r has no matching code' % language)
- return getattr(language, self.language_code)
-
- def get_language(self, code):
- """Get a :class:`~subliminal.language.Language` from a service code
-
- It uses the :data:`language_map` and if there's no match, uses the
- given code as ``language`` parameter for the :class:`~subliminal.language.Language`
- constructor
-
- .. note::
-
- A warning is emitted if the generated :class:`~subliminal.language.Language`
- is "Undetermined"
-
- """
- if code in self.language_map:
- return self.language_map[code]
- language = Language(code, strict=False)
- if language == Language('Undetermined'):
- logger.warning(u'Code %s could not be identified as a language for %s' % (code, self.__class__.__name__)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return language
-
- def query(self, *args):
- """Make the actual query"""
- raise NotImplementedError()
-
- def list(self, video, languages):
- """List subtitles
-
- As a service writer, you can either override this method or implement
- :meth:`list_checked` instead to have the languages pre-filtered for you
-
- """
- if not self.check_validity(video, languages):
- return []
- return self.list_checked(video, languages)
-
- def list_checked(self, video, languages):
- """List subtitles without having to check parameters for validity"""
- raise NotImplementedError()
-
- def download(self, subtitle):
- """Download a subtitle"""
- self.download_file(subtitle.link, subtitle.path)
- return subtitle
-
- @classmethod
- def check_validity(cls, video, languages):
- """Check for video and languages validity in the Service
-
- :param video: the video to check
- :type video: :class:`~subliminal.videos.video`
- :param languages: languages to check
- :type languages: :class:`~subliminal.language.Language`
- :rtype: bool
-
- """
- languages = (languages & cls.languages) - language_set(['Undetermined'])
- if not languages:
- logger.debug(u'No language available for service %s' % cls.__name__.lower()) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return False
- if cls.require_video and not video.exists or not isinstance(video, tuple(cls.videos)):
- logger.debug(u'%r is not valid for service %s' % (video, cls.__name__.lower())) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return False
- return True
-
- def download_file(self, url, filepath):
- """Attempt to download a file and remove it in case of failure
-
- :param string url: URL to download
- :param string filepath: destination path
-
- """
- logger.info(u'Downloading %s in %s' % (url, filepath)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- try:
- r = self.session.get(url, timeout = 10, headers = {'Referer': url, 'User-Agent': self.user_agent})
- with open(filepath, 'wb') as f:
- f.write(r.content)
- except Exception as e:
- logger.error(u'Download failed: %s' % e) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- if os.path.exists(filepath):
- os.remove(filepath)
- raise DownloadFailedError(str(e))
- logger.debug(u'Download finished') if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
-
- def download_zip_file(self, url, filepath):
- """Attempt to download a zip file and extract any subtitle file from it, if any.
- This cleans up after itself if anything fails.
-
- :param string url: URL of the zip file to download
- :param string filepath: destination path for the subtitle
-
- """
- logger.info(u'Downloading %s in %s' % (url, filepath)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- try:
- zippath = filepath + '.zip'
- r = self.session.get(url, timeout = 10, headers = {'Referer': url, 'User-Agent': self.user_agent})
- with open(zippath, 'wb') as f:
- f.write(r.content)
- if not zipfile.is_zipfile(zippath):
- # TODO: could check if maybe we already have a text file and
- # download it directly
- raise DownloadFailedError('Downloaded file is not a zip file')
- zipsub = zipfile.ZipFile(zippath)
- for subfile in zipsub.namelist():
- if os.path.splitext(subfile)[1] in EXTENSIONS:
- with open(filepath, 'wb') as f:
- f.write(zipsub.open(subfile).read())
- break
- else:
- zipsub.close()
- raise DownloadFailedError('No subtitles found in zip file')
- zipsub.close()
- os.remove(zippath)
- except Exception as e:
- logger.error(u'Download %s failed: %s' % (url, e)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- if os.path.exists(zippath):
- os.remove(zippath)
- if os.path.exists(filepath):
- os.remove(filepath)
- raise DownloadFailedError(str(e))
- logger.debug(u'Download finished') if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
-
-
-class ServiceConfig(object):
- """Configuration for any :class:`Service`
-
- :param bool multi: whether to download one subtitle per language or not
- :param string cache_dir: cache directory
-
- """
- def __init__(self, multi=False, cache_dir=None):
- self.multi = multi
- self.cache_dir = cache_dir
- self.cache = None
- if cache_dir is not None:
- self.cache = Cache(cache_dir)
-
- def __repr__(self):
- return 'ServiceConfig(%r, %s)' % (self.multi, self.cache.cache_dir)
diff --git a/lib/subliminal/services/addic7ed.py b/lib/subliminal/services/addic7ed.py
deleted file mode 100644
index b4a28510637961bdd3f324e3ec7f2ac9422b55c1..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/addic7ed.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2012 Olivier Leveau <olifozzy@gmail.com>
-# Copyright 2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..cache import cachedmethod
-from ..exceptions import DownloadFailedError
-from ..language import Language, language_set
-from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..utils import get_keywords, split_keyword
-from ..videos import Episode
-from bs4 import BeautifulSoup
-import logging
-import os
-import re
-import sys
-
-
-logger = logging.getLogger("subliminal")
-
-
-class Addic7ed(ServiceBase):
- server_url = 'http://www.addic7ed.com'
- site_url = 'http://www.addic7ed.com'
- api_based = False
- #TODO: Complete this
- languages = language_set(['ar', 'ca', 'de', 'el', 'en', 'es', 'eu', 'fr', 'ga', 'gl', 'he', 'hr', 'hu',
- 'it', 'nl', 'pl', 'pt', 'ro', 'ru', 'se', 'pb'])
- language_map = {'Portuguese (Brazilian)': Language('pob'), 'Greek': Language('gre'),
- 'Spanish (Latin America)': Language('spa'), 'Galego': Language('glg'),
- u'Català': Language('cat')}
- videos = [Episode]
- require_video = False
- required_features = ['permissive']
-
- @cachedmethod
- def get_series_id(self, name):
- """Get the show page and cache every show found in it"""
- r = self.session.get('%s/shows.php' % self.server_url)
- soup = BeautifulSoup(r.content, self.required_features)
- for html_series in soup.select('h3 > a'):
- series_name = html_series.text.lower()
- match = re.search('show/([0-9]+)', html_series['href'])
- if match is None:
- continue
- series_id = int(match.group(1))
- self.cache_for(self.get_series_id, args=(series_name,), result=series_id)
- return self.cached_value(self.get_series_id, args=(name,))
-
- def list_checked(self, video, languages):
- return self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
-
- def query(self, filepath, languages, keywords, series, season, episode):
-
- logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- self.init_cache()
- try:
- series_id = self.get_series_id(series.lower())
- except KeyError:
- logger.debug(u'Could not find series id for %s' % series) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- r = self.session.get('%s/show/%d&season=%d' % (self.server_url, series_id, season))
- soup = BeautifulSoup(r.content, self.required_features)
- subtitles = []
- for row in soup('tr', {'class': 'epeven completed'}):
- cells = row('td')
- if int(cells[0].text.strip()) != season or int(cells[1].text.strip()) != episode:
- continue
- if cells[6].text.strip():
- logger.debug(u'Skipping hearing impaired') if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- sub_status = cells[5].text.strip()
- if sub_status != 'Completed':
- logger.debug(u'Wrong subtitle status %s' % sub_status) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- sub_language = self.get_language(cells[3].text.strip())
- if sub_language not in languages:
- logger.debug(u'Language %r not in wanted languages %r' % (sub_language, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- sub_keywords = split_keyword(cells[4].text.strip().lower())
- #TODO: Maybe allow empty keywords here? (same in Subtitulos)
- if keywords and not keywords & sub_keywords:
- logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- sub_link = '%s/%s' % (self.server_url, cells[9].a['href'])
- sub_path = get_subtitle_path(filepath, sub_language, self.config.multi)
- subtitle = ResultSubtitle(sub_path, sub_language, self.__class__.__name__.lower(), sub_link, keywords=sub_keywords)
- subtitles.append(subtitle)
- return subtitles
-
- def download(self, subtitle):
- logger.info(u'Downloading %s in %s' % (subtitle.link, subtitle.path)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- try:
- r = self.session.get(subtitle.link, headers={'Referer': subtitle.link, 'User-Agent': self.user_agent})
- soup = BeautifulSoup(r.content, self.required_features)
- if soup.title is not None and u'Addic7ed.com' in soup.title.text.strip():
- raise DownloadFailedError('Download limit exceeded')
- with open(subtitle.path, 'wb') as f:
- f.write(r.content)
- except Exception as e:
- logger.error(u'Download failed: %s' % e) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- if os.path.exists(subtitle.path):
- os.remove(subtitle.path)
- raise DownloadFailedError(str(e))
- logger.debug(u'Download finished') if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return subtitle
-
-
-Service = Addic7ed
diff --git a/lib/subliminal/services/bierdopje.py b/lib/subliminal/services/bierdopje.py
deleted file mode 100644
index 44c2829fb9ef36a8533e2d72ba443aba34f17bf7..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/bierdopje.py
+++ /dev/null
@@ -1,104 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..cache import cachedmethod
-from ..exceptions import ServiceError
-from ..language import language_set
-from ..subtitles import get_subtitle_path, ResultSubtitle, EXTENSIONS
-from ..utils import to_unicode
-from ..videos import Episode
-from bs4 import BeautifulSoup
-import logging
-import urllib
-try:
- import cPickle as pickle
-except ImportError:
- import pickle
-import sys
-
-logger = logging.getLogger("subliminal")
-
-
-class BierDopje(ServiceBase):
- server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/'
- site_url = 'http://www.bierdopje.com'
- user_agent = 'Subliminal/0.6'
- api_based = True
- languages = language_set(['eng', 'dut'])
- videos = [Episode]
- require_video = False
- required_features = ['xml']
-
- @cachedmethod
- def get_show_id(self, series):
- r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower())))
- if r.status_code != 200:
- logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return None
- soup = BeautifulSoup(r.content, self.required_features)
- if soup.status.contents[0] == 'false':
- logger.debug(u'Could not find show %s' % series) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return None
- return int(soup.showid.contents[0])
-
- def load_cache(self):
- logger.debug(u'Loading showids from cache...') if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- with self.lock:
- with open(self.showids_cache, 'r') as f:
- self.showids = pickle.load(f)
-
- def query(self, filepath, season, episode, languages, tvdbid=None, series=None):
- self.init_cache()
- if series:
- request_id = self.get_show_id(series.lower())
- if request_id is None:
- return []
- request_source = 'showid'
- request_is_tvdbid = 'false'
- elif tvdbid:
- request_id = tvdbid
- request_source = 'tvdbid'
- request_is_tvdbid = 'true'
- else:
- raise ServiceError('One or more parameter missing')
- subtitles = []
- for language in languages:
- logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language.alpha2)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language.alpha2, request_is_tvdbid))
- if r.status_code != 200:
- logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- soup = BeautifulSoup(r.content, self.required_features)
- if soup.status.contents[0] == 'false':
- logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language.alpha2)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- path = get_subtitle_path(filepath, language, self.config.multi)
- for result in soup.results('result'):
- release = to_unicode(result.filename.contents[0])
- if not release.endswith(tuple(EXTENSIONS)):
- release += '.srt'
- subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result.downloadlink.contents[0],
- release=release)
- subtitles.append(subtitle)
- return subtitles
-
- def list_checked(self, video, languages):
- return self.query(video.path or video.release, video.season, video.episode, languages, video.tvdbid, video.series)
-
-
-Service = BierDopje
diff --git a/lib/subliminal/services/itasa.py b/lib/subliminal/services/itasa.py
deleted file mode 100644
index 5a9a8414ef310cb9e81bffac13db34a3b56a09d2..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/itasa.py
+++ /dev/null
@@ -1,217 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2012 Mr_Orange <mr_orange@hotmail.it>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..exceptions import DownloadFailedError, ServiceError
-from ..cache import cachedmethod
-from ..language import language_set, Language
-from ..subtitles import get_subtitle_path, ResultSubtitle, EXTENSIONS
-from ..utils import get_keywords
-from ..videos import Episode
-from bs4 import BeautifulSoup
-import logging
-import re
-import os
-from lib import requests
-import zipfile
-import StringIO
-import guessit
-import sys
-
-from sickbeard.common import Quality
-
-logger = logging.getLogger("subliminal")
-
-
-class Itasa(ServiceBase):
- server_url = 'http://www.italiansubs.net/'
- site_url = 'http://www.italiansubs.net/'
- api_based = False
- languages = language_set(['it'])
- videos = [Episode]
- require_video = False
- required_features = ['permissive']
- quality_dict = {Quality.SDTV : '',
- Quality.SDDVD : 'dvdrip',
- Quality.RAWHDTV : '1080i',
- Quality.HDTV : '720p',
- Quality.FULLHDTV : ('1080p','720p'),
- Quality.HDWEBDL : 'web-dl',
- Quality.FULLHDWEBDL : 'web-dl',
- Quality.HDBLURAY : ('bdrip', 'bluray'),
- Quality.FULLHDBLURAY : ('bdrip', 'bluray'),
- Quality.UNKNOWN : 'unknown' #Any subtitle will be downloaded
- }
-
- def init(self):
-
- super(Itasa, self).init()
- login_pattern = '<input type="hidden" name="return" value="([^\n\r\t ]+?)" /><input type="hidden" name="([^\n\r\t ]+?)" value="([^\n\r\t ]+?)" />'
-
- response = requests.get(self.server_url + 'index.php')
- if response.status_code != 200:
- raise ServiceError('Initiate failed')
-
- match = re.search(login_pattern, response.content, re.IGNORECASE | re.DOTALL)
- if not match:
- raise ServiceError('Can not find unique id parameter on page')
-
- login_parameter = {'username': 'sickbeard',
- 'passwd': 'subliminal',
- 'remember': 'yes',
- 'Submit': 'Login',
- 'remember': 'yes',
- 'option': 'com_user',
- 'task': 'login',
- 'silent': 'true',
- 'return': match.group(1),
- match.group(2): match.group(3)
- }
-
- self.session = requests.session()
- r = self.session.post(self.server_url + 'index.php', data=login_parameter)
- if not re.search('logouticon.png', r.content, re.IGNORECASE | re.DOTALL):
- raise ServiceError('Itasa Login Failed')
-
- @cachedmethod
- def get_series_id(self, name):
- """Get the show page and cache every show found in it"""
- r = self.session.get(self.server_url + 'index.php?option=com_remository&Itemid=9')
- soup = BeautifulSoup(r.content, self.required_features)
- all_series = soup.find('div', attrs = {'id' : 'remositorycontainerlist'})
- for tv_series in all_series.find_all(href=re.compile('func=select')):
- series_name = tv_series.text.lower().strip().replace(':','')
- match = re.search('&id=([0-9]+)', tv_series['href'])
- if match is None:
- continue
- series_id = int(match.group(1))
- self.cache_for(self.get_series_id, args=(series_name,), result=series_id)
- return self.cached_value(self.get_series_id, args=(name,))
-
- def get_episode_id(self, series, series_id, season, episode, quality):
- """Get the id subtitle for episode with the given quality"""
-
- season_link = None
- quality_link = None
- episode_id = None
-
- r = self.session.get(self.server_url + 'index.php?option=com_remository&Itemid=6&func=select&id=' + str(series_id))
- soup = BeautifulSoup(r.content, self.required_features)
- all_seasons = soup.find('div', attrs = {'id' : 'remositorycontainerlist'})
- for seasons in all_seasons.find_all(href=re.compile('func=select')):
- if seasons.text.lower().strip() == 'stagione %s' % str(season):
- season_link = seasons['href']
- break
-
- if not season_link:
- logger.debug(u'Could not find season %s for series %s' % (series, str(season))) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return None
-
- r = self.session.get(season_link)
- soup = BeautifulSoup(r.content, self.required_features)
-
- all_qualities = soup.find('div', attrs = {'id' : 'remositorycontainerlist'})
- for qualities in all_qualities.find_all(href=re.compile('func=select')):
- if qualities.text.lower().strip() in self.quality_dict[quality]:
- quality_link = qualities['href']
- r = self.session.get(qualities['href'])
- soup = BeautifulSoup(r.content, self.required_features)
- break
-
- #If we want SDTV we are just on the right page so quality link will be None
- if not quality == Quality.SDTV and not quality_link:
- logger.debug(u'Could not find a subtitle with required quality for series %s season %s' % (series, str(season))) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return None
-
- all_episodes = soup.find('div', attrs = {'id' : 'remositoryfilelisting'})
- for episodes in all_episodes.find_all(href=re.compile('func=fileinfo')):
- ep_string = "%(seasonnumber)dx%(episodenumber)02d" % {'seasonnumber': season, 'episodenumber': episode}
- if re.search(ep_string, episodes.text, re.I) or re.search('completa$', episodes.text, re.I):
- match = re.search('&id=([0-9]+)', episodes['href'])
- if match:
- episode_id = match.group(1)
- return episode_id
-
- return episode_id
-
- def list_checked(self, video, languages):
- return self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
-
- def query(self, filepath, languages, keywords, series, season, episode):
-
- logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- self.init_cache()
- try:
- series = series.lower().replace('(','').replace(')','')
- series_id = self.get_series_id(series)
- except KeyError:
- logger.debug(u'Could not find series id for %s' % series) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
-
- episode_id = self.get_episode_id(series, series_id, season, episode, Quality.nameQuality(filepath))
- if not episode_id:
- logger.debug(u'Could not find subtitle for series %s' % series) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
-
- r = self.session.get(self.server_url + 'index.php?option=com_remository&Itemid=6&func=fileinfo&id=' + episode_id)
- soup = BeautifulSoup(r.content)
-
- sub_link = soup.find('div', attrs = {'id' : 'remositoryfileinfo'}).find(href=re.compile('func=download'))['href']
- sub_language = self.get_language('it')
- path = get_subtitle_path(filepath, sub_language, self.config.multi)
- subtitle = ResultSubtitle(path, sub_language, self.__class__.__name__.lower(), sub_link)
-
- return [subtitle]
-
- def download(self, subtitle):
-
- logger.info(u'Downloading %s in %s' % (subtitle.link, subtitle.path)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- try:
- r = self.session.get(subtitle.link, headers={'Referer': self.server_url, 'User-Agent': self.user_agent})
- zipcontent = StringIO.StringIO(r.content)
- zipsub = zipfile.ZipFile(zipcontent)
-
-# if not zipsub.is_zipfile(zipcontent):
-# raise DownloadFailedError('Downloaded file is not a zip file')
-
- subfile = ''
- if len(zipsub.namelist()) == 1:
- subfile = zipsub.namelist()[0]
- else:
- #Season Zip Retrive Season and episode Numbers from path
- guess = guessit.guess_file_info(subtitle.path, 'episode')
- ep_string = "s%(seasonnumber)02de%(episodenumber)02d" % {'seasonnumber': guess['season'], 'episodenumber': guess['episodeNumber']}
- for file in zipsub.namelist():
- if re.search(ep_string, file, re.I):
- subfile = file
- break
- if os.path.splitext(subfile)[1] in EXTENSIONS:
- with open(subtitle.path, 'wb') as f:
- f.write(zipsub.open(subfile).read())
- else:
- zipsub.close()
- raise DownloadFailedError('No subtitles found in zip file')
-
- zipsub.close()
- except Exception as e:
- if os.path.exists(subtitle.path):
- os.remove(subtitle.path)
- raise DownloadFailedError(str(e))
-
- logger.debug(u'Download finished') if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
-
-Service = Itasa
\ No newline at end of file
diff --git a/lib/subliminal/services/opensubtitles.py b/lib/subliminal/services/opensubtitles.py
deleted file mode 100644
index 245b467b14a1de04e78b1ce2dd2b29b13a93f33f..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/opensubtitles.py
+++ /dev/null
@@ -1,163 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..exceptions import ServiceError, DownloadFailedError
-from ..language import Language, language_set
-from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..utils import to_unicode
-from ..videos import Episode, Movie
-import gzip
-import logging
-import os.path
-import xmlrpclib
-import sys
-
-logger = logging.getLogger("subliminal")
-
-
-class OpenSubtitles(ServiceBase):
- server_url = 'http://api.opensubtitles.org/xml-rpc'
- site_url = 'http://www.opensubtitles.org'
- api_based = True
- # Source: http://www.opensubtitles.org/addons/export_languages.php
- languages = language_set(['aar', 'abk', 'ace', 'ach', 'ada', 'ady', 'afa', 'afh', 'afr', 'ain', 'aka', 'akk',
- 'alb', 'ale', 'alg', 'alt', 'amh', 'ang', 'apa', 'ara', 'arc', 'arg', 'arm', 'arn',
- 'arp', 'art', 'arw', 'asm', 'ast', 'ath', 'aus', 'ava', 'ave', 'awa', 'aym', 'aze',
- 'bad', 'bai', 'bak', 'bal', 'bam', 'ban', 'baq', 'bas', 'bat', 'bej', 'bel', 'bem',
- 'ben', 'ber', 'bho', 'bih', 'bik', 'bin', 'bis', 'bla', 'bnt', 'bos', 'bra', 'bre',
- 'btk', 'bua', 'bug', 'bul', 'bur', 'byn', 'cad', 'cai', 'car', 'cat', 'cau', 'ceb',
- 'cel', 'cha', 'chb', 'che', 'chg', 'chi', 'chk', 'chm', 'chn', 'cho', 'chp', 'chr',
- 'chu', 'chv', 'chy', 'cmc', 'cop', 'cor', 'cos', 'cpe', 'cpf', 'cpp', 'cre', 'crh',
- 'crp', 'csb', 'cus', 'cze', 'dak', 'dan', 'dar', 'day', 'del', 'den', 'dgr', 'din',
- 'div', 'doi', 'dra', 'dua', 'dum', 'dut', 'dyu', 'dzo', 'efi', 'egy', 'eka', 'ell',
- 'elx', 'eng', 'enm', 'epo', 'est', 'ewe', 'ewo', 'fan', 'fao', 'fat', 'fij', 'fil',
- 'fin', 'fiu', 'fon', 'fre', 'frm', 'fro', 'fry', 'ful', 'fur', 'gaa', 'gay', 'gba',
- 'gem', 'geo', 'ger', 'gez', 'gil', 'gla', 'gle', 'glg', 'glv', 'gmh', 'goh', 'gon',
- 'gor', 'got', 'grb', 'grc', 'grn', 'guj', 'gwi', 'hai', 'hat', 'hau', 'haw', 'heb',
- 'her', 'hil', 'him', 'hin', 'hit', 'hmn', 'hmo', 'hrv', 'hun', 'hup', 'iba', 'ibo',
- 'ice', 'ido', 'iii', 'ijo', 'iku', 'ile', 'ilo', 'ina', 'inc', 'ind', 'ine', 'inh',
- 'ipk', 'ira', 'iro', 'ita', 'jav', 'jpn', 'jpr', 'jrb', 'kaa', 'kab', 'kac', 'kal',
- 'kam', 'kan', 'kar', 'kas', 'kau', 'kaw', 'kaz', 'kbd', 'kha', 'khi', 'khm', 'kho',
- 'kik', 'kin', 'kir', 'kmb', 'kok', 'kom', 'kon', 'kor', 'kos', 'kpe', 'krc', 'kro',
- 'kru', 'kua', 'kum', 'kur', 'kut', 'lad', 'lah', 'lam', 'lao', 'lat', 'lav', 'lez',
- 'lim', 'lin', 'lit', 'lol', 'loz', 'ltz', 'lua', 'lub', 'lug', 'lui', 'lun', 'luo',
- 'lus', 'mac', 'mad', 'mag', 'mah', 'mai', 'mak', 'mal', 'man', 'mao', 'map', 'mar',
- 'mas', 'may', 'mdf', 'mdr', 'men', 'mga', 'mic', 'min', 'mkh', 'mlg', 'mlt', 'mnc',
- 'mni', 'mno', 'moh', 'mon', 'mos', 'mun', 'mus', 'mwl', 'mwr', 'myn', 'myv', 'nah',
- 'nai', 'nap', 'nau', 'nav', 'nbl', 'nde', 'ndo', 'nds', 'nep', 'new', 'nia', 'nic',
- 'niu', 'nno', 'nob', 'nog', 'non', 'nor', 'nso', 'nub', 'nwc', 'nya', 'nym', 'nyn',
- 'nyo', 'nzi', 'oci', 'oji', 'ori', 'orm', 'osa', 'oss', 'ota', 'oto', 'paa', 'pag',
- 'pal', 'pam', 'pan', 'pap', 'pau', 'peo', 'per', 'phi', 'phn', 'pli', 'pol', 'pon',
- 'por', 'pra', 'pro', 'pus', 'que', 'raj', 'rap', 'rar', 'roa', 'roh', 'rom', 'rum',
- 'run', 'rup', 'rus', 'sad', 'sag', 'sah', 'sai', 'sal', 'sam', 'san', 'sas', 'sat',
- 'scn', 'sco', 'sel', 'sem', 'sga', 'sgn', 'shn', 'sid', 'sin', 'sio', 'sit', 'sla',
- 'slo', 'slv', 'sma', 'sme', 'smi', 'smj', 'smn', 'smo', 'sms', 'sna', 'snd', 'snk',
- 'sog', 'som', 'son', 'sot', 'spa', 'srd', 'srp', 'srr', 'ssa', 'ssw', 'suk', 'sun',
- 'sus', 'sux', 'swa', 'swe', 'syr', 'tah', 'tai', 'tam', 'tat', 'tel', 'tem', 'ter',
- 'tet', 'tgk', 'tgl', 'tha', 'tib', 'tig', 'tir', 'tiv', 'tkl', 'tlh', 'tli', 'tmh',
- 'tog', 'ton', 'tpi', 'tsi', 'tsn', 'tso', 'tuk', 'tum', 'tup', 'tur', 'tut', 'tvl',
- 'twi', 'tyv', 'udm', 'uga', 'uig', 'ukr', 'umb', 'urd', 'uzb', 'vai', 'ven', 'vie',
- 'vol', 'vot', 'wak', 'wal', 'war', 'was', 'wel', 'wen', 'wln', 'wol', 'xal', 'xho',
- 'yao', 'yap', 'yid', 'yor', 'ypk', 'zap', 'zen', 'zha', 'znd', 'zul', 'zun',
- 'pob', 'rum-MD'])
- language_map = {'mol': Language('rum-MD'), 'scc': Language('srp'),
- Language('rum-MD'): 'mol', Language('srp'): 'scc'}
- language_code = 'alpha3'
- videos = [Episode, Movie]
- require_video = False
- confidence_order = ['moviehash', 'imdbid', 'fulltext']
-
- def __init__(self, config=None):
- super(OpenSubtitles, self).__init__(config)
- self.server = xmlrpclib.ServerProxy(self.server_url)
- self.token = None
-
- def init(self):
- super(OpenSubtitles, self).init()
- result = self.server.LogIn('', '', 'eng', self.user_agent)
- if result['status'] != '200 OK':
- raise ServiceError('Login failed')
- self.token = result['token']
-
- def terminate(self):
- super(OpenSubtitles, self).terminate()
- if self.token:
- try:
- self.server.LogOut(self.token)
- except Exception as e:
- raise ServiceError(str(e))
-
-
- def query(self, filepath, languages, moviehash=None, size=None, imdbid=None, query=None):
- searches = []
- if moviehash and size:
- searches.append({'moviehash': moviehash, 'moviebytesize': size})
- if imdbid:
- searches.append({'imdbid': imdbid})
- if query:
- searches.append({'query': query})
- if not searches:
- raise ServiceError('One or more parameter missing')
- for search in searches:
- search['sublanguageid'] = ','.join(self.get_code(l) for l in languages)
- logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- results = self.server.SearchSubtitles(self.token, searches)
- if not results['data']:
- logger.debug(u'Could not find subtitles for %r with token %s' % (searches, self.token)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- subtitles = []
- for result in results['data']:
- language = self.get_language(result['SubLanguageID'])
- path = get_subtitle_path(filepath, language, self.config.multi)
- confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
- subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['SubDownloadLink'],
- release=to_unicode(result['SubFileName']), confidence=confidence)
- subtitles.append(subtitle)
- return subtitles
-
- def list_checked(self, video, languages):
- results = []
- if video.exists:
- results = self.query(video.path or video.release, languages, moviehash=video.hashes['OpenSubtitles'], size=str(video.size))
- elif video.imdbid:
- results = self.query(video.path or video.release, languages, imdbid=video.imdbid)
- elif isinstance(video, Episode):
- results = self.query(video.path or video.release, languages, query=video.series)
- elif isinstance(video, Movie):
- results = self.query(video.path or video.release, languages, query=video.title)
- return results
-
- def download(self, subtitle):
- #TODO: Use OpenSubtitles DownloadSubtitles method
- try:
- self.download_file(subtitle.link, subtitle.path + '.gz')
- with open(subtitle.path, 'wb') as dump:
- gz = gzip.open(subtitle.path + '.gz')
- dump.write(gz.read())
- gz.close()
- except Exception as e:
- if os.path.exists(subtitle.path):
- os.remove(subtitle.path)
- raise DownloadFailedError(str(e))
- finally:
- if os.path.exists(subtitle.path + '.gz'):
- os.remove(subtitle.path + '.gz')
- return subtitle
-
-
-Service = OpenSubtitles
diff --git a/lib/subliminal/services/podnapisi.py b/lib/subliminal/services/podnapisi.py
deleted file mode 100644
index d80e35d21e7665a367c2bd70b3bd7c3ad29ff57b..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/podnapisi.py
+++ /dev/null
@@ -1,112 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..exceptions import ServiceError, DownloadFailedError
-from ..language import language_set, Language
-from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..utils import to_unicode
-from ..videos import Episode, Movie
-from hashlib import md5, sha256
-import logging
-import xmlrpclib
-import sys
-
-
-logger = logging.getLogger("subliminal")
-
-
-class Podnapisi(ServiceBase):
- server_url = 'http://ssp.podnapisi.net:8000'
- site_url = 'http://www.podnapisi.net'
- api_based = True
- languages = language_set(['ar', 'be', 'bg', 'bs', 'ca', 'ca', 'cs', 'da', 'de', 'el', 'en',
- 'es', 'et', 'fa', 'fi', 'fr', 'ga', 'he', 'hi', 'hr', 'hu', 'id',
- 'is', 'it', 'ja', 'ko', 'lt', 'lv', 'mk', 'ms', 'nl', 'nn', 'pl',
- 'pt', 'ro', 'ru', 'sk', 'sl', 'sq', 'sr', 'sv', 'th', 'tr', 'uk',
- 'vi', 'zh', 'es-ar', 'pb'])
- language_map = {'jp': Language('jpn'), Language('jpn'): 'jp',
- 'gr': Language('gre'), Language('gre'): 'gr',
-# 'pb': Language('por-BR'), Language('por-BR'): 'pb',
- 'ag': Language('spa-AR'), Language('spa-AR'): 'ag',
- 'cyr': Language('srp')}
- videos = [Episode, Movie]
- require_video = True
-
- def __init__(self, config=None):
- super(Podnapisi, self).__init__(config)
- self.server = xmlrpclib.ServerProxy(self.server_url)
- self.token = None
-
- def init(self):
- super(Podnapisi, self).init()
- result = self.server.initiate(self.user_agent)
- if result['status'] != 200:
- raise ServiceError('Initiate failed')
- username = 'python_subliminal'
- password = sha256(md5('XWFXQ6gE5Oe12rv4qxXX').hexdigest() + result['nonce']).hexdigest()
- self.token = result['session']
- result = self.server.authenticate(self.token, username, password)
- if result['status'] != 200:
- raise ServiceError('Authenticate failed')
-
- def terminate(self):
- super(Podnapisi, self).terminate()
-
- def query(self, filepath, languages, moviehash):
- results = self.server.search(self.token, [moviehash])
- if results['status'] != 200:
- logger.error('Search failed with error code %d' % results['status']) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- if not results['results'] or not results['results'][moviehash]['subtitles']:
- logger.debug(u'Could not find subtitles for %r with token %s' % (moviehash, self.token)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- subtitles = []
- for result in results['results'][moviehash]['subtitles']:
- language = self.get_language(result['lang'])
- if language not in languages:
- continue
- path = get_subtitle_path(filepath, language, self.config.multi)
- subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), result['id'],
- release=to_unicode(result['release']), confidence=result['weight'])
- subtitles.append(subtitle)
- if not subtitles:
- return []
- # Convert weight to confidence
- max_weight = float(max([s.confidence for s in subtitles]))
- min_weight = float(min([s.confidence for s in subtitles]))
- for subtitle in subtitles:
- if max_weight == 0 and min_weight == 0:
- subtitle.confidence = 1.0
- else:
- subtitle.confidence = (subtitle.confidence - min_weight) / (max_weight - min_weight)
- return subtitles
-
- def list_checked(self, video, languages):
- results = self.query(video.path, languages, video.hashes['OpenSubtitles'])
- return results
-
- def download(self, subtitle):
- results = self.server.download(self.token, [subtitle.link])
- if results['status'] != 200:
- raise DownloadFailedError()
- subtitle.link = 'http://www.podnapisi.net/static/podnapisi/' + results['names'][0]['filename']
- self.download_file(subtitle.link, subtitle.path)
- return subtitle
-
-
-Service = Podnapisi
diff --git a/lib/subliminal/services/podnapisiweb.py b/lib/subliminal/services/podnapisiweb.py
deleted file mode 100644
index 0be0152965c842ee7bb543ec60d85db18291c8a3..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/podnapisiweb.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..exceptions import DownloadFailedError
-from ..language import Language, language_set
-from ..subtitles import ResultSubtitle
-from ..utils import get_keywords
-from ..videos import Episode, Movie
-from bs4 import BeautifulSoup
-import guessit
-import logging
-import re
-from subliminal.subtitles import get_subtitle_path
-import sys
-
-
-logger = logging.getLogger("subliminal")
-
-
-class PodnapisiWeb(ServiceBase):
- server_url = 'http://simple.podnapisi.net'
- site_url = 'http://www.podnapisi.net'
- api_based = True
- user_agent = 'Subliminal/0.6'
- videos = [Episode, Movie]
- require_video = False
- required_features = ['xml']
- languages = language_set(['Albanian', 'Arabic', 'Spanish (Argentina)', 'Belarusian', 'Bosnian', 'Portuguese (Brazil)', 'Bulgarian', 'Catalan',
- 'Chinese', 'Croatian', 'Czech', 'Danish', 'Dutch', 'English', 'Estonian', 'Persian',
- 'Finnish', 'French', 'German', 'gre', 'Kalaallisut', 'Hebrew', 'Hindi', 'Hungarian',
- 'Icelandic', 'Indonesian', 'Irish', 'Italian', 'Japanese', 'Kazakh', 'Korean', 'Latvian',
- 'Lithuanian', 'Macedonian', 'Malay', 'Norwegian', 'Polish', 'Portuguese', 'Romanian',
- 'Russian', 'Serbian', 'Sinhala', 'Slovak', 'Slovenian', 'Spanish', 'Swedish', 'Thai',
- 'Turkish', 'Ukrainian', 'Vietnamese'])
- language_map = {Language('Albanian'): 29, Language('Arabic'): 12, Language('Spanish (Argentina)'): 14, Language('Belarusian'): 50,
- Language('Bosnian'): 10, Language('Portuguese (Brazil)'): 48, Language('Bulgarian'): 33, Language('Catalan'): 53,
- Language('Chinese'): 17, Language('Croatian'): 38, Language('Czech'): 7, Language('Danish'): 24,
- Language('Dutch'): 23, Language('English'): 2, Language('Estonian'): 20, Language('Persian'): 52,
- Language('Finnish'): 31, Language('French'): 8, Language('German'): 5, Language('gre'): 16,
- Language('Kalaallisut'): 57, Language('Hebrew'): 22, Language('Hindi'): 42, Language('Hungarian'): 15,
- Language('Icelandic'): 6, Language('Indonesian'): 54, Language('Irish'): 49, Language('Italian'): 9,
- Language('Japanese'): 11, Language('Kazakh'): 58, Language('Korean'): 4, Language('Latvian'): 21,
- Language('Lithuanian'): 19, Language('Macedonian'): 35, Language('Malay'): 55,
- Language('Norwegian'): 3, Language('Polish'): 26, Language('Portuguese'): 32, Language('Romanian'): 13,
- Language('Russian'): 27, Language('Serbian'): 36, Language('Sinhala'): 56, Language('Slovak'): 37,
- Language('Slovenian'): 1, Language('Spanish'): 28, Language('Swedish'): 25, Language('Thai'): 44,
- Language('Turkish'): 30, Language('Ukrainian'): 46, Language('Vietnamese'): 51,
- 29: Language('Albanian'), 12: Language('Arabic'), 14: Language('Spanish (Argentina)'), 50: Language('Belarusian'),
- 10: Language('Bosnian'), 48: Language('Portuguese (Brazil)'), 33: Language('Bulgarian'), 53: Language('Catalan'),
- 17: Language('Chinese'), 38: Language('Croatian'), 7: Language('Czech'), 24: Language('Danish'),
- 23: Language('Dutch'), 2: Language('English'), 20: Language('Estonian'), 52: Language('Persian'),
- 31: Language('Finnish'), 8: Language('French'), 5: Language('German'), 16: Language('gre'),
- 57: Language('Kalaallisut'), 22: Language('Hebrew'), 42: Language('Hindi'), 15: Language('Hungarian'),
- 6: Language('Icelandic'), 54: Language('Indonesian'), 49: Language('Irish'), 9: Language('Italian'),
- 11: Language('Japanese'), 58: Language('Kazakh'), 4: Language('Korean'), 21: Language('Latvian'),
- 19: Language('Lithuanian'), 35: Language('Macedonian'), 55: Language('Malay'), 40: Language('Chinese'),
- 3: Language('Norwegian'), 26: Language('Polish'), 32: Language('Portuguese'), 13: Language('Romanian'),
- 27: Language('Russian'), 36: Language('Serbian'), 47: Language('Serbian'), 56: Language('Sinhala'),
- 37: Language('Slovak'), 1: Language('Slovenian'), 28: Language('Spanish'), 25: Language('Swedish'),
- 44: Language('Thai'), 30: Language('Turkish'), 46: Language('Ukrainian'), Language('Vietnamese'): 51}
-
- def list_checked(self, video, languages):
- if isinstance(video, Movie):
- return self.query(video.path or video.release, languages, video.title, year=video.year,
- keywords=get_keywords(video.guess))
- if isinstance(video, Episode):
- return self.query(video.path or video.release, languages, video.series, season=video.season,
- episode=video.episode, keywords=get_keywords(video.guess))
-
- def query(self, filepath, languages, title, season=None, episode=None, year=None, keywords=None):
- params = {'sXML': 1, 'sK': title, 'sJ': ','.join([str(self.get_code(l)) for l in languages])}
- if season is not None:
- params['sTS'] = season
- if episode is not None:
- params['sTE'] = episode
- if year is not None:
- params['sY'] = year
- if keywords is not None:
- params['sR'] = keywords
- r = self.session.get(self.server_url + '/ppodnapisi/search', params=params)
- if r.status_code != 200:
- logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- subtitles = []
- soup = BeautifulSoup(r.content, self.required_features)
- for sub in soup('subtitle'):
- if 'n' in sub.flags:
- logger.debug(u'Skipping hearing impaired') if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- language = self.get_language(sub.languageId.text)
- confidence = float(sub.rating.text) / 5.0
- sub_keywords = set()
- for release in sub.release.text.split():
- sub_keywords |= get_keywords(guessit.guess_file_info(release + '.srt', 'autodetect'))
- sub_path = get_subtitle_path(filepath, language, self.config.multi)
- subtitle = ResultSubtitle(sub_path, language, self.__class__.__name__.lower(),
- sub.url.text, confidence=confidence, keywords=sub_keywords)
- subtitles.append(subtitle)
- return subtitles
-
- def download(self, subtitle):
- r = self.session.get(subtitle.link)
- if r.status_code != 200:
- raise DownloadFailedError()
- soup = BeautifulSoup(r.content)
- self.download_zip_file(self.server_url + soup.find('a', href=re.compile('download'))['href'], subtitle.path)
- return subtitle
-
-
-Service = PodnapisiWeb
diff --git a/lib/subliminal/services/subscenter.py b/lib/subliminal/services/subscenter.py
deleted file mode 100644
index e20cccaa3302b1145250271110405a7f480b18f5..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/subscenter.py
+++ /dev/null
@@ -1,102 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2012 Ofir Brukner <ofirbrukner@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-import logging
-import re
-import json
-from . import ServiceBase
-from ..exceptions import ServiceError
-from ..language import language_set
-from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..videos import Episode, Movie
-from ..utils import to_unicode, get_keywords
-import sys
-
-
-logger = logging.getLogger("subliminal")
-
-
-class Subscenter(ServiceBase):
- server_url = 'http://subscenter.cinemast.com/he/'
- site_url = 'http://subscenter.cinemast.com/'
- api_based = False
- languages = language_set(['he', 'en'])
- videos = [Episode, Movie]
- require_video = False
- required_features = ['permissive']
-
- @staticmethod
- def slugify(string):
- new_string = string.replace(' ', '-').replace("'", '').replace(':', '').lower()
- # We remove multiple spaces by using this regular expression.
- return re.sub('-+', '-', new_string)
-
- def list_checked(self, video, languages):
- series = None
- season = None
- episode = None
- title = video.title
- if isinstance(video, Episode):
- series = video.series
- season = video.season
- episode = video.episode
- return self.query(video.path or video.release, languages, get_keywords(video.guess), series, season,
- episode, title)
-
- def query(self, filepath, languages=None, keywords=None, series=None, season=None, episode=None, title=None):
- logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- # Converts the title to Subscenter format by replacing whitespaces and removing specific chars.
- if series and season and episode:
- # Search for a TV show.
- kind = 'episode'
- slugified_series = self.slugify(series)
- url = self.server_url + 'cinemast/data/series/sb/' + slugified_series + '/' + str(season) + '/' + \
- str(episode) + '/'
- elif title:
- # Search for a movie.
- kind = 'movie'
- slugified_title = self.slugify(title)
- url = self.server_url + 'cinemast/data/movie/sb/' + slugified_title + '/'
- else:
- raise ServiceError('One or more parameters are missing')
- logger.debug('Searching subtitles %r', {'title': title, 'season': season, 'episode': episode}) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- response = self.session.get(url)
- if response.status_code != 200:
- raise ServiceError('Request failed with status code %d' % response.status_code)
-
- subtitles = []
- response_json = json.loads(response.content)
- for lang, lang_json in response_json.items():
- lang_obj = self.get_language(lang)
- if lang_obj in self.languages and lang_obj in languages:
- for group_data in lang_json.values():
- for quality in group_data.values():
- for sub in quality.values():
- release = sub.get('subtitle_version')
- sub_path = get_subtitle_path(filepath, lang_obj, self.config.multi)
- link = self.server_url + 'subtitle/download/' + lang + '/' + str(sub.get('id')) + \
- '/?v=' + release + '&key=' + str(sub.get('key'))
- subtitles.append(ResultSubtitle(sub_path, lang_obj, self.__class__.__name__.lower(),
- link, release=to_unicode(release)))
- return subtitles
-
- def download(self, subtitle):
- self.download_zip_file(subtitle.link, subtitle.path)
- return subtitle
-
-
-Service = Subscenter
\ No newline at end of file
diff --git a/lib/subliminal/services/subswiki.py b/lib/subliminal/services/subswiki.py
deleted file mode 100644
index 4e8039fe4c4c89ad8404bb4ac73472709f2e1fe5..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/subswiki.py
+++ /dev/null
@@ -1,101 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..exceptions import ServiceError
-from ..language import language_set, Language
-from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..utils import get_keywords, split_keyword
-from ..videos import Episode, Movie
-from bs4 import BeautifulSoup
-import logging
-import urllib
-import sys
-
-
-logger = logging.getLogger("subliminal")
-
-
-class SubsWiki(ServiceBase):
- server_url = 'http://www.subswiki.com'
- site_url = 'http://www.subswiki.com'
- api_based = False
- languages = language_set(['eng-US', 'eng-GB', 'eng', 'fre', 'pob', 'por', 'spa-ES', u'spa', u'ita', u'cat'])
- language_map = {u'Español': Language('spa'), u'Español (España)': Language('spa'), u'Español (Latinoamérica)': Language('spa'),
- u'Català': Language('cat'), u'Brazilian': Language('pob'), u'English (US)': Language('eng-US'),
- u'English (UK)': Language('eng-GB')}
- language_code = 'name'
- videos = [Episode, Movie]
- require_video = False
- required_features = ['permissive']
-
- def list_checked(self, video, languages):
- results = []
- if isinstance(video, Episode):
- results = self.query(video.path or video.release, languages, get_keywords(video.guess), series=video.series, season=video.season, episode=video.episode)
- elif isinstance(video, Movie) and video.year:
- results = self.query(video.path or video.release, languages, get_keywords(video.guess), movie=video.title, year=video.year)
- return results
-
- def query(self, filepath, languages, keywords=None, series=None, season=None, episode=None, movie=None, year=None):
- if series and season and episode:
- request_series = series.lower().replace(' ', '_')
- if isinstance(request_series, unicode):
- request_series = request_series.encode('utf-8')
- logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- r = self.session.get('%s/serie/%s/%s/%s/' % (self.server_url, urllib.quote(request_series), season, episode))
- if r.status_code == 404:
- logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- elif movie and year:
- request_movie = movie.title().replace(' ', '_')
- if isinstance(request_movie, unicode):
- request_movie = request_movie.encode('utf-8')
- logger.debug(u'Getting subtitles for %s (%d) with languages %r' % (movie, year, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- r = self.session.get('%s/film/%s_(%d)' % (self.server_url, urllib.quote(request_movie), year))
- if r.status_code == 404:
- logger.debug(u'Could not find subtitles for %s (%d) with languages %r' % (movie, year, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- else:
- raise ServiceError('One or more parameter missing')
- if r.status_code != 200:
- logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- soup = BeautifulSoup(r.content, self.required_features)
- subtitles = []
- for sub in soup('td', {'class': 'NewsTitle'}):
- sub_keywords = split_keyword(sub.b.string.lower())
- if keywords and not keywords & sub_keywords:
- logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- for html_language in sub.parent.parent.find_all('td', {'class': 'language'}):
- language = self.get_language(html_language.string.strip())
- if language not in languages:
- logger.debug(u'Language %r not in wanted languages %r' % (language, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- html_status = html_language.find_next_sibling('td')
- status = html_status.strong.string.strip()
- if status != 'Completado':
- logger.debug(u'Wrong subtitle status %s' % status) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- path = get_subtitle_path(filepath, language, self.config.multi)
- subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s%s' % (self.server_url, html_status.find_next('td').find('a')['href']))
- subtitles.append(subtitle)
- return subtitles
-
-
-Service = SubsWiki
diff --git a/lib/subliminal/services/subtitulos.py b/lib/subliminal/services/subtitulos.py
deleted file mode 100644
index 9beba36fb0af8cb6a287ba656d8306d22d53cc08..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/subtitulos.py
+++ /dev/null
@@ -1,91 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..language import language_set, Language
-from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..utils import get_keywords, split_keyword
-from ..videos import Episode
-from bs4 import BeautifulSoup
-import logging
-import re
-import unicodedata
-import urllib
-import sys
-
-
-logger = logging.getLogger("subliminal")
-
-
-class Subtitulos(ServiceBase):
- server_url = 'http://www.subtitulos.es'
- site_url = 'http://www.subtitulos.es'
- api_based = False
- languages = language_set(['eng-US', 'eng-GB', 'eng', 'fre', 'pob', 'por', 'spa-ES', u'spa', u'ita', u'cat'])
- language_map = {u'Español': Language('spa'), u'Español (España)': Language('spa'), #u'Español (Latinoamérica)': Language('spa'),
- u'Català': Language('cat'), u'Brazilian': Language('pob'), u'English (US)': Language('eng-US'),
- u'English (UK)': Language('eng-GB'), 'Galego': Language('glg')}
- language_code = 'name'
- videos = [Episode]
- require_video = False
- required_features = ['permissive']
- # the '.+' in the pattern for Version allows us to match both 'ó'
- # and the 'ó' char directly. This is because now BS4 converts the html
- # code chars into their equivalent unicode char
- release_pattern = re.compile('Versi.+n (.+) ([0-9]+).([0-9])+ megabytes')
- extra_keywords_pattern = re.compile("(?:con|para)\s(?:720p)?(?:\-|\s)?([A-Za-z]+)(?:\-|\s)?(?:720p)?(?:\s|\.)(?:y\s)?(?:720p)?(?:\-\s)?([A-Za-z]+)?(?:\-\s)?(?:720p)?(?:\.)?");
-
- def list_checked(self, video, languages):
- return self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
-
- def query(self, filepath, languages, keywords, series, season, episode):
- request_series = series.lower().replace(' ', '-').replace('&', '@').replace('(','').replace(')','')
- if isinstance(request_series, unicode):
- request_series = unicodedata.normalize('NFKD', request_series).encode('ascii', 'ignore')
- logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- r = self.session.get('%s/%s/%sx%.2d' % (self.server_url, urllib.quote(request_series), season, episode))
- if r.status_code == 404:
- logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- if r.status_code != 200:
- logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- soup = BeautifulSoup(r.content, self.required_features)
- subtitles = []
- for sub in soup('div', {'id': 'version'}):
- sub_keywords = split_keyword(self.release_pattern.search(sub.find('p', {'class': 'title-sub'}).contents[1]).group(1).lower())
- if keywords and not keywords & sub_keywords:
- logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- for html_language in sub.findAllNext('ul', {'class': 'sslist'}):
- language = self.get_language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
- if language not in languages:
- logger.debug(u'Language %r not in wanted languages %r' % (language, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- html_status = html_language.findNext('li', {'class': 'li-estado green'})
- status = html_status.contents[0].string.strip()
- if status != 'Completado':
- logger.debug(u'Wrong subtitle status %s' % status) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- continue
- path = get_subtitle_path(filepath, language, self.config.multi)
- subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'],
- keywords=sub_keywords)
- subtitles.append(subtitle)
- return subtitles
-
-
-Service = Subtitulos
diff --git a/lib/subliminal/services/thesubdb.py b/lib/subliminal/services/thesubdb.py
deleted file mode 100644
index 754fc3f774ff136741a653de7c86b2ce4607c513..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/thesubdb.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..language import language_set, Language
-from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..videos import Episode, Movie, UnknownVideo
-import logging
-import sys
-
-
-logger = logging.getLogger("subliminal")
-
-
-class TheSubDB(ServiceBase):
- server_url = 'http://api.thesubdb.com'
- site_url = 'http://www.thesubdb.com/'
- user_agent = 'SubDB/1.0 (subliminal/0.6; https://github.com/Diaoul/subliminal)'
- api_based = True
- # Source: http://api.thesubdb.com/?action=languages
- languages = language_set(['af', 'cs', 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'id', 'it',
- 'la', 'nl', 'no', 'oc', 'pl', 'pb', 'ro', 'ru', 'sl', 'sr', 'sv',
- 'tr'])
- videos = [Movie, Episode, UnknownVideo]
- require_video = True
-
- def list_checked(self, video, languages):
- return self.query(video.path, video.hashes['TheSubDB'], languages)
-
- def query(self, filepath, moviehash, languages):
- r = self.session.get(self.server_url, params={'action': 'search', 'hash': moviehash})
- if r.status_code == 404:
- logger.debug(u'Could not find subtitles for hash %s' % moviehash) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- if r.status_code != 200:
- logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- available_languages = language_set(r.content.split(','))
- #this is needed becase for theSubDB pt languages is Portoguese Brazil and not Portoguese#
- #So we are deleting pt language and adding pb language
- if Language('pt') in available_languages:
- available_languages = available_languages - language_set(['pt']) | language_set(['pb'])
- languages &= available_languages
- if not languages:
- logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- subtitles = []
- for language in languages:
- path = get_subtitle_path(filepath, language, self.config.multi)
- subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, language.alpha2))
- subtitles.append(subtitle)
- return subtitles
-
-
-Service = TheSubDB
diff --git a/lib/subliminal/services/tvsubtitles.py b/lib/subliminal/services/tvsubtitles.py
deleted file mode 100644
index 0f0a399dd37629d8bb99fea4e1fd45a347105dbd..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/tvsubtitles.py
+++ /dev/null
@@ -1,146 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2012 Nicolas Wack <wackou@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..cache import cachedmethod
-from ..language import language_set, Language
-from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..utils import get_keywords
-from ..videos import Episode
-from bs4 import BeautifulSoup
-import logging
-import re
-import sys
-
-
-logger = logging.getLogger("subliminal")
-
-
-def match(pattern, string):
- try:
- return re.search(pattern, string).group(1)
- except AttributeError:
- logger.debug(u'Could not match %r on %r' % (pattern, string)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return None
-
-
-class TvSubtitles(ServiceBase):
- server_url = 'http://www.tvsubtitles.net'
- site_url = 'http://www.tvsubtitles.net'
- api_based = False
- languages = language_set(['ar', 'bg', 'cs', 'da', 'de', 'el', 'en', 'es', 'fi', 'fr', 'hu',
- 'it', 'ja', 'ko', 'nl', 'pl', 'pt', 'ro', 'ru', 'sv', 'tr', 'uk',
- 'zh', 'pb'])
- #TODO: Find more exceptions
- language_map = {'gr': Language('gre'), 'cz': Language('cze'), 'ua': Language('ukr'),
- 'cn': Language('chi'), 'br': Language('pob')}
- videos = [Episode]
- require_video = False
- required_features = ['permissive']
-
- @cachedmethod
- def get_likely_series_id(self, name):
- r = self.session.post('%s/search.php' % self.server_url, data={'q': name})
- soup = BeautifulSoup(r.content, self.required_features)
- maindiv = soup.find('div', 'left')
- results = []
- for elem in maindiv.find_all('li'):
- sid = int(match('tvshow-([0-9]+)\.html', elem.a['href']))
- show_name = match('(.*) \(', elem.a.text)
- results.append((show_name, sid))
-
- if len(results):
- #TODO: pick up the best one in a smart way
- result = results[0]
- return result[1]
-
- @cachedmethod
- def get_episode_id(self, series_id, season, number):
- """Get the TvSubtitles id for the given episode. Raises KeyError if none
- could be found."""
- # download the page of the season, contains ids for all episodes
- episode_id = None
- r = self.session.get('%s/tvshow-%d-%d.html' % (self.server_url, series_id, season))
- soup = BeautifulSoup(r.content, self.required_features)
- table = soup.find('table', id='table5')
- for row in table.find_all('tr'):
- cells = row.find_all('td')
- if not cells:
- continue
- episode_number = match('x([0-9]+)', cells[0].text)
- if not episode_number:
- continue
- episode_number = int(episode_number)
- episode_id = int(match('episode-([0-9]+)', cells[1].a['href']))
- # we could just return the id of the queried episode, but as we
- # already downloaded the whole page we might as well fill in the
- # information for all the episodes of the season
- self.cache_for(self.get_episode_id, args=(series_id, season, episode_number), result=episode_id)
- # raises KeyError if not found
- return self.cached_value(self.get_episode_id, args=(series_id, season, number))
-
- # Do not cache this method in order to always check for the most recent
- # subtitles
- def get_sub_ids(self, episode_id):
- subids = []
- r = self.session.get('%s/episode-%d.html' % (self.server_url, episode_id))
- epsoup = BeautifulSoup(r.content, self.required_features)
- for subdiv in epsoup.find_all('a'):
- if 'href' not in subdiv.attrs or not subdiv['href'].startswith('/subtitle'):
- continue
- subid = int(match('([0-9]+)', subdiv['href']))
- lang = self.get_language(match('flags/(.*).gif', subdiv.img['src']))
- result = {'subid': subid, 'language': lang}
- for p in subdiv.find_all('p'):
- if 'alt' in p.attrs and p['alt'] == 'rip':
- result['rip'] = p.text.strip()
- if 'alt' in p.attrs and p['alt'] == 'release':
- result['release'] = p.text.strip()
- subids.append(result)
- return subids
-
- def list_checked(self, video, languages):
- return self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
-
- def query(self, filepath, languages, keywords, series, season, episode):
- logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- self.init_cache()
- sid = self.get_likely_series_id(series.lower())
- try:
- ep_id = self.get_episode_id(sid, season, episode)
- except KeyError:
- logger.debug(u'Could not find episode id for %s season %d episode %d' % (series, season, episode)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- subids = self.get_sub_ids(ep_id)
- # filter the subtitles with our queried languages
- subtitles = []
- for subid in subids:
- language = subid['language']
- if language not in languages:
- continue
- path = get_subtitle_path(filepath, language, self.config.multi)
- subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s/download-%d.html' % (self.server_url, subid['subid']),
- keywords=[subid['rip'], subid['release']])
- subtitles.append(subtitle)
- return subtitles
-
- def download(self, subtitle):
- self.download_zip_file(subtitle.link, subtitle.path)
- return subtitle
-
-
-Service = TvSubtitles
\ No newline at end of file
diff --git a/lib/subliminal/services/usub.py b/lib/subliminal/services/usub.py
deleted file mode 100644
index 593625f7aa38143f7d40881009bc932d32279121..0000000000000000000000000000000000000000
--- a/lib/subliminal/services/usub.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2013 Julien Goret <jgoret@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import ServiceBase
-from ..exceptions import ServiceError
-from ..language import language_set, Language
-from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..utils import get_keywords, split_keyword
-from ..videos import Episode
-from bs4 import BeautifulSoup
-import logging
-import urllib
-import sys
-
-logger = logging.getLogger("subliminal")
-
-class Usub(ServiceBase):
- server_url = 'http://www.u-sub.net/sous-titres'
- site_url = 'http://www.u-sub.net/'
- api_based = False
- languages = language_set(['fr'])
- videos = [Episode]
- require_video = False
- #required_features = ['permissive']
-
- def list_checked(self, video, languages):
- return self.query(video.path or video.release, languages, get_keywords(video.guess), series=video.series, season=video.season, episode=video.episode)
-
- def query(self, filepath, languages, keywords=None, series=None, season=None, episode=None):
-
- ## Check if we really got informations about our episode
- if series and season and episode:
- request_series = series.lower().replace(' ', '-')
- if isinstance(request_series, unicode):
- request_series = request_series.encode('utf-8')
- logger.debug(u'Getting subtitles for %s season %d episode %d with language %r' % (series, season, episode, languages)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- r = self.session.get('%s/%s/saison_%s' % (self.server_url, urllib.quote(request_series),season))
- if r.status_code == 404:
- print "Error 404"
- logger.debug(u'Could not find subtitles for %s' % (series)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
- else:
- print "One or more parameter missing"
- raise ServiceError('One or more parameter missing')
-
- ## Check if we didn't got an big and nasty http error
- if r.status_code != 200:
- print u'Request %s returned status code %d' % (r.url, r.status_code)
- logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return []
-
- ## Editing episode informations to be able to use it with our search
- if episode < 10 :
- episode_num='0'+str(episode)
- else :
- episode_num=str(episode)
- season_num = str(season)
- series_name = series.lower().replace(' ', '.')
- possible_episode_naming = [season_num+'x'+episode_num,season_num+episode_num]
-
-
- ## Actually parsing the page for the good subtitles
- soup = BeautifulSoup(r.content, self.required_features)
- subtitles = []
- subtitles_list = soup.find('table', {'id' : 'subtitles_list'})
- link_list = subtitles_list.findAll('a', {'class' : 'dl_link'})
-
- for link in link_list :
- link_url = link.get('href')
- splited_link = link_url.split('/')
- filename = splited_link[len(splited_link)-1]
- for episode_naming in possible_episode_naming :
- if episode_naming in filename :
- for language in languages:
- path = get_subtitle_path(filepath, language, self.config.multi)
- subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s' % (link_url))
- subtitles.append(subtitle)
- return subtitles
-
- def download(self, subtitle):
- ## All downloaded files are zip files
- self.download_zip_file(subtitle.link, subtitle.path)
- return subtitle
-
-
-Service = Usub
diff --git a/lib/subliminal/subtitle.py b/lib/subliminal/subtitle.py
new file mode 100644
index 0000000000000000000000000000000000000000..3522b8e9e9946712661759345b51a10b49b1c624
--- /dev/null
+++ b/lib/subliminal/subtitle.py
@@ -0,0 +1,285 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import logging
+import os.path
+import babelfish
+import chardet
+import guessit.matchtree
+import guessit.transfo
+import pysrt
+from .video import Episode, Movie
+
+
+logger = logging.getLogger(__name__)
+
+
+class Subtitle(object):
+ """Base class for subtitle
+
+ :param language: language of the subtitle
+ :type language: :class:`babelfish.Language`
+ :param bool hearing_impaired: `True` if the subtitle is hearing impaired, `False` otherwise
+ :param page_link: link to the web page from which the subtitle can be downloaded, if any
+ :type page_link: string or None
+
+ """
+ def __init__(self, language, hearing_impaired=False, page_link=None):
+ self.language = language
+ self.hearing_impaired = hearing_impaired
+ self.page_link = page_link
+
+ #: Content as bytes
+ self.content = None
+
+ #: Encoding to decode with when accessing :attr:`text`
+ self.encoding = None
+
+ @property
+ def guessed_encoding(self):
+ """Guessed encoding using the language, falling back on chardet"""
+ # always try utf-8 first
+ encodings = ['utf-8']
+
+ # add language-specific encodings
+ if self.language.alpha3 == 'zho':
+ encodings.extend(['gb18030', 'big5'])
+ elif self.language.alpha3 == 'jpn':
+ encodings.append('shift-jis')
+ elif self.language.alpha3 == 'ara':
+ encodings.append('windows-1256')
+ elif self.language.alpha3 == 'heb':
+ encodings.append('windows-1255')
+ elif self.language.alpha3 == 'tur':
+ encodings.extend(['iso-8859-9', 'windows-1254'])
+ elif self.language.alpha3 == 'pol':
+ # Eastern European Group 1
+ encodings.extend(['windows-1250'])
+ elif self.language.alpha3 == 'bul':
+ # Eastern European Group 2
+ encodings.extend(['windows-1251'])
+ else:
+ # Western European (windows-1252)
+ encodings.append('latin-1')
+
+ # try to decode
+ for encoding in encodings:
+ try:
+ self.content.decode(encoding)
+ return encoding
+ except UnicodeDecodeError:
+ pass
+
+ # fallback on chardet
+ logger.warning('Could not decode content with encodings %r', encodings)
+ return chardet.detect(self.content)['encoding']
+
+ @property
+ def text(self):
+ """Content as string
+
+ If :attr:`encoding` is None, the encoding is guessed with :attr:`guessed_encoding`
+
+ """
+ if not self.content:
+ return ''
+ return self.content.decode(self.encoding or self.guessed_encoding, errors='replace')
+
+ @property
+ def is_valid(self):
+ """Check if a subtitle text is a valid SubRip format"""
+ try:
+ pysrt.from_string(self.text, error_handling=pysrt.ERROR_RAISE)
+ return True
+ except pysrt.Error as e:
+ if e.args[0] > 80:
+ return True
+ except:
+ logger.exception('Unexpected error when validating subtitle')
+ return False
+
+ def compute_matches(self, video):
+ """Compute the matches of the subtitle against the `video`
+
+ :param video: the video to compute the matches against
+ :type video: :class:`~subliminal.video.Video`
+ :return: matches of the subtitle
+ :rtype: set
+
+ """
+ raise NotImplementedError
+
+ def compute_score(self, video):
+ """Compute the score of the subtitle against the `video`
+
+ There are equivalent matches so that a provider can match one element or its equivalent. This is
+ to give all provider a chance to have a score in the same range without hurting quality.
+
+ * Matching :class:`~subliminal.video.Video`'s `hashes` is equivalent to matching everything else
+ * Matching :class:`~subliminal.video.Episode`'s `season` and `episode`
+ is equivalent to matching :class:`~subliminal.video.Episode`'s `title`
+ * Matching :class:`~subliminal.video.Episode`'s `tvdb_id` is equivalent to matching
+ :class:`~subliminal.video.Episode`'s `series`
+
+ :param video: the video to compute the score against
+ :type video: :class:`~subliminal.video.Video`
+ :return: score of the subtitle
+ :rtype: int
+
+ """
+ score = 0
+ # compute matches
+ initial_matches = self.compute_matches(video)
+ matches = initial_matches.copy()
+ # hash is the perfect match
+ if 'hash' in matches:
+ score = video.scores['hash']
+ else:
+ # remove equivalences
+ if isinstance(video, Episode):
+ if 'imdb_id' in matches:
+ matches -= set(('series', 'tvdb_id', 'season', 'episode', 'title', 'year'))
+ if 'tvdb_id' in matches:
+ matches -= set(('series', 'year'))
+ if 'title' in matches:
+ matches -= set(('season', 'episode'))
+ # add other scores
+ score += sum((video.scores[match] for match in matches))
+ logger.info('Computed score %d with matches %r', score, initial_matches)
+ return score
+
+ def __repr__(self):
+ return '<%s [%s]>' % (self.__class__.__name__, self.language)
+
+
+def get_subtitle_path(video_path, language=None):
+ """Create the subtitle path from the given `video_path` and `language`
+
+ :param string video_path: path to the video
+ :param language: language of the subtitle to put in the path
+ :type language: :class:`babelfish.Language` or None
+ :return: path of the subtitle
+ :rtype: string
+
+ """
+ subtitle_path = os.path.splitext(video_path)[0]
+ if language is not None:
+ try:
+ return subtitle_path + '.%s.%s' % (language.alpha2, 'srt')
+ except babelfish.LanguageConvertError:
+ return subtitle_path + '.%s.%s' % (language.alpha3, 'srt')
+ return subtitle_path + '.srt'
+
+
+def compute_guess_matches(video, guess):
+ """Compute matches between a `video` and a `guess`
+
+ :param video: the video to compute the matches on
+ :type video: :class:`~subliminal.video.Video`
+ :param guess: the guess to compute the matches on
+ :type guess: :class:`guessit.Guess`
+ :return: matches of the `guess`
+ :rtype: set
+
+ """
+ matches = set()
+ if isinstance(video, Episode):
+ # series
+ if video.series and 'series' in guess and guess['series'].lower() == video.series.lower():
+ matches.add('series')
+ # season
+ if video.season and 'seasonNumber' in guess and guess['seasonNumber'] == video.season:
+ matches.add('season')
+ # episode
+ if video.episode and 'episodeNumber' in guess and guess['episodeNumber'] == video.episode:
+ matches.add('episode')
+ # year
+ if video.year == guess.get('year'): # count "no year" as an information
+ matches.add('year')
+ elif isinstance(video, Movie):
+ # year
+ if video.year and 'year' in guess and guess['year'] == video.year:
+ matches.add('year')
+ # title
+ if video.title and 'title' in guess and guess['title'].lower() == video.title.lower():
+ matches.add('title')
+ # release group
+ if video.release_group and 'releaseGroup' in guess and guess['releaseGroup'].lower() == video.release_group.lower():
+ matches.add('release_group')
+ # screen size
+ if video.resolution and 'screenSize' in guess and guess['screenSize'] == video.resolution:
+ matches.add('resolution')
+ # format
+ if video.format and 'format' in guess and guess['format'].lower() == video.format.lower():
+ matches.add('format')
+ # video codec
+ if video.video_codec and 'videoCodec' in guess and guess['videoCodec'] == video.video_codec:
+ matches.add('video_codec')
+ # audio codec
+ if video.audio_codec and 'audioCodec' in guess and guess['audioCodec'] == video.audio_codec:
+ matches.add('audio_codec')
+ return matches
+
+
+def compute_guess_properties_matches(video, string, propertytype):
+ """Compute matches between a `video` and properties of a certain property type
+
+ :param video: the video to compute the matches on
+ :type video: :class:`~subliminal.video.Video`
+ :param string string: the string to check for a certain property type
+ :param string propertytype: the type of property to check (as defined in guessit)
+ :return: matches of a certain property type (but will only be 1 match because we are checking for 1 property type)
+ :rtype: set
+
+ Supported property types: result of guessit.transfo.guess_properties.GuessProperties().supported_properties()
+ [u'audioProfile',
+ u'videoCodec',
+ u'container',
+ u'format',
+ u'episodeFormat',
+ u'videoApi',
+ u'screenSize',
+ u'videoProfile',
+ u'audioChannels',
+ u'other',
+ u'audioCodec']
+
+ """
+ matches = set()
+ # We only check for the property types relevant for us
+ if propertytype == 'screenSize' and video.resolution:
+ for prop in guess_properties(string, propertytype):
+ if prop.lower() == video.resolution.lower():
+ matches.add('resolution')
+ elif propertytype == 'format' and video.format:
+ for prop in guess_properties(string, propertytype):
+ if prop.lower() == video.format.lower():
+ matches.add('format')
+ elif propertytype == 'videoCodec' and video.video_codec:
+ for prop in guess_properties(string, propertytype):
+ if prop.lower() == video.video_codec.lower():
+ matches.add('video_codec')
+ elif propertytype == 'audioCodec' and video.audio_codec:
+ for prop in guess_properties(string, propertytype):
+ if prop.lower() == video.audio_codec.lower():
+ matches.add('audio_codec')
+ return matches
+
+
+def guess_properties(string, propertytype):
+ properties = set()
+ if string:
+ tree = guessit.matchtree.MatchTree(string)
+ guessit.transfo.guess_properties.GuessProperties().process(tree)
+ properties = set(n.guess[propertytype] for n in tree.nodes() if propertytype in n.guess)
+ return properties
+
+
+def fix_line_endings(content):
+ """Fix line ending of `content` by changing it to \n
+
+ :param bytes content: content of the subtitle
+ :return: the content with fixed line endings
+ :rtype: bytes
+
+ """
+ return content.replace(b'\r\n', b'\n').replace(b'\r', b'\n')
diff --git a/lib/subliminal/subtitles.py b/lib/subliminal/subtitles.py
deleted file mode 100644
index 117871b38d971c250ecd8c026f1e9763de158768..0000000000000000000000000000000000000000
--- a/lib/subliminal/subtitles.py
+++ /dev/null
@@ -1,149 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from .language import Language
-from .utils import to_unicode
-import os.path
-
-
-__all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path']
-
-#: Subtitles extensions
-EXTENSIONS = ['.srt', '.sub', '.txt', '.ass']
-
-
-class Subtitle(object):
- """Base class for subtitles
-
- :param string path: path to the subtitle
- :param language: language of the subtitle
- :type language: :class:`~subliminal.language.Language`
-
- """
- def __init__(self, path, language):
- if not isinstance(language, Language):
- raise TypeError('%r is not an instance of Language')
- self.path = path
- self.language = language
-
- @property
- def exists(self):
- """Whether the subtitle exists or not"""
- if self.path:
- return os.path.exists(self.path)
- return False
-
- def __unicode__(self):
- return to_unicode(self.path)
-
- def __str__(self):
- return unicode(self).encode('utf-8')
-
- def __repr__(self):
- return '%s(%s, %s)' % (self.__class__.__name__, self, self.language)
-
-
-class EmbeddedSubtitle(Subtitle):
- """Subtitle embedded in a container
-
- :param string path: path to the subtitle
- :param language: language of the subtitle
- :type language: :class:`~subliminal.language.Language`
- :param int track_id: id of the subtitle track in the container
-
- """
- def __init__(self, path, language, track_id):
- super(EmbeddedSubtitle, self).__init__(path, language)
- self.track_id = track_id
-
- @classmethod
- def from_enzyme(cls, path, subtitle):
- language = Language(subtitle.language, strict=False)
- return cls(path, language, subtitle.trackno)
-
-
-class ExternalSubtitle(Subtitle):
- """Subtitle in a file next to the video file"""
- @classmethod
- def from_path(cls, path):
- """Create an :class:`ExternalSubtitle` from path"""
- extension = None
- for e in EXTENSIONS:
- if path.endswith(e):
- extension = e
- break
- if extension is None:
- raise ValueError('Not a supported subtitle extension')
- language = Language(os.path.splitext(path[:len(path) - len(extension)])[1][1:], strict=False)
- return cls(path, language)
-
-
-class ResultSubtitle(ExternalSubtitle):
- """Subtitle found using :mod:`~subliminal.services`
-
- :param string path: path to the subtitle
- :param language: language of the subtitle
- :type language: :class:`~subliminal.language.Language`
- :param string service: name of the service
- :param string link: download link for the subtitle
- :param string release: release name of the video
- :param float confidence: confidence that the subtitle matches the video according to the service
- :param set keywords: keywords that describe the subtitle
-
- """
- def __init__(self, path, language, service, link, release=None, confidence=1, keywords=None):
- super(ResultSubtitle, self).__init__(path, language)
- self.service = service
- self.link = link
- self.release = release
- self.confidence = confidence
- self.keywords = keywords or set()
-
- @property
- def single(self):
- """Whether this is a single subtitle or not. A single subtitle does not have
- a language indicator in its file name
-
- :rtype: bool
-
- """
- return self.language == Language('Undetermined')
-
- def __repr__(self):
- if not self.release:
- return 'ResultSubtitle(%s, %s, %s, %.2f)' % (self.path, self.language, self.service, self.confidence)
- return 'ResultSubtitle(%s, %s, %s, %.2f, release=%s)' % (self.path, self.language, self.service, self.confidence, self.release.encode('ascii', 'ignore'))
-
-
-def get_subtitle_path(video_path, language, multi):
- """Create the subtitle path from the given video path using language if multi
-
- :param string video_path: path to the video
- :param language: language of the subtitle
- :type language: :class:`~subliminal.language.Language`
- :param bool multi: whether to use multi language naming or not
- :return: path of the subtitle
- :rtype: string
-
- """
- if not os.path.exists(video_path):
- path = os.path.splitext(os.path.basename(video_path))[0]
- else:
- path = os.path.splitext(video_path)[0]
- if multi and language:
- return path + '.%s%s' % (language.alpha2, EXTENSIONS[0])
- return path + '%s' % EXTENSIONS[0]
diff --git a/lib/subliminal/tasks.py b/lib/subliminal/tasks.py
deleted file mode 100644
index bccf9ab53835710297d07ca8565fc7e209d57eb4..0000000000000000000000000000000000000000
--- a/lib/subliminal/tasks.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-__all__ = ['Task', 'ListTask', 'DownloadTask', 'StopTask']
-
-
-class Task(object):
- """Base class for tasks to use in subliminal"""
- pass
-
-
-class ListTask(Task):
- """List task used by the worker to search for subtitles
-
- :param video: video to search subtitles for
- :type video: :class:`~subliminal.videos.Video`
- :param list languages: languages to search for
- :param string service: name of the service to use
- :param config: configuration for the service
- :type config: :class:`~subliminal.services.ServiceConfig`
-
- """
- def __init__(self, video, languages, service, config):
- super(ListTask, self).__init__()
- self.video = video
- self.service = service
- self.languages = languages
- self.config = config
-
- def __repr__(self):
- return 'ListTask(%r, %r, %s, %r)' % (self.video, self.languages, self.service, self.config)
-
-
-class DownloadTask(Task):
- """Download task used by the worker to download subtitles
-
- :param video: video to download subtitles for
- :type video: :class:`~subliminal.videos.Video`
- :param subtitles: subtitles to download in order of preference
- :type subtitles: list of :class:`~subliminal.subtitles.Subtitle`
-
- """
- def __init__(self, video, subtitles):
- super(DownloadTask, self).__init__()
- self.video = video
- self.subtitles = subtitles
-
- def __repr__(self):
- return 'DownloadTask(%r, %r)' % (self.video, self.subtitles)
-
-
-class StopTask(Task):
- """Stop task that will stop the worker"""
- pass
diff --git a/lib/subliminal/tests/__init__.py b/lib/subliminal/tests/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6cef7800c72744f406499ac3769df7cc7e7210c9
--- /dev/null
+++ b/lib/subliminal/tests/__init__.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from unittest import TextTestRunner, TestSuite
+from subliminal import cache_region
+from . import test_providers, test_subliminal
+
+
+cache_region.configure('dogpile.cache.memory', expiration_time=60 * 30) # @UndefinedVariable
+suite = TestSuite([test_providers.suite(), test_subliminal.suite()])
+
+
+if __name__ == '__main__':
+ TextTestRunner().run(suite)
diff --git a/lib/subliminal/tests/common.py b/lib/subliminal/tests/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..bd1608d456ec3f2f51c092e24c2ca3cfa21a091c
--- /dev/null
+++ b/lib/subliminal/tests/common.py
@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from subliminal import Movie, Episode
+
+
+MOVIES = [Movie('Man of Steel (2013)/man.of.steel.2013.720p.bluray.x264-felony.mkv', 'Man of Steel',
+ format='BluRay', release_group='felony', resolution='720p', video_codec='h264', audio_codec='DTS',
+ imdb_id=770828, size=7033732714, year=2013,
+ hashes={'opensubtitles': '5b8f8f4e41ccb21e', 'thesubdb': 'ad32876133355929d814457537e12dc2'})]
+
+EPISODES = [Episode('The Big Bang Theory/Season 07/The.Big.Bang.Theory.S07E05.720p.HDTV.X264-DIMENSION.mkv',
+ 'The Big Bang Theory', 7, 5, format='HDTV', release_group='DIMENSION', resolution='720p',
+ video_codec='h264', audio_codec='AC3', imdb_id=3229392, size=501910737,
+ title='The Workplace Proximity', year=2007, tvdb_id=80379,
+ hashes={'opensubtitles': '6878b3ef7c1bd19e', 'thesubdb': '9dbbfb7ba81c9a6237237dae8589fccc'}),
+ Episode('Game of Thrones/Season 03/Game.of.Thrones.S03E10.Mhysa.720p.WEB-DL.DD5.1.H.264-NTb.mkv',
+ 'Game of Thrones', 3, 10, format='WEB-DL', release_group='NTb', resolution='720p',
+ video_codec='h264', audio_codec='AC3', imdb_id=2178796, size=2142810931, title='Mhysa',
+ tvdb_id=121361,
+ hashes={'opensubtitles': 'b850baa096976c22', 'thesubdb': 'b1f899c77f4c960b84b8dbf840d4e42d'}),
+ Episode('Dallas.S01E03.mkv', 'Dallas', 1, 3),
+ Episode('Dallas.2012.S01E03.mkv', 'Dallas', 1, 3, year=2012)]
diff --git a/lib/subliminal/tests/test_providers.py b/lib/subliminal/tests/test_providers.py
new file mode 100644
index 0000000000000000000000000000000000000000..e98d9ad321c1dae3bd0dbf3975f41dab01ba0eda
--- /dev/null
+++ b/lib/subliminal/tests/test_providers.py
@@ -0,0 +1,475 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import os
+from unittest import TestCase, TestSuite, TestLoader, TextTestRunner
+from babelfish import Language
+from subliminal import provider_manager
+from subliminal.tests.common import MOVIES, EPISODES
+
+
+class ProviderTestCase(TestCase):
+ provider_name = ''
+
+ def setUp(self):
+ self.Provider = provider_manager[self.provider_name]
+
+
+class Addic7edProviderTestCase(ProviderTestCase):
+ provider_name = 'addic7ed'
+
+ def test_find_show_id(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('the big bang')
+ self.assertEqual(show_id, 126)
+
+ def test_find_show_id_no_year(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('dallas')
+ self.assertEqual(show_id, 802)
+
+ def test_find_show_id_year(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('dallas', 2012)
+ self.assertEqual(show_id, 2559)
+
+ def test_find_show_id_error(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('the big how i met your mother')
+ self.assertIsNone(show_id)
+
+ def test_get_show_ids(self):
+ with self.Provider() as provider:
+ show_ids = provider.get_show_ids()
+ self.assertIn('the big bang theory', show_ids)
+ self.assertEqual(show_ids['the big bang theory'], 126)
+
+ def test_get_show_ids_no_year(self):
+ with self.Provider() as provider:
+ show_ids = provider.get_show_ids()
+ self.assertIn('dallas', show_ids)
+ self.assertEqual(show_ids['dallas'], 802)
+
+ def test_get_show_ids_year(self):
+ with self.Provider() as provider:
+ show_ids = provider.get_show_ids()
+ self.assertIn('dallas (2012)', show_ids)
+ self.assertEqual(show_ids['dallas (2012)'], 2559)
+
+ def test_query_episode_0(self):
+ video = EPISODES[0]
+ languages = {Language('tur'), Language('rus'), Language('heb'), Language('ita'), Language('fra'),
+ Language('ron'), Language('nld'), Language('eng'), Language('deu'), Language('ell'),
+ Language('por', 'BR'), Language('bul'), Language('por'), Language('msa')}
+ matches = {frozenset(['series', 'resolution', 'season']),
+ frozenset(['series', 'episode', 'season', 'title']),
+ frozenset(['series', 'release_group', 'season']),
+ frozenset(['series', 'episode', 'season', 'release_group', 'title']),
+ frozenset(['series', 'season']),
+ frozenset(['series', 'season', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(video.series, video.season, video.year)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_query_episode_1(self):
+ video = EPISODES[1]
+ languages = {Language('ind'), Language('spa'), Language('hrv'), Language('ita'), Language('fra'),
+ Language('cat'), Language('ell'), Language('nld'), Language('eng'), Language('fas'),
+ Language('por'), Language('nor'), Language('deu'), Language('ron'), Language('por', 'BR'),
+ Language('bul')}
+ matches = {frozenset(['series', 'episode', 'resolution', 'season', 'title', 'year']),
+ frozenset(['series', 'resolution', 'season', 'year']),
+ frozenset(['series', 'resolution', 'season', 'year', 'format']),
+ frozenset(['series', 'episode', 'season', 'title', 'year']),
+ frozenset(['series', 'episode', 'season', 'title', 'year', 'format']),
+ frozenset(['series', 'release_group', 'season', 'year']),
+ frozenset(['series', 'release_group', 'season', 'year', 'format']),
+ frozenset(['series', 'resolution', 'release_group', 'season', 'year']),
+ frozenset(['series', 'resolution', 'release_group', 'season', 'year', 'format']),
+ frozenset(['series', 'episode', 'season', 'release_group', 'title', 'year', 'format']),
+ frozenset(['series', 'season', 'year']),
+ frozenset(['series', 'season', 'year', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(video.series, video.season, video.year)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_query_episode_year(self):
+ video_no_year = EPISODES[2]
+ video_year = EPISODES[3]
+ with self.Provider() as provider:
+ subtitles_no_year = provider.query(video_no_year.series, video_no_year.season, video_no_year.year)
+ subtitles_year = provider.query(video_year.series, video_year.season, video_year.year)
+ self.assertNotEqual(subtitles_no_year, subtitles_year)
+
+ def test_list_subtitles(self):
+ video = EPISODES[0]
+ languages = {Language('eng'), Language('fra')}
+ matches = {frozenset(['series', 'episode', 'season', 'release_group', 'title']),
+ frozenset(['series', 'episode', 'season', 'title'])}
+ with self.Provider() as provider:
+ subtitles = provider.list_subtitles(video, languages)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_download_subtitle(self):
+ video = EPISODES[0]
+ languages = {Language('eng'), Language('fra')}
+ with self.Provider() as provider:
+ subtitles = provider.list_subtitles(video, languages)
+ provider.download_subtitle(subtitles[0])
+ self.assertIsNotNone(subtitles[0].content)
+ self.assertTrue(subtitles[0].is_valid)
+
+
+class OpenSubtitlesProviderTestCase(ProviderTestCase):
+ provider_name = 'opensubtitles'
+
+ def test_query_movie_0_query(self):
+ video = MOVIES[0]
+ languages = {Language('eng')}
+ matches = {frozenset([]),
+ frozenset(['imdb_id', 'resolution', 'title', 'year']),
+ frozenset(['imdb_id', 'resolution', 'title', 'year', 'format']),
+ frozenset(['imdb_id', 'title', 'year']),
+ frozenset(['imdb_id', 'title', 'year', 'format']),
+ frozenset(['imdb_id', 'video_codec', 'title', 'year', 'format']),
+ frozenset(['imdb_id', 'resolution', 'title', 'video_codec', 'year', 'format']),
+ frozenset(['imdb_id', 'title', 'year', 'video_codec', 'resolution', 'release_group', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(languages, query=video.title)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_query_episode_0_query(self):
+ video = EPISODES[0]
+ languages = {Language('eng')}
+ matches = {frozenset(['series', 'episode', 'season', 'imdb_id', 'format']),
+ frozenset(['series', 'imdb_id', 'video_codec', 'episode', 'season', 'format']),
+ frozenset(['episode', 'title', 'series', 'imdb_id', 'video_codec', 'season'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(languages, query=os.path.split(video.name)[1])
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_query_episode_year(self):
+ video_no_year = EPISODES[2]
+ video_year = EPISODES[3]
+ languages = {Language('eng')}
+ with self.Provider() as provider:
+ subtitles_no_year = provider.query(languages, query=os.path.split(video_no_year.name)[1])
+ subtitles_year = provider.query(languages, query=os.path.split(video_year.name)[1])
+ self.assertNotEqual(subtitles_no_year, subtitles_year)
+
+ def test_query_episode_1_query(self):
+ video = EPISODES[1]
+ languages = {Language('eng'), Language('fra')}
+ matches = {frozenset(['episode', 'title', 'series', 'imdb_id', 'video_codec', 'season', 'year', 'format']),
+ frozenset(['series', 'imdb_id', 'video_codec', 'episode', 'season', 'year']),
+ frozenset(['episode', 'video_codec', 'series', 'imdb_id', 'resolution', 'season', 'year']),
+ frozenset(['series', 'imdb_id', 'resolution', 'episode', 'season', 'year']),
+ frozenset(['series', 'episode', 'season', 'imdb_id', 'year']),
+ frozenset(['series', 'episode', 'season', 'imdb_id', 'year', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(languages, query=os.path.split(video.name)[1])
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_query_movie_0_imdb_id(self):
+ video = MOVIES[0]
+ languages = {Language('eng'), Language('fra')}
+ matches = {frozenset(['imdb_id', 'video_codec', 'title', 'year', 'format']),
+ frozenset(['imdb_id', 'resolution', 'title', 'video_codec', 'year']),
+ frozenset(['imdb_id', 'resolution', 'title', 'video_codec', 'year', 'format']),
+ frozenset(['imdb_id', 'title', 'year', 'video_codec', 'resolution', 'release_group', 'format']),
+ frozenset(['imdb_id', 'title', 'year']),
+ frozenset(['imdb_id', 'title', 'year', 'format']),
+ frozenset(['imdb_id', 'resolution', 'title', 'year']),
+ frozenset(['imdb_id', 'resolution', 'title', 'year', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(languages, imdb_id=video.imdb_id)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_query_episode_0_imdb_id(self):
+ video = EPISODES[0]
+ languages = {Language('eng'), Language('fra')}
+ matches = {frozenset(['series', 'episode', 'season', 'imdb_id', 'format']),
+ frozenset(['episode', 'release_group', 'video_codec', 'series', 'imdb_id', 'resolution', 'season', 'format']),
+ frozenset(['series', 'imdb_id', 'video_codec', 'episode', 'season', 'format']),
+ frozenset(['episode', 'title', 'series', 'imdb_id', 'video_codec', 'season'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(languages, imdb_id=video.imdb_id)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_query_movie_0_hash(self):
+ video = MOVIES[0]
+ languages = {Language('eng')}
+ matches = {frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'imdb_id', 'format']),
+ frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'release_group', 'imdb_id', 'format']),
+ frozenset(['year', 'video_codec', 'imdb_id', 'hash', 'title', 'format']),
+ frozenset([]),
+ frozenset(['year', 'resolution', 'imdb_id', 'hash', 'title', 'format']),
+ frozenset(['year', 'imdb_id', 'hash', 'title']),
+ frozenset(['year', 'imdb_id', 'hash', 'title', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(languages, hash=video.hashes['opensubtitles'], size=video.size)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_query_episode_0_hash(self):
+ video = EPISODES[0]
+ languages = {Language('eng')}
+ matches = {frozenset(['series', 'hash', 'format']),
+ frozenset(['episode', 'season', 'series', 'imdb_id', 'video_codec', 'hash', 'format']),
+ frozenset(['series', 'episode', 'season', 'hash', 'imdb_id', 'format']),
+ frozenset(['series', 'resolution', 'hash', 'video_codec', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(languages, hash=video.hashes['opensubtitles'], size=video.size)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_list_subtitles(self):
+ video = MOVIES[0]
+ languages = {Language('eng'), Language('fra')}
+ matches = {frozenset(['title', 'video_codec', 'year', 'resolution', 'release_group', 'imdb_id', 'format']),
+ frozenset(['imdb_id', 'year', 'title']),
+ frozenset(['imdb_id', 'year', 'title', 'format']),
+ frozenset(['year', 'video_codec', 'imdb_id', 'resolution', 'title']),
+ frozenset(['year', 'video_codec', 'imdb_id', 'resolution', 'title', 'format']),
+ frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'release_group', 'imdb_id', 'format']),
+ frozenset(['year', 'video_codec', 'imdb_id', 'hash', 'title', 'format']),
+ frozenset([]),
+ frozenset(['year', 'resolution', 'imdb_id', 'hash', 'title', 'format']),
+ frozenset(['hash', 'title', 'video_codec', 'year', 'resolution', 'imdb_id', 'format']),
+ frozenset(['year', 'imdb_id', 'hash', 'title']),
+ frozenset(['year', 'imdb_id', 'hash', 'title', 'format']),
+ frozenset(['video_codec', 'imdb_id', 'year', 'title', 'format']),
+ frozenset(['year', 'imdb_id', 'resolution', 'title']),
+ frozenset(['year', 'imdb_id', 'resolution', 'title', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.list_subtitles(video, languages)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_download_subtitle(self):
+ video = MOVIES[0]
+ languages = {Language('eng'), Language('fra')}
+ with self.Provider() as provider:
+ subtitles = provider.list_subtitles(video, languages)
+ provider.download_subtitle(subtitles[0])
+ self.assertIsNotNone(subtitles[0].content)
+ self.assertTrue(subtitles[0].is_valid)
+
+
+class PodnapisiProviderTestCase(ProviderTestCase):
+ provider_name = 'podnapisi'
+
+ def test_query_movie_0(self):
+ video = MOVIES[0]
+ language = Language('eng')
+ matches = {frozenset(['video_codec', 'title', 'resolution', 'year']),
+ frozenset(['title', 'resolution', 'year']),
+ frozenset(['video_codec', 'title', 'year']),
+ frozenset(['title', 'year']),
+ frozenset(['title']),
+ frozenset(['video_codec', 'title', 'resolution', 'release_group', 'year', 'format']),
+ frozenset(['video_codec', 'title', 'resolution', 'audio_codec', 'year', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(language, title=video.title, year=video.year)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, {language})
+
+ def test_query_episode_0(self):
+ video = EPISODES[0]
+ language = Language('eng')
+ matches = {frozenset(['episode', 'series', 'season', 'video_codec', 'resolution', 'release_group', 'format']),
+ frozenset(['season', 'video_codec', 'episode', 'resolution', 'series'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(language, series=video.series, season=video.season, episode=video.episode,
+ year=video.year)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, {language})
+
+ def test_query_episode_1(self):
+ video = EPISODES[1]
+ language = Language('eng')
+ matches = {frozenset(['episode', 'release_group', 'series', 'video_codec', 'resolution', 'season', 'year', 'format']),
+ frozenset(['episode', 'series', 'video_codec', 'resolution', 'season', 'year']),
+ frozenset(['season', 'video_codec', 'episode', 'series', 'year'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(language, series=video.series, season=video.season, episode=video.episode,
+ year=video.year)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, {language})
+
+ def test_list_subtitles(self):
+ video = MOVIES[0]
+ languages = {Language('eng'), Language('fra')}
+ matches = {frozenset(['video_codec', 'title', 'resolution', 'year']),
+ frozenset(['title', 'resolution', 'year']),
+ frozenset(['video_codec', 'title', 'year']),
+ frozenset(['video_codec', 'title', 'year', 'format']),
+ frozenset(['title', 'year']),
+ frozenset(['title']),
+ frozenset(['video_codec', 'title', 'resolution', 'release_group', 'year', 'format']),
+ frozenset(['video_codec', 'title', 'resolution', 'audio_codec', 'year', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.list_subtitles(video, languages)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_download_subtitle(self):
+ video = MOVIES[0]
+ languages = {Language('eng'), Language('fra')}
+ with self.Provider() as provider:
+ subtitles = provider.list_subtitles(video, languages)
+ provider.download_subtitle(subtitles[0])
+ self.assertIsNotNone(subtitles[0].content)
+ self.assertTrue(subtitles[0].is_valid)
+
+
+class TheSubDBProviderTestCase(ProviderTestCase):
+ provider_name = 'thesubdb'
+
+ def test_query_episode_0(self):
+ video = EPISODES[0]
+ languages = {Language('eng'), Language('spa'), Language('por')}
+ matches = {frozenset(['hash'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(video.hashes['thesubdb'])
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_query_episode_1(self):
+ video = EPISODES[1]
+ languages = {Language('eng'), Language('por')}
+ matches = {frozenset(['hash'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(video.hashes['thesubdb'])
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_list_subtitles(self):
+ video = MOVIES[0]
+ languages = {Language('eng'), Language('por')}
+ matches = {frozenset(['hash'])}
+ with self.Provider() as provider:
+ subtitles = provider.list_subtitles(video, languages)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_download_subtitle(self):
+ video = MOVIES[0]
+ languages = {Language('eng'), Language('por')}
+ with self.Provider() as provider:
+ subtitles = provider.list_subtitles(video, languages)
+ provider.download_subtitle(subtitles[0])
+ provider.download_subtitle(subtitles[1])
+ self.assertIsNotNone(subtitles[0].content)
+ self.assertTrue(subtitles[0].is_valid)
+
+
+class TVsubtitlesProviderTestCase(ProviderTestCase):
+ provider_name = 'tvsubtitles'
+
+ def test_find_show_id(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('the big bang')
+ self.assertEqual(show_id, 154)
+
+ def test_find_show_id_ambiguous(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('new girl')
+ self.assertEqual(show_id, 977)
+
+ def test_find_show_id_no_dots(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('marvel\'s agents of s h i e l d')
+ self.assertEqual(show_id, 1340)
+
+ def test_find_show_id_no_year_dallas(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('dallas')
+ self.assertEqual(show_id, 646)
+
+ def test_find_show_id_no_year_house_of_cards(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('house of cards')
+ self.assertEqual(show_id, 352)
+
+ def test_find_show_id_year_dallas(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('dallas', 2012)
+ self.assertEqual(show_id, 1127)
+
+ def test_find_show_id_year_house_of_cards(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('house of cards', 2013)
+ self.assertEqual(show_id, 1246)
+
+ def test_find_show_id_error(self):
+ with self.Provider() as provider:
+ show_id = provider.find_show_id('the big gaming')
+ self.assertIsNone(show_id)
+
+ def test_find_episode_ids(self):
+ with self.Provider() as provider:
+ episode_ids = provider.find_episode_ids(154, 5)
+ self.assertEqual(set(episode_ids.keys()), set(range(1, 25)))
+
+ def test_query_episode_0(self):
+ video = EPISODES[0]
+ languages = {Language('fra'), Language('por'), Language('hun'), Language('ron'), Language('eng')}
+ matches = {frozenset(['series', 'episode', 'season', 'video_codec', 'format']),
+ frozenset(['series', 'episode', 'season', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(video.series, video.season, video.episode, video.year)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_query_episode_1(self):
+ video = EPISODES[1]
+ languages = {Language('fra'), Language('ell'), Language('ron'), Language('eng'), Language('hun'),
+ Language('por'), Language('por', 'BR'), Language('jpn')}
+ matches = {frozenset(['series', 'episode', 'resolution', 'season', 'year']),
+ frozenset(['series', 'episode', 'season', 'video_codec', 'year']),
+ frozenset(['series', 'episode', 'season', 'year'])}
+ with self.Provider() as provider:
+ subtitles = provider.query(video.series, video.season, video.episode, video.year)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_list_subtitles(self):
+ video = EPISODES[0]
+ languages = {Language('eng'), Language('fra')}
+ matches = {frozenset(['series', 'episode', 'season', 'format'])}
+ with self.Provider() as provider:
+ subtitles = provider.list_subtitles(video, languages)
+ self.assertEqual({frozenset(subtitle.compute_matches(video)) for subtitle in subtitles}, matches)
+ self.assertEqual({subtitle.language for subtitle in subtitles}, languages)
+
+ def test_download_subtitle(self):
+ video = EPISODES[0]
+ languages = {Language('hun')}
+ with self.Provider() as provider:
+ subtitles = provider.list_subtitles(video, languages)
+ provider.download_subtitle(subtitles[0])
+ self.assertIsNotNone(subtitles[0].content)
+ self.assertTrue(subtitles[0].is_valid)
+
+
+def suite():
+ suite = TestSuite()
+ suite.addTest(TestLoader().loadTestsFromTestCase(Addic7edProviderTestCase))
+ suite.addTest(TestLoader().loadTestsFromTestCase(OpenSubtitlesProviderTestCase))
+ suite.addTest(TestLoader().loadTestsFromTestCase(PodnapisiProviderTestCase))
+ suite.addTest(TestLoader().loadTestsFromTestCase(TheSubDBProviderTestCase))
+ suite.addTest(TestLoader().loadTestsFromTestCase(TVsubtitlesProviderTestCase))
+ return suite
+
+
+if __name__ == '__main__':
+ TextTestRunner().run(suite())
diff --git a/lib/subliminal/tests/test_subliminal.py b/lib/subliminal/tests/test_subliminal.py
new file mode 100644
index 0000000000000000000000000000000000000000..a991d81fc50180c5fefa9614612fefa4b3e31eba
--- /dev/null
+++ b/lib/subliminal/tests/test_subliminal.py
@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import os
+import shutil
+from unittest import TestCase, TestSuite, TestLoader, TextTestRunner
+from babelfish import Language
+from subliminal import list_subtitles, download_subtitles, save_subtitles, download_best_subtitles, scan_video
+from subliminal.tests.common import MOVIES, EPISODES
+
+
+TEST_DIR = 'test_data'
+
+
+class ApiTestCase(TestCase):
+ def setUp(self):
+ os.mkdir(TEST_DIR)
+
+ def tearDown(self):
+ shutil.rmtree(TEST_DIR)
+
+ def test_list_subtitles_movie_0(self):
+ videos = [MOVIES[0]]
+ languages = {Language('eng')}
+ subtitles = list_subtitles(videos, languages)
+ self.assertEqual(len(subtitles), len(videos))
+ self.assertGreater(len(subtitles[videos[0]]), 0)
+
+ def test_list_subtitles_movie_0_por_br(self):
+ videos = [MOVIES[0]]
+ languages = {Language('por', 'BR')}
+ subtitles = list_subtitles(videos, languages)
+ self.assertEqual(len(subtitles), len(videos))
+ self.assertGreater(len(subtitles[videos[0]]), 0)
+
+ def test_list_subtitles_episodes(self):
+ videos = [EPISODES[0], EPISODES[1]]
+ languages = {Language('eng'), Language('fra')}
+ subtitles = list_subtitles(videos, languages)
+ self.assertEqual(len(subtitles), len(videos))
+ self.assertGreater(len(subtitles[videos[0]]), 0)
+
+ def test_download_subtitles(self):
+ videos = [EPISODES[0]]
+ for video in videos:
+ video.name = os.path.join(TEST_DIR, os.path.split(video.name)[1])
+ languages = {Language('eng')}
+ subtitles = list_subtitles(videos, languages)
+ download_subtitles(subtitles[videos[0]][:5])
+ self.assertGreaterEqual(len([s for s in subtitles[videos[0]] if s.content is not None]), 4)
+
+ def test_download_best_subtitles(self):
+ videos = [EPISODES[0], EPISODES[1]]
+ for video in videos:
+ video.name = os.path.join(TEST_DIR, os.path.split(video.name)[1])
+ languages = {Language('eng'), Language('fra')}
+ subtitles = download_best_subtitles(videos, languages)
+ for video in videos:
+ self.assertIn(video, subtitles)
+ self.assertEqual(len(subtitles[video]), 2)
+
+ def test_save_subtitles(self):
+ videos = [EPISODES[0], EPISODES[1]]
+ for video in videos:
+ video.name = os.path.join(TEST_DIR, os.path.split(video.name)[1])
+ languages = {Language('eng'), Language('fra')}
+ subtitles = list_subtitles(videos, languages)
+
+ # make a list of subtitles to download (one per language per video)
+ subtitles_to_download = []
+ for video, video_subtitles in subtitles.items():
+ video_subtitle_languages = set()
+ for video_subtitle in video_subtitles:
+ if video_subtitle.language in video_subtitle_languages:
+ continue
+ subtitles_to_download.append(video_subtitle)
+ video_subtitle_languages.add(video_subtitle.language)
+ if video_subtitle_languages == languages:
+ break
+ self.assertEqual(len(subtitles_to_download), 4)
+
+ # download
+ download_subtitles(subtitles_to_download)
+ save_subtitles(subtitles)
+ for video in videos:
+ self.assertTrue(os.path.exists(os.path.splitext(video.name)[0] + '.en.srt'))
+ self.assertTrue(os.path.exists(os.path.splitext(video.name)[0] + '.fr.srt'))
+
+ def test_save_subtitles_single(self):
+ videos = [EPISODES[0], EPISODES[1]]
+ for video in videos:
+ video.name = os.path.join(TEST_DIR, os.path.split(video.name)[1])
+ languages = {Language('eng'), Language('fra')}
+ subtitles = download_best_subtitles(videos, languages)
+ save_subtitles(subtitles, single=True)
+ for video in videos:
+ self.assertIn(video, subtitles)
+ self.assertEqual(len(subtitles[video]), 2)
+ self.assertTrue(os.path.exists(os.path.splitext(video.name)[0] + '.srt'))
+
+ def test_download_best_subtitles_min_score(self):
+ videos = [MOVIES[0]]
+ for video in videos:
+ video.name = os.path.join(TEST_DIR, os.path.split(video.name)[1])
+ languages = {Language('eng'), Language('fra')}
+ subtitles = download_best_subtitles(videos, languages, min_score=1000)
+ self.assertEqual(len(subtitles), 0)
+
+ def test_download_best_subtitles_hearing_impaired(self):
+ videos = [MOVIES[0]]
+ for video in videos:
+ video.name = os.path.join(TEST_DIR, os.path.split(video.name)[1])
+ languages = {Language('eng')}
+ subtitles = download_best_subtitles(videos, languages, hearing_impaired=True)
+ self.assertTrue(subtitles[videos[0]][0].hearing_impaired)
+
+
+class VideoTestCase(TestCase):
+ def setUp(self):
+ os.mkdir(TEST_DIR)
+ for video in MOVIES + EPISODES:
+ open(os.path.join(TEST_DIR, os.path.split(video.name)[1]), 'w').close()
+
+ def tearDown(self):
+ shutil.rmtree(TEST_DIR)
+
+ def test_scan_video_movie(self):
+ video = MOVIES[0]
+ scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+ self.assertEqual(scanned_video.name, os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+ self.assertEqual(scanned_video.title.lower(), video.title.lower())
+ self.assertEqual(scanned_video.year, video.year)
+ self.assertEqual(scanned_video.video_codec, video.video_codec)
+ self.assertEqual(scanned_video.format, video.format)
+ self.assertEqual(scanned_video.resolution, video.resolution)
+ self.assertEqual(scanned_video.release_group, video.release_group)
+ self.assertEqual(scanned_video.subtitle_languages, set())
+ self.assertEqual(scanned_video.hashes, {})
+ self.assertIsNone(scanned_video.audio_codec)
+ self.assertIsNone(scanned_video.imdb_id)
+ self.assertEqual(scanned_video.size, 0)
+
+ def test_scan_video_episode(self):
+ video = EPISODES[0]
+ scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+ self.assertEqual(scanned_video.name, os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+ self.assertEqual(scanned_video.series, video.series)
+ self.assertEqual(scanned_video.season, video.season)
+ self.assertEqual(scanned_video.episode, video.episode)
+ self.assertEqual(scanned_video.video_codec, video.video_codec)
+ self.assertEqual(scanned_video.format, video.format)
+ self.assertEqual(scanned_video.resolution, video.resolution)
+ self.assertEqual(scanned_video.release_group, video.release_group)
+ self.assertEqual(scanned_video.subtitle_languages, set())
+ self.assertEqual(scanned_video.hashes, {})
+ self.assertIsNone(scanned_video.title)
+ self.assertIsNone(scanned_video.tvdb_id)
+ self.assertIsNone(scanned_video.imdb_id)
+ self.assertIsNone(scanned_video.audio_codec)
+ self.assertEqual(scanned_video.size, 0)
+
+ def test_scan_video_subtitle_language_und(self):
+ video = EPISODES[0]
+ open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.srt', 'w').close()
+ scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+ self.assertEqual(scanned_video.subtitle_languages, {Language('und')})
+
+ def test_scan_video_subtitles_language_eng(self):
+ video = EPISODES[0]
+ open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.en.srt', 'w').close()
+ scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+ self.assertEqual(scanned_video.subtitle_languages, {Language('eng')})
+
+ def test_scan_video_subtitles_languages(self):
+ video = EPISODES[0]
+ open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.en.srt', 'w').close()
+ open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.fr.srt', 'w').close()
+ open(os.path.join(TEST_DIR, os.path.splitext(os.path.split(video.name)[1])[0]) + '.srt', 'w').close()
+ scanned_video = scan_video(os.path.join(TEST_DIR, os.path.split(video.name)[1]))
+ self.assertEqual(scanned_video.subtitle_languages, {Language('eng'), Language('fra'), Language('und')})
+
+
+def suite():
+ suite = TestSuite()
+ suite.addTest(TestLoader().loadTestsFromTestCase(ApiTestCase))
+ suite.addTest(TestLoader().loadTestsFromTestCase(VideoTestCase))
+ return suite
+
+
+if __name__ == '__main__':
+ TextTestRunner().run(suite())
diff --git a/lib/subliminal/utils.py b/lib/subliminal/utils.py
deleted file mode 100644
index e4fe4e8583045efa4c892869016bf25eee8721fd..0000000000000000000000000000000000000000
--- a/lib/subliminal/utils.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-import re
-
-
-__all__ = ['get_keywords', 'split_keyword', 'to_unicode']
-
-
-def get_keywords(guess):
- """Retrieve keywords from guessed informations
-
- :param guess: guessed informations
- :type guess: :class:`guessit.guess.Guess`
- :return: lower case alphanumeric keywords
- :rtype: set
-
- """
- keywords = set()
- for k in ['releaseGroup', 'screenSize', 'videoCodec', 'format']:
- if k in guess:
- keywords = keywords | split_keyword(guess[k].lower())
- return keywords
-
-
-def split_keyword(keyword):
- """Split a keyword in multiple ones on any non-alphanumeric character
-
- :param string keyword: keyword
- :return: keywords
- :rtype: set
-
- """
- split = set(re.findall(r'\w+', keyword))
- return split
-
-
-def to_unicode(data):
- """Convert a basestring to unicode
-
- :param basestring data: data to decode
- :return: data as unicode
- :rtype: unicode
-
- """
- if not isinstance(data, basestring):
- raise ValueError('Basestring expected')
- if isinstance(data, unicode):
- return data
- for encoding in ('utf-8', 'latin-1'):
- try:
- return unicode(data, encoding)
- except UnicodeDecodeError:
- pass
- return unicode(data, 'utf-8', 'replace')
diff --git a/lib/subliminal/video.py b/lib/subliminal/video.py
new file mode 100644
index 0000000000000000000000000000000000000000..09e36a9ec3b371e2a05dd781335836b7c0f0f832
--- /dev/null
+++ b/lib/subliminal/video.py
@@ -0,0 +1,411 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals, division
+import datetime
+import hashlib
+import logging
+import os
+import struct
+import babelfish
+import enzyme
+import guessit
+
+
+logger = logging.getLogger(__name__)
+
+#: Video extensions
+VIDEO_EXTENSIONS = ('.3g2', '.3gp', '.3gp2', '.3gpp', '.60d', '.ajp', '.asf', '.asx', '.avchd', '.avi', '.bik',
+ '.bix', '.box', '.cam', '.dat', '.divx', '.dmf', '.dv', '.dvr-ms', '.evo', '.flc', '.fli',
+ '.flic', '.flv', '.flx', '.gvi', '.gvp', '.h264', '.m1v', '.m2p', '.m2ts', '.m2v', '.m4e',
+ '.m4v', '.mjp', '.mjpeg', '.mjpg', '.mkv', '.moov', '.mov', '.movhd', '.movie', '.movx', '.mp4',
+ '.mpe', '.mpeg', '.mpg', '.mpv', '.mpv2', '.mxf', '.nsv', '.nut', '.ogg', '.ogm', '.omf', '.ps',
+ '.qt', '.ram', '.rm', '.rmvb', '.swf', '.ts', '.vfw', '.vid', '.video', '.viv', '.vivo', '.vob',
+ '.vro', '.wm', '.wmv', '.wmx', '.wrap', '.wvx', '.wx', '.x264', '.xvid')
+
+#: Subtitle extensions
+SUBTITLE_EXTENSIONS = ('.srt', '.sub', '.smi', '.txt', '.ssa', '.ass', '.mpl')
+
+
+class Video(object):
+ """Base class for videos
+
+ Represent a video, existing or not, with various properties that defines it.
+ Each property has an associated score based on equations that are described in
+ subclasses.
+
+ :param string name: name or path of the video
+ :param string format: format of the video (HDTV, WEB-DL, ...)
+ :param string release_group: release group of the video
+ :param string resolution: screen size of the video stream (480p, 720p, 1080p or 1080i)
+ :param string video_codec: codec of the video stream
+ :param string audio_codec: codec of the main audio stream
+ :param int imdb_id: IMDb id of the video
+ :param dict hashes: hashes of the video file by provider names
+ :param int size: byte size of the video file
+ :param set subtitle_languages: existing subtitle languages
+
+ """
+ scores = {}
+
+ def __init__(self, name, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None,
+ imdb_id=None, hashes=None, size=None, subtitle_languages=None):
+ self.name = name
+ self.format = format
+ self.release_group = release_group
+ self.resolution = resolution
+ self.video_codec = video_codec
+ self.audio_codec = audio_codec
+ self.imdb_id = imdb_id
+ self.hashes = hashes or {}
+ self.size = size
+ self.subtitle_languages = subtitle_languages or set()
+
+ @classmethod
+ def fromguess(cls, name, guess):
+ if guess['type'] == 'episode':
+ return Episode.fromguess(name, guess)
+ if guess['type'] == 'movie':
+ return Movie.fromguess(name, guess)
+ raise ValueError('The guess must be an episode or a movie guess')
+
+ @classmethod
+ def fromname(cls, name):
+ return cls.fromguess(os.path.split(name)[1], guessit.guess_file_info(name))
+
+ def __repr__(self):
+ return '<%s [%r]>' % (self.__class__.__name__, self.name)
+
+ def __hash__(self):
+ return hash(self.name)
+
+
+class Episode(Video):
+ """Episode :class:`Video`
+
+ Scores are defined by a set of equations, see :func:`~subliminal.score.get_episode_equations`
+
+ :param string series: series of the episode
+ :param int season: season number of the episode
+ :param int episode: episode number of the episode
+ :param string title: title of the episode
+ :param int year: year of series
+ :param int tvdb_id: TheTVDB id of the episode
+
+ """
+ scores = {'format': 3, 'video_codec': 2, 'tvdb_id': 48, 'title': 12, 'imdb_id': 60, 'audio_codec': 1, 'year': 24,
+ 'resolution': 2, 'season': 6, 'release_group': 6, 'series': 24, 'episode': 6, 'hash': 74}
+
+ def __init__(self, name, series, season, episode, format=None, release_group=None, resolution=None, video_codec=None,
+ audio_codec=None, imdb_id=None, hashes=None, size=None, subtitle_languages=None, title=None,
+ year=None, tvdb_id=None):
+ super(Episode, self).__init__(name, format, release_group, resolution, video_codec, audio_codec, imdb_id, hashes,
+ size, subtitle_languages)
+ self.series = series
+ self.season = season
+ self.episode = episode
+ self.title = title
+ self.year = year
+ self.tvdb_id = tvdb_id
+
+ @classmethod
+ def fromguess(cls, name, guess):
+ if guess['type'] != 'episode':
+ raise ValueError('The guess must be an episode guess')
+ if 'series' not in guess or 'season' not in guess or 'episodeNumber' not in guess:
+ raise ValueError('Insufficient data to process the guess')
+ return cls(name, guess['series'], guess['season'], guess['episodeNumber'], format=guess.get('format'),
+ release_group=guess.get('releaseGroup'), resolution=guess.get('screenSize'),
+ video_codec=guess.get('videoCodec'), audio_codec=guess.get('audioCodec'),
+ title=guess.get('title'), year=guess.get('year'))
+
+ @classmethod
+ def fromname(cls, name):
+ return cls.fromguess(os.path.split(name)[1], guessit.guess_episode_info(name))
+
+ def __repr__(self):
+ if self.year is None:
+ return '<%s [%r, %dx%d]>' % (self.__class__.__name__, self.series, self.season, self.episode)
+ return '<%s [%r, %d, %dx%d]>' % (self.__class__.__name__, self.series, self.year, self.season, self.episode)
+
+
+class Movie(Video):
+ """Movie :class:`Video`
+
+ Scores are defined by a set of equations, see :func:`~subliminal.score.get_movie_equations`
+
+ :param string title: title of the movie
+ :param int year: year of the movie
+
+ """
+ scores = {'format': 3, 'video_codec': 2, 'title': 13, 'imdb_id': 34, 'audio_codec': 1, 'year': 7, 'resolution': 2,
+ 'release_group': 6, 'hash': 34}
+
+ def __init__(self, name, title, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None,
+ imdb_id=None, hashes=None, size=None, subtitle_languages=None, year=None):
+ super(Movie, self).__init__(name, format, release_group, resolution, video_codec, audio_codec, imdb_id, hashes,
+ size, subtitle_languages)
+ self.title = title
+ self.year = year
+
+ @classmethod
+ def fromguess(cls, name, guess):
+ if guess['type'] != 'movie':
+ raise ValueError('The guess must be a movie guess')
+ if 'title' not in guess:
+ raise ValueError('Insufficient data to process the guess')
+ return cls(name, guess['title'], format=guess.get('format'), release_group=guess.get('releaseGroup'),
+ resolution=guess.get('screenSize'), video_codec=guess.get('videoCodec'),
+ audio_codec=guess.get('audioCodec'),year=guess.get('year'))
+
+ @classmethod
+ def fromname(cls, name):
+ return cls.fromguess(os.path.split(name)[1], guessit.guess_movie_info(name))
+
+ def __repr__(self):
+ if self.year is None:
+ return '<%s [%r]>' % (self.__class__.__name__, self.title)
+ return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year)
+
+
+def scan_subtitle_languages(path):
+ """Search for subtitles with alpha2 extension from a video `path` and return their language
+
+ :param string path: path to the video
+ :return: found subtitle languages
+ :rtype: set
+
+ """
+ language_extensions = tuple('.' + c for c in babelfish.language_converters['alpha2'].codes)
+ dirpath, filename = os.path.split(path)
+ subtitles = set()
+ for p in os.listdir(dirpath):
+ if not isinstance(p, bytes) and p.startswith(os.path.splitext(filename)[0]) and p.endswith(SUBTITLE_EXTENSIONS):
+ if os.path.splitext(p)[0].endswith(language_extensions):
+ subtitles.add(babelfish.Language.fromalpha2(os.path.splitext(p)[0][-2:]))
+ else:
+ subtitles.add(babelfish.Language('und'))
+ logger.debug('Found subtitles %r', subtitles)
+ return subtitles
+
+
+def scan_video(path, subtitles=True, embedded_subtitles=True):
+ """Scan a video and its subtitle languages from a video `path`
+
+ :param string path: absolute path to the video
+ :param bool subtitles: scan for subtitles with the same name
+ :param bool embedded_subtitles: scan for embedded subtitles
+ :return: the scanned video
+ :rtype: :class:`Video`
+ :raise: ValueError if cannot guess enough information from the path
+
+ """
+ dirpath, filename = os.path.split(path)
+ logger.info('Scanning video %r in %r', filename, dirpath)
+ video = Video.fromguess(path, guessit.guess_file_info(path))
+ video.size = os.path.getsize(path)
+ if video.size > 10485760:
+ logger.debug('Size is %d', video.size)
+ video.hashes['opensubtitles'] = hash_opensubtitles(path)
+ video.hashes['thesubdb'] = hash_thesubdb(path)
+ logger.debug('Computed hashes %r', video.hashes)
+ else:
+ logger.warning('Size is lower than 10MB: hashes not computed')
+ if subtitles:
+ video.subtitle_languages |= scan_subtitle_languages(path)
+ # enzyme
+ try:
+ if filename.endswith('.mkv'):
+ with open(path, 'rb') as f:
+ mkv = enzyme.MKV(f)
+ if mkv.video_tracks:
+ video_track = mkv.video_tracks[0]
+ # resolution
+ if video_track.height in (480, 720, 1080):
+ if video_track.interlaced:
+ video.resolution = '%di' % video_track.height
+ logger.debug('Found resolution %s with enzyme', video.resolution)
+ else:
+ video.resolution = '%dp' % video_track.height
+ logger.debug('Found resolution %s with enzyme', video.resolution)
+ # video codec
+ if video_track.codec_id == 'V_MPEG4/ISO/AVC':
+ video.video_codec = 'h264'
+ logger.debug('Found video_codec %s with enzyme', video.video_codec)
+ elif video_track.codec_id == 'V_MPEG4/ISO/SP':
+ video.video_codec = 'DivX'
+ logger.debug('Found video_codec %s with enzyme', video.video_codec)
+ elif video_track.codec_id == 'V_MPEG4/ISO/ASP':
+ video.video_codec = 'XviD'
+ logger.debug('Found video_codec %s with enzyme', video.video_codec)
+ else:
+ logger.warning('MKV has no video track')
+ if mkv.audio_tracks:
+ audio_track = mkv.audio_tracks[0]
+ # audio codec
+ if audio_track.codec_id == 'A_AC3':
+ video.audio_codec = 'AC3'
+ logger.debug('Found audio_codec %s with enzyme', video.audio_codec)
+ elif audio_track.codec_id == 'A_DTS':
+ video.audio_codec = 'DTS'
+ logger.debug('Found audio_codec %s with enzyme', video.audio_codec)
+ elif audio_track.codec_id == 'A_AAC':
+ video.audio_codec = 'AAC'
+ logger.debug('Found audio_codec %s with enzyme', video.audio_codec)
+ else:
+ logger.warning('MKV has no audio track')
+ if mkv.subtitle_tracks:
+ # embedded subtitles
+ if embedded_subtitles:
+ embedded_subtitle_languages = set()
+ for st in mkv.subtitle_tracks:
+ if st.language:
+ try:
+ embedded_subtitle_languages.add(babelfish.Language.fromalpha3b(st.language))
+ except babelfish.Error:
+ logger.error('Embedded subtitle track language %r is not a valid language', st.language)
+ embedded_subtitle_languages.add(babelfish.Language('und'))
+ elif st.name:
+ try:
+ embedded_subtitle_languages.add(babelfish.Language.fromname(st.name))
+ except babelfish.Error:
+ logger.debug('Embedded subtitle track name %r is not a valid language', st.name)
+ embedded_subtitle_languages.add(babelfish.Language('und'))
+ else:
+ embedded_subtitle_languages.add(babelfish.Language('und'))
+ logger.debug('Found embedded subtitle %r with enzyme', embedded_subtitle_languages)
+ video.subtitle_languages |= embedded_subtitle_languages
+ else:
+ logger.debug('MKV has no subtitle track')
+ except enzyme.Error:
+ logger.exception('Parsing video metadata with enzyme failed')
+ return video
+
+
+def scan_videos(paths, subtitles=True, embedded_subtitles=True, age=None):
+ """Scan `paths` for videos and their subtitle languages
+
+ :params paths: absolute paths to scan for videos
+ :type paths: list of string
+ :param bool subtitles: scan for subtitles with the same name
+ :param bool embedded_subtitles: scan for embedded subtitles
+ :param age: age of the video, if any
+ :type age: datetime.timedelta or None
+ :return: the scanned videos
+ :rtype: list of :class:`Video`
+
+ """
+ videos = []
+ # scan files
+ for filepath in [p for p in paths if os.path.isfile(p)]:
+ if age is not None:
+ try:
+ video_age = datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(filepath))
+ except ValueError:
+ logger.exception('Error while getting video age, skipping it')
+ continue
+ if video_age > age:
+ logger.info('Skipping video %r: older than %r', filepath, age)
+ continue
+ try:
+ videos.append(scan_video(filepath, subtitles, embedded_subtitles))
+ except ValueError as e:
+ logger.error('Skipping video: %s', e)
+ continue
+ # scan directories
+ for path in [p for p in paths if os.path.isdir(p)]:
+ logger.info('Scanning directory %r', path)
+ for dirpath, dirnames, filenames in os.walk(path):
+ # skip badly encoded directories
+ if isinstance(dirpath, bytes):
+ logger.error('Skipping badly encoded directory %r', dirpath.decode('utf-8', errors='replace'))
+ continue
+ # skip badly encoded and hidden sub directories
+ for dirname in list(dirnames):
+ if isinstance(dirname, bytes):
+ logger.error('Skipping badly encoded dirname %r in %r', dirname.decode('utf-8', errors='replace'),
+ dirpath)
+ dirnames.remove(dirname)
+ elif dirname.startswith('.'):
+ logger.debug('Skipping hidden dirname %r in %r', dirname, dirpath)
+ dirnames.remove(dirname)
+ # scan for videos
+ for filename in filenames:
+ # skip badly encoded files
+ if isinstance(filename, bytes):
+ logger.error('Skipping badly encoded filename %r in %r', filename.decode('utf-8', errors='replace'),
+ dirpath)
+ continue
+ # filter videos
+ if not filename.endswith(VIDEO_EXTENSIONS):
+ continue
+ # skip hidden files
+ if filename.startswith('.'):
+ logger.debug('Skipping hidden filename %r in %r', filename, dirpath)
+ continue
+ filepath = os.path.join(dirpath, filename)
+ # skip links
+ if os.path.islink(filepath):
+ logger.debug('Skipping link %r in %r', filename, dirpath)
+ continue
+ if age is not None:
+ try:
+ video_age = datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(filepath))
+ except ValueError:
+ logger.exception('Error while getting video age, skipping it')
+ continue
+ if video_age > age:
+ logger.info('Skipping video %r: older than %r', filepath, age)
+ continue
+ try:
+ video = scan_video(filepath, subtitles, embedded_subtitles)
+ except ValueError as e:
+ logger.error('Skipping video: %s', e)
+ continue
+ videos.append(video)
+ return videos
+
+
+def hash_opensubtitles(video_path):
+ """Compute a hash using OpenSubtitles' algorithm
+
+ :param string video_path: path of the video
+ :return: the hash
+ :rtype: string
+
+ """
+ bytesize = struct.calcsize(b'<q')
+ with open(video_path, 'rb') as f:
+ filesize = os.path.getsize(video_path)
+ filehash = filesize
+ if filesize < 65536 * 2:
+ return None
+ for _ in range(65536 // bytesize):
+ filebuffer = f.read(bytesize)
+ (l_value,) = struct.unpack(b'<q', filebuffer)
+ filehash += l_value
+ filehash = filehash & 0xFFFFFFFFFFFFFFFF # to remain as 64bit number
+ f.seek(max(0, filesize - 65536), 0)
+ for _ in range(65536 // bytesize):
+ filebuffer = f.read(bytesize)
+ (l_value,) = struct.unpack(b'<q', filebuffer)
+ filehash += l_value
+ filehash = filehash & 0xFFFFFFFFFFFFFFFF
+ returnedhash = '%016x' % filehash
+ return returnedhash
+
+
+def hash_thesubdb(video_path):
+ """Compute a hash using TheSubDB's algorithm
+
+ :param string video_path: path of the video
+ :return: the hash
+ :rtype: string
+
+ """
+ readsize = 64 * 1024
+ if os.path.getsize(video_path) < readsize:
+ return None
+ with open(video_path, 'rb') as f:
+ data = f.read(readsize)
+ f.seek(-readsize, os.SEEK_END)
+ data += f.read(readsize)
+ return hashlib.md5(data).hexdigest()
diff --git a/lib/subliminal/videos.py b/lib/subliminal/videos.py
deleted file mode 100644
index ca33f065e2dcddb8363febfe3d6d6b338a557ddd..0000000000000000000000000000000000000000
--- a/lib/subliminal/videos.py
+++ /dev/null
@@ -1,312 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin <diaoulael@gmail.com>
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see <http://www.gnu.org/licenses/>.
-from . import subtitles
-from .language import Language
-from .utils import to_unicode
-import enzyme.core
-import guessit
-import hashlib
-import logging
-import mimetypes
-import os
-import struct
-import sys
-
-from sickbeard import encodingKludge as ek
-import sickbeard
-
-
-__all__ = ['EXTENSIONS', 'MIMETYPES', 'Video', 'Episode', 'Movie', 'UnknownVideo',
- 'scan', 'hash_opensubtitles', 'hash_thesubdb']
-logger = logging.getLogger("subliminal")
-
-#: Video extensions
-EXTENSIONS = ['.avi', '.mkv', '.mpg', '.mp4', '.m4v', '.mov', '.ogm', '.ogv', '.wmv',
- '.divx', '.asf']
-
-#: Video mimetypes
-MIMETYPES = ['video/mpeg', 'video/mp4', 'video/quicktime', 'video/x-ms-wmv', 'video/x-msvideo',
- 'video/x-flv', 'video/x-matroska', 'video/x-matroska-3d']
-
-
-class Video(object):
- """Base class for videos
-
- :param string path: path
- :param guess: guessed informations
- :type guess: :class:`~guessit.guess.Guess`
- :param string imdbid: imdbid
-
- """
- def __init__(self, path, guess, imdbid=None):
- self.guess = guess
- self.imdbid = imdbid
- self._path = None
- self.hashes = {}
-
- if sys.platform == 'win32':
- if isinstance(path, str):
- path = unicode(path.encode('utf-8'))
- else:
- if isinstance(path, unicode):
- path = path.encode('utf-8')
-
- self.release = path
-
- if os.path.exists(path):
- self._path = path
- self.size = os.path.getsize(self._path)
- self._compute_hashes()
-
- @classmethod
- def from_path(cls, path):
- """Create a :class:`Video` subclass guessing all informations from the given path
-
- :param string path: path
- :return: video object
- :rtype: :class:`Episode` or :class:`Movie` or :class:`UnknownVideo`
-
- """
- guess = guessit.guess_file_info(path, 'autodetect')
- result = None
- if guess['type'] == 'episode' and 'series' in guess and 'season' in guess and 'episodeNumber' in guess:
- title = None
- if 'title' in guess:
- title = guess['title']
- result = Episode(path, guess['series'], guess['season'], guess['episodeNumber'], title, guess)
- if guess['type'] == 'movie' and 'title' in guess:
- year = None
- if 'year' in guess:
- year = guess['year']
- result = Movie(path, guess['title'], year, guess)
- if not result:
- result = UnknownVideo(path, guess)
- if not isinstance(result, cls):
- raise ValueError('Video is not of requested type')
- return result
-
- @property
- def exists(self):
- """Whether the video exists or not"""
- if self._path:
- return os.path.exists(self._path)
- return False
-
- @property
- def path(self):
- """Path to the video"""
- return self._path
-
- @path.setter
- def path(self, value):
- if not os.path.exists(value):
- raise ValueError('Path does not exists')
- self._path = value
- self.size = os.path.getsize(self._path)
- self._compute_hashes()
-
- def _compute_hashes(self):
- """Compute different hashes"""
- self.hashes['OpenSubtitles'] = hash_opensubtitles(self.path)
- self.hashes['TheSubDB'] = hash_thesubdb(self.path)
-
- def scan(self):
- """Scan and return associated subtitles
-
- :return: associated subtitles
- :rtype: list of :class:`~subliminal.subtitles.Subtitle`
-
- """
- if not self.exists:
- return []
- basepath = os.path.splitext(self.path)[0]
- results = []
- if not sickbeard.EMBEDDED_SUBTITLES_ALL:
- video_infos = None
- try:
- video_infos = enzyme.parse(self.path)
- logger.debug(u'Succeeded parsing %s with enzyme: %r' % (self.path, video_infos)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- except:
- logger.debug(u'Failed parsing %s with enzyme' % self.path) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- if isinstance(video_infos, enzyme.core.AVContainer):
- results.extend([subtitles.EmbeddedSubtitle.from_enzyme(self.path, s) for s in video_infos.subtitles])
- # cannot use glob here because it chokes if there are any square
- # brackets inside the filename, so we have to use basic string
- # startswith/endswith comparisons
- folder, basename = os.path.split(basepath)
- if folder == '':
- folder = '.'
- existing = [f for f in os.listdir(folder) if f.startswith(basename)]
- if sickbeard.SUBTITLES_DIR:
- subsDir = ek.ek(os.path.join, folder, sickbeard.SUBTITLES_DIR)
- if ek.ek(os.path.isdir, subsDir):
- existing.extend([f for f in os.listdir(subsDir) if f.startswith(basename)])
- for path in existing:
- for ext in subtitles.EXTENSIONS:
- if path.endswith(ext):
- language = Language(path[len(basename) + 1:-len(ext)], strict=False)
- results.append(subtitles.ExternalSubtitle(path, language))
- return results
-
- def __unicode__(self):
- return to_unicode(self.path or self.release)
-
- def __str__(self):
- return unicode(self).encode('utf-8')
-
- def __repr__(self):
- return '%s(%s)' % (self.__class__.__name__, self)
-
- def __hash__(self):
- return hash(self.path or self.release)
-
-
-class Episode(Video):
- """Episode :class:`Video`
-
- :param string path: path
- :param string series: series
- :param int season: season number
- :param int episode: episode number
- :param string title: title
- :param guess: guessed informations
- :type guess: :class:`~guessit.guess.Guess`
- :param string tvdbid: tvdbid
- :param string imdbid: imdbid
-
- """
- def __init__(self, path, series, season, episode, title=None, guess=None, tvdbid=None, imdbid=None):
- super(Episode, self).__init__(path, guess, imdbid)
- self.series = series
- self.title = title
- self.season = season
- self.episode = episode
- self.tvdbid = tvdbid
-
-
-class Movie(Video):
- """Movie :class:`Video`
-
- :param string path: path
- :param string title: title
- :param int year: year
- :param guess: guessed informations
- :type guess: :class:`~guessit.guess.Guess`
- :param string imdbid: imdbid
-
- """
- def __init__(self, path, title, year=None, guess=None, imdbid=None):
- super(Movie, self).__init__(path, guess, imdbid)
- self.title = title
- self.year = year
-
-
-class UnknownVideo(Video):
- """Unknown video"""
- pass
-
-
-def scan(entry, max_depth=3, scan_filter=None, depth=0):
- """Scan a path for videos and subtitles
-
- :param string entry: path
- :param int max_depth: maximum folder depth
- :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
- :param int depth: starting depth
- :return: found videos and subtitles
- :rtype: list of (:class:`Video`, [:class:`~subliminal.subtitles.Subtitle`])
-
- """
-
- if sys.platform == 'win32':
- if isinstance(entry, str):
- entry = unicode(entry.encode('utf-8'))
- else:
- if isinstance(entry, unicode):
- entry = entry.encode('utf-8')
-
- if depth > max_depth and max_depth != 0: # we do not want to search the whole file system except if max_depth = 0
- return []
- if os.path.isdir(entry): # a dir? recurse
- logger.debug(u'Scanning directory %s with depth %d/%d' % (entry, depth, max_depth)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- result = []
- for e in os.listdir(entry):
- result.extend(scan(os.path.join(entry, e), max_depth, scan_filter, depth + 1))
- return result
- if os.path.isfile(entry) or depth == 0:
- logger.debug(u'Scanning file %s with depth %d/%d' % (entry, depth, max_depth)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- if depth != 0: # trust the user: only check for valid format if recursing
- if mimetypes.guess_type(entry)[0] not in MIMETYPES and os.path.splitext(entry)[1] not in EXTENSIONS:
- return []
- if scan_filter is not None and scan_filter(entry):
- return []
- video = Video.from_path(entry)
- return [(video, video.scan())]
- logger.warning(u'Scanning entry %s failed with depth %d/%d' % (entry, depth, max_depth)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return [] # anything else
-
-
-def hash_opensubtitles(path):
- """Compute a hash using OpenSubtitles' algorithm
-
- :param string path: path
- :return: hash
- :rtype: string
-
- """
- longlongformat = 'q' # long long
- bytesize = struct.calcsize(longlongformat)
- with open(path, 'rb') as f:
- filesize = os.path.getsize(path)
- filehash = filesize
- if filesize < 65536 * 2:
- return None
- for _ in range(65536 / bytesize):
- filebuffer = f.read(bytesize)
- (l_value,) = struct.unpack(longlongformat, filebuffer)
- filehash += l_value
- filehash = filehash & 0xFFFFFFFFFFFFFFFF # to remain as 64bit number
- f.seek(max(0, filesize - 65536), 0)
- for _ in range(65536 / bytesize):
- filebuffer = f.read(bytesize)
- (l_value,) = struct.unpack(longlongformat, filebuffer)
- filehash += l_value
- filehash = filehash & 0xFFFFFFFFFFFFFFFF
- returnedhash = '%016x' % filehash
- logger.debug(u'Computed OpenSubtitle hash %s for %s' % (returnedhash, path)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return returnedhash
-
-
-def hash_thesubdb(path):
- """Compute a hash using TheSubDB's algorithm
-
- :param string path: path
- :return: hash
- :rtype: string
-
- """
- readsize = 64 * 1024
- if os.path.getsize(path) < readsize:
- return None
- with open(path, 'rb') as f:
- data = f.read(readsize)
- f.seek(-readsize, os.SEEK_END)
- data += f.read(readsize)
- returnedhash = hashlib.md5(data).hexdigest()
- logger.debug(u'Computed TheSubDB hash %s for %s' % (returnedhash, path)) if sys.platform != 'win32' else logger.debug('Log line suppressed on windows')
- return returnedhash
diff --git a/lib/tmdb_api/tmdb_api.py b/lib/tmdb_api/tmdb_api.py
index d0ef11fe6d9e32554a4655f7c24b0fa8f21dea77..2ddf02f12b1fbcd4e3ef940f34dab115e72b605f 100644
--- a/lib/tmdb_api/tmdb_api.py
+++ b/lib/tmdb_api/tmdb_api.py
@@ -7,7 +7,7 @@ Created by Celia Oakley on 2013-10-31.
"""
import json
-import lib.requests as requests
+import requests
class TMDB:
def __init__(self, api_key, version=3):
diff --git a/lib/trakt/trakt.py b/lib/trakt/trakt.py
index 8ce2e072842aa0aec8fbeec3aa7cfa423f5ba530..d9f9069ac6e7697fb7b5ae3d93ec36e66dcf1f87 100644
--- a/lib/trakt/trakt.py
+++ b/lib/trakt/trakt.py
@@ -1,4 +1,5 @@
-from lib import requests
+import requests
+import certifi
import json
import sickbeard
import time
@@ -8,7 +9,8 @@ from exceptions import traktException, traktAuthException, traktServerBusy
class TraktAPI():
def __init__(self, disable_ssl_verify=False, timeout=30):
- self.verify = not disable_ssl_verify
+ self.session = requests.Session()
+ self.verify = certifi.where() if not disable_ssl_verify else False
self.timeout = timeout if timeout else None
self.auth_url = sickbeard.TRAKT_OAUTH_URL
self.api_url = sickbeard.TRAKT_API_URL
@@ -19,21 +21,21 @@ class TraktAPI():
}
def traktToken(self, trakt_pin=None, refresh=False, count=0):
-
+
if count > 3:
sickbeard.TRAKT_ACCESS_TOKEN = ''
return False
elif count > 0:
time.sleep(2)
-
-
-
+
+
+
data = {
'client_id': sickbeard.TRAKT_API_KEY,
'client_secret': sickbeard.TRAKT_API_SECRET,
'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob'
}
-
+
if refresh:
data['grant_type'] = 'refresh_token'
data['refresh_token'] = sickbeard.TRAKT_REFRESH_TOKEN
@@ -41,11 +43,11 @@ class TraktAPI():
data['grant_type'] = 'authorization_code'
if not None == trakt_pin:
data['code'] = trakt_pin
-
+
headers = {
'Content-Type': 'application/json'
- }
-
+ }
+
resp = self.traktRequest('oauth/token', data=data, headers=headers, url=self.auth_url , method='POST', count=count)
if 'access_token' in resp:
@@ -54,34 +56,32 @@ class TraktAPI():
sickbeard.TRAKT_REFRESH_TOKEN = resp['refresh_token']
return True
return False
-
+
def validateAccount(self):
-
+
resp = self.traktRequest('users/settings')
-
+
if 'account' in resp:
return True
return False
-
- def traktRequest(self, path, data=None, headers=None, url=None, method='GET',count=0):
+
+ def traktRequest(self, path, data=None, headers=None, url=None, method='GET', count=0):
if None == url:
- url = self.api_url + path
- else:
- url = url + path
-
+ url = self.api_url
+
count = count + 1
-
+
if None == headers:
headers = self.headers
-
+
if None == sickbeard.TRAKT_ACCESS_TOKEN:
logger.log(u'You must get a Trakt TOKEN. Check your Trakt settings', logger.WARNING)
- raise traktAuthException(e)
-
+ return {}
+
headers['Authorization'] = 'Bearer ' + sickbeard.TRAKT_ACCESS_TOKEN
try:
- resp = requests.request(method, url, headers=headers, timeout=self.timeout,
+ resp = self.session.request(method, url + path, headers=headers, timeout=self.timeout,
data=json.dumps(data) if data else [], verify=self.verify)
# check for http errors and raise if any are present
@@ -98,16 +98,16 @@ class TraktAPI():
elif code == 502:
# Retry the request, cloudflare had a proxying issue
logger.log(u'Retrying trakt api request: %s' % path, logger.WARNING)
- return self.traktRequest(path, data, headers, method)
+ return self.traktRequest(path, data, headers, url, method)
elif code == 401:
logger.log(u'Unauthorized. Please check your Trakt settings', logger.WARNING)
- if self.traktToken(refresh=True,count=count):
- return self.traktRequest(path, data, url, method)
+ if self.traktToken(refresh=True, count=count):
+ return self.traktRequest(path, data, headers, url, method)
raise traktAuthException(e)
elif code in (500,501,503,504,520,521,522):
#http://docs.trakt.apiary.io/#introduction/status-codes
logger.log(u'Trakt may have some issues and it\'s unavailable. Try again later please', logger.WARNING)
- raise traktServerBusy(e)
+ return {}
else:
raise traktException(e)
diff --git a/lib/tvdb_api/tvdb_api.py b/lib/tvdb_api/tvdb_api.py
index 6bd904ef48d0a115cecd0e0005e92ad851fc7cce..7ae10ac9f2c0db550b05e18f572b2b2a68ea3606 100644
--- a/lib/tvdb_api/tvdb_api.py
+++ b/lib/tvdb_api/tvdb_api.py
@@ -21,8 +21,8 @@ import warnings
import logging
import zipfile
import datetime as dt
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
import xmltodict
try:
@@ -479,6 +479,8 @@ class Tvdb:
else:
raise ValueError("Invalid value for Cache %r (type was %s)" % (cache, type(cache)))
+ self.config['session'] = requests.Session()
+
self.config['banners_enabled'] = banners
self.config['actors_enabled'] = actors
@@ -563,7 +565,8 @@ class Tvdb:
# get response from TVDB
if self.config['cache_enabled']:
- session = CacheControl(cache=caches.FileCache(self.config['cache_location']))
+
+ session = CacheControl(sess=self.config['session'], cache=caches.FileCache(self.config['cache_location']), cache_etags=False)
if self.config['proxy']:
log().debug("Using proxy for URL: %s" % url)
session.proxies = {
@@ -571,7 +574,7 @@ class Tvdb:
"https": self.config['proxy'],
}
- resp = session.get(url.strip(), cache_auto=True, params=params)
+ resp = session.get(url.strip(), params=params)
else:
resp = requests.get(url.strip(), params=params)
diff --git a/lib/tvrage_api/tvrage_api.py b/lib/tvrage_api/tvrage_api.py
index 16cb78b16b7aad90123d50542a2627e80f50f856..98fe034bfdee52afe84d784742448eaed718de5e 100644
--- a/lib/tvrage_api/tvrage_api.py
+++ b/lib/tvrage_api/tvrage_api.py
@@ -23,8 +23,8 @@ import tempfile
import warnings
import logging
import datetime as dt
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
import xmltodict
try:
@@ -327,6 +327,8 @@ class TVRage:
else:
raise ValueError("Invalid value for Cache %r (type was %s)" % (cache, type(cache)))
+ self.config['session'] = requests.Session()
+
if self.config['debug_enabled']:
warnings.warn("The debug argument to tvrage_api.__init__ will be removed in the next version. "
"To enable debug messages, use the following code before importing: "
@@ -399,7 +401,7 @@ class TVRage:
# get response from TVRage
if self.config['cache_enabled']:
- session = CacheControl(cache=caches.FileCache(self.config['cache_location']))
+ session = CacheControl(sess=self.config['session'], cache=caches.FileCache(self.config['cache_location']), cache_etags=False)
if self.config['proxy']:
log().debug("Using proxy for URL: %s" % url)
session.proxies = {
@@ -407,7 +409,7 @@ class TVRage:
"https": self.config['proxy'],
}
- resp = session.get(url.strip(), cache_auto=True, params=params)
+ resp = session.get(url.strip(), params=params)
else:
resp = requests.get(url.strip(), params=params)
diff --git a/sickbeard/__init__.py b/sickbeard/__init__.py
index 4546404c35870069ac9f372b5c38516cc698d2f5..4acc548366715bb1dc17dea142254c5ba9e837a5 100755
--- a/sickbeard/__init__.py
+++ b/sickbeard/__init__.py
@@ -37,8 +37,8 @@ from github import Github
from sickbeard import providers, metadata, config, webserveInit
from sickbeard.providers.generic import GenericProvider
from providers import btn, newznab, womble, thepiratebay, torrentleech, kat, iptorrents, \
- omgwtfnzbs, scc, hdtorrents, torrentday, hdbits, hounddawgs, nextgen, speedcd, nyaatorrents, animenzb, torrentbytes, animezb, \
- freshontv, morethantv, bitsoup, t411, tokyotoshokan, shazbat, rarbg, alpharatio, tntvillage, binsearch, scenetime
+ omgwtfnzbs, scc, hdtorrents, torrentday, hdbits, hounddawgs, nextgen, speedcd, nyaatorrents, animenzb, bluetigers, fnt, torrentbytes, animezb, \
+ freshontv, morethantv, bitsoup, t411, tokyotoshokan, shazbat, rarbg, alpharatio, tntvillage, binsearch, scenetime, btdigg
from sickbeard.config import CheckSection, check_setting_int, check_setting_str, check_setting_float, ConfigMigrator, \
naming_ep_type
from sickbeard import searchBacklog, showUpdater, versionChecker, properFinder, autoPostProcesser, \
@@ -57,7 +57,7 @@ from sickbeard.helpers import ex
from lib.configobj import ConfigObj
-from lib import requests
+import requests
requests.packages.urllib3.disable_warnings()
PID = None
@@ -387,6 +387,7 @@ PUSHOVER_NOTIFY_ONDOWNLOAD = False
PUSHOVER_NOTIFY_ONSUBTITLEDOWNLOAD = False
PUSHOVER_USERKEY = None
PUSHOVER_APIKEY = None
+PUSHOVER_DEVICE = None
USE_LIBNOTIFY = False
LIBNOTIFY_NOTIFY_ONSNATCH = False
@@ -424,6 +425,7 @@ TRAKT_ACCESS_TOKEN = None
TRAKT_REFRESH_TOKEN = None
TRAKT_REMOVE_WATCHLIST = False
TRAKT_REMOVE_SERIESLIST = False
+TRAKT_REMOVE_SHOW_FROM_SICKRAGE = False
TRAKT_SYNC_WATCHLIST = False
TRAKT_METHOD_ADD = None
TRAKT_START_PAUSED = False
@@ -550,7 +552,7 @@ def initialize(consoleLogging=True):
TORRENT_USERNAME, TORRENT_PASSWORD, TORRENT_HOST, TORRENT_PATH, TORRENT_SEED_TIME, TORRENT_PAUSED, TORRENT_HIGH_BANDWIDTH, TORRENT_LABEL, TORRENT_LABEL_ANIME, TORRENT_VERIFY_CERT, TORRENT_RPCURL, TORRENT_AUTH_TYPE, \
USE_KODI, KODI_ALWAYS_ON, KODI_NOTIFY_ONSNATCH, KODI_NOTIFY_ONDOWNLOAD, KODI_NOTIFY_ONSUBTITLEDOWNLOAD, KODI_UPDATE_FULL, KODI_UPDATE_ONLYFIRST, \
KODI_UPDATE_LIBRARY, KODI_HOST, KODI_USERNAME, KODI_PASSWORD, BACKLOG_FREQUENCY, \
- USE_TRAKT, TRAKT_USERNAME, TRAKT_ACCESS_TOKEN, TRAKT_REFRESH_TOKEN, TRAKT_REMOVE_WATCHLIST, TRAKT_SYNC_WATCHLIST, TRAKT_METHOD_ADD, TRAKT_START_PAUSED, traktCheckerScheduler, traktRollingScheduler, TRAKT_USE_RECOMMENDED, TRAKT_SYNC, TRAKT_SYNC_REMOVE, TRAKT_DEFAULT_INDEXER, TRAKT_REMOVE_SERIESLIST, TRAKT_DISABLE_SSL_VERIFY, TRAKT_TIMEOUT, TRAKT_BLACKLIST_NAME, TRAKT_USE_ROLLING_DOWNLOAD, TRAKT_ROLLING_NUM_EP, TRAKT_ROLLING_ADD_PAUSED, TRAKT_ROLLING_FREQUENCY, \
+ USE_TRAKT, TRAKT_USERNAME, TRAKT_ACCESS_TOKEN, TRAKT_REFRESH_TOKEN, TRAKT_REMOVE_WATCHLIST, TRAKT_SYNC_WATCHLIST, TRAKT_REMOVE_SHOW_FROM_SICKRAGE, TRAKT_METHOD_ADD, TRAKT_START_PAUSED, traktCheckerScheduler, traktRollingScheduler, TRAKT_USE_RECOMMENDED, TRAKT_SYNC, TRAKT_SYNC_REMOVE, TRAKT_DEFAULT_INDEXER, TRAKT_REMOVE_SERIESLIST, TRAKT_DISABLE_SSL_VERIFY, TRAKT_TIMEOUT, TRAKT_BLACKLIST_NAME, TRAKT_USE_ROLLING_DOWNLOAD, TRAKT_ROLLING_NUM_EP, TRAKT_ROLLING_ADD_PAUSED, TRAKT_ROLLING_FREQUENCY, \
USE_PLEX, PLEX_NOTIFY_ONSNATCH, PLEX_NOTIFY_ONDOWNLOAD, PLEX_NOTIFY_ONSUBTITLEDOWNLOAD, PLEX_UPDATE_LIBRARY, USE_PLEX_CLIENT, PLEX_CLIENT_USERNAME, PLEX_CLIENT_PASSWORD, \
PLEX_SERVER_HOST, PLEX_SERVER_TOKEN, PLEX_HOST, PLEX_USERNAME, PLEX_PASSWORD, DEFAULT_BACKLOG_FREQUENCY, MIN_BACKLOG_FREQUENCY, BACKLOG_STARTUP, SKIP_REMOVED_FILES, \
showUpdateScheduler, __INITIALIZED__, INDEXER_DEFAULT_LANGUAGE, EP_DEFAULT_DELETED_STATUS, LAUNCH_BROWSER, UPDATE_SHOWS_ON_START, UPDATE_SHOWS_ON_SNATCH, TRASH_REMOVE_SHOW, TRASH_ROTATE_LOGS, SORT_ARTICLE, showList, loadingShowList, \
@@ -571,7 +573,7 @@ def initialize(consoleLogging=True):
EXTRA_SCRIPTS, USE_TWITTER, TWITTER_USERNAME, TWITTER_PASSWORD, TWITTER_PREFIX, DAILYSEARCH_FREQUENCY, \
USE_BOXCAR, BOXCAR_USERNAME, BOXCAR_PASSWORD, BOXCAR_NOTIFY_ONDOWNLOAD, BOXCAR_NOTIFY_ONSUBTITLEDOWNLOAD, BOXCAR_NOTIFY_ONSNATCH, \
USE_BOXCAR2, BOXCAR2_ACCESSTOKEN, BOXCAR2_NOTIFY_ONDOWNLOAD, BOXCAR2_NOTIFY_ONSUBTITLEDOWNLOAD, BOXCAR2_NOTIFY_ONSNATCH, \
- USE_PUSHOVER, PUSHOVER_USERKEY, PUSHOVER_APIKEY, PUSHOVER_NOTIFY_ONDOWNLOAD, PUSHOVER_NOTIFY_ONSUBTITLEDOWNLOAD, PUSHOVER_NOTIFY_ONSNATCH, \
+ USE_PUSHOVER, PUSHOVER_USERKEY, PUSHOVER_APIKEY, PUSHOVER_DEVICE, PUSHOVER_NOTIFY_ONDOWNLOAD, PUSHOVER_NOTIFY_ONSUBTITLEDOWNLOAD, PUSHOVER_NOTIFY_ONSNATCH, \
USE_LIBNOTIFY, LIBNOTIFY_NOTIFY_ONSNATCH, LIBNOTIFY_NOTIFY_ONDOWNLOAD, LIBNOTIFY_NOTIFY_ONSUBTITLEDOWNLOAD, USE_NMJ, NMJ_HOST, NMJ_DATABASE, NMJ_MOUNT, USE_NMJv2, NMJv2_HOST, NMJv2_DATABASE, NMJv2_DBLOC, USE_SYNOINDEX, \
USE_SYNOLOGYNOTIFIER, SYNOLOGYNOTIFIER_NOTIFY_ONSNATCH, SYNOLOGYNOTIFIER_NOTIFY_ONDOWNLOAD, SYNOLOGYNOTIFIER_NOTIFY_ONSUBTITLEDOWNLOAD, \
USE_EMAIL, EMAIL_HOST, EMAIL_PORT, EMAIL_TLS, EMAIL_USER, EMAIL_PASSWORD, EMAIL_FROM, EMAIL_NOTIFY_ONSNATCH, EMAIL_NOTIFY_ONDOWNLOAD, EMAIL_NOTIFY_ONSUBTITLEDOWNLOAD, EMAIL_LIST, \
@@ -654,7 +656,7 @@ def initialize(consoleLogging=True):
# git_remote
GIT_REMOTE = check_setting_str(CFG, 'General', 'git_remote', 'origin')
GIT_REMOTE_URL = check_setting_str(CFG, 'General', 'git_remote_url',
- 'https://github.com/SiCKRAGETV/SickRage.git')
+ 'git@github.com:SiCKRAGETV/SickRage.git')
# current commit hash
CUR_COMMIT_HASH = check_setting_str(CFG, 'General', 'cur_commit_hash', '')
@@ -709,9 +711,6 @@ def initialize(consoleLogging=True):
except Exception as e:
logger.log(u"Restore: Unable to remove the restore directory: {0}".format(str(e)), logger.ERROR)
- # clean cache folders
- if CACHE_DIR:
- helpers.clearCache()
GUI_NAME = check_setting_str(CFG, 'GUI', 'gui_name', 'slick')
@@ -940,7 +939,7 @@ def initialize(consoleLogging=True):
USE_PLEX_CLIENT = bool(check_setting_int(CFG, 'Plex', 'use_plex_client', 0))
PLEX_CLIENT_USERNAME = check_setting_str(CFG, 'Plex', 'plex_client_username', '', censor_log=True)
PLEX_CLIENT_PASSWORD = check_setting_str(CFG, 'Plex', 'plex_client_password', '', censor_log=True)
-
+
USE_GROWL = bool(check_setting_int(CFG, 'Growl', 'use_growl', 0))
GROWL_NOTIFY_ONSNATCH = bool(check_setting_int(CFG, 'Growl', 'growl_notify_onsnatch', 0))
GROWL_NOTIFY_ONDOWNLOAD = bool(check_setting_int(CFG, 'Growl', 'growl_notify_ondownload', 0))
@@ -980,22 +979,21 @@ def initialize(consoleLogging=True):
USE_BOXCAR2 = bool(check_setting_int(CFG, 'Boxcar2', 'use_boxcar2', 0))
BOXCAR2_NOTIFY_ONSNATCH = bool(check_setting_int(CFG, 'Boxcar2', 'boxcar2_notify_onsnatch', 0))
BOXCAR2_NOTIFY_ONDOWNLOAD = bool(check_setting_int(CFG, 'Boxcar2', 'boxcar2_notify_ondownload', 0))
- BOXCAR2_NOTIFY_ONSUBTITLEDOWNLOAD = bool(
- check_setting_int(CFG, 'Boxcar2', 'boxcar2_notify_onsubtitledownload', 0))
+ BOXCAR2_NOTIFY_ONSUBTITLEDOWNLOAD = bool(check_setting_int(CFG, 'Boxcar2', 'boxcar2_notify_onsubtitledownload', 0))
BOXCAR2_ACCESSTOKEN = check_setting_str(CFG, 'Boxcar2', 'boxcar2_accesstoken', '', censor_log=True)
USE_PUSHOVER = bool(check_setting_int(CFG, 'Pushover', 'use_pushover', 0))
PUSHOVER_NOTIFY_ONSNATCH = bool(check_setting_int(CFG, 'Pushover', 'pushover_notify_onsnatch', 0))
PUSHOVER_NOTIFY_ONDOWNLOAD = bool(check_setting_int(CFG, 'Pushover', 'pushover_notify_ondownload', 0))
- PUSHOVER_NOTIFY_ONSUBTITLEDOWNLOAD = bool(
- check_setting_int(CFG, 'Pushover', 'pushover_notify_onsubtitledownload', 0))
+ PUSHOVER_NOTIFY_ONSUBTITLEDOWNLOAD = bool(check_setting_int(CFG, 'Pushover', 'pushover_notify_onsubtitledownload', 0))
PUSHOVER_USERKEY = check_setting_str(CFG, 'Pushover', 'pushover_userkey', '', censor_log=True)
PUSHOVER_APIKEY = check_setting_str(CFG, 'Pushover', 'pushover_apikey', '', censor_log=True)
+ PUSHOVER_DEVICE = check_setting_str(CFG, 'Pushover', 'pushover_device', '')
+
USE_LIBNOTIFY = bool(check_setting_int(CFG, 'Libnotify', 'use_libnotify', 0))
LIBNOTIFY_NOTIFY_ONSNATCH = bool(check_setting_int(CFG, 'Libnotify', 'libnotify_notify_onsnatch', 0))
LIBNOTIFY_NOTIFY_ONDOWNLOAD = bool(check_setting_int(CFG, 'Libnotify', 'libnotify_notify_ondownload', 0))
- LIBNOTIFY_NOTIFY_ONSUBTITLEDOWNLOAD = bool(
- check_setting_int(CFG, 'Libnotify', 'libnotify_notify_onsubtitledownload', 0))
+ LIBNOTIFY_NOTIFY_ONSUBTITLEDOWNLOAD = bool(check_setting_int(CFG, 'Libnotify', 'libnotify_notify_onsubtitledownload', 0))
USE_NMJ = bool(check_setting_int(CFG, 'NMJ', 'use_nmj', 0))
NMJ_HOST = check_setting_str(CFG, 'NMJ', 'nmj_host', '')
@@ -1023,6 +1021,7 @@ def initialize(consoleLogging=True):
TRAKT_REFRESH_TOKEN = check_setting_str(CFG, 'Trakt', 'trakt_refresh_token', '', censor_log=True)
TRAKT_REMOVE_WATCHLIST = bool(check_setting_int(CFG, 'Trakt', 'trakt_remove_watchlist', 0))
TRAKT_REMOVE_SERIESLIST = bool(check_setting_int(CFG, 'Trakt', 'trakt_remove_serieslist', 0))
+ TRAKT_REMOVE_SHOW_FROM_SICKRAGE = bool(check_setting_int(CFG, 'Trakt', 'trakt_remove_show_from_sickrage', 0))
TRAKT_SYNC_WATCHLIST = bool(check_setting_int(CFG, 'Trakt', 'trakt_sync_watchlist', 0))
TRAKT_METHOD_ADD = check_setting_int(CFG, 'Trakt', 'trakt_method_add', 0)
TRAKT_START_PAUSED = bool(check_setting_int(CFG, 'Trakt', 'trakt_start_paused', 0))
@@ -1039,7 +1038,7 @@ def initialize(consoleLogging=True):
TRAKT_ROLLING_FREQUENCY = check_setting_int(CFG, 'Trakt', 'trakt_rolling_frequency', 8)
if TRAKT_ROLLING_FREQUENCY < 4:
TRAKT_ROLLING_FREQUENCY = 4
-
+
CheckSection(CFG, 'pyTivo')
USE_PYTIVO = bool(check_setting_int(CFG, 'pyTivo', 'use_pytivo', 0))
PYTIVO_NOTIFY_ONSNATCH = bool(check_setting_int(CFG, 'pyTivo', 'pytivo_notify_onsnatch', 0))
@@ -1956,6 +1955,7 @@ def save_config():
new_config['Pushover']['pushover_notify_onsubtitledownload'] = int(PUSHOVER_NOTIFY_ONSUBTITLEDOWNLOAD)
new_config['Pushover']['pushover_userkey'] = PUSHOVER_USERKEY
new_config['Pushover']['pushover_apikey'] = PUSHOVER_APIKEY
+ new_config['Pushover']['pushover_device'] = PUSHOVER_DEVICE
new_config['Libnotify'] = {}
new_config['Libnotify']['use_libnotify'] = int(USE_LIBNOTIFY)
@@ -1992,6 +1992,7 @@ def save_config():
new_config['Trakt']['trakt_refresh_token'] = TRAKT_REFRESH_TOKEN
new_config['Trakt']['trakt_remove_watchlist'] = int(TRAKT_REMOVE_WATCHLIST)
new_config['Trakt']['trakt_remove_serieslist'] = int(TRAKT_REMOVE_SERIESLIST)
+ new_config['Trakt']['trakt_remove_show_from_sickrage'] = int(TRAKT_REMOVE_SHOW_FROM_SICKRAGE)
new_config['Trakt']['trakt_sync_watchlist'] = int(TRAKT_SYNC_WATCHLIST)
new_config['Trakt']['trakt_method_add'] = int(TRAKT_METHOD_ADD)
new_config['Trakt']['trakt_start_paused'] = int(TRAKT_START_PAUSED)
@@ -2077,7 +2078,7 @@ def save_config():
new_config['GUI']['poster_sortby'] = POSTER_SORTBY
new_config['GUI']['poster_sortdir'] = POSTER_SORTDIR
new_config['GUI']['filter_row'] = int(FILTER_ROW)
-
+
new_config['Subtitles'] = {}
new_config['Subtitles']['use_subtitles'] = int(USE_SUBTITLES)
new_config['Subtitles']['subtitles_languages'] = ','.join(SUBTITLES_LANGUAGES)
diff --git a/sickbeard/clients/generic.py b/sickbeard/clients/generic.py
index 3dfc58dde954a368363ced1c354d0eae3e2d755d..6e2bf18e716b0e2832edbad8eeaefce24df9a905 100644
--- a/sickbeard/clients/generic.py
+++ b/sickbeard/clients/generic.py
@@ -8,8 +8,8 @@ from sickbeard import logger
from sickbeard.exceptions import ex
from sickbeard.clients import http_error_code
from lib.bencode import bencode, bdecode
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from lib.bencode.BTL import BTFailure
class GenericClient(object):
@@ -25,7 +25,7 @@ class GenericClient(object):
self.response = None
self.auth = None
self.last_time = time.time()
- self.session = requests.session()
+ self.session = requests.Session()
self.session.auth = (self.username, self.password)
def _request(self, method='get', params={}, data=None, files=None):
diff --git a/sickbeard/clients/qbittorrent.py b/sickbeard/clients/qbittorrent.py
index 5923c274cc75b298107b5a839f074d978a0447aa..d53d31f4fd33e6353881fb7ab83b8568f13a62f6 100644
--- a/sickbeard/clients/qbittorrent.py
+++ b/sickbeard/clients/qbittorrent.py
@@ -19,8 +19,8 @@
import sickbeard
from sickbeard import logger
from sickbeard.clients.generic import GenericClient
-from lib import requests
-from lib.requests.auth import HTTPDigestAuth
+import requests
+from requests.auth import HTTPDigestAuth
class qbittorrentAPI(GenericClient):
def __init__(self, host=None, username=None, password=None):
diff --git a/sickbeard/dailysearcher.py b/sickbeard/dailysearcher.py
index 66657c38615e895df764aef159a5d8cedec6de21..482746490e606fd99e003339a03cb28ad454b5e4 100644
--- a/sickbeard/dailysearcher.py
+++ b/sickbeard/dailysearcher.py
@@ -30,6 +30,8 @@ from sickbeard import helpers
from sickbeard import exceptions
from sickbeard import network_timezones
from sickbeard.exceptions import ex
+from sickbeard.common import SKIPPED
+from common import Quality, qualityPresetStrings, statusStrings
class DailySearcher():
@@ -84,34 +86,20 @@ class DailySearcher():
# if an error occured assume the episode hasn't aired yet
continue
+ UpdateWantedList = 0
ep = show.getEpisode(int(sqlEp["season"]), int(sqlEp["episode"]))
with ep.lock:
if ep.show.paused:
ep.status = common.SKIPPED
+ elif ep.season == 0:
+ logger.log(u"New episode " + ep.prettyName() + " airs today, setting status to SKIPPED because is a special season")
+ ep.status = common.SKIPPED
+ elif sickbeard.TRAKT_USE_ROLLING_DOWNLOAD and sickbeard.USE_TRAKT:
+ ep.status = common.SKIPPED
+ UpdateWantedList = 1
else:
- myDB = db.DBConnection()
- sql_selection="SELECT show_name, indexer_id, season, episode, paused FROM (SELECT * FROM tv_shows s,tv_episodes e WHERE s.indexer_id = e.showid) T1 WHERE T1.paused = 0 and T1.episode_id IN (SELECT T2.episode_id FROM tv_episodes T2 WHERE T2.showid = T1.indexer_id and T2.status in (?) ORDER BY T2.season,T2.episode LIMIT 1) and airdate is not null and indexer_id = ? ORDER BY T1.show_name,season,episode"
- results = myDB.select(sql_selection, [common.SKIPPED, sqlEp["showid"]])
- if not sickbeard.TRAKT_USE_ROLLING_DOWNLOAD:
- if ep.season == 0:
- logger.log(u"New episode " + ep.prettyName() + " airs today, setting status to SKIPPED, due to trakt integration")
- ep.status = common.SKIPPED
- else:
- logger.log(u"New episode " + ep.prettyName() + " airs today, setting status to WANTED")
- ep.status = common.WANTED
- else:
- sn_sk = results[0]["season"]
- ep_sk = results[0]["episode"]
- if (int(sn_sk)*100+int(ep_sk)) < (int(sqlEp["season"])*100+int(sqlEp["episode"])):
- logger.log(u"New episode " + ep.prettyName() + " airs today, setting status to SKIPPED, due to trakt integration")
- ep.status = common.SKIPPED
- else:
- if ep.season == 0:
- logger.log(u"New episode " + ep.prettyName() + " airs today, setting status to SKIPPED, due to trakt integration")
- ep.status = common.SKIPPED
- else:
- logger.log(u"New episode " + ep.prettyName() + " airs today, setting status to WANTED")
- ep.status = common.WANTED
+ logger.log(u"New episode " + ep.prettyName() + " airs today, setting status to WANTED")
+ ep.status = ep.show.default_ep_status
sql_l.append(ep.get_sql())
else:
@@ -121,6 +109,8 @@ class DailySearcher():
myDB = db.DBConnection()
myDB.mass_action(sql_l)
+ sickbeard.traktRollingScheduler.action.updateWantedList()
+
# queue episode for daily search
dailysearch_queue_item = sickbeard.search_queue.DailySearchQueueItem()
sickbeard.searchQueueScheduler.action.add_item(dailysearch_queue_item)
diff --git a/sickbeard/db.py b/sickbeard/db.py
index 79162cdfadfadf5b5ca6e970ec7014d575c635d8..346b3725953292f48946ef76ef904e5d2f74e2fd 100644
--- a/sickbeard/db.py
+++ b/sickbeard/db.py
@@ -23,7 +23,7 @@ import re
import sqlite3
import time
import threading
-
+import chardet
import sickbeard
from sickbeard import encodingKludge as ek
@@ -58,8 +58,6 @@ class DBConnection(object):
self.connection = sqlite3.connect(dbFilename(self.filename, self.suffix), 20, check_same_thread=False)
self.connection.text_factory = self._unicode_text_factory
- self.connection.isolation_level = None
- self.connection.cursor().execute('''PRAGMA locking_mode = EXCLUSIVE''')
db_cons[self.filename] = self.connection
else:
self.connection = db_cons[self.filename]
@@ -108,7 +106,7 @@ class DBConnection(object):
def mass_action(self, querylist=[], logTransaction=False, fetchall=False):
# remove None types
- querylist = [i for i in querylist if i is not None]
+ querylist = [i for i in querylist if i is not None and len(i)]
sqlResult = []
attempt = 0
@@ -125,7 +123,7 @@ class DBConnection(object):
if logTransaction:
logger.log(qu[0] + " with args " + str(qu[1]), logger.DEBUG)
sqlResult.append(self.execute(qu[0], qu[1], fetchall=fetchall))
-
+ self.connection.commit()
logger.log(u"Transaction with " + str(len(querylist)) + u" queries executed", logger.DEBUG)
# finished
@@ -168,6 +166,7 @@ class DBConnection(object):
logger.log(self.filename + ": " + query + " with args " + str(args), logger.DB)
sqlResult = self.execute(query, args, fetchall=fetchall, fetchone=fetchone)
+ self.connection.commit()
# get out of the connection attempt loop since we were successful
break
@@ -230,9 +229,22 @@ class DBConnection(object):
def _unicode_text_factory(self, x):
try:
- return unicode(x, 'utf-8')
- except:
- return unicode(x, sickbeard.SYS_ENCODING,errors="ignore")
+ x = unicode(x)
+ except UnicodeDecodeError:
+ try:
+ x = unicode(x, chardet.detect(x).get('encoding'))
+ except UnicodeDecodeError:
+ try:
+ x = unicode(x, sickbeard.SYS_ENCODING)
+ except UnicodeDecodeError:
+ try:
+ x = unicode(x, 'utf-8')
+ except UnicodeDecodeError:
+ try:
+ x = unicode(x, 'latin-1')
+ except UnicodeDecodeError:
+ x = unicode(x, sickbeard.SYS_ENCODING, errors="ignore")
+ return x
def _dict_factory(self, cursor, row):
d = {}
diff --git a/sickbeard/encodingKludge.py b/sickbeard/encodingKludge.py
index fadb28d0ff178d05a0f080d2936eeab2f9fc4a7c..4cb8ac9832d4fe88518b13f178e381b7cfeade7f 100644
--- a/sickbeard/encodingKludge.py
+++ b/sickbeard/encodingKludge.py
@@ -20,42 +20,33 @@ import os
import chardet
import sickbeard
-def fixStupidEncodings(x, silent=False):
- if type(x) == str:
+def _toUnicode(x):
+ if isinstance(x, str):
try:
- return x.decode(sickbeard.SYS_ENCODING)
+ x = unicode(x)
except UnicodeDecodeError:
- logger.log(u"Unable to decode value: " + repr(x), logger.ERROR)
- return None
- elif type(x) == unicode:
- return x
- else:
- logger.log(
- u"Unknown value passed in, ignoring it: " + str(type(x)) + " (" + repr(x) + ":" + repr(type(x)) + ")",
- logger.DEBUG if silent else logger.ERROR)
- return None
-
-def _toUnicode(x):
- try:
- if not isinstance(x, unicode):
- if chardet.detect(x).get('encoding') == 'utf-8':
- x = x.decode('utf-8')
- elif isinstance(x, str):
- x = x.decode(sickbeard.SYS_ENCODING)
- finally:
- return x
+ try:
+ x = unicode(x, chardet.detect(x).get('encoding'))
+ except UnicodeDecodeError:
+ try:
+ x = unicode(x, sickbeard.SYS_ENCODING)
+ except UnicodeDecodeError:
+ pass
+ return x
def ss(x):
x = _toUnicode(x)
try:
+ x = x.encode(sickbeard.SYS_ENCODING)
+ except UnicodeDecodeError, UnicodeEncodeError:
try:
+ x = x.encode('utf-8')
+ except UnicodeDecodeError, UnicodeEncodeError:
try:
- x = x.encode(sickbeard.SYS_ENCODING)
- except:
- x = x.encode(sickbeard.SYS_ENCODING, 'ignore')
- except:
- x = x.encode('utf-8', 'ignore')
+ x = x.encode(sickbeard.SYS_ENCODING, 'replace')
+ except UnicodeDecodeError, UnicodeEncodeError:
+ x = x.encode('utf-8', 'ignore')
finally:
return x
@@ -77,4 +68,4 @@ def ek(func, *args, **kwargs):
elif isinstance(result, str):
return _toUnicode(result)
else:
- return result
\ No newline at end of file
+ return result
diff --git a/sickbeard/exceptions.py b/sickbeard/exceptions.py
index 1e9114e8ffa0a20db35dd6961818b805c83143ca..58fc9df2878ad081b752a2de6235e2b5d6880757 100644
--- a/sickbeard/exceptions.py
+++ b/sickbeard/exceptions.py
@@ -33,18 +33,15 @@ def ex(e):
if arg is not None:
if isinstance(arg, (str, unicode)):
fixed_arg = ek.ss(arg)
-
else:
try:
fixed_arg = u"error " + ek.ss(str(arg))
-
except:
fixed_arg = None
if fixed_arg:
if not e_message:
e_message = fixed_arg
-
else:
e_message = e_message + " : " + fixed_arg
diff --git a/sickbeard/helpers.py b/sickbeard/helpers.py
index 03ca043b52856eff54b9f8cad4e8c6ddba87240f..f8e6d514f570849015b3602cda3bb60d55ea2e18 100644
--- a/sickbeard/helpers.py
+++ b/sickbeard/helpers.py
@@ -38,11 +38,14 @@ import datetime
import errno
import ast
import operator
+from contextlib import closing
import sickbeard
import subliminal
+import babelfish
+
import adba
-from lib import requests
+import requests
import certifi
import xmltodict
@@ -57,6 +60,7 @@ from sickbeard import notifiers
from sickbeard import clients
from lib.cachecontrol import CacheControl, caches
+
from itertools import izip, cycle
import shutil
@@ -67,6 +71,10 @@ shutil.copyfile = lib.shutil_custom.copyfile_custom
urllib._urlopener = classes.SickBeardURLopener()
+def fixGlob(path):
+ path = re.sub(r'\[', '[[]', path)
+ return re.sub(r'(?<!\[)\]', '[]]', path)
+
def indentXML(elem, level=0):
'''
Does our pretty printing, makes Matt very happy
@@ -511,7 +519,7 @@ def rename_ep_file(cur_path, new_path, old_path_length=0):
# Check if the language extracted from filename is a valid language
try:
- language = subliminal.language.Language(sublang, strict=True)
+ language = babelfish.language.Language(sublang, strict=True)
cur_file_ext = '.' + sublang + cur_file_ext
except ValueError:
pass
@@ -1283,32 +1291,55 @@ def codeDescription(status_code):
return 'unknown'
-def headURL(url, params=None, headers={}, timeout=30, session=None, json=False, proxyGlypeProxySSLwarning=None):
+def _setUpSession(session, headers):
"""
- Checks if URL is valid, without reading it
+ Returns a session initialized with default cache and parameter settings
"""
# request session
cache_dir = sickbeard.CACHE_DIR or _getTempDir()
- session = CacheControl(sess=session, cache=caches.FileCache(os.path.join(cache_dir, 'sessions')))
+ session = CacheControl(sess=session, cache=caches.FileCache(os.path.join(cache_dir, 'sessions')), cache_etags=False)
+
+ # request session clear residual referer
+ if 'Referer' in session.headers and not 'Referer' in headers:
+ session.headers.pop('Referer')
# request session headers
session.headers.update({'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate'})
session.headers.update(headers)
- # request session paramaters
+ # request session ssl verify
+ session.verify = certifi.where()
+
+ # request session allow redirects
+ session.allow_redirects = True
+
+ # request session proxies
+ if not 'Referer' in session.headers and sickbeard.PROXY_SETTING:
+ logger.log("Using proxy for url: " + url, logger.DEBUG)
+ scheme, address = urllib2.splittype(sickbeard.PROXY_SETTING)
+ address = sickbeard.PROXY_SETTING if scheme else 'http://' + sickbeard.PROXY_SETTING
+ session.proxies = {
+ "http": address,
+ "https": address,
+ }
+ session.headers.update({'Referer': address})
+
+ if 'Content-Type' in session.headers:
+ session.headers.pop('Content-Type')
+
+ return session
+
+def headURL(url, params=None, headers={}, timeout=30, session=None, json=False, proxyGlypeProxySSLwarning=None):
+ """
+ Checks if URL is valid, without reading it
+ """
+
+ session = _setUpSession(session, headers)
session.params = params
try:
- # request session proxies
- if sickbeard.PROXY_SETTING:
- logger.log("Using proxy for url: " + url, logger.DEBUG)
- session.proxies = {
- "http": sickbeard.PROXY_SETTING,
- "https": sickbeard.PROXY_SETTING,
- }
-
- resp = session.head(url)
+ resp = session.head(url, timeout=timeout)
if not resp.ok:
logger.log(u"Requested url " + url + " returned status code is " + str(
@@ -1327,46 +1358,30 @@ def headURL(url, params=None, headers={}, timeout=30, session=None, json=False,
return resp.status_code == 200
except requests.exceptions.HTTPError, e:
- logger.log(u"HTTP error " + str(e.errno) + " in headURL " + url, logger.WARNING)
+ logger.log(u"HTTP error in headURL {0}. Error: {1}".format(url,e.errno), logger.WARNING)
except requests.exceptions.ConnectionError, e:
- logger.log(u"Connection error " + str(e.message) + " in headURL " + url, logger.WARNING)
+ logger.log(u"Connection error to {0}. Error: {1}".format(url,e.message), logger.WARNING)
except requests.exceptions.Timeout, e:
- logger.log(u"Connection timed out " + str(e.message) + " in headURL " + url, logger.WARNING)
- except Exception:
- logger.log(u"Unknown exception in headURL " + url + ": " + traceback.format_exc(), logger.WARNING)
+ logger.log(u"Connection timed out accessing {0}. Error: {1}".format(url,e.message), logger.WARNING)
+ except Exception as e:
+ logger.log(u"Unknown exception in headURL {0}. Error: {1}".format(url,e.message), logger.WARNING)
+ logger.log(traceback.format_exc(), logger.WARNING)
return False
+
def getURL(url, post_data=None, params={}, headers={}, timeout=30, session=None, json=False, proxyGlypeProxySSLwarning=None):
"""
Returns a byte-string retrieved from the url provider.
"""
- # request session
- cache_dir = sickbeard.CACHE_DIR or _getTempDir()
- session = CacheControl(sess=session, cache=caches.FileCache(os.path.join(cache_dir, 'sessions')))
-
- # request session headers
- session.headers.update({'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate'})
- session.headers.update(headers)
-
- # request session ssl verify
- session.verify = certifi.where()
-
- # request session paramaters
+ session = _setUpSession(session, headers)
session.params = params
try:
- # request session proxies
- if sickbeard.PROXY_SETTING:
- logger.log("Using proxy for url: " + url, logger.DEBUG)
- session.proxies = {
- "http": sickbeard.PROXY_SETTING,
- "https": sickbeard.PROXY_SETTING,
- }
-
# decide if we get or post data to server
if post_data:
+ session.headers.update({'Content-Type': 'application/x-www-form-urlencoded'})
resp = session.post(url, data=post_data, timeout=timeout)
else:
resp = session.get(url, timeout=timeout)
@@ -1386,60 +1401,44 @@ def getURL(url, post_data=None, params={}, headers={}, timeout=30, session=None,
return
except requests.exceptions.HTTPError, e:
- logger.log(u"HTTP error " + str(e.errno) + " while loading URL " + url, logger.WARNING)
+ logger.log(u"HTTP error in getURL {0}. Error: {1}".format(url,e.errno), logger.WARNING)
return
except requests.exceptions.ConnectionError, e:
- logger.log(u"Connection error " + str(e.message) + " while loading URL " + url, logger.WARNING)
+ logger.log(u"Connection error to {0}. Error: {1}".format(url,e.message), logger.WARNING)
return
except requests.exceptions.Timeout, e:
- logger.log(u"Connection timed out " + str(e.message) + " while loading URL " + url, logger.WARNING)
+ logger.log(u"Connection timed out accessing {0}. Error: {1}".format(url,e.message), logger.WARNING)
return
- except Exception:
- logger.log(u"Unknown exception while loading URL " + url + ": " + traceback.format_exc(), logger.WARNING)
+ except Exception as e:
+ logger.log(u"Unknown exception in getURL {0}. Error: {1}".format(url,e.message), logger.WARNING)
+ logger.log(traceback.format_exc(), logger.WARNING)
return
return resp.content if not json else resp.json()
-def download_file(url, filename, session=None):
- # create session
- cache_dir = sickbeard.CACHE_DIR or _getTempDir()
- session = CacheControl(sess=session, cache=caches.FileCache(os.path.join(cache_dir, 'sessions')))
-
- # request session headers
- session.headers.update({'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate'})
- # request session ssl verify
- session.verify = certifi.where()
+def download_file(url, filename, session=None, headers={}):
- # request session streaming
+ session = _setUpSession(session, headers)
session.stream = True
- # request session proxies
- if sickbeard.PROXY_SETTING:
- logger.log("Using proxy for url: " + url, logger.DEBUG)
- session.proxies = {
- "http": sickbeard.PROXY_SETTING,
- "https": sickbeard.PROXY_SETTING,
- }
-
try:
- resp = session.get(url)
-
- if not resp.ok:
- logger.log(u"Requested url " + url + " returned status code is " + str(
- resp.status_code) + ': ' + codeDescription(resp.status_code), logger.DEBUG)
- return False
+ with closing(session.get(url)) as resp:
+ if not resp.ok:
+ logger.log(u"Requested url " + url + " returned status code is " + str(
+ resp.status_code) + ': ' + codeDescription(resp.status_code), logger.DEBUG)
+ return False
- try:
- with open(filename, 'wb') as fp:
- for chunk in resp.iter_content(chunk_size=1024):
- if chunk:
- fp.write(chunk)
- fp.flush()
+ try:
+ with open(filename, 'wb') as fp:
+ for chunk in resp.iter_content(chunk_size=1024):
+ if chunk:
+ fp.write(chunk)
+ fp.flush()
- chmodAsParent(filename)
- except:
- logger.log(u"Problem setting permissions or writing file to: %s" % filename, logger.WARNING)
+ chmodAsParent(filename)
+ except:
+ logger.log(u"Problem setting permissions or writing file to: %s" % filename, logger.WARNING)
except requests.exceptions.HTTPError, e:
_remove_file_failed(filename)
@@ -1465,39 +1464,6 @@ def download_file(url, filename, session=None):
return True
-def clearCache(force=False):
- update_datetime = datetime.datetime.now()
-
- # clean out cache directory, remove everything > 12 hours old
- if sickbeard.CACHE_DIR:
- logger.log(u"Trying to clean cache folder " + sickbeard.CACHE_DIR, logger.DEBUG)
-
- # Does our cache_dir exists
- if not ek.ek(os.path.isdir, sickbeard.CACHE_DIR):
- logger.log(u"Can't clean " + sickbeard.CACHE_DIR + " if it doesn't exist", logger.WARNING)
- else:
- max_age = datetime.timedelta(hours=12)
-
- # Get all our cache files
- exclude = ['rss', 'images']
- for cache_root, cache_dirs, cache_files in os.walk(sickbeard.CACHE_DIR, topdown=True):
- cache_dirs[:] = [d for d in cache_dirs if d not in exclude]
-
- for file in cache_files:
- cache_file = ek.ek(os.path.join, cache_root, file)
-
- if ek.ek(os.path.isfile, cache_file):
- cache_file_modified = datetime.datetime.fromtimestamp(
- ek.ek(os.path.getmtime, cache_file))
-
- if force or (update_datetime - cache_file_modified > max_age):
- try:
- ek.ek(os.remove, cache_file)
- except OSError, e:
- logger.log(u"Unable to clean " + cache_root + ": " + repr(e) + " / " + str(e),
- logger.WARNING)
- break
-
def get_size(start_path='.'):
total_size = 0
@@ -1650,22 +1616,22 @@ def isFileLocked(file, writeLockCheck=False):
@param file: the file being checked
@param writeLockCheck: when true will check if the file is locked for writing (prevents move operations)
'''
- if(not(os.path.exists(file))):
+ if not ek.ek(os.path.exists, file):
return True
try:
- f = open(file, 'r')
+ f = ek.ek(open, file, 'r')
f.close()
except IOError:
return True
if(writeLockCheck):
lockFile = file + ".lckchk"
- if(os.path.exists(lockFile)):
- os.remove(lockFile)
+ if ek.ek(os.path.exists, lockFile):
+ ek.ek(os.remove, lockFile)
try:
- os.rename(file, lockFile)
+ ek.ek(os.rename, file, lockFile)
time.sleep(1)
- os.rename(lockFile, file)
+ ek.ek(os.rename, lockFile, file)
except (OSError, IOError):
return True
diff --git a/sickbeard/history.py b/sickbeard/history.py
index 2df2590477ba97bff30202590549780b4e298834..3f4f042cbbdbb88dd477ac57eefeec10444eb47d 100644
--- a/sickbeard/history.py
+++ b/sickbeard/history.py
@@ -77,8 +77,9 @@ def logDownload(episode, filename, new_ep_quality, release_group=None, version=-
def logSubtitle(showid, season, episode, status, subtitleResult):
- resource = subtitleResult.path
- provider = subtitleResult.service
+ resource = subtitleResult.language.alpha3
+ provider = subtitleResult.provider_name
+
status, quality = Quality.splitCompositeStatus(status)
action = Quality.compositeStatus(SUBTITLED, quality)
diff --git a/sickbeard/indexers/indexer_config.py b/sickbeard/indexers/indexer_config.py
index b06fd1cf8508766a88d3d1dcb11b06d262ae3eeb..35c15e9771e9dd2eb9cb493c174389afd5501d0f 100644
--- a/sickbeard/indexers/indexer_config.py
+++ b/sickbeard/indexers/indexer_config.py
@@ -1,5 +1,6 @@
from lib.tvdb_api.tvdb_api import Tvdb
from lib.tvrage_api.tvrage_api import TVRage
+import requests
INDEXER_TVDB = 1
INDEXER_TVRAGE = 2
@@ -25,6 +26,7 @@ indexerConfig[INDEXER_TVDB] = {
'language': 'en',
'useZip': True,
},
+ 'session': requests.Session()
}
indexerConfig[INDEXER_TVRAGE] = {
@@ -34,6 +36,7 @@ indexerConfig[INDEXER_TVRAGE] = {
'api_params': {'apikey': 'Uhewg1Rr0o62fvZvUIZt',
'language': 'en',
},
+ 'session': requests.Session()
}
# TVDB Indexer Settings
diff --git a/sickbeard/metadata/helpers.py b/sickbeard/metadata/helpers.py
index ac6df9dcb21280889c2cb6fcbbd48fdcdfc1b1c2..6f3eb3ff48145bb9094aec88dbe9a64c69a7404f 100644
--- a/sickbeard/metadata/helpers.py
+++ b/sickbeard/metadata/helpers.py
@@ -18,8 +18,9 @@
from sickbeard import helpers
from sickbeard import logger
+import requests
-
+meta_session = requests.Session()
def getShowImage(url, imgNum=None):
image_data = None # @UnusedVariable
@@ -34,7 +35,7 @@ def getShowImage(url, imgNum=None):
logger.log(u"Fetching image from " + tempURL, logger.DEBUG)
- image_data = helpers.getURL(tempURL)
+ image_data = helpers.getURL(tempURL, session=meta_session)
if image_data is None:
logger.log(u"There was an error trying to retrieve the image, aborting", logger.WARNING)
return
diff --git a/sickbeard/notifiers/boxcar2.py b/sickbeard/notifiers/boxcar2.py
index 06f13a2f4ba201ace8084ecfd80ac6ff803a3ff0..61593cbc1b4dd93fbad9768ef52a50895037b2f1 100755
--- a/sickbeard/notifiers/boxcar2.py
+++ b/sickbeard/notifiers/boxcar2.py
@@ -60,10 +60,10 @@ class Boxcar2Notifier:
# send the request to boxcar2
try:
req = urllib2.Request(curUrl)
- handle = urllib2.urlopen(req, data)
+ handle = urllib2.urlopen(req, data,timeout=60)
handle.close()
- except urllib2.HTTPError, e:
+ except Exception as e:
# if we get an error back that doesn't have an error code then who knows what's really happening
if not hasattr(e, 'code'):
logger.log("Boxcar2 notification failed." + ex(e), logger.ERROR)
diff --git a/sickbeard/notifiers/kodi.py b/sickbeard/notifiers/kodi.py
index 4f9e55c90e458b145ef997031564ea6563e4173d..e577b3d823677e03551154785f8dc178e39c1092 100644
--- a/sickbeard/notifiers/kodi.py
+++ b/sickbeard/notifiers/kodi.py
@@ -429,23 +429,39 @@ class KODINotifier:
# if we're doing per-show
if showName:
+ showName = urllib.unquote_plus(showName)
tvshowid = -1
+ path = ''
+
logger.log(u"Updating library in KODI via JSON method for show " + showName, logger.DEBUG)
+ # let's try letting kodi filter the shows
+ showsCommand = '{"jsonrpc":"2.0","method":"VideoLibrary.GetTVShows","params":{"filter":{"field":"title","operator":"is","value":"%s"},"properties":["title",]},"id":"SickRage"}'
+
# get tvshowid by showName
- showsCommand = '{"jsonrpc":"2.0","method":"VideoLibrary.GetTVShows","id":1}'
- showsResponse = self._send_to_kodi_json(showsCommand, host)
+ showsResponse = self._send_to_kodi_json(showsCommand % showName, host)
if showsResponse and "result" in showsResponse and "tvshows" in showsResponse["result"]:
shows = showsResponse["result"]["tvshows"]
else:
- logger.log(u"KODI: No tvshows in KODI TV show list", logger.DEBUG)
- return False
+ # fall back to retrieving the entire show list
+ showsCommand = '{"jsonrpc":"2.0","method":"VideoLibrary.GetTVShows","id":1}'
+ showsResponse = self._send_to_kodi_json(showsCommand, host)
+
+ if showsResponse and "result" in showsResponse and "tvshows" in showsResponse["result"]:
+ shows = showsResponse["result"]["tvshows"]
+ else:
+ logger.log(u"KODI: No tvshows in KODI TV show list", logger.DEBUG)
+ return False
for show in shows:
- if (show["label"] == showName):
+ if ("label" in show and show["label"] == showName) or ("title" in show and show["title"] == showName):
tvshowid = show["tvshowid"]
- break # exit out of loop otherwise the label and showname will not match up
+ # set the path is we have it already
+ if "file" in show:
+ path = show["file"]
+
+ break
# this can be big, so free some memory
del shows
@@ -455,16 +471,19 @@ class KODINotifier:
logger.log(u'Exact show name not matched in KODI TV show list', logger.DEBUG)
return False
- # lookup tv-show path
- pathCommand = '{"jsonrpc":"2.0","method":"VideoLibrary.GetTVShowDetails","params":{"tvshowid":%d, "properties": ["file"]},"id":1}' % (
- tvshowid)
- pathResponse = self._send_to_kodi_json(pathCommand, host)
- path = pathResponse["result"]["tvshowdetails"]["file"]
+ # lookup tv-show path if we don't already know it
+ if not len(path):
+ pathCommand = '{"jsonrpc":"2.0","method":"VideoLibrary.GetTVShowDetails","params":{"tvshowid":%d, "properties": ["file"]},"id":1}' % (
+ tvshowid)
+ pathResponse = self._send_to_kodi_json(pathCommand, host)
+
+ path = pathResponse["result"]["tvshowdetails"]["file"]
+
logger.log(u"Received Show: " + showName + " with ID: " + str(tvshowid) + " Path: " + path,
logger.DEBUG)
- if (len(path) < 1):
+ if not len(path):
logger.log(u"No valid path found for " + showName + " with ID: " + str(tvshowid) + " on " + host,
logger.WARNING)
return False
@@ -490,7 +509,7 @@ class KODINotifier:
else:
logger.log(u"Doing Full Library KODI update on host: " + host, logger.DEBUG)
updateCommand = '{"jsonrpc":"2.0","method":"VideoLibrary.Scan","id":1}'
- request = self._send_to_kodi_json(updateCommand, host, sickbeard.KODI_USERNAME, sickbeard.KODI_PASSWORD)
+ request = self._send_to_kodi_json(updateCommand, host)
if not request:
logger.log(u"KODI Full Library update failed on: " + host, logger.ERROR)
diff --git a/sickbeard/notifiers/plex.py b/sickbeard/notifiers/plex.py
index 5e39138df4fbba67c1a37c1ecd44ef0272998da3..c67aa8f119dbed59dd70dcf551471e748ad874ad 100644
--- a/sickbeard/notifiers/plex.py
+++ b/sickbeard/notifiers/plex.py
@@ -26,7 +26,6 @@ import sickbeard
from sickbeard import logger
from sickbeard import common
from sickbeard.exceptions import ex
-from sickbeard.encodingKludge import fixStupidEncodings
try:
import xml.etree.cElementTree as etree
@@ -67,7 +66,7 @@ class PLEXNotifier:
enc_command = urllib.urlencode(command)
logger.log(u'PLEX: Encoded API command: ' + enc_command, logger.DEBUG)
- url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, enc_command)
+ url = u'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, enc_command)
try:
req = urllib2.Request(url)
# if we have a password, use authentication
@@ -89,7 +88,7 @@ class PLEXNotifier:
return 'OK'
except (urllib2.URLError, IOError), e:
- logger.log(u'PLEX: Warning: Couldn\'t contact Plex at ' + fixStupidEncodings(url) + ' ' + ex(e), logger.WARNING)
+ logger.log(u'PLEX: Warning: Couldn\'t contact Plex at ' + url + ' ' + ex(e), logger.WARNING)
return False
def _notify_pmc(self, message, title='SickRage', host=None, username=None, password=None, force=False):
@@ -227,6 +226,12 @@ class PLEXNotifier:
logger.log(u'PLEX: Error while trying to contact Plex Media Server: ' + ex(e), logger.WARNING)
hosts_failed.append(cur_host)
continue
+ except Exception as e:
+ if 'invalid token' in str(e):
+ logger.log(u'PLEX: Please set TOKEN in Plex settings: ', logger.ERROR)
+ else:
+ logger.log(u'PLEX: Error while trying to contact Plex Media Server: ' + ex(e), logger.ERROR)
+ continue
sections = media_container.findall('.//Directory')
if not sections:
diff --git a/sickbeard/notifiers/pushover.py b/sickbeard/notifiers/pushover.py
index 59033726e64ac896a8541b0210103a45d43e1861..31fd191fcd70f9ee3461245ca6902c3257340b38 100644
--- a/sickbeard/notifiers/pushover.py
+++ b/sickbeard/notifiers/pushover.py
@@ -57,17 +57,22 @@ class PushoverNotifier:
# send the request to pushover
try:
+ args = {
+ "token": apiKey,
+ "user": userKey,
+ "title": title.encode('utf-8'),
+ "message": msg.encode('utf-8'),
+ "timestamp": int(time.time()),
+ "retry": 60,
+ "expire": 3600,
+ }
+
+ if sickbeard.PUSHOVER_DEVICE:
+ args["device"] = sickbeard.PUSHOVER_DEVICE
+
conn = httplib.HTTPSConnection("api.pushover.net:443")
conn.request("POST", "/1/messages.json",
- urllib.urlencode({
- "token": apiKey,
- "user": userKey,
- "title": title.encode('utf-8'),
- "message": msg.encode('utf-8'),
- 'timestamp': int(time.time()),
- "retry": 60,
- "expire": 3600,
- }), {"Content-type": "application/x-www-form-urlencoded"})
+ urllib.urlencode(args), {"Content-type": "application/x-www-form-urlencoded"})
except urllib2.HTTPError, e:
# if we get an error back that doesn't have an error code then who knows what's really happening
@@ -142,7 +147,6 @@ class PushoverNotifier:
logger.log("Sending notification for " + message, logger.DEBUG)
- # self._sendPushover(message, title, userKey, apiKey)
return self._sendPushover(message, title, userKey, apiKey)
diff --git a/sickbeard/postProcessor.py b/sickbeard/postProcessor.py
index 2db0681627ac64e88398c9b2a2a2e90e8cd620c9..06b262e50a57a2865ad0a2c5356bb3536b03db10 100644
--- a/sickbeard/postProcessor.py
+++ b/sickbeard/postProcessor.py
@@ -37,7 +37,7 @@ from sickbeard import notifiers
from sickbeard import show_name_helpers
from sickbeard import failed_history
from sickbeard import name_cache
-
+from sickbeard import subtitles
from sickbeard import encodingKludge as ek
from sickbeard.exceptions import ex
@@ -187,7 +187,8 @@ class PostProcessor(object):
filelist = ek.ek(recursive_glob, ek.ek(os.path.dirname, file_path), base_name + '*') # just create the list of all files starting with the basename
else: # this is called when PP, so we need to do the filename check case-insensitive
filelist = []
- checklist = ek.ek(glob.glob, ek.ek(os.path.join, ek.ek(os.path.dirname, file_path), '*')) # get a list of all the files in the folder
+
+ checklist = ek.ek(glob.glob, helpers.fixGlob(ek.ek(os.path.join, ek.ek(os.path.dirname, file_path), '*'))) # get a list of all the files in the folder
for filefound in checklist: # loop through all the files in the folder, and check if they are the same name even when the cases don't match
file_name = filefound.rpartition('.')[0]
if not base_name_only:
@@ -299,7 +300,7 @@ class PostProcessor(object):
# check if file have subtitles language
if os.path.splitext(cur_extension)[1][1:] in common.subtitleExtensions:
cur_lang = os.path.splitext(cur_extension)[0]
- if cur_lang in sickbeard.SUBTITLES_LANGUAGES:
+ if cur_lang in subtitles.wantedSubtitles():
cur_extension = cur_lang + os.path.splitext(cur_extension)[1]
# replace .nfo with .nfo-orig to avoid conflicts
@@ -966,7 +967,7 @@ class PostProcessor(object):
else:
cur_ep.status = common.Quality.compositeStatus(common.DOWNLOADED, new_ep_quality)
- cur_ep.subtitles = []
+ cur_ep.subtitles = u''
cur_ep.subtitles_searchcount = 0
diff --git a/sickbeard/processTV.py b/sickbeard/processTV.py
index a1d49d71bca084db46160cfc7f3608486696aca7..deb05df71b173200a77541b56b6f4959a118e8fa 100644
--- a/sickbeard/processTV.py
+++ b/sickbeard/processTV.py
@@ -437,10 +437,6 @@ def already_postprocessed(dirName, videofile, force, result):
if force:
return False
- #Needed for accessing DB with a unicode DirName
- if not isinstance(dirName, unicode):
- dirName = unicode(dirName, 'utf_8')
-
# Avoid processing the same dir again if we use a process method <> move
myDB = db.DBConnection()
sqlResult = myDB.select("SELECT * FROM tv_episodes WHERE release_name = ?", [dirName])
@@ -449,10 +445,6 @@ def already_postprocessed(dirName, videofile, force, result):
return True
else:
- # This is needed for video whose name differ from dirName
- if not isinstance(videofile, unicode):
- videofile = unicode(videofile, 'utf_8')
-
sqlResult = myDB.select("SELECT * FROM tv_episodes WHERE release_name = ?", [videofile.rpartition('.')[0]])
if sqlResult:
#result.output += logHelper(u"You're trying to post process a video that's already been processed, skipping", logger.DEBUG)
diff --git a/sickbeard/providers/__init__.py b/sickbeard/providers/__init__.py
index a20125b0c13ddfda6b91fdfaf458d497a9a93768..79fb54854f3efd8169401f459f7e226e60b29e31 100644
--- a/sickbeard/providers/__init__.py
+++ b/sickbeard/providers/__init__.py
@@ -44,7 +44,10 @@ __all__ = ['womble',
'rarbg',
'tntvillage',
'binsearch',
+ 'bluetigers',
+ 'fnt',
'scenetime',
+ 'btdigg',
]
import sickbeard
diff --git a/sickbeard/providers/alpharatio.py b/sickbeard/providers/alpharatio.py
index f1d7c24d9cdde428ff9a523d278ede66bbb73ac5..8154c3933f936cc6a8ec7fca0ac0bfa60d2ea120 100644
--- a/sickbeard/providers/alpharatio.py
+++ b/sickbeard/providers/alpharatio.py
@@ -34,8 +34,8 @@ from sickbeard import show_name_helpers
from sickbeard.common import Overview
from sickbeard.exceptions import ex
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
diff --git a/sickbeard/providers/bitsoup.py b/sickbeard/providers/bitsoup.py
index 078349743328346b3151d0ad5959ba134b3c539d..8805bbc3dcb2e7316c5a49539aad1bf68a26f781 100644
--- a/sickbeard/providers/bitsoup.py
+++ b/sickbeard/providers/bitsoup.py
@@ -21,8 +21,8 @@ import traceback
import datetime
import sickbeard
import generic
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
import urllib
from sickbeard.common import Quality
@@ -88,7 +88,7 @@ class BitSoupProvider(generic.TorrentProvider):
}
if not self.session:
- self.session = requests.session()
+ self.session = requests.Session()
try:
response = self.session.post(self.urls['login'], data=login_params, timeout=30)
diff --git a/sickbeard/providers/bluetigers.py b/sickbeard/providers/bluetigers.py
new file mode 100644
index 0000000000000000000000000000000000000000..82de4a61e17ac1a4bbecf60f86c888621c435cf6
--- /dev/null
+++ b/sickbeard/providers/bluetigers.py
@@ -0,0 +1,276 @@
+# -*- coding: latin-1 -*-
+# Author: raver2046 <raver2046@gmail.com>
+# URL: http://code.google.com/p/sickbeard/
+#
+# This file is part of Sick Beard.
+#
+# Sick Beard is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Sick Beard is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Sick Beard. If not, see <http://www.gnu.org/licenses/>.
+
+import traceback
+import re
+import datetime
+import time
+from requests.auth import AuthBase
+import sickbeard
+import generic
+import urllib
+import requests
+from requests import exceptions
+from sickbeard.bs4_parser import BS4Parser
+from sickbeard.common import Quality
+from sickbeard import logger
+from sickbeard import tvcache
+from sickbeard import show_name_helpers
+from sickbeard import db
+from sickbeard import helpers
+from lib.unidecode import unidecode
+from sickbeard import classes
+from sickbeard.helpers import sanitizeSceneName
+from sickbeard.exceptions import ex
+
+
+class BLUETIGERSProvider(generic.TorrentProvider):
+ def __init__(self):
+ generic.TorrentProvider.__init__(self, "BLUETIGERS")
+
+ self.supportsBacklog = True
+ self.enabled = False
+ self.username = None
+ self.password = None
+ self.ratio = None
+ self.token = None
+ self.tokenLastUpdate = None
+
+ self.cache = BLUETIGERSCache(self)
+
+ self.urls = {'base_url': 'https://www.bluetigers.ca/',
+ 'search': 'https://www.bluetigers.ca/torrents-search.php?search=%s%s',
+ 'login': 'https://www.bluetigers.ca/account-login.php',
+ 'download': 'https://www.bluetigers.ca/torrents-details.php?id=%s&hit=1',
+ }
+
+ self.url = self.urls['base_url']
+ self.categories = "&c16=1&c10=1&c130=1&c131=1&c17=1&c18=1&c19=1"
+
+ def isEnabled(self):
+ return self.enabled
+
+ def imageName(self):
+ return 'BLUETIGERS.png'
+
+ def getQuality(self, item, anime=False):
+ quality = Quality.sceneQuality(item[0], anime)
+ return quality
+
+ def _doLogin(self):
+
+
+ if any(requests.utils.dict_from_cookiejar(self.session.cookies).values()):
+ return True
+
+
+ login_params = {'username': self.username,
+ 'password': self.password,
+ 'take_login' : '1'
+ }
+
+ if not self.session:
+ self.session = requests.Session()
+
+ logger.log('Performing authentication to BLUETIGERS', logger.DEBUG)
+ try:
+ response = self.session.post(self.urls['login'], data=login_params, timeout=30)
+ except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e:
+ logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR)
+ return False
+
+ if re.search('/account-logout.php', response.text):
+ logger.log(u'Login to ' + self.name + ' was successful.', logger.DEBUG)
+ return True
+ else:
+ logger.log(u'Login to ' + self.name + ' was unsuccessful.', logger.DEBUG)
+ return False
+
+ return True
+
+ def _get_season_search_strings(self, ep_obj):
+
+ search_string = {'Season': []}
+ for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ if ep_obj.show.air_by_date or ep_obj.show.sports:
+ ep_string = show_name + '.' + str(ep_obj.airdate).split('-')[0]
+ elif ep_obj.show.anime:
+ ep_string = show_name + '.' + "%d" % ep_obj.scene_absolute_number
+ else:
+ ep_string = show_name + '.S%02d' % int(ep_obj.scene_season) # 1) showName.SXX
+
+ search_string['Season'].append(ep_string)
+
+ return [search_string]
+
+ def _get_episode_search_strings(self, ep_obj, add_string=''):
+
+ search_string = {'Episode': []}
+
+ if not ep_obj:
+ return []
+
+ if self.show.air_by_date:
+ for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ ep_string = sanitizeSceneName(show_name) + '.' + \
+ str(ep_obj.airdate).replace('-', '|')
+ search_string['Episode'].append(ep_string)
+ elif self.show.sports:
+ for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ ep_string = sanitizeSceneName(show_name) + '.' + \
+ str(ep_obj.airdate).replace('-', '|') + '|' + \
+ ep_obj.airdate.strftime('%b')
+ search_string['Episode'].append(ep_string)
+ elif self.show.anime:
+ for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ ep_string = sanitizeSceneName(show_name) + '.' + \
+ "%i" % int(ep_obj.scene_absolute_number)
+ search_string['Episode'].append(ep_string)
+ else:
+ for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ ep_string = show_name_helpers.sanitizeSceneName(show_name) + '.' + \
+ sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season,
+ 'episodenumber': ep_obj.scene_episode} + ' %s' % add_string
+
+ search_string['Episode'].append(re.sub('\s+', '.', ep_string))
+
+ return [search_string]
+
+ def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None):
+
+ logger.log(u"_doSearch started with ..." + str(search_params), logger.DEBUG)
+
+ results = []
+ items = {'Season': [], 'Episode': [], 'RSS': []}
+
+ for mode in search_params.keys():
+
+ for search_string in search_params[mode]:
+
+ if isinstance(search_string, unicode):
+ search_string = unidecode(search_string)
+
+
+ searchURL = self.urls['search'] % (urllib.quote(search_string), self.categories)
+
+ logger.log(u"Search string: " + searchURL, logger.DEBUG)
+
+ data = self.getURL(searchURL)
+ if not data:
+ continue
+
+ try:
+ with BS4Parser(data, features=["html5lib", "permissive"]) as html:
+ result_linkz = html.findAll('a', href=re.compile("torrents-details"))
+
+ if not result_linkz:
+ logger.log(u"The Data returned from " + self.name + " do not contains any torrent",
+ logger.DEBUG)
+ continue
+
+ if result_linkz:
+ for link in result_linkz:
+ title = link.text
+ logger.log(u"BLUETIGERS TITLE TEMP: " + title, logger.DEBUG)
+ download_url = self.urls['base_url'] + "/" + link['href']
+ download_url = download_url.replace("torrents-details","download")
+ logger.log(u"BLUETIGERS downloadURL: " + download_url, logger.DEBUG)
+
+ if not title or not download_url:
+ continue
+
+ item = title, download_url
+ logger.log(u"Found result: " + title.replace(' ','.') + " (" + download_url + ")", logger.DEBUG)
+
+ items[mode].append(item)
+
+ except Exception, e:
+ logger.log(u"Failed parsing " + self.name + " Traceback: " + traceback.format_exc(), logger.ERROR)
+
+ results += items[mode]
+
+ return results
+
+ def _get_title_and_url(self, item):
+
+ title, url = item
+
+ if title:
+ title = self._clean_title_from_provider(title)
+
+ if url:
+ url = str(url).replace('&', '&')
+
+ return title, url
+
+ def findPropers(self, search_date=datetime.datetime.today()):
+
+ results = []
+
+ myDB = db.DBConnection()
+ sqlResults = myDB.select(
+ 'SELECT s.show_name, e.showid, e.season, e.episode, e.status, e.airdate FROM tv_episodes AS e' +
+ ' INNER JOIN tv_shows AS s ON (e.showid = s.indexer_id)' +
+ ' WHERE e.airdate >= ' + str(search_date.toordinal()) +
+ ' AND (e.status IN (' + ','.join([str(x) for x in Quality.DOWNLOADED]) + ')' +
+ ' OR (e.status IN (' + ','.join([str(x) for x in Quality.SNATCHED]) + ')))'
+ )
+
+ if not sqlResults:
+ return []
+
+ for sqlshow in sqlResults:
+ self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
+ if self.show:
+ curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
+ searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
+
+ for item in self._doSearch(searchString[0]):
+ title, url = self._get_title_and_url(item)
+ results.append(classes.Proper(title, url, datetime.datetime.today(), self.show))
+
+ return results
+
+ def seedRatio(self):
+ return self.ratio
+
+
+class BLUETIGERSAuth(AuthBase):
+ """Attaches HTTP Authentication to the given Request object."""
+ def __init__(self, token):
+ self.token = token
+
+ def __call__(self, r):
+ r.headers['Authorization'] = self.token
+ return r
+
+
+class BLUETIGERSCache(tvcache.TVCache):
+ def __init__(self, provider):
+ tvcache.TVCache.__init__(self, provider)
+
+ # Only poll BLUETIGERS every 10 minutes max
+ self.minTime = 10
+
+ def _getRSSData(self):
+ search_params = {'RSS': ['']}
+ return {'entries': self.provider._doSearch(search_params)}
+
+
+provider = BLUETIGERSProvider()
diff --git a/sickbeard/providers/btdigg.py b/sickbeard/providers/btdigg.py
new file mode 100644
index 0000000000000000000000000000000000000000..2b63a96d5e7f05d93dc3a24db26fde39892bdd63
--- /dev/null
+++ b/sickbeard/providers/btdigg.py
@@ -0,0 +1,166 @@
+# Author: Jodi Jones <venom@gen-x.co.nz>
+# URL: http://code.google.com/p/sickbeard/
+#
+#Ported to sickrage by: matigonkas
+#
+# This file is part of Sick Beard.
+#
+# Sick Beard is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Sick Beard is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Sick Beard. If not, see <http://www.gnu.org/licenses/>.
+
+import datetime
+import generic
+
+from sickbeard.common import Quality
+from sickbeard import logger
+from sickbeard import tvcache
+from sickbeard import helpers
+from sickbeard import show_name_helpers
+from sickbeard import db
+from sickbeard.common import WANTED
+from sickbeard.exceptions import ex
+from sickbeard.config import naming_ep_type
+
+class BTDIGGProvider(generic.TorrentProvider):
+
+ def __init__(self):
+ generic.TorrentProvider.__init__(self, "BTDigg")
+
+ self.supportsBacklog = True
+ self.url = 'https://api.btdigg.org/'
+
+ self.cache = BTDiggCache(self)
+
+ def isEnabled(self):
+ return self.enabled
+
+
+ def imageName(self):
+ return 'btdigg.png'
+
+
+ def _get_airbydate_season_range(self, season):
+ if season == None:
+ return ()
+ year, month = map(int, season.split('-'))
+ min_date = datetime.date(year, month, 1)
+ if month == 12:
+ max_date = datetime.date(year, month, 31)
+ else:
+ max_date = datetime.date(year, month+1, 1) - datetime.timedelta(days=1)
+ return (min_date, max_date)
+
+
+ def _get_season_search_strings(self, show, season=None):
+ search_string = []
+
+ if not (show and season):
+ return []
+
+ myDB = db.DBConnection()
+
+ if show.air_by_date:
+ (min_date, max_date) = self._get_airbydate_season_range(season)
+ sqlResults = myDB.select("SELECT DISTINCT airdate FROM tv_episodes WHERE showid = ? AND airdate >= ? AND airdate <= ? AND status = ?", [show.tvdbid, min_date.toordinal(), max_date.toordinal(), WANTED])
+ else:
+ sqlResults = myDB.select("SELECT DISTINCT season FROM tv_episodes WHERE showid = ? AND season = ? AND status = ?", [show.tvdbid, season, WANTED])
+
+ for sqlEp in sqlResults:
+ for show_name in set(show_name_helpers.allPossibleShowNames(show)):
+ if show.air_by_date:
+ ep_string = show_name_helpers.sanitizeSceneName(show_name) +' '+ str(datetime.date.fromordinal(sqlEp["airdate"])).replace('-', '.')
+ search_string.append(ep_string)
+ else:
+ ep_string = show_name_helpers.sanitizeSceneName(show_name) + ' S%02d' % sqlEp["season"]
+ search_string.append(ep_string)
+
+ return search_string
+
+
+ def _get_episode_search_strings(self, ep_obj, add_string=''):
+
+ if not ep_obj:
+ return []
+
+ search_string = []
+
+ for show_name in set(show_name_helpers.allPossibleShowNames(ep_obj.show)):
+ ep_string = show_name_helpers.sanitizeSceneName(show_name)
+ if ep_obj.show.air_by_date:
+ ep_string += ' ' + str(ep_obj.airdate).replace('-', '.')
+ else:
+ ep_string += ' ' + naming_ep_type[2] % {'seasonnumber': ep_obj.season, 'episodenumber': ep_obj.episode}
+
+ if len(add_string):
+ ep_string += ' %s' % add_string
+
+ search_string.append(ep_string)
+
+ return search_string
+
+
+ def _get_title_and_url(self, item):
+ title, url, size = item
+ if title:
+ title = self._clean_title_from_provider(title)
+
+ if url:
+ url = str(url).replace('&', '&')
+
+ return (title, url)
+
+
+ def _get_size(self, item):
+ title, url, size = item
+ logger.log(u'Size: %s' % size, logger.DEBUG)
+
+ return size
+
+
+ def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None):
+
+ logger.log("Performing Search: {0}".format(search_params))
+
+ # TODO: Make order configurable. 0: weight, 1: req, 2: added, 3: size, 4: files, 5
+ searchUrl = self.url + "api/private-341ada3245790954/s02?q=" + search_params + "&p=0&order=1"
+
+ jdata = self.getURL(searchUrl, json=True)
+ if not jdata:
+ logger.log("No data returned to be parsed!!!")
+ return []
+
+ logger.log("URL to be parsed: " + searchUrl, logger.DEBUG)
+
+ results = []
+
+ for torrent in jdata:
+ if not torrent['ff']:
+ results.append((torrent['name'], torrent['magnet'], torrent['size']))
+
+ return results
+
+
+class BTDiggCache(tvcache.TVCache):
+ def __init__(self, provider):
+
+ tvcache.TVCache.__init__(self, provider)
+
+ # set this 0 to suppress log line, since we aren't updating it anyways
+ self.minTime = 0
+
+ def _getRSSData(self):
+ # no rss for btdigg, can't search with empty string
+ # newest results are always > 1 day since added anyways
+ return {'entries': {}}
+
+provider = BTDIGGProvider()
diff --git a/sickbeard/providers/fnt.py b/sickbeard/providers/fnt.py
new file mode 100644
index 0000000000000000000000000000000000000000..a62ac6813140ade3f33a9568c13efccd9608afca
--- /dev/null
+++ b/sickbeard/providers/fnt.py
@@ -0,0 +1,300 @@
+# -*- coding: latin-1 -*-
+# Author: raver2046 <raver2046@gmail.com> from djoole <bobby.djoole@gmail.com>
+# URL: http://code.google.com/p/sickbeard/
+#
+# This file is part of Sick Beard.
+#
+# Sick Beard is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Sick Beard is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Sick Beard. If not, see <http://www.gnu.org/licenses/>.
+
+import traceback
+import re
+import datetime
+import time
+from requests.auth import AuthBase
+import sickbeard
+import generic
+import urllib
+import requests
+from requests import exceptions
+from sickbeard.bs4_parser import BS4Parser
+from sickbeard.common import Quality
+from sickbeard import logger
+from sickbeard import tvcache
+from sickbeard import show_name_helpers
+from sickbeard import db
+from sickbeard import helpers
+from sickbeard import classes
+from lib.unidecode import unidecode
+from sickbeard.helpers import sanitizeSceneName
+from sickbeard.exceptions import ex
+
+
+class FNTProvider(generic.TorrentProvider):
+ def __init__(self):
+ generic.TorrentProvider.__init__(self, "FNT")
+
+ self.supportsBacklog = True
+ self.enabled = False
+ self.username = None
+ self.password = None
+ self.ratio = None
+ self.minseed = None
+ self.minleech = None
+
+ self.cache = FNTCache(self)
+
+ self.urls = {'base_url': 'https://fnt.nu',
+ 'search': 'https://www.fnt.nu/torrents/recherche/?afficher=1&recherche=%s%s',
+ 'login': 'https://fnt.nu/account-login.php',
+ 'download': 'https://fnt.nu/download.php?id=%s&dl=oui',
+ }
+
+ self.url = self.urls['base_url']
+ self.categories = "&afficher=1&c118=1&c129=1&c119=1&c120=1&c121=1&c126=1&c137=1&c138=1&c146=1&c122=1&c110=1&c109=1&c135=1&c148=1&c153=1&c149=1&c150=1&c154=1&c155=1&c156=1&c114=1&visible=1&freeleech=0&nuke=1&3D=0&sort=size&order=desc"
+
+ def isEnabled(self):
+ return self.enabled
+
+ def imageName(self):
+ return 'FNT.png'
+
+ def getQuality(self, item, anime=False):
+ quality = Quality.sceneQuality(item[0], anime)
+ return quality
+
+ def _doLogin(self):
+
+
+ if any(requests.utils.dict_from_cookiejar(self.session.cookies).values()):
+ return True
+
+ login_params = {'username': self.username,
+ 'password': self.password,
+ 'submit' : 'Se loguer'
+ }
+
+ if not self.session:
+ self.session = requests.Session()
+
+ logger.log('Performing authentication to FNT', logger.DEBUG)
+ try:
+ response = self.session.post(self.urls['login'], data=login_params, timeout=30)
+ except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e:
+ logger.log(u'Unable to connect to ' + self.name + ' provider: ' + ex(e), logger.ERROR)
+ return False
+
+ if re.search('/account-logout.php', response.text):
+ logger.log(u'Login to ' + self.name + ' was successful.', logger.DEBUG)
+ return True
+ else:
+ logger.log(u'Login to ' + self.name + ' was unsuccessful.', logger.DEBUG)
+ return False
+
+ return True
+
+ def _get_season_search_strings(self, ep_obj):
+
+ search_string = {'Season': []}
+ for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ if ep_obj.show.air_by_date or ep_obj.show.sports:
+ ep_string = show_name + '.' + str(ep_obj.airdate).split('-')[0]
+ elif ep_obj.show.anime:
+ ep_string = show_name + '.' + "%d" % ep_obj.scene_absolute_number
+ else:
+ ep_string = show_name + '.S%02d' % int(ep_obj.scene_season) # 1) showName.SXX
+
+ search_string['Season'].append(ep_string)
+
+ return [search_string]
+
+ def _get_episode_search_strings(self, ep_obj, add_string=''):
+
+ search_string = {'Episode': []}
+
+ if not ep_obj:
+ return []
+
+ if self.show.air_by_date:
+ for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ ep_string = sanitizeSceneName(show_name) + '.' + \
+ str(ep_obj.airdate).replace('-', '|')
+ search_string['Episode'].append(ep_string)
+ elif self.show.sports:
+ for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ ep_string = sanitizeSceneName(show_name) + '.' + \
+ str(ep_obj.airdate).replace('-', '|') + '|' + \
+ ep_obj.airdate.strftime('%b')
+ search_string['Episode'].append(ep_string)
+ elif self.show.anime:
+ for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ ep_string = sanitizeSceneName(show_name) + '.' + \
+ "%i" % int(ep_obj.scene_absolute_number)
+ search_string['Episode'].append(ep_string)
+ else:
+ for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ ep_string = show_name_helpers.sanitizeSceneName(show_name) + '.' + \
+ sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season,
+ 'episodenumber': ep_obj.scene_episode} + ' %s' % add_string
+
+ search_string['Episode'].append(re.sub('\s+', '.', ep_string))
+
+ return [search_string]
+
+ def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None):
+
+ logger.log(u"_doSearch started with ..." + str(search_params), logger.DEBUG)
+
+ results = []
+ items = {'Season': [], 'Episode': [], 'RSS': []}
+
+ for mode in search_params.keys():
+
+
+ for search_string in search_params[mode]:
+
+ if isinstance(search_string, unicode):
+ search_string = unidecode(search_string)
+
+
+ searchURL = self.urls['search'] % (urllib.quote(search_string), self.categories)
+
+ logger.log(u"Search string: " + searchURL, logger.DEBUG)
+
+
+ data = self.getURL(searchURL)
+ if not data:
+ continue
+
+ try:
+ with BS4Parser(data, features=["html5lib", "permissive"]) as html:
+ result_table = html.find('table', {'id': 'tablealign3bis'})
+
+ if not result_table:
+ logger.log(u"The Data returned from " + self.name + " do not contains any torrent",
+ logger.DEBUG)
+ continue
+
+ if result_table:
+ rows = result_table.findAll("tr" , {"class" : "ligntorrent"} )
+
+ for row in rows:
+ link = row.findAll('td')[1].find("a" , href=re.compile("fiche_film") )
+
+ if link:
+
+ try:
+ title = link.text
+ logger.log(u"FNT TITLE : " + title, logger.DEBUG)
+ download_url = self.urls['base_url'] + "/" + row.find("a",href=re.compile("download\.php"))['href']
+ except (AttributeError, TypeError):
+ continue
+
+ if not title or not download_url:
+ continue
+
+ try:
+ id = download_url.replace(self.urls['base_url'] + "/" + 'download.php?id=', '').replace('&dl=oui', '').replace('&dl=oui', '')
+ logger.log(u"FNT id du torrent " + str(id), logger.DEBUG)
+ defailseedleech = link['mtcontent']
+ seeders = int(defailseedleech.split("<font color='#00b72e'>")[1].split("</font>")[0])
+ logger.log(u"FNT seeders : " + str(seeders), logger.DEBUG)
+ leechers = int(defailseedleech.split("<font color='red'>")[1].split("</font>")[0])
+ logger.log(u"FNT leechers : " + str(leechers), logger.DEBUG)
+ except:
+ logger.log(u"Unable to parse torrent id & seeders leechers " + self.name + " Traceback: " + traceback.format_exc(), logger.DEBUG)
+ continue
+
+ #Filter unseeded torrent
+ if mode != 'RSS' and (seeders < self.minseed or leechers < self.minleech):
+ continue
+
+ item = title, download_url , id, seeders, leechers
+ logger.log(u"Found result: " + title.replace(' ','.') + " (" + download_url + ")", logger.DEBUG)
+
+ items[mode].append(item)
+
+ except Exception, e:
+ logger.log(u"Failed parsing " + self.name + " Traceback: " + traceback.format_exc(), logger.ERROR)
+
+ results += items[mode]
+
+ return results
+
+ def _get_title_and_url(self, item):
+
+ title, url, id, seeders, leechers = item
+
+ if title:
+ title = self._clean_title_from_provider(title)
+
+ if url:
+ url = str(url).replace('&', '&')
+
+ return title, url
+
+ def findPropers(self, search_date=datetime.datetime.today()):
+
+ results = []
+
+ myDB = db.DBConnection()
+ sqlResults = myDB.select(
+ 'SELECT s.show_name, e.showid, e.season, e.episode, e.status, e.airdate FROM tv_episodes AS e' +
+ ' INNER JOIN tv_shows AS s ON (e.showid = s.indexer_id)' +
+ ' WHERE e.airdate >= ' + str(search_date.toordinal()) +
+ ' AND (e.status IN (' + ','.join([str(x) for x in Quality.DOWNLOADED]) + ')' +
+ ' OR (e.status IN (' + ','.join([str(x) for x in Quality.SNATCHED]) + ')))'
+ )
+
+ if not sqlResults:
+ return []
+
+ for sqlshow in sqlResults:
+ self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"]))
+ if self.show:
+ curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"]))
+ searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK')
+
+ for item in self._doSearch(searchString[0]):
+ title, url = self._get_title_and_url(item)
+ results.append(classes.Proper(title, url, datetime.datetime.today(), self.show))
+
+ return results
+
+ def seedRatio(self):
+ return self.ratio
+
+
+class FNTAuth(AuthBase):
+ """Attaches HTTP Authentication to the given Request object."""
+ def __init__(self, token):
+ self.token = token
+
+ def __call__(self, r):
+ r.headers['Authorization'] = self.token
+ return r
+
+
+class FNTCache(tvcache.TVCache):
+ def __init__(self, provider):
+ tvcache.TVCache.__init__(self, provider)
+
+ # Only poll FNT every 10 minutes max
+ self.minTime = 10
+
+ def _getRSSData(self):
+ search_params = {'RSS': ['']}
+ return {'entries': self.provider._doSearch(search_params)}
+
+
+provider = FNTProvider()
diff --git a/sickbeard/providers/freshontv.py b/sickbeard/providers/freshontv.py
index 16a3de6455f6d0defcae732725f6bd45d990bbd7..13a26c0e98151053adedb24962500193cfd64da1 100644
--- a/sickbeard/providers/freshontv.py
+++ b/sickbeard/providers/freshontv.py
@@ -32,8 +32,8 @@ from sickbeard import helpers
from sickbeard import show_name_helpers
from sickbeard.exceptions import ex, AuthException
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
diff --git a/sickbeard/providers/generic.py b/sickbeard/providers/generic.py
index f40d366416227f01f599996743a65f66ce2b2576..86398aa4c378cdb90cb62dbec35617e4b61ec522 100644
--- a/sickbeard/providers/generic.py
+++ b/sickbeard/providers/generic.py
@@ -24,9 +24,10 @@ import os
import re
import itertools
import urllib
+import random
import sickbeard
-from lib import requests
+import requests
from sickbeard import helpers, classes, logger, db
from sickbeard.common import MULTI_EP_RESULT, SEASON_RESULT, USER_AGENT
@@ -66,9 +67,18 @@ class GenericProvider:
self.cache = tvcache.TVCache(self)
- self.session = requests.session()
+ self.session = requests.Session()
- self.headers = {'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT}
+ self.headers = {'User-Agent': USER_AGENT}
+
+ self.btCacheURLS = [
+ 'http://torcache.net/torrent/{torrent_hash}.torrent',
+ 'http://zoink.ch/torrent/{torrent_name}.torrent',
+ 'http://torrage.com/torrent/{torrent_hash}.torrent',
+ 'http://itorrents.org/torrent/{torrent_hash}.torrent',
+ ]
+
+ random.shuffle(self.btCacheURLS)
def getID(self):
return GenericProvider.makeID(self.name)
@@ -128,8 +138,11 @@ class GenericProvider:
if self.proxy.isEnabled():
self.headers.update({'Referer': self.proxy.getProxyURL()})
- # GlypeProxy SSL warning message
self.proxyGlypeProxySSLwarning = self.proxy.getProxyURL() + 'includes/process.php?action=sslagree&submit=Continue anyway...'
+ else:
+ if 'Referer' in self.headers:
+ self.headers.pop('Referer')
+ self.proxyGlypeProxySSLwarning = None
return helpers.getURL(self.proxy._buildURL(url), post_data=post_data, params=params, headers=self.headers, timeout=timeout,
session=self.session, json=json, proxyGlypeProxySSLwarning=self.proxyGlypeProxySSLwarning)
@@ -150,11 +163,7 @@ class GenericProvider:
logger.log("Unable to extract torrent hash from link: " + ex(result.url), logger.ERROR)
return (urls, filename)
- urls = [
- 'http://torcache.net/torrent/' + torrent_hash + '.torrent',
- 'http://zoink.ch/torrent/' + torrent_name + '.torrent',
- 'http://torrage.com/torrent/' + torrent_hash + '.torrent',
- ]
+ urls = [x.format(torrent_hash=torrent_hash, torrent_name=torrent_name) for x in self.btCacheURLS]
except:
urls = [result.url]
else:
@@ -184,11 +193,15 @@ class GenericProvider:
if self.proxy.isEnabled():
self.headers.update({'Referer': self.proxy.getProxyURL()})
- # GlypeProxy SSL warning message
self.proxyGlypeProxySSLwarning = self.proxy.getProxyURL() + 'includes/process.php?action=sslagree&submit=Continue anyway...'
+ else:
+ if 'Referer' in self.headers:
+ self.headers.pop('Referer')
+ self.proxyGlypeProxySSLwarning = None
for url in urls:
- if helpers.headURL(self.proxy._buildURL(url), session=self.session, proxyGlypeProxySSLwarning=self.proxyGlypeProxySSLwarning):
+ if helpers.headURL(self.proxy._buildURL(url), session=self.session, headers=self.headers,
+ proxyGlypeProxySSLwarning=self.proxyGlypeProxySSLwarning):
return url
return u''
@@ -204,9 +217,14 @@ class GenericProvider:
urls, filename = self._makeURL(result)
+ if self.proxy.isEnabled():
+ self.headers.update({'Referer': self.proxy.getProxyURL()})
+ elif 'Referer' in self.headers:
+ self.headers.pop('Referer')
+
for url in urls:
logger.log(u"Downloading a result from " + self.name + " at " + url)
- if helpers.download_file(url, filename, session=self.session):
+ if helpers.download_file(self.proxy._buildURL(url), filename, session=self.session, headers=self.headers):
if self._verify_download(filename):
logger.log(u"Saved result to " + filename, logger.INFO)
return True
diff --git a/sickbeard/providers/hdtorrents.py b/sickbeard/providers/hdtorrents.py
index 2f6d63590ab66a14ca5aa8c90e45e52eb039197f..6bea0529f325183c0fa6fc9597850ab0731ce37b 100644
--- a/sickbeard/providers/hdtorrents.py
+++ b/sickbeard/providers/hdtorrents.py
@@ -19,7 +19,6 @@
import re
import traceback
-import datetime
import urlparse
import sickbeard
import generic
@@ -33,12 +32,14 @@ from sickbeard import helpers
from sickbeard import show_name_helpers
from sickbeard.exceptions import ex, AuthException
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
-from sickbeard.bs4_parser import BS4Parser
+import requests
+from requests import exceptions
+from bs4 import BeautifulSoup as soup
+#from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
-
+from requests.auth import AuthBase
+from datetime import datetime
class HDTorrentsProvider(generic.TorrentProvider):
def __init__(self):
@@ -48,20 +49,19 @@ class HDTorrentsProvider(generic.TorrentProvider):
self.supportsBacklog = True
self.enabled = False
- self._uid = None
- self._hash = None
+ #self._uid = None
+ #self._hash = None
+ self.session = requests.Session()
self.username = None
self.password = None
self.ratio = None
self.minseed = None
self.minleech = None
- self.urls = {'base_url': 'https://hdts.ru/index.php',
- 'login': 'https://hdts.ru/login.php',
- 'detail': 'https://www.hdts.ru/details.php?id=%s',
- 'search': 'https://hdts.ru/torrents.php?search=%s&active=1&options=0%s',
- 'download': 'https://www.sceneaccess.eu/%s',
- 'home': 'https://www.hdts.ru/%s'
+ self.urls = {'base_url': 'https://hd-torrents.org',
+ 'login': 'https://hd-torrents.org/login.php',
+ 'search': 'https://hd-torrents.org/torrents.php?search=%s&active=1&options=0%s',
+ 'home': 'https://hd-torrents.org/%s'
}
self.url = self.urls['base_url']
@@ -70,7 +70,7 @@ class HDTorrentsProvider(generic.TorrentProvider):
self.categories = "&category[]=59&category[]=60&category[]=30&category[]=38"
- self.cookies = None
+ #self.cookies = None
def isEnabled(self):
return self.enabled
@@ -78,11 +78,6 @@ class HDTorrentsProvider(generic.TorrentProvider):
def imageName(self):
return 'hdtorrents.png'
- def getQuality(self, item, anime=False):
-
- quality = Quality.sceneQuality(item[0])
- return quality
-
def _checkAuth(self):
if not self.username or not self.password:
@@ -95,9 +90,10 @@ class HDTorrentsProvider(generic.TorrentProvider):
if any(requests.utils.dict_from_cookiejar(self.session.cookies).values()):
return True
- if self._uid and self._hash:
+ # requests automatically handles cookies.
+ #if self._uid and self._hash:
- requests.utils.add_dict_to_cookiejar(self.session.cookies, self.cookies)
+ # requests.utils.add_dict_to_cookiejar(self.session.cookies, self.cookies)
else:
@@ -117,179 +113,140 @@ class HDTorrentsProvider(generic.TorrentProvider):
logger.log(u'Invalid username or password for ' + self.name + ' Check your settings', logger.ERROR)
return False
- self._uid = requests.utils.dict_from_cookiejar(self.session.cookies)['uid']
- self._hash = requests.utils.dict_from_cookiejar(self.session.cookies)['pass']
-
- self.cookies = {'uid': self._uid,
- 'pass': self._hash
- }
+ #self._uid = requests.utils.dict_from_cookiejar(self.session.cookies)['uid']
+ #self._hash = requests.utils.dict_from_cookiejar(self.session.cookies)['pass']
+ #self.cookies = {'uid': self._uid,
+ # 'pass': self._hash
+ #}
return True
def _get_season_search_strings(self, ep_obj):
+ if not ep_obj:
+ return search_strings
- search_string = {'Season': []}
+ search_strings = []
for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
if ep_obj.show.air_by_date or ep_obj.show.sports:
ep_string = show_name + ' ' + str(ep_obj.airdate).split('-')[0]
elif ep_obj.show.anime:
ep_string = show_name + ' ' + "%d" % ep_obj.scene_absolute_number
else:
- ep_string = show_name + ' S%02d' % int(ep_obj.scene_season) #1) showName SXX
+ ep_string = show_name + ' S%02d' % ep_obj.scene_season
- search_string['Season'].append(ep_string)
+ search_strings.append(ep_string)
- return [search_string]
+ return [search_strings]
def _get_episode_search_strings(self, ep_obj, add_string=''):
- search_string = {'Episode': []}
-
if not ep_obj:
- return []
+ return search_strings
- if self.show.air_by_date:
- for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ search_strings = []
+ for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ if self.show.air_by_date:
ep_string = sanitizeSceneName(show_name) + ' ' + \
str(ep_obj.airdate).replace('-', '|')
- search_string['Episode'].append(ep_string)
- elif self.show.sports:
- for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ elif self.show.sports:
ep_string = sanitizeSceneName(show_name) + ' ' + \
str(ep_obj.airdate).replace('-', '|') + '|' + \
ep_obj.airdate.strftime('%b')
- search_string['Episode'].append(ep_string)
- elif self.show.anime:
- for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ elif self.show.anime:
ep_string = sanitizeSceneName(show_name) + ' ' + \
"%i" % int(ep_obj.scene_absolute_number)
- search_string['Episode'].append(ep_string)
- else:
- for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
+ else:
ep_string = show_name_helpers.sanitizeSceneName(show_name) + ' ' + \
sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season,
- 'episodenumber': ep_obj.scene_episode} + ' %s' % add_string
+ 'episodenumber': ep_obj.scene_episode}
+ if add_string:
+ ep_string += ' %s' % add_string
- search_string['Episode'].append(re.sub('\s+', ' ', ep_string))
+ search_strings.append(ep_string)
- return [search_string]
+ return [search_strings]
def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None):
results = []
- items = {'Season': [], 'Episode': [], 'RSS': []}
if not self._doLogin():
return results
- for mode in search_params.keys():
- for search_string in search_params[mode]:
+ for search_string in search_params if search_params else '':
+ if isinstance(search_string, unicode):
+ search_string = unidecode(search_string)
- if isinstance(search_string, unicode):
- search_string = unidecode(search_string)
- if search_string == '':
- continue
- search_string = str(search_string).replace('.', ' ')
- searchURL = self.urls['search'] % (urllib.quote(search_string), self.categories)
-
- logger.log(u"Search string: " + searchURL, logger.DEBUG)
-
- data = self.getURL(searchURL)
- if not data:
- continue
-
- # Remove HDTorrents NEW list
- split_data = data.partition('<!-- Show New Torrents After Last Visit -->\n\n\n\n')
- data = split_data[2]
+ searchURL = self.urls['search'] % (urllib.quote_plus(search_string.replace('.', ' ')), self.categories)
+ logger.log(u"Search string: " + searchURL, logger.DEBUG)
+ data = self.getURL(searchURL)
+ if not data:
+ logger.log(u'No grabs for you', logger.DEBUG)
+ continue
- try:
- with BS4Parser(data, features=["html5lib", "permissive"]) as html:
- #Get first entry in table
- entries = html.find_all('td', attrs={'align': 'center'})
+ html = soup(data)
+ if not html:
+ continue
- if html.find(text='No torrents here...'):
- logger.log(u"No results found for: " + search_string + " (" + searchURL + ")", logger.DEBUG)
- continue
+ empty = html.find('No torrents here')
+ if empty:
+ continue
- if not entries:
- logger.log(u"The Data returned from " + self.name + " do not contains any torrent",
- logger.DEBUG)
- continue
+ tables = html.find('table', attrs={'class': 'mainblockcontenttt'})
+ if not tables:
+ continue
- try:
- title = entries[22].find('a')['title'].replace('History - ','').replace('Blu-ray', 'bd50')
- url = self.urls['home'] % entries[15].find('a')['href']
- download_url = self.urls['home'] % entries[15].find('a')['href']
- id = entries[23].find('div')['id']
- seeders = int(entries[20].get_text())
- leechers = int(entries[21].get_text())
- except (AttributeError, TypeError):
- continue
+ torrents = tables.findChildren('tr')
+ if not torrents:
+ continue
- if mode != 'RSS' and (seeders < self.minseed or leechers < self.minleech):
- continue
-
- if not title or not download_url:
- continue
-
- item = title, download_url, id, seeders, leechers
- logger.log(u"Found result: " + title.replace(' ','.') + " (" + searchURL + ")", logger.DEBUG)
-
- items[mode].append(item)
-
- #Now attempt to get any others
- result_table = html.find('table', attrs={'class': 'mainblockcontenttt'})
-
- if not result_table:
- continue
-
- entries = result_table.find_all('td', attrs={'align': 'center', 'class': 'listas'})
-
- if not entries:
+ # Skip column headers
+ for result in torrents[1:]:
+ try:
+ cells = result.findChildren('td', attrs={'class': re.compile(r'(green|yellow|red|mainblockcontent)')})
+ if not cells:
+ continue
+
+ title = url = seeders = leechers = None
+ size = 0
+ for cell in cells:
+ if None is title and cell.get('title') and cell.get('title') in 'Download':
+ title = re.search('f=(.*).torrent', cell.a['href']).group(1).replace('+', '.')
+ url = self.urls['home'] % cell.a['href']
+ if None is seeders and cell.get('class')[0] and cell.get('class')[0] in 'green' 'yellow' 'red':
+ seeders = int(cell.text)
+ elif None is leechers and cell.get('class')[0] and cell.get('class')[0] in 'green' 'yellow' 'red':
+ leechers = int(cell.text)
+
+ # Skip torrents released before the episode aired (fakes)
+ if re.match('..:..:.. ..:..:....', cells[6].text):
+ if (datetime.strptime(cells[6].text, '%H:%M:%S %m/%d/%Y') -
+ datetime.combine(ep_obj.airdate, datetime.min.time())).days < 0:
continue
- for result in entries:
- block2 = result.find_parent('tr').find_next_sibling('tr')
- if not block2:
- continue
- cells = block2.find_all('td')
+ # Need size for failed downloads handling
+ if re.match('[0-9]+,?\.?[0-9]* [KkMmGg]+[Bb]+', cells[7].text):
+ size = self._convertSize(cells[7].text)
- try:
- title = cells[1].find('b').get_text().strip('\t ').replace('Blu-ray', 'bd50')
- url = self.urls['home'] % cells[4].find('a')['href']
- download_url = self.urls['home'] % cells[4].find('a')['href']
- detail = cells[1].find('a')['href']
- id = detail.replace('details.php?id=', '')
- seeders = int(cells[9].get_text())
- leechers = int(cells[10].get_text())
- except (AttributeError, TypeError):
- continue
-
- if mode != 'RSS' and (seeders < self.minseed or leechers < self.minleech):
- continue
-
- if not title or not download_url:
- continue
-
- item = title, download_url, id, seeders, leechers
- logger.log(u"Found result: " + title.replace(' ','.') + " (" + searchURL + ")", logger.DEBUG)
-
- items[mode].append(item)
+ except (AttributeError, TypeError, KeyError, ValueError):
+ continue
- except Exception, e:
- logger.log(u"Failed parsing " + self.name + " Traceback: " + traceback.format_exc(), logger.ERROR)
+ if not title or not url or not seeders or not leechers or not size or \
+ seeders < self.minseed or leechers < self.minleech:
+ continue
- #For each search mode sort all the items by seeders
- items[mode].sort(key=lambda tup: tup[3], reverse=True)
+ item = title, url, seeders, leechers, size
+ logger.log(u"Found result: " + title + " (" + searchURL + ")", logger.DEBUG)
- results += items[mode]
+ results.append(item)
+ results.sort(key=lambda tup: tup[3], reverse=True)
return results
def _get_title_and_url(self, item):
- title, url, id, seeders, leechers = item
+ title, url, seeders, leechers, size = item
if title:
title = self._clean_title_from_provider(title)
@@ -299,7 +256,13 @@ class HDTorrentsProvider(generic.TorrentProvider):
return (title, url)
- def findPropers(self, search_date=datetime.datetime.today()):
+ def _get_size(self, item):
+
+ title, url, seeders, leechers, size = item
+
+ return size
+
+ def findPropers(self, search_date=datetime.today()):
results = []
@@ -324,19 +287,31 @@ class HDTorrentsProvider(generic.TorrentProvider):
for item in self._doSearch(proper_searchString[0]):
title, url = self._get_title_and_url(item)
- results.append(classes.Proper(title, url, datetime.datetime.today(), self.show))
-
+ results.append(classes.Proper(title, url, datetime.today(), self.show))
+
repack_searchString = self._get_episode_search_strings(curEp, add_string='REPACK')
for item in self._doSearch(repack_searchString[0]):
title, url = self._get_title_and_url(item)
- results.append(classes.Proper(title, url, datetime.datetime.today(), self.show))
+ results.append(classes.Proper(title, url, datetime.today(), self.show))
return results
def seedRatio(self):
return self.ratio
+ def _convertSize(self, size):
+ size, modifier = size.split(' ')
+ size = float(size)
+ if modifier in 'KB':
+ size = size * 1024
+ elif modifier in 'MB':
+ size = size * 1024**2
+ elif modifier in 'GB':
+ size = size * 1024**3
+ elif modifier in 'TB':
+ size = size * 1024**4
+ return size
class HDTorrentsCache(tvcache.TVCache):
def __init__(self, provider):
@@ -347,7 +322,7 @@ class HDTorrentsCache(tvcache.TVCache):
self.minTime = 20
def _getRSSData(self):
- search_params = {'RSS': []}
+ search_params = []
return {'entries': self.provider._doSearch(search_params)}
diff --git a/sickbeard/providers/hounddawgs.py b/sickbeard/providers/hounddawgs.py
index 07721c4e09c62a9f0cf9816f04c381c2494ecf4c..5f24eb469b549d3ae4dac374b46f4a8561ea8be6 100644
--- a/sickbeard/providers/hounddawgs.py
+++ b/sickbeard/providers/hounddawgs.py
@@ -33,8 +33,8 @@ from sickbeard import show_name_helpers
from sickbeard.common import Overview
from sickbeard.exceptions import ex
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
diff --git a/sickbeard/providers/iptorrents.py b/sickbeard/providers/iptorrents.py
index a65af362fe5b39630a99e9f5cbe9fa8c7f1b8e7c..485459f3acbac95d36cf837b1583b663200d9b44 100644
--- a/sickbeard/providers/iptorrents.py
+++ b/sickbeard/providers/iptorrents.py
@@ -34,8 +34,8 @@ from sickbeard import helpers
from sickbeard import show_name_helpers
from sickbeard.exceptions import ex, AuthException
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
@@ -116,12 +116,12 @@ class IPTorrentsProvider(generic.TorrentProvider):
if self.show.air_by_date:
for show_name in set(allPossibleShowNames(self.show)):
ep_string = sanitizeSceneName(show_name) + ' ' + \
- str(ep_obj.airdate).replace('-', '|')
+ str(ep_obj.airdate).replace('-', ' ')
search_string['Episode'].append(ep_string)
elif self.show.sports:
for show_name in set(allPossibleShowNames(self.show)):
ep_string = sanitizeSceneName(show_name) + ' ' + \
- str(ep_obj.airdate).replace('-', '|') + '|' + \
+ str(ep_obj.airdate).replace('-', ' ') + '|' + \
ep_obj.airdate.strftime('%b')
search_string['Episode'].append(ep_string)
elif self.show.anime:
diff --git a/sickbeard/providers/morethantv.py b/sickbeard/providers/morethantv.py
index 83e43f43eb3e11fa01b4018a23929a75f6dd4665..69d8c64b346aecc60ec013c5178dfbd408587d38 100644
--- a/sickbeard/providers/morethantv.py
+++ b/sickbeard/providers/morethantv.py
@@ -35,8 +35,8 @@ from sickbeard import helpers
from sickbeard import show_name_helpers
from sickbeard.exceptions import ex, AuthException
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
diff --git a/sickbeard/providers/newznab.py b/sickbeard/providers/newznab.py
index 41ce16c4aa3f78f1af7c9dfd0d21224eaa19fd6d..85db5e81c84cc5b77b9d7a091a38ed6a728c3bb8 100644
--- a/sickbeard/providers/newznab.py
+++ b/sickbeard/providers/newznab.py
@@ -118,6 +118,7 @@ class NewznabProvider(generic.NZBProvider):
try:
for category in data.feed.categories:
if category.get('name') == 'TV':
+ return_categories.append(category)
for subcat in category.subcats:
return_categories.append(subcat)
except:
@@ -254,7 +255,7 @@ class NewznabProvider(generic.NZBProvider):
self._checkAuth()
params = {"t": "tvsearch",
- "maxage": sickbeard.USENET_RETENTION,
+ "maxage": (4, age)[age],
"limit": 100,
"attrs": "rageid",
"offset": 0}
@@ -267,14 +268,14 @@ class NewznabProvider(generic.NZBProvider):
else:
params['cat'] = self.catIDs
- params['maxage'] = (4, age)[age]
-
if search_params:
params.update(search_params)
if self.needs_auth and self.key:
params['apikey'] = self.key
+ params['maxage'] = min(params['maxage'], sickbeard.USENET_RETENTION)
+
results = []
offset = total = 0
diff --git a/sickbeard/providers/nextgen.py b/sickbeard/providers/nextgen.py
index dd4a3fc78237439ea781edf9654781ef905b4d03..8d2640ac2a6e7f9182429dfeb4afccf9db9ef792 100644
--- a/sickbeard/providers/nextgen.py
+++ b/sickbeard/providers/nextgen.py
@@ -35,8 +35,8 @@ from sickbeard import show_name_helpers
from sickbeard.common import Overview
from sickbeard.exceptions import ex
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import sanitizeSceneName
diff --git a/sickbeard/providers/rarbg.py b/sickbeard/providers/rarbg.py
index 110dcf98082c0141349d86046b2b20d3abf2d0f2..69a8ab22dec1d776793f99d3da1992604d0f523f 100644
--- a/sickbeard/providers/rarbg.py
+++ b/sickbeard/providers/rarbg.py
@@ -26,8 +26,8 @@ import datetime
import json
import time
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
import sickbeard
from sickbeard.common import Quality, USER_AGENT
@@ -40,7 +40,7 @@ from sickbeard import helpers
from sickbeard import classes
from sickbeard.exceptions import ex
from sickbeard.helpers import sanitizeSceneName
-from lib.requests.exceptions import RequestException
+from requests.exceptions import RequestException
from sickbeard.indexers.indexer_config import INDEXER_TVDB,INDEXER_TVRAGE
@@ -54,7 +54,6 @@ class RarbgProvider(generic.TorrentProvider):
generic.TorrentProvider.__init__(self, "Rarbg")
self.enabled = False
- self.session = None
self.supportsBacklog = True
self.ratio = None
self.minseed = None
@@ -105,14 +104,13 @@ class RarbgProvider(generic.TorrentProvider):
if self.token and self.tokenExpireDate and datetime.datetime.now() < self.tokenExpireDate:
return True
- self.session = requests.Session()
resp_json = None
try:
response = self.session.get(self.urls['token'], timeout=30, headers=self.headers)
response.raise_for_status()
resp_json = response.json()
- except (RequestException, BaseSSLError) as e:
+ except (RequestException) as e:
logger.log(u'Unable to connect to {name} provider: {error}'.format(name=self.name, error=ex(e)), logger.ERROR)
return False
diff --git a/sickbeard/providers/rsstorrent.py b/sickbeard/providers/rsstorrent.py
index 807114e6f6083551374150c9da1e65203604cefd..a69cc2e70b097d3d248a70033368c7ffd5ac2ad9 100644
--- a/sickbeard/providers/rsstorrent.py
+++ b/sickbeard/providers/rsstorrent.py
@@ -28,7 +28,7 @@ from sickbeard import logger
from sickbeard import tvcache
from sickbeard.exceptions import ex
-from lib import requests
+import requests
from lib.bencode import bdecode
diff --git a/sickbeard/providers/scc.py b/sickbeard/providers/scc.py
index e15c963761bc43fd7a147f5cd676b1a2cbeaf5c4..57e18c5d0cba768e3a3b03417134056b82884c7f 100644
--- a/sickbeard/providers/scc.py
+++ b/sickbeard/providers/scc.py
@@ -35,8 +35,8 @@ from sickbeard import helpers
from sickbeard import show_name_helpers
from sickbeard.exceptions import ex
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
@@ -191,7 +191,10 @@ class SCCProvider(generic.TorrentProvider):
#Continue only if at least one Release is found
if len(torrent_rows) < 2:
- logger.log(u'The Data returned from %s%s does not contain any torrent' % (self.name, ('', ' (%s)' % html.title)[html.title]), logger.DEBUG)
+ info = u'The Data returned from %s does not contain any torrent' % self.name
+ if html.title:
+ info += ' (%s)' % html.title
+ logger.log(info, logger.DEBUG)
continue
for result in torrent_table.find_all('tr')[1:]:
diff --git a/sickbeard/providers/scenetime.py b/sickbeard/providers/scenetime.py
index a2911829bf6d93d67bfa06c0de2d9f75ed2e6149..7fb7e29541f87770956c6836347612ed08b0e021 100644
--- a/sickbeard/providers/scenetime.py
+++ b/sickbeard/providers/scenetime.py
@@ -32,8 +32,8 @@ from sickbeard import helpers
from sickbeard import show_name_helpers
from sickbeard.exceptions import ex
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
diff --git a/sickbeard/providers/speedcd.py b/sickbeard/providers/speedcd.py
index a37be409315d080647f71999d8f58710ff98b972..15242804f4918d9b14fc60de5dfd1aea9437eb21 100644
--- a/sickbeard/providers/speedcd.py
+++ b/sickbeard/providers/speedcd.py
@@ -33,8 +33,8 @@ from sickbeard import show_name_helpers
from sickbeard.common import Overview
from sickbeard.exceptions import ex
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.helpers import sanitizeSceneName
diff --git a/sickbeard/providers/t411.py b/sickbeard/providers/t411.py
index e48778a15ec7d882abdeb00e2393e914495d9b4e..2a11ba34f5d54902fe607e42b3e288f1fd358296 100644
--- a/sickbeard/providers/t411.py
+++ b/sickbeard/providers/t411.py
@@ -21,12 +21,12 @@ import traceback
import re
import datetime
import time
-from lib.requests.auth import AuthBase
+from requests.auth import AuthBase
import sickbeard
import generic
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.common import Quality
from sickbeard import logger
diff --git a/sickbeard/providers/thepiratebay.py b/sickbeard/providers/thepiratebay.py
index 24d9aa11c2e805c99bb1a9c9c0ecee91380c489a..370a80b2dd1bac807a8f4c60fda02025deb33339 100644
--- a/sickbeard/providers/thepiratebay.py
+++ b/sickbeard/providers/thepiratebay.py
@@ -39,8 +39,8 @@ from sickbeard.show_name_helpers import allPossibleShowNames, sanitizeSceneName
from sickbeard.common import Overview
from sickbeard.exceptions import ex
from sickbeard import encodingKludge as ek
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from lib.unidecode import unidecode
diff --git a/sickbeard/providers/tntvillage.py b/sickbeard/providers/tntvillage.py
index d4a325907ab9cb387b43122af326333fddfcf919..d8b2b9aa2feb969e0fad7c8eade4c17a9349e9b4 100644
--- a/sickbeard/providers/tntvillage.py
+++ b/sickbeard/providers/tntvillage.py
@@ -30,8 +30,8 @@ from sickbeard import helpers
from sickbeard import show_name_helpers
from sickbeard.exceptions import ex, AuthException
from sickbeard import clients
-from lib import requests
-from lib.requests.exceptions import RequestException
+import requests
+from requests.exceptions import RequestException
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
@@ -116,6 +116,11 @@ class TNTVillageProvider(generic.TorrentProvider):
'download' : 'http://forum.tntvillage.scambioetico.org/index.php?act=Attach&type=post&id=%s',
}
+ self.sub_string = [
+ 'sub',
+ 'softsub',
+ ]
+
self.url = self.urls['base_url']
self.cache = TNTVillageCache(self)
@@ -146,7 +151,7 @@ class TNTVillageProvider(generic.TorrentProvider):
login_params = {'UserName': self.username,
'PassWord': self.password,
- 'CookieDate': 1,
+ 'CookieDate': 0,
'submit': 'Connettiti al Forum',
}
@@ -296,12 +301,28 @@ class TNTVillageProvider(generic.TorrentProvider):
span_tag = (torrent_rows.find_all('td'))[1].find('b').find('span')
name = str(span_tag)
- name = name.split('sub')[0]
- if re.search("ita", name, re.I):
- logger.log(u"Found Italian release", logger.DEBUG)
- is_italian=1
+ subFound=0
+
+ for sub in self.sub_string:
+
+ if not re.search(sub, name, re.I):
+ continue
+ else:
+ subFound = 1
+ name = name.split(sub)
+
+ if re.search("ita", name[0], re.I):
+ logger.log(u"Found Italian release", logger.DEBUG)
+ is_italian=1
+ break
+
+ if not subFound:
+ if re.search("ita", name, re.I):
+ logger.log(u"Found Italian release", logger.DEBUG)
+ is_italian=1
+
return is_italian
def _is_season_pack(self, name):
diff --git a/sickbeard/providers/torrentbytes.py b/sickbeard/providers/torrentbytes.py
index a3351e32e8d7f5c27d46f9e00f255cf05b3ecaf2..94845b99954f5914d00cc00baa1f0a92e8e6da7f 100644
--- a/sickbeard/providers/torrentbytes.py
+++ b/sickbeard/providers/torrentbytes.py
@@ -32,8 +32,8 @@ from sickbeard import helpers
from sickbeard import show_name_helpers
from sickbeard.exceptions import ex
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
diff --git a/sickbeard/providers/torrentday.py b/sickbeard/providers/torrentday.py
index d4e447a03cb79e54437532a819f3c6295d564cc0..76e65166b0209df3b3afc06b3708f9fd1aa7b515 100644
--- a/sickbeard/providers/torrentday.py
+++ b/sickbeard/providers/torrentday.py
@@ -29,8 +29,8 @@ from sickbeard import helpers
from sickbeard import show_name_helpers
from sickbeard.exceptions import ex
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.helpers import sanitizeSceneName
diff --git a/sickbeard/providers/torrentleech.py b/sickbeard/providers/torrentleech.py
index 385f1fd1acbd22ff2663f28ba20e36ac3e2b4775..687ce5baf837b7a3a66715283a2bd30214eacb7d 100644
--- a/sickbeard/providers/torrentleech.py
+++ b/sickbeard/providers/torrentleech.py
@@ -34,8 +34,8 @@ from sickbeard import show_name_helpers
from sickbeard.common import Overview
from sickbeard.exceptions import ex
from sickbeard import clients
-from lib import requests
-from lib.requests import exceptions
+import requests
+from requests import exceptions
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
from sickbeard.helpers import sanitizeSceneName
diff --git a/sickbeard/scene_exceptions.py b/sickbeard/scene_exceptions.py
index 02c2c089d0823d3f994e107e02a954327b13db2d..ae1e920d0b63e122c4633d0b9e54b1d37c18e805 100644
--- a/sickbeard/scene_exceptions.py
+++ b/sickbeard/scene_exceptions.py
@@ -29,6 +29,7 @@ from sickbeard import logger
from sickbeard import db
from sickbeard import encodingKludge as ek
import os
+import requests
exception_dict = {}
anidb_exception_dict = {}
@@ -175,7 +176,7 @@ def retrieve_exceptions():
loc = sickbeard.indexerApi(indexer).config['scene_loc']
if loc.startswith("http"):
- data = helpers.getURL(loc)
+ data = helpers.getURL(loc, session=sickbeard.indexerApi(indexer).session)
else:
loc = helpers.real_path(ek.ek(os.path.join, ek.ek(os.path.dirname, __file__), loc))
with open(loc, 'r') as file:
@@ -293,8 +294,11 @@ def _anidb_exceptions_fetcher():
return anidb_exception_dict
+xem_session = requests.Session()
+
def _xem_exceptions_fetcher():
global xem_exception_dict
+ global xem_session
if shouldRefresh('xem'):
for indexer in sickbeard.indexerApi().indexers:
@@ -303,7 +307,7 @@ def _xem_exceptions_fetcher():
url = "http://thexem.de/map/allNames?origin=%s&seasonNumbers=1" % sickbeard.indexerApi(indexer).config[
'xem_origin']
- parsedJSON = helpers.getURL(url, json=True)
+ parsedJSON = helpers.getURL(url, session=xem_session, json=True)
if not parsedJSON:
logger.log(u"Check scene exceptions update failed for " + sickbeard.indexerApi(
indexer).name + ", Unable to get URL: " + url, logger.ERROR)
diff --git a/sickbeard/scene_numbering.py b/sickbeard/scene_numbering.py
index bb3f2be32e3584a428517d7d7d7101a0f778e750..4bf5a84b64328eea332bf3f9f3e9669ec1f951bf 100644
--- a/sickbeard/scene_numbering.py
+++ b/sickbeard/scene_numbering.py
@@ -496,7 +496,8 @@ def xem_refresh(indexer_id, indexer, force=False):
{'indexer_id': indexer_id})
try:
- parsedJSON = sickbeard.helpers.getURL(url, json=True)
+ from .scene_exceptions import xem_session
+ parsedJSON = sickbeard.helpers.getURL(url, session=xem_session, json=True)
if not parsedJSON or parsedJSON == '':
logger.log(u'No XEM data for show "%s on %s"' % (indexer_id, sickbeard.indexerApi(indexer).name,), logger.INFO)
return
diff --git a/sickbeard/search.py b/sickbeard/search.py
index 734ec0e127a15e6b7ed3cf45aeb941f6fcff1f6f..5c3b8e8f7325a22c7a6a10f8daecc5821a8f1027 100644
--- a/sickbeard/search.py
+++ b/sickbeard/search.py
@@ -242,6 +242,7 @@ def pickBestResult(results, show):
if len(cur_result.url) and cur_result.provider:
cur_result.url = cur_result.provider.headURL(cur_result)
if not len(cur_result.url):
+ logger.log('Skipping %s, URL check failed. Bad result from provider.' % cur_result.name,logger.INFO)
continue
if cur_result.quality in bestQualities and (not bestResult or bestResult.quality < cur_result.quality or bestResult not in bestQualities):
diff --git a/sickbeard/showUpdater.py b/sickbeard/showUpdater.py
index 6ceeefcd69bf3ab467dd4bdb32aafc1305b5e348..ad9001e40d41f9bf606af7aa014dde37012514fe 100644
--- a/sickbeard/showUpdater.py
+++ b/sickbeard/showUpdater.py
@@ -51,9 +51,6 @@ class ShowUpdater():
logger.log(u"Doing full update on all shows")
- # clean out cache directory, remove everything > 12 hours old
- sickbeard.helpers.clearCache()
-
# select 10 'Ended' tv_shows updated more than 90 days ago to include in this update
stale_should_update = []
stale_update_date = (update_date - datetime.timedelta(days=90)).toordinal()
diff --git a/sickbeard/show_queue.py b/sickbeard/show_queue.py
index 0da740f5cf638ea5df96e06e86fa4b6c2f2b5569..c87a325b56483e42b8b615de4864d6e1c36c1549 100644
--- a/sickbeard/show_queue.py
+++ b/sickbeard/show_queue.py
@@ -282,8 +282,8 @@ class QueueItemAdd(ShowQueueItem):
self._finishEarly()
return
except Exception, e:
- logger.log(u"Unable to find show ID:" + str(self.indexer_id) + " on Indexer: " + str(
- sickbeard.indexerApi(self.indexer).name), logger.ERROR)
+ logger.log(u"Show name with ID %s don't exist in %s anymore. Please change/delete your local .nfo file or remove it from your TRAKT watchlist" %
+ (self.indexer_id,sickbeard.indexerApi(self.indexer).name) , logger.ERROR)
ui.notifications.error("Unable to add show",
"Unable to look up the show in " + self.showDir + " on " + str(sickbeard.indexerApi(
self.indexer).name) + " using ID " + str(
@@ -420,6 +420,9 @@ class QueueItemAdd(ShowQueueItem):
self.show.indexer):
self.show.scene = 1
+ # After initial add, set back to WANTED.
+ self.show.default_ep_status = WANTED
+
self.finish()
def _finishEarly(self):
diff --git a/sickbeard/subtitles.py b/sickbeard/subtitles.py
index a3ba16d383960642d3433ed706624509b4514f27..0d9f17f41c481855a44019892e401eea010b8a16 100644
--- a/sickbeard/subtitles.py
+++ b/sickbeard/subtitles.py
@@ -26,39 +26,57 @@ from sickbeard import helpers
from sickbeard import encodingKludge as ek
from sickbeard import db
from sickbeard import history
-from lib import subliminal
+import subliminal
+import babelfish
+
+subliminal.cache_region.configure('dogpile.cache.memory')
+
+provider_urls = {'addic7ed': 'http://www.addic7ed.com',
+ 'opensubtitles': 'http://www.opensubtitles.org',
+ 'podnapisi': 'http://www.podnapisi.net',
+ 'thesubdb': 'http://www.thesubdb.com',
+ 'tvsubtitles': 'http://www.tvsubtitles.net'
+ }
SINGLE = 'und'
def sortedServiceList():
- servicesMapping = dict([(x.lower(), x) for x in subliminal.core.SERVICES])
-
newList = []
+ lmgtfy = 'http://lmgtfy.com/?q='
- # add all services in the priority list, in order
curIndex = 0
for curService in sickbeard.SUBTITLES_SERVICES_LIST:
- if curService in servicesMapping:
- curServiceDict = {'id': curService, 'image': curService+'.png', 'name': servicesMapping[curService], 'enabled': sickbeard.SUBTITLES_SERVICES_ENABLED[curIndex] == 1, 'api_based': __import__('lib.subliminal.services.' + curService, globals=globals(), locals=locals(), fromlist=['Service'], level=-1).Service.api_based, 'url': __import__('lib.subliminal.services.' + curService, globals=globals(), locals=locals(), fromlist=['Service'], level=-1).Service.site_url}
- newList.append(curServiceDict)
+ if curService in subliminal.provider_manager.available_providers:
+ newList.append({'name': curService,
+ 'url': provider_urls[curService] if curService in provider_urls else lmgtfy % curService,
+ 'image': curService + '.png',
+ 'enabled': sickbeard.SUBTITLES_SERVICES_ENABLED[curIndex] == 1
+ })
curIndex += 1
- # add any services that are missing from that list
- for curService in servicesMapping.keys():
- if curService not in [x['id'] for x in newList]:
- curServiceDict = {'id': curService, 'image': curService+'.png', 'name': servicesMapping[curService], 'enabled': False, 'api_based': __import__('lib.subliminal.services.' + curService, globals=globals(), locals=locals(), fromlist=['Service'], level=-1).Service.api_based, 'url': __import__('lib.subliminal.services.' + curService, globals=globals(), locals=locals(), fromlist=['Service'], level=-1).Service.site_url}
- newList.append(curServiceDict)
+ for curService in subliminal.provider_manager.available_providers:
+ if curService not in [x['name'] for x in newList]:
+ newList.append({'name': curService,
+ 'url': provider_urls[curService] if curService in provider_urls else lmgtfy % curService,
+ 'image': curService + '.png',
+ 'enabled': False,
+ })
return newList
-
+
def getEnabledServiceList():
return [x['name'] for x in sortedServiceList() if x['enabled']]
-
+
def isValidLanguage(language):
- return subliminal.language.language_list(language)
+ try:
+ langObj = babelfish.Language.fromietf(language)
+ except:
+ return False
+ return True
-def getLanguageName(selectLang):
- return subliminal.language.Language(selectLang).name
+def getLanguageName(language):
+ return babelfish.Language.fromietf(language).name
+# TODO: Filter here for non-languages in sickbeard.SUBTITLES_LANGUAGES
def wantedLanguages(sqlLike = False):
wantedLanguages = sorted(sickbeard.SUBTITLES_LANGUAGES)
if sqlLike:
@@ -67,25 +85,31 @@ def wantedLanguages(sqlLike = False):
def subtitlesLanguages(video_path):
"""Return a list detected subtitles for the given video file"""
- video = subliminal.videos.Video.from_path(video_path)
- subtitles = video.scan()
- languages = set()
- for subtitle in subtitles:
- if subtitle.language and subtitle.language.alpha2:
- languages.add(subtitle.language.alpha2)
- else:
- languages.add(SINGLE)
- return list(languages)
-
-# Return a list with languages that have alpha2 code
+ resultList = []
+ languages = subliminal.video.scan_subtitle_languages(video_path)
+
+ for language in languages:
+ if hasattr(language, 'alpha3') and language.alpha3:
+ resultList.append(language.alpha3)
+ elif hasattr(language, 'alpha2') and language.alpha2:
+ resultList.append(language.alpha2)
+
+ defaultLang = wantedLanguages()
+ if len(resultList) is 1 and len(defaultLang) is 1:
+ return defaultLang
+
+ return sorted(resultList)
+
+# TODO: Return only languages our providers allow
def subtitleLanguageFilter():
- return [language for language in subliminal.language.LANGUAGES if language[2] != ""]
+ return [language for language in babelfish.LANGUAGE_MATRIX if hasattr(language, 'alpha2') and language.alpha2]
class SubtitlesFinder():
"""
The SubtitlesFinder will be executed every hour but will not necessarly search
and download subtitles. Only if the defined rule is true
"""
+
def run(self, force=False):
if not sickbeard.USE_SUBTITLES:
return
@@ -107,11 +131,18 @@ class SubtitlesFinder():
# you have 5 minutes to understand that one. Good luck
myDB = db.DBConnection()
- sqlResults = myDB.select('SELECT s.show_name, e.showid, e.season, e.episode, e.status, e.subtitles, e.subtitles_searchcount AS searchcount, e.subtitles_lastsearch AS lastsearch, e.location, (? - e.airdate) AS airdate_daydiff FROM tv_episodes AS e INNER JOIN tv_shows AS s ON (e.showid = s.indexer_id) WHERE s.subtitles = 1 AND e.subtitles NOT LIKE (?) AND ((e.subtitles_searchcount <= 2 AND (? - e.airdate) > 7) OR (e.subtitles_searchcount <= 7 AND (? - e.airdate) <= 7)) AND (e.status IN ('+','.join([str(x) for x in Quality.DOWNLOADED])+') OR (e.status IN ('+','.join([str(x) for x in Quality.SNATCHED + Quality.SNATCHED_PROPER])+') AND e.location != ""))', [today, wantedLanguages(True), today, today])
+
+ sqlResults = myDB.select('SELECT s.show_name, e.showid, e.season, e.episode, e.status, e.subtitles, ' +
+ 'e.subtitles_searchcount AS searchcount, e.subtitles_lastsearch AS lastsearch, e.location, (? - e.airdate) AS airdate_daydiff ' +
+ 'FROM tv_episodes AS e INNER JOIN tv_shows AS s ON (e.showid = s.indexer_id) ' +
+ 'WHERE s.subtitles = 1 AND e.subtitles NOT LIKE (?) ' +
+ 'AND (e.subtitles_searchcount <= 2 OR (e.subtitles_searchcount <= 7 AND airdate_daydiff <= 7)) ' +
+ 'AND e.location != ""', [today, wantedLanguages(True)])
+
if len(sqlResults) == 0:
logger.log('No subtitles to download', logger.INFO)
return
-
+
rules = self._getRules()
now = datetime.datetime.now()
for epToSub in sqlResults:
@@ -119,7 +150,7 @@ class SubtitlesFinder():
if not ek.ek(os.path.isfile, epToSub['location']):
logger.log('Episode file does not exist, cannot download subtitles for episode %dx%d of show %s' % (epToSub['season'], epToSub['episode'], epToSub['show_name']), logger.DEBUG)
continue
-
+
# Old shows rule
throwaway = datetime.datetime.strptime('20110101', '%Y%m%d')
if ((epToSub['airdate_daydiff'] > 7 and epToSub['searchcount'] < 2 and now - datetime.datetime.strptime(epToSub['lastsearch'], '%Y-%m-%d %H:%M:%S') > datetime.timedelta(hours=rules['old'][epToSub['searchcount']])) or
@@ -136,15 +167,20 @@ class SubtitlesFinder():
if isinstance(epObj, str):
logger.log(u'Episode not found', logger.DEBUG)
return
-
+
previous_subtitles = epObj.subtitles
try:
- subtitles = epObj.downloadSubtitles()
- except:
+ epObj.downloadSubtitles()
+ except Exception as e:
logger.log(u'Unable to find subtitles', logger.DEBUG)
+ logger.log(str(e), logger.DEBUG)
return
+ newSubtitles = frozenset(epObj.subtitles).difference(previous_subtitles)
+ if newSubtitles:
+ logger.log(u'Downloaded subtitles for S%02dE%02d in %s' % (epToSub["season"], epToSub["episode"], ', '.join(newSubtitles)))
+
def _getRules(self):
"""
Define the hours to wait between 2 subtitles search depending on:
diff --git a/sickbeard/traktChecker.py b/sickbeard/traktChecker.py
index 55ed11e980e16ebc108118a090c5738ef7249d30..80d2fff963ff427c61e267aab310169d7cb3593a 100644
--- a/sickbeard/traktChecker.py
+++ b/sickbeard/traktChecker.py
@@ -228,6 +228,8 @@ class TraktChecker():
if sickbeard.TRAKT_SYNC_WATCHLIST and sickbeard.USE_TRAKT:
logger.log(u"Sync SickRage with Trakt Watchlist", logger.DEBUG)
+ self.removeShowFromSickRage()
+
if self._getShowWatchlist():
self.addShowToTraktWatchList()
self.updateShows()
@@ -306,6 +308,25 @@ class TraktChecker():
logger.log(u"SHOW_WATCHLIST::ADD::FINISH - Look for Shows to Add to Trakt Watchlist", logger.DEBUG)
+ def removeShowFromSickRage(self):
+ if sickbeard.TRAKT_SYNC_WATCHLIST and sickbeard.USE_TRAKT and sickbeard.TRAKT_REMOVE_SHOW_FROM_SICKRAGE:
+ logger.log(u"SHOW_SICKRAGE::REMOVE::START - Look for Shows to remove from SickRage", logger.DEBUG)
+
+ if sickbeard.showList is not None:
+ for show in sickbeard.showList:
+ if show.status == "Ended":
+ try:
+ progress = self.trakt_api.traktRequest("shows/" + show.imdbid + "/progress/watched") or []
+ except traktException as e:
+ logger.log(u"Could not connect to Trakt service: %s" % ex(e), logger.WARNING)
+ return
+
+ if progress['aired'] == progress['completed']:
+ show.deleteShow(full=True)
+ logger.log(u"Show: " + show.name + " has been removed from SickRage", logger.DEBUG)
+
+ logger.log(u"SHOW_SICKRAGE::REMOVE::FINISH - Trakt Show Watchlist", logger.DEBUG)
+
def updateShows(self):
logger.log(u"SHOW_WATCHLIST::CHECK::START - Trakt Show Watchlist", logger.DEBUG)
@@ -619,7 +640,7 @@ class TraktChecker():
showList.append(show)
post_data = {'shows': showList}
return post_data
-
+
class TraktRolling():
def __init__(self):
diff --git a/sickbeard/tv.py b/sickbeard/tv.py
index c70d7efb4f3e058613c6860a7127739809852f20..cd8e2f069aaf172803707394de9a9998402f57ea 100644
--- a/sickbeard/tv.py
+++ b/sickbeard/tv.py
@@ -32,7 +32,7 @@ import xml.etree.cElementTree as etree
from name_parser.parser import NameParser, InvalidNameException, InvalidShowException
-from lib import subliminal
+import subliminal
try:
from lib.send2trash import send2trash
@@ -40,7 +40,7 @@ except ImportError:
pass
from lib.imdb import imdb
-
+import logging
from sickbeard import db
from sickbeard import helpers, exceptions, logger
from sickbeard.exceptions import ex
@@ -53,7 +53,7 @@ from sickbeard.blackandwhitelist import BlackAndWhiteList
from sickbeard import sbdatetime
from sickbeard import network_timezones
from dateutil.tz import *
-from lib.subliminal.exceptions import ServiceError
+from subliminal.exceptions import Error as ServiceError
from sickbeard import encodingKludge as ek
@@ -66,6 +66,8 @@ from common import NAMING_DUPLICATE, NAMING_EXTEND, NAMING_LIMITED_EXTEND, NAMIN
import shutil
import lib.shutil_custom
+import babelfish
+
shutil.copyfile = lib.shutil_custom.copyfile_custom
@@ -412,6 +414,8 @@ class TVShow(object):
# get file list
mediaFiles = helpers.listMediaFiles(self._location)
+ logger.log(u"%s: Found files: %s" %
+ (self.indexerid, mediaFiles), logger.DEBUG)
# create TVEpisodes from each media file (if possible)
sql_l = []
@@ -454,12 +458,13 @@ class TVShow(object):
try:
curEpisode.refreshSubtitles()
except:
- logger.log(str(self.indexerid) + ": Could not refresh subtitles", logger.ERROR)
+ logger.log("%s: Could not refresh subtitles" % self.indexerid, logger.ERROR)
logger.log(traceback.format_exc(), logger.DEBUG)
sql_l.append(curEpisode.get_sql())
- if len(sql_l) > 0:
+
+ if sql_l:
myDB = db.DBConnection()
myDB.mass_action(sql_l)
@@ -1109,21 +1114,22 @@ class TVShow(object):
with curEp.lock:
curEp.airdateModifyStamp()
- if len(sql_l) > 0:
+ if sql_l:
myDB = db.DBConnection()
myDB.mass_action(sql_l)
def downloadSubtitles(self, force=False):
+ # TODO: Add support for force option
if not ek.ek(os.path.isdir, self._location):
logger.log(str(self.indexerid) + ": Show dir doesn't exist, can't download subtitles", logger.DEBUG)
return
- logger.log(str(self.indexerid) + ": Downloading subtitles", logger.DEBUG)
+ logger.log("%s: Downloading subtitles" % self.indexerid, logger.DEBUG)
try:
episodes = self.getAllEpisodes(has_location=True)
- if not len(episodes) > 0:
- logger.log(str(self.indexerid) + ": No episodes to download subtitles for " + self.name, logger.DEBUG)
+ if not episodes:
+ logger.log("%s: No episodes to download subtitles for %s" % (self.indexerid, self.name), logger.DEBUG)
return
for episode in episodes:
@@ -1422,79 +1428,139 @@ class TVEpisode(object):
self.subtitles = subtitles.subtitlesLanguages(self.location)
def downloadSubtitles(self, force=False):
- # TODO: Add support for force option
if not ek.ek(os.path.isfile, self.location):
- logger.log(
- str(self.show.indexerid) + ": Episode file doesn't exist, can't download subtitles for episode " + str(
- self.season) + "x" + str(self.episode), logger.DEBUG)
+ logger.log(u"%s: Episode file doesn't exist, can't download subtitles for S%02dE%02d" %
+ (self.show.indexerid, self.season, self.episode), logger.DEBUG)
return
- logger.log(str(self.show.indexerid) + ": Downloading subtitles for episode " + str(self.season) + "x" + str(
- self.episode), logger.DEBUG)
+
+ logger.log(u"%s: Downloading subtitles for S%02dE%02d" % (self.show.indexerid, self.season, self.episode), logger.DEBUG)
previous_subtitles = self.subtitles
- added_subtitles = []
+
+ #logging.getLogger('subliminal.api').addHandler(logging.StreamHandler())
+ #logging.getLogger('subliminal.api').setLevel(logging.DEBUG)
+ #logging.getLogger('subliminal').addHandler(logging.StreamHandler())
+ #logging.getLogger('subliminal').setLevel(logging.DEBUG)
try:
- need_languages = set(sickbeard.SUBTITLES_LANGUAGES) - set(self.subtitles)
- subtitles = subliminal.download_subtitles([self.location], languages=need_languages,
- services=sickbeard.subtitles.getEnabledServiceList(), force=force,
- multi=sickbeard.SUBTITLES_MULTI, cache_dir=sickbeard.CACHE_DIR)
-
- if sickbeard.SUBTITLES_DIR:
- for video in subtitles:
- subs_new_path = ek.ek(os.path.join, os.path.dirname(video.path), sickbeard.SUBTITLES_DIR)
+ languages = set()
+ for language in frozenset(subtitles.wantedLanguages()).difference(self.subtitles):
+ languages.add(babelfish.Language.fromietf(language))
+
+ if not languages:
+ logger.log(u'%s: No missing subtitles for S%02dE%02d' % (self.show.indexerid, self.season, self.episode), logger.DEBUG)
+ return
+
+ providers = sickbeard.subtitles.getEnabledServiceList()
+ vname = self.location
+ create_link = False
+ if self.release_name:
+ try:
+ dir, name = ek.ek(os.path.split, self.location)
+ tmp = ek.ek(os.path.join, dir, self.release_name)
+ if ek.ek(os.path.splitext, tmp)[-1] is not ek.ek(os.path.splitext, self.location)[-1]:
+ tmp += ek.ek(os.path.splitext, self.location)[-1]
+
+ create_link = not ek.ek(os.path.exists, tmp)
+ if create_link:
+ ek.ek(helpers.link, self.location, tmp)
+ vname = tmp
+ except Exception:
+ create_link = False
+ vname = self.location
+ pass
+ else:
+ logger.log(u'%s: No release name for S%02dE%02d, using existing file name' %
+ (self.show.indexerid, self.season, self.episode), logger.DEBUG)
+
+ video = None
+ try:
+ video = subliminal.scan_video(vname, subtitles=not force, embedded_subtitles=not sickbeard.EMBEDDED_SUBTITLES_ALL or not force)
+ except Exception:
+ logger.log(u'%s: Exception caught in subliminal.scan_video for S%02dE%02d' %
+ (self.show.indexerid, self.season, self.episode), logger.DEBUG)
+ if create_link and vname is not self.location:
+ ek.ek(os.unlink, vname)
+ return
+ pass
+
+ if create_link and vname is not self.location:
+ ek.ek(os.unlink, vname)
+
+ video.tvdb_id = self.show.indexerid
+ video.imdb_id = self.show.imdbid
+ if not video.title and self.name:
+ video.title = u'' + self.name
+ if not video.release_group and self.release_group:
+ video.release_group = self.release_group
+
+ # TODO: Add gui option for hearing_impaired parameter ?
+ foundSubs = subliminal.download_best_subtitles([video], languages=languages, providers=providers, single=not sickbeard.SUBTITLES_MULTI, hearing_impaired=False)
+ if not foundSubs:
+ logger.log(u'%s: No subtitles found for S%02dE%02d on any provider' % (self.show.indexerid, self.season, self.episode), logger.DEBUG)
+ return
+
+ if len(sickbeard.SUBTITLES_DIR):
+ # absolute path (GUI 'Browse' button, or typed absolute path) - sillyness?
+ if ek.ek(os.path.isdir, sickbeard.SUBTITLES_DIR):
+ subs_new_path = ek.ek(os.path.join, sickbeard.SUBTITLES_DIR, self.show.title)
+ dir_exists = True
+ else:
+ # relative to the folder the episode is in - sillyness?
+ subs_new_path = ek.ek(os.path.join, os.path.dirname(self.location), sickbeard.SUBTITLES_DIR)
dir_exists = helpers.makeDir(subs_new_path)
- if not dir_exists:
- logger.log(u"Unable to create subtitles folder " + subs_new_path, logger.ERROR)
- else:
- helpers.chmodAsParent(subs_new_path)
- for subtitle in subtitles.get(video):
- added_subtitles.append(subtitle.language.alpha2)
- new_file_path = ek.ek(os.path.join, subs_new_path, os.path.basename(subtitle.path))
- helpers.moveFile(subtitle.path, new_file_path)
- helpers.chmodAsParent(new_file_path)
+ if not dir_exists:
+ logger.log(u'Unable to create subtitles folder ' + subs_new_path, logger.ERROR)
+ else:
+ helpers.chmodAsParent(subs_new_path)
else:
- for video in subtitles:
- for subtitle in subtitles.get(video):
- added_subtitles.append(subtitle.language.alpha2)
- helpers.chmodAsParent(subtitle.path)
- except ServiceError as e:
- logger.log("Service is unavailable: {0}".format(str(e)), logger.INFO)
- return
+ # let subliminal save the subtitles next to the episode
+ subs_new_path = None
+
+ subliminal.save_subtitles(foundSubs, directory=subs_new_path, single=not sickbeard.SUBTITLES_MULTI)
+
+ # rename the subtitles if needed
+ if create_link:
+ for video, subs in foundSubs.items():
+ for sub in subs:
+ path = subliminal.subtitle.get_subtitle_path(video.name, sub.language if sickbeard.SUBTITLES_MULTI else None)
+ if subs_new_path:
+ path = ek.ek(os.path.join, subs_new_path, ek.ek(os.path.split, path)[1])
+ new_path = path.replace(ek.ek(os.path.splitext, release_name)[0],
+ ek.ek(os.path.splitext, ek.ek(os.path.basename, self.location))[0])
+ if ek.ek(os.path.exists, path) and not ek.ek(os.path.exists, newpath):
+ ek.ek(os.rename, path, newpath)
+
except Exception as e:
- logger.log("Error occurred when downloading subtitles: " + str(e), logger.ERROR)
+ logger.log("Error occurred when downloading subtitles: " + traceback.format_exc(), logger.ERROR)
return
- if sickbeard.SUBTITLES_MULTI:
- self.refreshSubtitles()
- else:
- self.subtitles = added_subtitles
+ self.refreshSubtitles()
- self.subtitles_searchcount = self.subtitles_searchcount + 1 if self.subtitles_searchcount else 1 # added the if because sometime it raise an error
+ self.subtitles_searchcount += 1 if self.subtitles_searchcount else 1
self.subtitles_lastsearch = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
self.saveToDB()
- newsubtitles = set(self.subtitles).difference(set(previous_subtitles))
-
- if newsubtitles:
- subtitleList = ", ".join(subliminal.language.Language(x).name for x in newsubtitles)
- logger.log(str(self.show.indexerid) + u": Downloaded " + subtitleList + " subtitles for episode " + str(
- self.season) + "x" + str(self.episode), logger.DEBUG)
+ newSubtitles = frozenset(self.subtitles).difference(previous_subtitles)
+ if newSubtitles:
+ subtitleList = ", ".join([babelfish.Language.fromietf(newSub).name for newSub in newSubtitles])
+ logger.log(u"%s: Downloaded %s subtitles for S%02dE%02d" %
+ (self.show.indexerid, subtitleList, self.season, self.episode), logger.DEBUG)
notifiers.notify_subtitle_download(self.prettyName(), subtitleList)
-
else:
- logger.log(
- str(self.show.indexerid) + u": No subtitles downloaded for episode " + str(self.season) + "x" + str(
- self.episode), logger.DEBUG)
+ logger.log(u"%s: No subtitles downloaded for S%02dE%02d" %
+ (self.show.indexerid, self.season, self.episode), logger.DEBUG)
if sickbeard.SUBTITLES_HISTORY:
- for video in subtitles:
- for subtitle in subtitles.get(video):
- history.logSubtitle(self.show.indexerid, self.season, self.episode, self.status, subtitle)
+ for video, subs in foundSubs.items():
+ for sub in subs:
+ logger.log(u'history.logSubtitle %s, %s' % (sub.provider_name, sub.language.alpha3), logger.DEBUG)
+ history.logSubtitle(self.show.indexerid, self.season, self.episode, self.status, sub)
+
+ return self.subtitles
- return subtitles
def checkForMetaFiles(self):
@@ -1978,60 +2044,62 @@ class TVEpisode(object):
forceSave: If True it will create SQL queue even if no data has been changed since the
last save (aka if the record is not dirty).
"""
+ try:
+ if not self.dirty and not forceSave:
+ logger.log(str(self.show.indexerid) + u": Not creating SQL queue - record is not dirty", logger.DEBUG)
+ return
- if not self.dirty and not forceSave:
- logger.log(str(self.show.indexerid) + u": Not creating SQL queue - record is not dirty", logger.DEBUG)
- return
-
- myDB = db.DBConnection()
- rows = myDB.select(
- 'SELECT episode_id, subtitles FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?',
- [self.show.indexerid, self.season, self.episode])
-
- epID = None
- if rows:
- epID = int(rows[0]['episode_id'])
-
- if epID:
- # use a custom update method to get the data into the DB for existing records.
- # Multi or added subtitle or removed subtitles
- if sickbeard.SUBTITLES_MULTI or not rows[0]['subtitles'] or not self.subtitles:
- return [
- "UPDATE tv_episodes SET indexerid = ?, indexer = ?, name = ?, description = ?, subtitles = ?, "
- "subtitles_searchcount = ?, subtitles_lastsearch = ?, airdate = ?, hasnfo = ?, hastbn = ?, status = ?, "
- "location = ?, file_size = ?, release_name = ?, is_proper = ?, showid = ?, season = ?, episode = ?, "
- "absolute_number = ?, version = ?, release_group = ? WHERE episode_id = ?",
- [self.indexerid, self.indexer, self.name, self.description, ",".join([sub for sub in self.subtitles]),
- self.subtitles_searchcount, self.subtitles_lastsearch, self.airdate.toordinal(), self.hasnfo,
- self.hastbn,
- self.status, self.location, self.file_size, self.release_name, self.is_proper, self.show.indexerid,
- self.season, self.episode, self.absolute_number, self.version, self.release_group, epID]]
+ myDB = db.DBConnection()
+ rows = myDB.select(
+ 'SELECT episode_id, subtitles FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?',
+ [self.show.indexerid, self.season, self.episode])
+
+ epID = None
+ if rows:
+ epID = int(rows[0]['episode_id'])
+
+ if epID:
+ # use a custom update method to get the data into the DB for existing records.
+ # Multi or added subtitle or removed subtitles
+ if sickbeard.SUBTITLES_MULTI or not rows[0]['subtitles'] or not self.subtitles:
+ return [
+ "UPDATE tv_episodes SET indexerid = ?, indexer = ?, name = ?, description = ?, subtitles = ?, "
+ "subtitles_searchcount = ?, subtitles_lastsearch = ?, airdate = ?, hasnfo = ?, hastbn = ?, status = ?, "
+ "location = ?, file_size = ?, release_name = ?, is_proper = ?, showid = ?, season = ?, episode = ?, "
+ "absolute_number = ?, version = ?, release_group = ? WHERE episode_id = ?",
+ [self.indexerid, self.indexer, self.name, self.description, ",".join(self.subtitles),
+ self.subtitles_searchcount, self.subtitles_lastsearch, self.airdate.toordinal(), self.hasnfo,
+ self.hastbn,
+ self.status, self.location, self.file_size, self.release_name, self.is_proper, self.show.indexerid,
+ self.season, self.episode, self.absolute_number, self.version, self.release_group, epID]]
+ else:
+ # Don't update the subtitle language when the srt file doesn't contain the alpha2 code, keep value from subliminal
+ return [
+ "UPDATE tv_episodes SET indexerid = ?, indexer = ?, name = ?, description = ?, "
+ "subtitles_searchcount = ?, subtitles_lastsearch = ?, airdate = ?, hasnfo = ?, hastbn = ?, status = ?, "
+ "location = ?, file_size = ?, release_name = ?, is_proper = ?, showid = ?, season = ?, episode = ?, "
+ "absolute_number = ?, version = ?, release_group = ? WHERE episode_id = ?",
+ [self.indexerid, self.indexer, self.name, self.description,
+ self.subtitles_searchcount, self.subtitles_lastsearch, self.airdate.toordinal(), self.hasnfo,
+ self.hastbn,
+ self.status, self.location, self.file_size, self.release_name, self.is_proper, self.show.indexerid,
+ self.season, self.episode, self.absolute_number, self.version, self.release_group, epID]]
else:
- # Don't update the subtitle language when the srt file doesn't contain the alpha2 code, keep value from subliminal
+ # use a custom insert method to get the data into the DB.
return [
- "UPDATE tv_episodes SET indexerid = ?, indexer = ?, name = ?, description = ?, "
- "subtitles_searchcount = ?, subtitles_lastsearch = ?, airdate = ?, hasnfo = ?, hastbn = ?, status = ?, "
- "location = ?, file_size = ?, release_name = ?, is_proper = ?, showid = ?, season = ?, episode = ?, "
- "absolute_number = ?, version = ?, release_group = ? WHERE episode_id = ?",
- [self.indexerid, self.indexer, self.name, self.description,
- self.subtitles_searchcount, self.subtitles_lastsearch, self.airdate.toordinal(), self.hasnfo,
- self.hastbn,
- self.status, self.location, self.file_size, self.release_name, self.is_proper, self.show.indexerid,
- self.season, self.episode, self.absolute_number, self.version, self.release_group, epID]]
- else:
- # use a custom insert method to get the data into the DB.
- return [
- "INSERT OR IGNORE INTO tv_episodes (episode_id, indexerid, indexer, name, description, subtitles, "
- "subtitles_searchcount, subtitles_lastsearch, airdate, hasnfo, hastbn, status, location, file_size, "
- "release_name, is_proper, showid, season, episode, absolute_number, version, release_group) VALUES "
- "((SELECT episode_id FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?)"
- ",?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);",
- [self.show.indexerid, self.season, self.episode, self.indexerid, self.indexer, self.name,
- self.description,
- ",".join([sub for sub in self.subtitles]), self.subtitles_searchcount, self.subtitles_lastsearch,
- self.airdate.toordinal(), self.hasnfo, self.hastbn, self.status, self.location, self.file_size,
- self.release_name, self.is_proper, self.show.indexerid, self.season, self.episode,
- self.absolute_number, self.version, self.release_group]]
+ "INSERT OR IGNORE INTO tv_episodes (episode_id, indexerid, indexer, name, description, subtitles, "
+ "subtitles_searchcount, subtitles_lastsearch, airdate, hasnfo, hastbn, status, location, file_size, "
+ "release_name, is_proper, showid, season, episode, absolute_number, version, release_group) VALUES "
+ "((SELECT episode_id FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?)"
+ ",?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);",
+ [self.show.indexerid, self.season, self.episode, self.indexerid, self.indexer, self.name,
+ self.description, ",".join(self.subtitles), self.subtitles_searchcount, self.subtitles_lastsearch,
+ self.airdate.toordinal(), self.hasnfo, self.hastbn, self.status, self.location, self.file_size,
+ self.release_name, self.is_proper, self.show.indexerid, self.season, self.episode,
+ self.absolute_number, self.version, self.release_group]]
+ except Exception as e:
+ logger.log(u"Error while updating database: %s" %
+ (repr(e)), logger.ERROR)
def saveToDB(self, forceSave=False):
"""
@@ -2053,7 +2121,7 @@ class TVEpisode(object):
"indexer": self.indexer,
"name": self.name,
"description": self.description,
- "subtitles": ",".join([sub for sub in self.subtitles]),
+ "subtitles": ",".join(self.subtitles),
"subtitles_searchcount": self.subtitles_searchcount,
"subtitles_lastsearch": self.subtitles_lastsearch,
"airdate": self.airdate.toordinal(),
@@ -2536,6 +2604,7 @@ class TVEpisode(object):
related_files = postProcessor.PostProcessor(self.location).list_associated_files(
self.location, base_name_only=True, subfolders=True)
+ #This is wrong. Cause of pp not moving subs.
if self.show.subtitles and sickbeard.SUBTITLES_DIR != '':
related_subs = postProcessor.PostProcessor(self.location).list_associated_files(sickbeard.SUBTITLES_DIR,
subtitles_only=True, subfolders=True)
diff --git a/sickbeard/tvcache.py b/sickbeard/tvcache.py
index 6b3db41511c30ba961de401a44e62328c736240b..03b2312868ad894265b015695a2d5126d3910eff 100644
--- a/sickbeard/tvcache.py
+++ b/sickbeard/tvcache.py
@@ -144,12 +144,11 @@ class TVCache():
elif sickbeard.PROXY_SETTING:
logger.log("Using proxy for url: " + url, logger.DEBUG)
scheme, address = urllib2.splittype(sickbeard.PROXY_SETTING)
- if not scheme:
- scheme = 'http'
- address = 'http://' + sickbeard.PROXY_SETTING
- else:
- address = sickbeard.PROXY_SETTING
- handlers = [urllib2.ProxyHandler({scheme: address})]
+ address = sickbeard.PROXY_SETTING if scheme else 'http://' + sickbeard.PROXY_SETTING
+ handlers = [urllib2.ProxyHandler({'http': address, 'https': address})]
+ self.provider.headers.update({'Referer': address})
+ elif 'Referer' in self.provider.headers:
+ self.provider.headers.pop('Referer')
return RSSFeeds(self.providerID).getFeed(
self.provider.proxy._buildURL(url),
diff --git a/sickbeard/versionChecker.py b/sickbeard/versionChecker.py
index f4ada5650afd1f6728afb3f1edfe74873163a86f..29a6551fc1c18d969dc46adf3b40ee4f0bc1270b 100644
--- a/sickbeard/versionChecker.py
+++ b/sickbeard/versionChecker.py
@@ -33,8 +33,8 @@ from sickbeard import ui
from sickbeard import logger, helpers
from sickbeard.exceptions import ex
from sickbeard import encodingKludge as ek
-from lib import requests
-from lib.requests.exceptions import RequestException
+import requests
+from requests.exceptions import RequestException
import shutil
import lib.shutil_custom
@@ -68,7 +68,7 @@ class CheckVersion():
if sickbeard.AUTO_UPDATE:
logger.log(u"New update found for SickRage, starting auto-updater ...")
ui.notifications.message('New update found for SickRage, starting auto-updater')
- if self.safe_to_update() == True and self._runbackup() == True:
+ if self.run_backup_if_safe() is True:
if sickbeard.versionCheckScheduler.action.update():
logger.log(u"Update was successful!")
ui.notifications.message('Update was successful')
@@ -77,6 +77,9 @@ class CheckVersion():
logger.log(u"Update failed!")
ui.notifications.message('Update failed!')
+ def run_backup_if_safe(self):
+ return self.safe_to_update() is True and self._runbackup() is True
+
def _runbackup(self):
# Do a system backup before update
logger.log(u"Config backup in progress...")
@@ -293,15 +296,28 @@ class GitUpdateManager(UpdateManager):
self.github_org = self.get_github_org()
self.github_repo = self.get_github_repo()
- self.branch = sickbeard.BRANCH
- if sickbeard.BRANCH == '':
- self.branch = self._find_installed_branch()
+ sickbeard.BRANCH = self.branch = self._find_installed_branch()
self._cur_commit_hash = None
self._newest_commit_hash = None
self._num_commits_behind = 0
self._num_commits_ahead = 0
+ def get_cur_commit_hash(self):
+ return self._cur_commit_hash
+
+ def get_newest_commit_hash(self):
+ return self._newest_commit_hash
+
+ def get_cur_version(self):
+ return self._run_git(self._git_path, "describe --abbrev=0 " + self._cur_commit_hash)[0]
+
+ def get_newest_version(self):
+ return self._run_git(self._git_path, "describe --abbrev=0 " + self._newest_commit_hash)[0]
+
+ def get_num_commits_behind(self):
+ return self._num_commits_behind
+
def _git_error(self):
error_message = 'Unable to find your git executable - Shutdown SickRage and EITHER set git_path in your config.ini OR delete your .git folder and run from source to enable updates.'
sickbeard.NEWEST_VERSION_STRING = error_message
@@ -360,7 +376,7 @@ class GitUpdateManager(UpdateManager):
output = err = exit_status = None
if not git_path:
- logger.log(u"No git specified, can't use git commands", logger.ERROR)
+ logger.log(u"No git specified, can't use git commands", logger.WARNING)
exit_status = 1
return (output, err, exit_status)
@@ -393,7 +409,7 @@ class GitUpdateManager(UpdateManager):
exit_status = 1
elif exit_status == 128 or 'fatal:' in output or err:
- logger.log(cmd + u" returned : " + str(output), logger.ERROR)
+ logger.log(cmd + u" returned : " + str(output), logger.WARNING)
exit_status = 128
else:
@@ -429,6 +445,7 @@ class GitUpdateManager(UpdateManager):
if exit_status == 0 and branch_info:
branch = branch_info.strip().replace('refs/heads/', '', 1)
if branch:
+ sickbeard.BRANCH = branch
return branch
return ""
@@ -491,7 +508,7 @@ class GitUpdateManager(UpdateManager):
sickbeard.NEWEST_VERSION_STRING = None
if self._num_commits_ahead:
- logger.log(u"Local branch is ahead of " + self.branch + ". Automatic update not possible.", logger.ERROR)
+ logger.log(u"Local branch is ahead of " + self.branch + ". Automatic update not possible.", logger.WARNING)
newest_text = "Local branch is ahead of " + self.branch + ". Automatic update not possible."
elif self._num_commits_behind > 0:
@@ -526,7 +543,7 @@ class GitUpdateManager(UpdateManager):
try:
self._check_github_for_update()
except Exception, e:
- logger.log(u"Unable to contact github, can't check for update: " + repr(e), logger.ERROR)
+ logger.log(u"Unable to contact github, can't check for update: " + repr(e), logger.WARNING)
return False
if self._num_commits_behind > 0:
@@ -585,6 +602,7 @@ class GitUpdateManager(UpdateManager):
def list_remote_branches(self):
# update remote origin url
self.update_remote_origin()
+ sickbeard.BRANCH = self._find_installed_branch()
branches, err, exit_status = self._run_git(self._git_path, 'ls-remote --heads %s' % sickbeard.GIT_REMOTE) # @UnusedVariable
if exit_status == 0 and branches:
@@ -593,7 +611,7 @@ class GitUpdateManager(UpdateManager):
return []
def update_remote_origin(self):
- self._run_git(self._git_path, 'config remote.origin.url %s' % sickbeard.GIT_REMOTE_URL)
+ self._run_git(self._git_path, 'config remote.%s.url %s' % (sickbeard.GIT_REMOTE, sickbeard.GIT_REMOTE_URL))
class SourceUpdateManager(UpdateManager):
def __init__(self):
@@ -613,13 +631,28 @@ class SourceUpdateManager(UpdateManager):
return "master"
else:
return sickbeard.CUR_COMMIT_BRANCH
-
+
+ def get_cur_commit_hash(self):
+ return self._cur_commit_hash
+
+ def get_newest_commit_hash(self):
+ return self._newest_commit_hash
+
+ def get_cur_version(self):
+ return ""
+
+ def get_newest_version(self):
+ return ""
+
+ def get_num_commits_behind(self):
+ return self._num_commits_behind
+
def need_update(self):
# need this to run first to set self._newest_commit_hash
try:
self._check_github_for_update()
except Exception, e:
- logger.log(u"Unable to contact github, can't check for update: " + repr(e), logger.ERROR)
+ logger.log(u"Unable to contact github, can't check for update: " + repr(e), logger.WARNING)
return False
if self.branch != self._find_installed_branch():
@@ -719,7 +752,7 @@ class SourceUpdateManager(UpdateManager):
urllib.urlretrieve(tar_download_url, tar_download_path)
if not ek.ek(os.path.isfile, tar_download_path):
- logger.log(u"Unable to retrieve new version from " + tar_download_url + ", can't update", logger.ERROR)
+ logger.log(u"Unable to retrieve new version from " + tar_download_url + ", can't update", logger.WARNING)
return False
if not ek.ek(tarfile.is_tarfile, tar_download_path):
diff --git a/sickbeard/webapi.py b/sickbeard/webapi.py
index 2aa55de1b22dc6f44dc5127b7daa0e8fd4028ce1..0c62e0d4c7e72a95cd060ffaece781774df2abd2 100644
--- a/sickbeard/webapi.py
+++ b/sickbeard/webapi.py
@@ -28,6 +28,7 @@ import traceback
import sickbeard
+from versionChecker import CheckVersion
from sickbeard import db, logger, exceptions, history, ui, helpers
from sickbeard import encodingKludge as ek
from sickbeard import search_queue
@@ -45,7 +46,8 @@ try:
except ImportError:
from lib import simplejson as json
-from lib import subliminal
+import subliminal
+import babelfish
from tornado.web import RequestHandler
@@ -752,7 +754,7 @@ class CMD_ComingEpisodes(ApiCall):
# Safety Measure to convert rows in sql_results to dict.
# This should not be required as the DB connections should only be returning dict results not sqlite3.row_type
dict_results = [dict(row) for row in sql_results]
-
+
for ep in dict_results:
"""
Missed: yesterday... (less than 1week)
@@ -1082,8 +1084,8 @@ class CMD_SubtitleSearch(ApiCall):
# return the correct json value
if previous_subtitles != epObj.subtitles:
status = 'New subtitles downloaded: %s' % ' '.join([
- "<img src='" + sickbeard.WEB_ROOT + "/images/flags/" + subliminal.language.Language(
- x).alpha2 + ".png' alt='" + subliminal.language.Language(x).name + "'/>" for x in
+ "<img src='" + sickbeard.WEB_ROOT + "/images/flags/" + babelfish.language.Language(
+ x).alpha2 + ".png' alt='" + babelfish.language.Language(x).name + "'/>" for x in
sorted(list(set(epObj.subtitles).difference(previous_subtitles)))])
response = _responds(RESULT_SUCCESS, msg='New subtitles found')
else:
@@ -1482,6 +1484,35 @@ class CMD_SickBeardAddRootDir(ApiCall):
sickbeard.ROOT_DIRS = root_dirs_new
return _responds(RESULT_SUCCESS, _getRootDirs(), msg="Root directories updated")
+class CMD_SickBeardCheckVersion(ApiCall):
+ _help = {"desc": "check if a new version of SickRage is available"}
+
+ def __init__(self, args, kwargs):
+ # required
+ # optional
+ # super, missing, help
+ ApiCall.__init__(self, args, kwargs)
+
+ def run(self):
+ checkversion = CheckVersion()
+ needs_update = checkversion.check_for_new_version()
+
+ data = {
+ "current_version": {
+ "branch": checkversion.get_branch(),
+ "commit": checkversion.updater.get_cur_commit_hash(),
+ "version": checkversion.updater.get_cur_version(),
+ },
+ "latest_version": {
+ "branch": checkversion.get_branch(),
+ "commit": checkversion.updater.get_newest_commit_hash(),
+ "version": checkversion.updater.get_newest_version(),
+ },
+ "commits_offset": checkversion.updater.get_num_commits_behind(),
+ "needs_update": needs_update,
+ }
+
+ return _responds(RESULT_SUCCESS, data)
class CMD_SickBeardCheckScheduler(ApiCall):
_help = {"desc": "query the scheduler"}
@@ -1868,6 +1899,27 @@ class CMD_SickBeardShutdown(ApiCall):
sickbeard.events.put(sickbeard.events.SystemEvent.SHUTDOWN)
return _responds(RESULT_SUCCESS, msg="SickRage is shutting down...")
+class CMD_SickBeardUpdate(ApiCall):
+ _help = {"desc": "update SickRage to the latest version available"}
+
+ def __init__(self, args, kwargs):
+ # required
+ # optional
+ # super, missing, help
+ ApiCall.__init__(self, args, kwargs)
+
+ def run(self):
+ checkversion = CheckVersion()
+
+ if checkversion.check_for_new_version():
+ if checkversion.run_backup_if_safe():
+ checkversion.update()
+
+ return _responds(RESULT_SUCCESS, msg="SickRage is updating ...")
+
+ return _responds(RESULT_FAILURE, msg="SickRage could not backup config ...")
+
+ return _responds(RESULT_FAILURE, msg="SickRage is already up to date")
class CMD_Show(ApiCall):
_help = {"desc": "display information for a given show",
@@ -2867,6 +2919,7 @@ _functionMaper = {"help": CMD_Help,
"sb": CMD_SickBeard,
"postprocess": CMD_PostProcess,
"sb.addrootdir": CMD_SickBeardAddRootDir,
+ "sb.checkversion": CMD_SickBeardCheckVersion,
"sb.checkscheduler": CMD_SickBeardCheckScheduler,
"sb.deleterootdir": CMD_SickBeardDeleteRootDir,
"sb.getdefaults": CMD_SickBeardGetDefaults,
@@ -2879,6 +2932,7 @@ _functionMaper = {"help": CMD_Help,
"sb.searchtvdb": CMD_SickBeardSearchTVDB,
"sb.searchtvrage": CMD_SickBeardSearchTVRAGE,
"sb.setdefaults": CMD_SickBeardSetDefaults,
+ "sb.update": CMD_SickBeardUpdate,
"sb.shutdown": CMD_SickBeardShutdown,
"show": CMD_Show,
"show.addexisting": CMD_ShowAddExisting,
diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py
index aeef2a29bbd0733c5a50639d13a7c85126a5305e..39a53c10e4220d07f540ccea5f50527c9d07e849 100644
--- a/sickbeard/webserve.py
+++ b/sickbeard/webserve.py
@@ -54,10 +54,11 @@ from sickbeard.scene_numbering import get_scene_numbering, set_scene_numbering,
from lib.dateutil import tz, parser as dateutil_parser
from lib.unrar2 import RarFile
-from lib import adba, subliminal
+import adba, subliminal
from lib.trakt import TraktAPI
from lib.trakt.exceptions import traktException
from versionChecker import CheckVersion
+import babelfish
try:
import json
@@ -80,7 +81,7 @@ from tornado.concurrent import run_on_executor
from concurrent.futures import ThreadPoolExecutor
route_locks = {}
-
+import chardet
class html_entities(CheetahFilter):
def filter(self, val, **dummy_kw):
@@ -90,13 +91,23 @@ class html_entities(CheetahFilter):
filtered = ''
elif isinstance(val, str):
try:
- filtered = val.decode(sickbeard.SYS_ENCODING).encode('ascii', 'xmlcharrefreplace')
- except UnicodeDecodeError as e:
- logger.log(u'Unable to decode using {0}, trying utf-8. Error is: {1}'.format(sickbeard.SYS_ENCODING, ex(e)),logger.DEBUG)
+ filtered = unicode(val).encode('ascii', 'xmlcharrefreplace')
+ except UnicodeDecodeError, UnicodeEncodeError:
try:
- filtered = val.decode('utf-8').encode('ascii', 'xmlcharrefreplace')
- except UnicodeDecodeError as e:
- logger.log(u'Unable to decode using utf-8, Error is {0}.'.format(ex(e)),logger.ERROR)
+ filtered = unicode(val, chardet.detect(val).get('encoding')).encode('ascii', 'xmlcharrefreplace')
+ except (UnicodeDecodeError, UnicodeEncodeError) as e:
+ try:
+ filtered = unicode(val, sickbeard.SYS_ENCODING).encode('ascii', 'xmlcharrefreplace')
+ except (UnicodeDecodeError, UnicodeEncodeError) as e:
+ logger.log(u'Unable to decode using {0}, trying utf-8. Error is: {1}'.format(sickbeard.SYS_ENCODING, ex(e)), logger.DEBUG)
+ try:
+ filtered = unicode(val, 'utf-8').encode('ascii', 'xmlcharrefreplace')
+ except (UnicodeDecodeError, UnicodeEncodeError) as e:
+ try:
+ logger.log(u'Unable to decode using utf-8, trying latin-1. Error is: {1}'.format(ex(e)), logger.DEBUG)
+ filtered = unicode(val, 'latin-1').encode('ascii', 'xmlcharrefreplace')
+ except UnicodeDecodeError, UnicodeEncodeError:
+ logger.log(u'Unable to decode using latin-1, Error is {0}.'.format(ex(e)),logger.ERROR)
else:
filtered = self.filter(str(val))
@@ -1447,7 +1458,7 @@ class Home(WebRoot):
showObj.sports = sports
showObj.subtitles = subtitles
showObj.air_by_date = air_by_date
- #showObj.default_ep_status = int(defaultEpStatus)
+ showObj.default_ep_status = int(defaultEpStatus)
if not directCall:
showObj.lang = indexer_lang
@@ -2012,24 +2023,23 @@ class Home(WebRoot):
return json.dumps({'result': 'failure'})
# try do download subtitles for that episode
- previous_subtitles = set(subliminal.language.Language(x) for x in ep_obj.subtitles)
+ previous_subtitles = ep_obj.subtitles
try:
- ep_obj.subtitles = set(x.language for x in ep_obj.downloadSubtitles().values()[0])
+ ep_obj.downloadSubtitles()
except:
return json.dumps({'result': 'failure'})
# return the correct json value
- if previous_subtitles != ep_obj.subtitles:
+ newSubtitles = frozenset(ep_obj.subtitles).difference(previous_subtitles)
+ if newSubtitles:
+ newLangs = [babelfish.Language.fromietf(newSub) for newSub in newSubtitles]
status = 'New subtitles downloaded: %s' % ' '.join([
- "<img src='" + sickbeard.WEB_ROOT + "/images/flags/" + x.alpha2 +
- ".png' alt='" + x.name + "'/>" for x in
- sorted(list(ep_obj.subtitles.difference(previous_subtitles)))])
+ "<img src='" + sickbeard.WEB_ROOT + "/images/flags/" + newLang.alpha3 +
+ ".png' alt='" + newLang.name + "'/>" for newLang in newLangs])
else:
status = 'No subtitles downloaded'
ui.notifications.message('Subtitles Search', status)
- return json.dumps({'result': status, 'subtitles': ','.join(sorted([x.alpha2 for x in
- ep_obj.subtitles.union(
- previous_subtitles)]))})
+ return json.dumps({'result': status, 'subtitles': ','.join(ep_obj.subtitles)})
def setSceneNumbering(self, show, indexer, forSeason=None, forEpisode=None, forAbsolute=None, sceneSeason=None,
sceneEpisode=None, sceneAbsolute=None):
@@ -2885,10 +2895,9 @@ class Manage(Home, WebRoot):
result = {}
for cur_result in cur_show_results:
if whichSubs == 'all':
- if len(set(cur_result["subtitles"].split(',')).intersection(set(subtitles.wantedLanguages()))) >= len(
- subtitles.wantedLanguages()):
+ if not frozenset(subtitles.wantedLanguages()).difference(cur_result["subtitles"].split(',')):
continue
- elif whichSubs in cur_result["subtitles"].split(','):
+ elif whichSubs in cur_result["subtitles"]:
continue
cur_season = int(cur_result["season"])
@@ -2902,9 +2911,7 @@ class Manage(Home, WebRoot):
result[cur_season][cur_episode]["name"] = cur_result["name"]
- result[cur_season][cur_episode]["subtitles"] = ",".join(
- subliminal.language.Language(subtitle).alpha2 for subtitle in cur_result["subtitles"].split(',')) if not \
- cur_result["subtitles"] == '' else ''
+ result[cur_season][cur_episode]["subtitles"] = cur_result["subtitles"]
return json.dumps(result)
@@ -2920,17 +2927,19 @@ class Manage(Home, WebRoot):
myDB = db.DBConnection()
status_results = myDB.select(
- "SELECT show_name, tv_shows.indexer_id as indexer_id, tv_episodes.subtitles subtitles FROM tv_episodes, tv_shows WHERE tv_shows.subtitles = 1 AND tv_episodes.status LIKE '%4' AND tv_episodes.season != 0 AND tv_episodes.showid = tv_shows.indexer_id ORDER BY show_name")
+ "SELECT show_name, tv_shows.indexer_id as indexer_id, tv_episodes.subtitles subtitles " +
+ "FROM tv_episodes, tv_shows " +
+ "WHERE tv_shows.subtitles = 1 AND tv_episodes.status LIKE '%4' AND tv_episodes.season != 0 " +
+ "AND tv_episodes.showid = tv_shows.indexer_id ORDER BY show_name")
ep_counts = {}
show_names = {}
sorted_show_ids = []
for cur_status_result in status_results:
if whichSubs == 'all':
- if len(set(cur_status_result["subtitles"].split(',')).intersection(
- set(subtitles.wantedLanguages()))) >= len(subtitles.wantedLanguages()):
+ if not frozenset(subtitles.wantedLanguages()).difference(cur_status_result["subtitles"].split(',')):
continue
- elif whichSubs in cur_status_result["subtitles"].split(','):
+ elif whichSubs in cur_status_result["subtitles"]:
continue
cur_indexer_id = int(cur_status_result["indexer_id"])
@@ -3368,33 +3377,33 @@ class Manage(Home, WebRoot):
sickbeard.showQueueScheduler.action.downloadSubtitles(showObj)
subtitles.append(showObj.name)
- if len(errors) > 0:
+ if errors:
ui.notifications.error("Errors encountered",
'<br >\n'.join(errors))
messageDetail = ""
- if len(updates) > 0:
+ if updates:
messageDetail += "<br /><b>Updates</b><br /><ul><li>"
messageDetail += "</li><li>".join(updates)
messageDetail += "</li></ul>"
- if len(refreshes) > 0:
+ if refreshes:
messageDetail += "<br /><b>Refreshes</b><br /><ul><li>"
messageDetail += "</li><li>".join(refreshes)
messageDetail += "</li></ul>"
- if len(renames) > 0:
+ if renames:
messageDetail += "<br /><b>Renames</b><br /><ul><li>"
messageDetail += "</li><li>".join(renames)
messageDetail += "</li></ul>"
- if len(subtitles) > 0:
+ if subtitles:
messageDetail += "<br /><b>Subtitles</b><br /><ul><li>"
messageDetail += "</li><li>".join(subtitles)
messageDetail += "</li></ul>"
- if len(updates + refreshes + renames + subtitles) > 0:
+ if updates + refreshes + renames + subtitles:
ui.notifications.message("The following actions were queued:",
messageDetail)
@@ -4642,13 +4651,13 @@ class ConfigNotifications(Config):
use_boxcar2=None, boxcar2_notify_onsnatch=None, boxcar2_notify_ondownload=None,
boxcar2_notify_onsubtitledownload=None, boxcar2_accesstoken=None,
use_pushover=None, pushover_notify_onsnatch=None, pushover_notify_ondownload=None,
- pushover_notify_onsubtitledownload=None, pushover_userkey=None, pushover_apikey=None,
+ pushover_notify_onsubtitledownload=None, pushover_userkey=None, pushover_apikey=None, pushover_device=None,
use_libnotify=None, libnotify_notify_onsnatch=None, libnotify_notify_ondownload=None,
libnotify_notify_onsubtitledownload=None,
use_nmj=None, nmj_host=None, nmj_database=None, nmj_mount=None, use_synoindex=None,
use_nmjv2=None, nmjv2_host=None, nmjv2_dbloc=None, nmjv2_database=None,
use_trakt=None, trakt_username=None, trakt_pin=None,
- trakt_remove_watchlist=None, trakt_sync_watchlist=None, trakt_method_add=None,
+ trakt_remove_watchlist=None, trakt_sync_watchlist=None, trakt_remove_show_from_sickrage=None, trakt_method_add=None,
trakt_start_paused=None, trakt_use_recommended=None, trakt_sync=None, trakt_sync_remove=None,
trakt_default_indexer=None, trakt_remove_serieslist=None, trakt_disable_ssl_verify=None, trakt_timeout=None, trakt_blacklist_name=None,
trakt_use_rolling_download=None, trakt_rolling_num_ep=None, trakt_rolling_add_paused=None, trakt_rolling_frequency=None,
@@ -4741,6 +4750,7 @@ class ConfigNotifications(Config):
sickbeard.PUSHOVER_NOTIFY_ONSUBTITLEDOWNLOAD = config.checkbox_to_value(pushover_notify_onsubtitledownload)
sickbeard.PUSHOVER_USERKEY = pushover_userkey
sickbeard.PUSHOVER_APIKEY = pushover_apikey
+ sickbeard.PUSHOVER_DEVICE = pushover_device
sickbeard.USE_LIBNOTIFY = config.checkbox_to_value(use_libnotify)
sickbeard.LIBNOTIFY_NOTIFY_ONSNATCH = config.checkbox_to_value(libnotify_notify_onsnatch)
@@ -4769,6 +4779,7 @@ class ConfigNotifications(Config):
sickbeard.TRAKT_USERNAME = trakt_username
sickbeard.TRAKT_REMOVE_WATCHLIST = config.checkbox_to_value(trakt_remove_watchlist)
sickbeard.TRAKT_REMOVE_SERIESLIST = config.checkbox_to_value(trakt_remove_serieslist)
+ sickbeard.TRAKT_REMOVE_SHOW_FROM_SICKRAGE = config.checkbox_to_value(trakt_remove_show_from_sickrage)
sickbeard.TRAKT_SYNC_WATCHLIST = config.checkbox_to_value(trakt_sync_watchlist)
sickbeard.TRAKT_METHOD_ADD = int(trakt_method_add)
sickbeard.TRAKT_START_PAUSED = config.checkbox_to_value(trakt_start_paused)
@@ -4858,8 +4869,7 @@ class ConfigSubtitles(Config):
config.change_SUBTITLES_FINDER_FREQUENCY(subtitles_finder_frequency)
config.change_USE_SUBTITLES(use_subtitles)
- sickbeard.SUBTITLES_LANGUAGES = [lang.alpha2 for lang in subtitles.isValidLanguage(
- subtitles_languages.replace(' ', '').split(','))] if subtitles_languages != '' else ''
+ sickbeard.SUBTITLES_LANGUAGES = [lang.strip() for lang in subtitles_languages.split(',') if subtitles.isValidLanguage(lang.strip())] if subtitles_languages else []
sickbeard.SUBTITLES_DIR = subtitles_dir
sickbeard.SUBTITLES_HISTORY = config.checkbox_to_value(subtitles_history)
sickbeard.EMBEDDED_SUBTITLES_ALL = config.checkbox_to_value(embedded_subtitles_all)
diff --git a/tests/ssl_sni_tests.py b/tests/ssl_sni_tests.py
index cce6a23edbf0f96406c847ca6f10643c991ec85b..700743a2b63122fe2ac7595c2a1ed16cdfe2ad9d 100644
--- a/tests/ssl_sni_tests.py
+++ b/tests/ssl_sni_tests.py
@@ -22,36 +22,29 @@ import sys, os.path
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-from lib import requests
-from sickbeard.providers.torrentday import provider as torrentday
-from sickbeard.providers.rarbg import provider as rarbg
-from sickbeard.providers.scc import provider as sceneaccess
-
-enabled_sni = True
-if sys.version_info < (2, 7, 9):
- try:
- import cryptography
- except ImportError:
- try:
- from OpenSSL.version import __version__ as pyOpenSSL_Version
- if int(pyOpenSSL_Version.replace('.', '')[:3]) > 13:
- raise ImportError
- except ImportError:
- enabled_sni = False
-
+import requests
+import sickbeard.providers as providers
+import certifi
+from sickbeard.exceptions import ex
class SNI_Tests(unittest.TestCase):
def test_SNI_URLS(self):
- if not enabled_sni:
- print('\nSNI is disabled with pyOpenSSL >= 0.14 when the cryptography module is missing,\n' +
- 'you will encounter SSL errors with HTTPS! To fix this issue:\n' +
- 'pip install pyopenssl==0.13.1 (easy) or pip install cryptography (pita)')
- print
- else:
- for provider in [ torrentday, rarbg, sceneaccess ]:
- #print 'Checking ' + provider.name
- self.assertEqual(requests.get(provider.url).status_code, 200)
-
+ print ''
+ #Just checking all providers - we should make this error on non-existent urls.
+ for provider in providers.makeProviderList():
+ print 'Checking %s' % provider.name
+ try:
+ requests.head(provider.url, verify=certifi.where(), timeout=5)
+ except requests.exceptions.Timeout:
+ pass
+ except requests.exceptions.SSLError as error:
+ if u'SSL3_GET_SERVER_CERTIFICATE' not in ex(error.message):
+ print 'SSLError on %s: %s' % (provider.name, ex(error.message))
+ raise
+ else:
+ print 'Cannot verify certificate for %s' % provider.name
+ except Exception:
+ pass
if __name__ == "__main__":
suite = unittest.TestLoader().loadTestsFromTestCase(SNI_Tests)
diff --git a/tests/torrent_tests.py b/tests/torrent_tests.py
index cf8e39c994e5abf5a117fb00d63e4ce1e171b74b..d1b4a7063c3990e1834ab4e588aec53f48a0b70c 100644
--- a/tests/torrent_tests.py
+++ b/tests/torrent_tests.py
@@ -29,6 +29,7 @@ import urlparse
import test_lib as test
from bs4 import BeautifulSoup
from sickbeard.helpers import getURL
+import requests
class TorrentBasicTests(test.SickbeardTestDBCase):
@@ -36,7 +37,7 @@ class TorrentBasicTests(test.SickbeardTestDBCase):
self.url = 'http://kickass.to/'
searchURL = 'http://kickass.to/usearch/American%20Dad%21%20S08%20-S08E%20category%3Atv/?field=seeders&sorder=desc'
- html = getURL(searchURL)
+ html = getURL(searchURL, session=requests.Session())
if not html:
return