diff --git a/src/Jackett.Console/ConsoleOptions.cs b/src/Jackett.Console/ConsoleOptions.cs
index 635eb2e2ac4dc6b6d89e8c6b1e6eb5684d1e46bf..74982506bbc13e008eead3763dedb66506b62c52 100644
--- a/src/Jackett.Console/ConsoleOptions.cs
+++ b/src/Jackett.Console/ConsoleOptions.cs
@@ -27,6 +27,9 @@ namespace Jackett.Console
[Option('c', "UseClient", HelpText = "Override web client selection. [automatic(Default)/libcurl/safecurl/httpclient]")]
public string Client { get; set; }
+ [Option('j', "ProxyConnection", HelpText = "use proxy - e.g. 127.0.0.1:8888")]
+ public string ProxyConnection { get; set; }
+
[Option('s', "Start", HelpText = "Start the Jacket Windows service (Must be admin)")]
public bool StartService { get; set; }
diff --git a/src/Jackett.Console/Program.cs b/src/Jackett.Console/Program.cs
index a5b5ad94119f1ff243547f34da39aa87d8c5df04..64abc0a3a814c2298e3c3b925610e46042525e8b 100644
--- a/src/Jackett.Console/Program.cs
+++ b/src/Jackett.Console/Program.cs
@@ -65,6 +65,11 @@ namespace JackettConsole
if (options.Client != null)
Startup.ClientOverride = options.Client.ToLowerInvariant();
+ // Use Proxy
+ if (options.ProxyConnection != null)
+ {
+ Startup.ProxyConnection = options.ProxyConnection.ToLowerInvariant();
+ }
// Logging
if (options.Logging)
Startup.LogRequests = true;
diff --git a/src/Jackett/CurlHelper.cs b/src/Jackett/CurlHelper.cs
index c2a94cdd9e9c886fe573c19c5277f4351482f4c0..fe58143e4b15dde2cec63ec46b52230cc1460ee9 100644
--- a/src/Jackett/CurlHelper.cs
+++ b/src/Jackett/CurlHelper.cs
@@ -85,6 +85,7 @@ namespace Jackett
using (var easy = new CurlEasy())
{
+
easy.Url = curlRequest.Url;
easy.BufferSize = 64 * 1024;
easy.UserAgent = BrowserUtil.ChromeUserAgent;
@@ -139,7 +140,12 @@ namespace Jackett
{
easy.SetOpt(CurlOption.SslVerifyhost, false);
easy.SetOpt(CurlOption.SslVerifyPeer, false);
- }
+ }
+
+ if (Startup.ProxyConnection != null)
+ {
+ easy.SetOpt(CurlOption.Proxy, Startup.ProxyConnection);
+ }
easy.Perform();
@@ -155,6 +161,15 @@ namespace Jackett
var headerBytes = Combine(headerBuffers.ToArray());
var headerString = Encoding.UTF8.GetString(headerBytes);
+ if (Startup.ProxyConnection != null)
+ {
+ var firstcrlf = headerString.IndexOf("\r\n\r\n");
+ var secondcrlf = headerString.IndexOf("\r\n\r\n", firstcrlf + 1);
+ if (secondcrlf > 0)
+ {
+ headerString = headerString.Substring(firstcrlf + 4, secondcrlf - (firstcrlf));
+ }
+ }
var headerParts = headerString.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
var headers = new List<string[]>();
var headerCount = 0;
diff --git a/src/Jackett/Indexers/BaseIndexer.cs b/src/Jackett/Indexers/BaseIndexer.cs
index 84cdbef320d8246dd19b4f9dbf01b2d1673fc6e0..c80c9479ad4aff95906967047cdd4b3c593fb854 100644
--- a/src/Jackett/Indexers/BaseIndexer.cs
+++ b/src/Jackett/Indexers/BaseIndexer.cs
@@ -147,28 +147,28 @@ namespace Jackett.Indexers
}
}
- protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
+ protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
{
var byteResult = new WebClientByteResult();
// Map to byte
Mapper.Map(response, byteResult);
- await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies);
+ await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies, accumulateCookies);
// Map to string
Mapper.Map(byteResult, response);
}
- protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
+ protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
{
// Follow up to 5 redirects
for (int i = 0; i < 5; i++)
{
if (!response.IsRedirect)
break;
- await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies);
- if (response.Cookies != null && overrideCookies != null)
+ await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies, accumulateCookies);
+ if (accumulateCookies)
{
- response.Cookies = overrideCookies + " " + response.Cookies;
- overrideCookies = response.Cookies;
+ CookieHeader = ResolveCookies(response.Cookies);
+ response.Cookies = CookieHeader;
}
if (overrideCookies != null && response.Cookies == null)
{
@@ -177,16 +177,39 @@ namespace Jackett.Indexers
}
}
- private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
+ private String ResolveCookies(String incomingCookies = "")
+ {
+ var redirRequestCookies = (CookieHeader != "" ? CookieHeader + " " : "") + incomingCookies;
+ System.Text.RegularExpressions.Regex expression = new System.Text.RegularExpressions.Regex(@"([^\s]+)=([^=]+)(?:\s|$)");
+ Dictionary<string, string> cookieDIctionary = new Dictionary<string, string>();
+ var matches = expression.Match(redirRequestCookies);
+ while (matches.Success)
+ {
+ if (matches.Groups.Count > 2) cookieDIctionary[matches.Groups[1].Value] = matches.Groups[2].Value;
+ matches = matches.NextMatch();
+ }
+ return string.Join(" ", cookieDIctionary.Select(kv => kv.Key.ToString() + "=" + kv.Value.ToString()).ToArray());
+
+ }
+
+ private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
{
if (incomingResponse.IsRedirect)
{
+ var redirRequestCookies = "";
+ if (accumulateCookies)
+ {
+ redirRequestCookies = ResolveCookies((CookieHeader != "" ? CookieHeader + " " : "") + (overrideCookies != null ? overrideCookies : ""));
+ } else
+ {
+ redirRequestCookies = (overrideCookies != null ? overrideCookies : "");
+ }
// Do redirect
var redirectedResponse = await webclient.GetBytes(new WebRequest()
{
Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
Referer = referrer,
- Cookies = overrideCookies ?? CookieHeader
+ Cookies = redirRequestCookies
});
Mapper.Map(redirectedResponse, incomingResponse);
}
@@ -371,30 +394,20 @@ namespace Jackett.Indexers
var response = await webclient.GetString(request);
if (accumulateCookies)
{
- response.Cookies = (request.Cookies == null ? "" : request.Cookies + " ") + response.Cookies;
+ response.Cookies = ResolveCookies((request.Cookies == null ? "" : request.Cookies + " ") + response.Cookies);
}
var firstCallCookies = response.Cookies;
if (response.IsRedirect)
{
- await FollowIfRedirect(response, request.Url, redirectUrlOverride, response.Cookies);
+ await FollowIfRedirect(response, request.Url, redirectUrlOverride, response.Cookies, accumulateCookies);
}
if (returnCookiesFromFirstCall)
{
- response.Cookies = firstCallCookies + (accumulateCookies ? " " + response.Cookies : "");
- }
- // resolve cookie conflicts - really no need for this as the webclient will handle it
- System.Text.RegularExpressions.Regex expression = new System.Text.RegularExpressions.Regex(@"([^\s]+)=([^=]+)(?:\s|$)");
- Dictionary<string, string> cookieDIctionary = new Dictionary<string, string>();
- var matches = expression.Match(response.Cookies);
- while (matches.Success)
- {
- if (matches.Groups.Count > 2) cookieDIctionary[matches.Groups[1].Value] = matches.Groups[2].Value;
- matches = matches.NextMatch();
+ response.Cookies = ResolveCookies(firstCallCookies + (accumulateCookies ? " " + response.Cookies : ""));
}
- response.Cookies = string.Join(" ", cookieDIctionary.Select(kv => kv.Key.ToString() + "=" + kv.Value.ToString()).ToArray());
-
+
return response;
}
diff --git a/src/Jackett/Indexers/BitSoup.cs b/src/Jackett/Indexers/BitSoup.cs
index 7af224584d8150eec6e07b14ed4e8c918f50d56a..accc6fb2c5ccdc0c123335e688b96867251a0bd6 100644
--- a/src/Jackett/Indexers/BitSoup.cs
+++ b/src/Jackett/Indexers/BitSoup.cs
@@ -1,46 +1,53 @@
-using CsQuery;
-using Jackett.Models;
-using Jackett.Services;
-using Jackett.Utils;
-using Jackett.Utils.Clients;
-using Newtonsoft.Json.Linq;
-using NLog;
-using System;
-using System.Collections.Generic;
-using System.Collections.Specialized;
-using System.Globalization;
-using System.Linq;
-using System.Net;
-using System.Net.Http;
-using System.Text;
-using System.Threading.Tasks;
-using System.Web;
-using Jackett.Models.IndexerConfig;
-
-namespace Jackett.Indexers
-{
- public class BitSoup : BaseIndexer, IIndexer
- {
- private string BrowseUrl { get { return SiteLink + "browse.php"; } }
- private string LoginUrl { get { return SiteLink + "takelogin.php"; } }
-
- new ConfigurationDataBasicLogin configData
- {
- get { return (ConfigurationDataBasicLogin)base.configData; }
- set { base.configData = value; }
- }
-
- public BitSoup(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps)
- : base(name: "BitSoup",
- description: "SoupieBits",
- link: "https://www.bitsoup.me/",
- caps: TorznabUtil.CreateDefaultTorznabTVCaps(),
- manager: i,
- client: wc,
- logger: l,
- p: ps,
- configData: new ConfigurationDataBasicLogin())
+using CsQuery;
+using Jackett.Models;
+using Jackett.Services;
+using Jackett.Utils;
+using Jackett.Utils.Clients;
+using Newtonsoft.Json.Linq;
+using NLog;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Text;
+using System.Threading.Tasks;
+using System.Web;
+using Jackett.Models.IndexerConfig;
+using System.Collections.Specialized;
+using System.Text.RegularExpressions;
+
+namespace Jackett.Indexers
+{
+ public class BitSoup : BaseIndexer, IIndexer
+ {
+ private string UseLink { get { return (this.configData.AlternateLink.Value != "" ? this.configData.AlternateLink.Value : SiteLink); } }
+ private string BrowseUrl { get { return UseLink + "browse.php"; } }
+ private string LoginUrl { get { return UseLink + "takelogin.php"; } }
+ private string LoginReferer { get { return UseLink + "login.php"; } }
+ private List<String> KnownURLs = new List<String>{ "https://www.bitsoup.me/","https://www.bitsoup.org/"};
+
+ new NxtGnConfigurationData configData
+ {
+ get { return (NxtGnConfigurationData)base.configData; }
+ set { base.configData = value; }
+ }
+
+ public BitSoup(IIndexerManagerService i, IWebClient wc, Logger l, IProtectionService ps)
+ : base(name: "BitSoup",
+ description: "SoupieBits",
+ link: "https://www.bitsoup.me/",
+ caps: TorznabUtil.CreateDefaultTorznabTVCaps(),
+ manager: i,
+ client: wc,
+ logger: l,
+ p: ps,
+ configData: new NxtGnConfigurationData())
{
+ this.configData.DisplayText.Value = this.DisplayName + " has multiple URLs. The default (" + this.SiteLink + ") can be changed by entering a new value in the box below.";
+ this.configData.DisplayText.Value += "The following are some known URLs for " + this.DisplayName;
+ this.configData.DisplayText.Value += "<ul><li>" + String.Join("</li><li>", this.KnownURLs.ToArray()) + "</li></ul>";
//AddCategoryMapping("624", TorznabCatType.Console);
//AddCategoryMapping("307", TorznabCatType.ConsoleNDS);
@@ -134,45 +141,58 @@ namespace Jackett.Indexers
//AddCategoryMapping("1", TorznabCatType.BooksTechnical);
//AddCategoryMapping("1", TorznabCatType.BooksOther);
//AddCategoryMapping("1", TorznabCatType.BooksForeign);
- }
-
- public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
- {
- configData.LoadValuesFromJson(configJson);
- var pairs = new Dictionary<string, string> {
- { "username", configData.Username.Value },
- { "password", configData.Password.Value },
- { "returnto", "/" },
- { "login", "Log in!" }
- };
-
- var loginPage = await RequestStringWithCookies(SiteLink, string.Empty);
-
- var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, SiteLink, SiteLink, true);
- await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () =>
- {
- CQ dom = result.Content;
- var messageEl = dom["body > div"].First();
- var errorMessage = messageEl.Text().Trim();
- throw new ExceptionWithConfigData(errorMessage, configData);
- });
- return IndexerConfigurationStatus.RequiresTesting;
- }
-
- public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
- {
- var releases = new List<ReleaseInfo>();
- var searchString = query.GetQueryString();
- var searchUrl = BrowseUrl;
- var trackerCats = MapTorznabCapsToTrackers(query);
- var queryCollection = new NameValueCollection();
-
-
-
- if (!string.IsNullOrWhiteSpace(searchString))
+ }
+
+ public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
+ {
+ configData.LoadValuesFromJson(configJson);
+ if (configData.AlternateLink.Value != null && configData.AlternateLink.Value != "")
+ {
+ if (!configData.AlternateLink.Value.EndsWith("/"))
+ {
+ configData.AlternateLink.Value = null;
+ throw new Exception("AlternateLink must end with a slash.");
+ }
+ var match = Regex.Match(configData.AlternateLink.Value, "^https?:\\/\\/(?:[\\w]+\\.)+(?:[a-zA-Z]+)\\/$");
+ if (!match.Success)
+ {
+ configData.AlternateLink.Value = null;
+ throw new Exception("AlternateLink must be a valid url.");
+ }
+ }
+ var pairs = new Dictionary<string, string> {
+ { "username", configData.Username.Value },
+ { "password", configData.Password.Value },
+
+ };
+
+ var loginPage = await RequestStringWithCookies(UseLink, string.Empty);
+
+ var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginReferer, true);
+ await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () =>
+ {
+ CQ dom = result.Content;
+ var messageEl = dom["body > table.statusbar1 > tbody > tr > td > table > tbody > tr > td > table > tbody > tr > td"].First();
+ var errorMessage = messageEl.Text().Trim();
+ throw new ExceptionWithConfigData(errorMessage, configData);
+ });
+ return IndexerConfigurationStatus.RequiresTesting;
+ }
+
+ public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
+ {
+ var releases = new List<ReleaseInfo>();
+ var searchString = query.GetQueryString();
+ var searchUrl = BrowseUrl;
+ var trackerCats = MapTorznabCapsToTrackers(query);
+ var queryCollection = new NameValueCollection();
+
+
+
+ if (!string.IsNullOrWhiteSpace(searchString))
{
queryCollection.Add("search", searchString);
- queryCollection.Add("incldead", "0");
+ queryCollection.Add("incldead", "0");
queryCollection.Add("cat", "0");
// Tracker cannot search multi categories
// so we either search "all"
@@ -190,37 +210,37 @@ namespace Jackett.Indexers
searchUrl += "?" + queryCollection.GetQueryString();
await ProcessPage(releases, searchUrl);
}
- }
+ }
- }
- else
+ }
+ else
{
- queryCollection.Add("search", "");
- queryCollection.Add("cat", "0");
- searchUrl += "?" + queryCollection.GetQueryString();
- await ProcessPage(releases, searchUrl);
- }
-
- return releases;
- }
-
- private async Task ProcessPage(List<ReleaseInfo> releases, string searchUrl)
- {
- var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl);
- var results = response.Content;
- try
- {
- CQ dom = results;
-
- var rows = dom["table.koptekst tr"];
- foreach (var row in rows.Skip(1))
- {
- var release = new ReleaseInfo();
-
+ queryCollection.Add("search", "");
+ queryCollection.Add("cat", "0");
+ searchUrl += "?" + queryCollection.GetQueryString();
+ await ProcessPage(releases, searchUrl);
+ }
+
+ return releases;
+ }
+
+ private async Task ProcessPage(List<ReleaseInfo> releases, string searchUrl)
+ {
+ var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl);
+ var results = response.Content;
+ try
+ {
+ CQ dom = results;
+
+ var rows = dom["table.koptekst tr"];
+ foreach (var row in rows.Skip(1))
+ {
+ var release = new ReleaseInfo();
+
release.Title = row.Cq().Find("td:eq(1) a").First().Text().Trim();
- release.Comments = new Uri(SiteLink + row.Cq().Find("td:eq(1) a").First().Attr("href"));
-
- release.Link = new Uri(SiteLink + row.Cq().Find("td:eq(2) a").First().Attr("href"));
+ release.Comments = new Uri(UseLink + row.Cq().Find("td:eq(1) a").First().Attr("href"));
+
+ release.Link = new Uri(UseLink + row.Cq().Find("td:eq(2) a").First().Attr("href"));
release.Description = release.Title;
var cat = row.Cq().Find("td:eq(0) a").First().Attr("href").Substring(15);
release.Category = MapTrackerCatToNewznab(cat);
@@ -234,13 +254,31 @@ namespace Jackett.Indexers
release.Seeders = ParseUtil.CoerceInt(row.Cq().Find("td:eq(10)").First().Text().Trim());
release.Peers = ParseUtil.CoerceInt(row.Cq().Find("td:eq(11)").First().Text().Trim()) + release.Seeders;
- releases.Add(release);
- }
- }
- catch (Exception ex)
- {
- OnParseError(results, ex);
- }
- }
- }
-}
+ releases.Add(release);
+ }
+ }
+ catch (Exception ex)
+ {
+ OnParseError(results, ex);
+ }
+ }
+
+ public class NxtGnConfigurationData : ConfigurationData
+ {
+ public StringItem Username { get; private set; }
+ public StringItem Password { get; private set; }
+ public DisplayItem DisplayText { get; private set; }
+ public StringItem AlternateLink { get; set; }
+
+
+ public NxtGnConfigurationData()
+ {
+ Username = new StringItem { Name = "Username" };
+ Password = new StringItem { Name = "Password" };
+ DisplayText = new DisplayItem("") { Name = "" };
+ AlternateLink = new StringItem { Name = "AlternateLinks" };
+ }
+
+ }
+ }
+}
diff --git a/src/Jackett/Indexers/XSpeeds.cs b/src/Jackett/Indexers/XSpeeds.cs
index 1c74b4df45802ddc93bc6390a2839c4fe627e8f0..5abe5a01b18dba4e653fab5d19e58c3f9b5cddd8 100644
--- a/src/Jackett/Indexers/XSpeeds.cs
+++ b/src/Jackett/Indexers/XSpeeds.cs
@@ -127,6 +127,9 @@ namespace Jackett.Indexers
if (string.IsNullOrWhiteSpace(searchString))
{
var rssPage = await RequestStringWithCookiesAndRetry(string.Format(RSSUrl, configData.RSSKey.Value));
+ if (rssPage.Content.EndsWith("\0")) {
+ rssPage.Content = rssPage.Content.Substring(0, rssPage.Content.Length - 1);
+ }
var rssDoc = XDocument.Parse(rssPage.Content);
foreach (var item in rssDoc.Descendants("item"))
@@ -170,6 +173,11 @@ namespace Jackett.Indexers
}
else
{
+ if (searchString.Length < 3)
+ {
+ OnParseError("", new Exception("Minimum search length is 3"));
+ return releases;
+ }
var searchParams = new Dictionary<string, string> {
{ "do", "search" },
{ "keywords", searchString },
diff --git a/src/Jackett/Models/IndexerConfig/ConfigurationData.cs b/src/Jackett/Models/IndexerConfig/ConfigurationData.cs
index 499a6879128186ab072a5e4ed79a6fc096f4773a..134968542926d5c905ff86dfc6963e2a4cd6fe48 100644
--- a/src/Jackett/Models/IndexerConfig/ConfigurationData.cs
+++ b/src/Jackett/Models/IndexerConfig/ConfigurationData.cs
@@ -132,7 +132,7 @@ namespace Jackett.Models.IndexerConfig
if (!forDisplay)
{
properties = properties
- .Where(p => p.ItemType == ItemType.HiddenData || p.ItemType == ItemType.InputBool || p.ItemType == ItemType.InputString || p.ItemType == ItemType.Recaptcha)
+ .Where(p => p.ItemType == ItemType.HiddenData || p.ItemType == ItemType.InputBool || p.ItemType == ItemType.InputString || p.ItemType == ItemType.Recaptcha || p.ItemType == ItemType.DisplayInfo)
.ToArray();
}
diff --git a/src/Jackett/Startup.cs b/src/Jackett/Startup.cs
index 7426088f0cece811088e3c0ea07e643a9129bc44..d462836f33b21eff36182a420d8ad40badb95f1f 100644
--- a/src/Jackett/Startup.cs
+++ b/src/Jackett/Startup.cs
@@ -40,6 +40,11 @@ namespace Jackett
set;
}
+ public static string ProxyConnection
+ {
+ get;
+ set;
+ }
public static bool? DoSSLFix
{
get;
diff --git a/src/Jackett/Utils/Clients/HttpWebClient.cs b/src/Jackett/Utils/Clients/HttpWebClient.cs
index e4848ec148511d59c952b912960f8dace220a3bd..2e2602d05b634e74bdacf2d1144e56fd648e8465 100644
--- a/src/Jackett/Utils/Clients/HttpWebClient.cs
+++ b/src/Jackett/Utils/Clients/HttpWebClient.cs
@@ -62,18 +62,28 @@ namespace Jackett.Utils.Clients
}
}
}
-
+ var useProxy = false;
+ WebProxy proxyServer = null;
+ if (Startup.ProxyConnection != null)
+ {
+ proxyServer = new WebProxy(Startup.ProxyConnection, false);
+ useProxy = true;
+ }
var client = new HttpClient(new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
+ Proxy = proxyServer,
+ UseProxy = useProxy
});
+
- if(webRequest.EmulateBrowser)
+ if (webRequest.EmulateBrowser)
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
else
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
+
HttpResponseMessage response = null;
var request = new HttpRequestMessage();
request.Headers.ExpectContinue = false;
@@ -158,6 +168,7 @@ namespace Jackett.Utils.Clients
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable<string> cookieHeaders;
var responseCookies = new List<Tuple<string, string>>();
+
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
foreach (var value in cookieHeaders)
diff --git a/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs b/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs
index 39eeec4e1aadbac75d6d477e5406b14ba02b59bd..1bb92ee0d9ea550284bc5ddbda776e8c15f53f16 100644
--- a/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs
+++ b/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs
@@ -104,6 +104,28 @@ namespace Jackett.Utils.Clients
case "location":
result.RedirectingTo = header[1];
break;
+ case "refresh":
+ if (response.Status == System.Net.HttpStatusCode.ServiceUnavailable)
+ {
+ //"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
+ var redirval = "";
+ var value = header[1];
+ var start = value.IndexOf("=");
+ var end = value.IndexOf(";");
+ var len = value.Length;
+ if (start > -1)
+ {
+ redirval = value.Substring(start + 1);
+ result.RedirectingTo = redirval;
+ // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
+ // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
+ // it shoudln't include service unavailable..only if we have this redirect header.
+ result.Status = System.Net.HttpStatusCode.Redirect;
+ var redirtime = Int32.Parse(value.Substring(0, end));
+ System.Threading.Thread.Sleep(redirtime * 1000);
+ }
+ }
+ break;
}
}
}
diff --git a/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs b/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs
index 7c124ddc5547a6a2dfc2dc66a620e044d3fbb364..cecef59eaa38158c0a71651f8c29d6ae07ddeb5b 100644
--- a/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs
+++ b/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs
@@ -50,6 +50,11 @@ namespace Jackett.Utils.Clients
private async Task<WebClientByteResult> Run(WebRequest request)
{
var args = new StringBuilder();
+ if (Startup.ProxyConnection != null)
+ {
+ args.AppendFormat("-x " + Startup.ProxyConnection + " ");
+ }
+
args.AppendFormat("--url \"{0}\" ", request.Url);
if (request.EmulateBrowser)
@@ -86,11 +91,16 @@ namespace Jackett.Utils.Clients
// https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html
args.Append("--cipher " + SSLFix.CipherList);
}
-
+ if (Startup.IgnoreSslErrors == true)
+ {
+ args.Append("-k ");
+ }
+ args.Append("-H \"Accept-Language: en-US,en\" ");
+ args.Append("-H \"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\" ");
string stdout = null;
await Task.Run(() =>
{
- stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString(), true);
+ stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString() , true);
});
var outputData = File.ReadAllBytes(tempFile);
@@ -101,6 +111,16 @@ namespace Jackett.Utils.Clients
if (headSplit < 0)
throw new Exception("Invalid response");
var headers = stdout.Substring(0, headSplit);
+ if (Startup.ProxyConnection != null)
+ {
+ // the proxy provided headers too so we need to split headers again
+ var headSplit1 = stdout.IndexOf("\r\n\r\n",headSplit + 4);
+ if (headSplit1 > 0)
+ {
+ headers = stdout.Substring(headSplit + 4,headSplit1 - (headSplit + 4));
+ headSplit = headSplit1;
+ }
+ }
var headerCount = 0;
var cookieBuilder = new StringBuilder();
var cookies = new List<Tuple<string, string>>();
@@ -131,6 +151,24 @@ namespace Jackett.Utils.Clients
case "location":
result.RedirectingTo = value.Trim();
break;
+ case "refresh":
+ //"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
+ var redirval = "";
+ var start = value.IndexOf("=");
+ var end = value.IndexOf(";");
+ var len = value.Length;
+ if (start > -1)
+ {
+ redirval = value.Substring(start + 1);
+ result.RedirectingTo = redirval;
+ // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
+ // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
+ // it shoudln't include service unavailable..only if we have this redirect header.
+ result.Status = System.Net.HttpStatusCode.Redirect;
+ var redirtime = Int32.Parse(value.Substring(0, end));
+ System.Threading.Thread.Sleep(redirtime * 1000);
+ }
+ break;
}
}
}