Example 1 with WebResult
use of com.biglybt.core.metasearch.impl.web.WebResult in project BiglyBT by BiglySoftware.
the class RegexEngine method searchSupport.
@Override
protected Result[] searchSupport(final SearchParameter[] searchParameters, Map searchContext, final int desired_max_matches, final int o_absolute_max_matches, final String headers, final ResultListener listener) throws SearchException {
debugStart();
final pageDetails page_details = getWebPageContent(searchParameters, searchContext, headers, false);
final String page = page_details.getContent();
if (listener != null) {
listener.contentReceived(this, page);
}
debugLog("pattern: " + pattern_str);
try {
TimeLimitedTask task = new TimeLimitedTask("MetaSearch:regexpr", 30 * 1000, Thread.NORM_PRIORITY - 1, new TimeLimitedTask.task() {
@Override
public Object run() throws Exception {
int max_matches = o_absolute_max_matches;
if (max_matches < 0 || max_matches > 1024) {
max_matches = 1024;
}
String searchQuery = null;
for (int i = 0; i < searchParameters.length; i++) {
if (searchParameters[i].getMatchPattern().equals("s")) {
searchQuery = searchParameters[i].getValue();
}
}
FieldMapping[] mappings = getMappings();
try {
List<WebResult> results = new ArrayList<>();
for (int pat_num = 0; pat_num < patterns.length; pat_num++) {
if (results.size() > 0) {
break;
}
Pattern pattern = patterns[pat_num];
Matcher m = pattern.matcher(page);
while (m.find()) {
if (max_matches >= 0) {
if (--max_matches < 0) {
break;
}
}
String[] groups = new String[m.groupCount()];
for (int i = 0; i < groups.length; i++) {
groups[i] = m.group(i + 1);
}
if (listener != null) {
listener.matchFound(RegexEngine.this, groups);
}
debugLog("Found match:");
WebResult result = new WebResult(RegexEngine.this, getRootPage(), getBasePage(), getDateParser(), searchQuery);
int fields_matched = 0;
for (int i = 0; i < mappings.length; i++) {
String fieldFrom = mappings[i].getName();
String fieldContent = null;
Matcher matcher = patternVariable.matcher(fieldFrom);
if (matcher.find()) {
fieldContent = fieldFrom;
do {
String key = matcher.group();
key = key.substring(2, key.length() - 1);
String[] keys = key.split(",", -1);
try {
int groupNo = Integer.parseInt(keys[0]);
// Default: Replace ${1} with groups[0]
String replaceWith = groups[groupNo - 1];
if (keys.length > 1) {
String[] commands = keys[1].split("\\+");
int keyPos = 2;
for (String command : commands) {
try {
if (DEBUG_MAPPINGS) {
System.out.println("command " + command);
}
if (command.equals("replace")) {
if (keyPos + 2 > keys.length) {
if (DEBUG_MAPPINGS) {
System.out.println("not enough keys. have " + keys.length + "; need " + (keyPos + 3));
}
break;
}
String simpleReplace = keys[keyPos];
keyPos++;
String simpleReplacement = keys[keyPos];
keyPos++;
replaceWith = replaceWith.replaceAll(simpleReplace, simpleReplacement);
} else if (command.equals("ucase")) {
replaceWith = replaceWith.toUpperCase();
} else if (command.equals("lcase")) {
replaceWith = replaceWith.toLowerCase();
} else if (command.equals("urldecode")) {
replaceWith = UrlUtils.decode(replaceWith);
}
if (DEBUG_MAPPINGS) {
System.out.println("replaceWith now " + replaceWith);
}
} catch (Exception e) {
if (DEBUG_MAPPINGS) {
System.out.println(e.toString());
}
}
}
}
fieldContent = fieldContent.replaceFirst(variablePattern, replaceWith);
} catch (Exception e) {
}
} while (matcher.find());
} else {
try {
int groupNo = Integer.parseInt(fieldFrom);
fieldContent = groups[groupNo - 1];
} catch (Exception e) {
// In "Debug/Test" mode, we should fire an exception / notification
}
}
if (fieldContent != null) {
int fieldTo = mappings[i].getField();
debugLog(" " + fieldTo + "=" + fieldContent);
fields_matched++;
switch(fieldTo) {
case FIELD_NAME:
result.setNameFromHTML(fieldContent);
break;
case FIELD_SIZE:
result.setSizeFromHTML(fieldContent);
break;
case FIELD_PEERS:
result.setNbPeersFromHTML(fieldContent);
break;
case FIELD_SEEDS:
result.setNbSeedsFromHTML(fieldContent);
break;
case FIELD_CATEGORY:
result.setCategoryFromHTML(fieldContent);
break;
case FIELD_DATE:
result.setPublishedDateFromHTML(fieldContent);
break;
case FIELD_CDPLINK:
result.setCDPLink(fieldContent);
break;
case FIELD_TORRENTLINK:
result.setTorrentLink(fieldContent);
break;
case FIELD_PLAYLINK:
result.setPlayLink(fieldContent);
break;
case FIELD_DOWNLOADBTNLINK:
result.setDownloadButtonLink(fieldContent);
break;
case FIELD_COMMENTS:
result.setCommentsFromHTML(fieldContent);
break;
case FIELD_VOTES:
result.setVotesFromHTML(fieldContent);
break;
case FIELD_SUPERSEEDS:
result.setNbSuperSeedsFromHTML(fieldContent);
break;
case FIELD_PRIVATE:
result.setPrivateFromHTML(fieldContent);
break;
case FIELD_DRMKEY:
result.setDrmKey(fieldContent);
break;
case FIELD_VOTES_DOWN:
result.setVotesDownFromHTML(fieldContent);
break;
case FIELD_HASH:
result.setHash(fieldContent);
break;
default:
fields_matched--;
break;
}
}
}
if (fields_matched > 0) {
if (result.getHash() == null) {
String downloadLink = result.getDownloadLink();
String possibleMagnet = UrlUtils.parseTextForMagnets(downloadLink);
byte[] hash = UrlUtils.getHashFromMagnetURI(possibleMagnet);
if (hash != null) {
result.setHash(ByteFormatter.nicePrint(hash, true));
}
}
results.add(result);
}
}
}
if (results.size() == 0 && isNeedsAuth()) {
if (page_details.getInitialURL().getProtocol().equalsIgnoreCase("http") && page_details.getFinalURL().getProtocol().equalsIgnoreCase("https")) {
throw new SearchLoginException("login possibly required");
}
}
return (Result[]) results.toArray(new Result[results.size()]);
} catch (Throwable e) {
log("Failed process result", e);
if (e instanceof SearchException) {
throw ((SearchException) e);
}
throw new SearchException(e);
}
}
});
Result[] res = (Result[]) task.run();
debugLog("success: found " + res.length + " results");
return (res);
} catch (Throwable e) {
debugLog("failed: " + Debug.getNestedExceptionMessageAndStack(e));
if (e instanceof SearchException) {
throw ((SearchException) e);
}
throw (new SearchException("Regex matching failed", e));
}
}
Also used :
Pattern(java.util.regex.Pattern)
Matcher(java.util.regex.Matcher)
IOException(java.io.IOException)
WebResult(com.biglybt.core.metasearch.impl.web.WebResult)
JSONObject(org.json.simple.JSONObject)
ArrayList(java.util.ArrayList)
List(java.util.List)
TimeLimitedTask(com.biglybt.core.util.TimeLimitedTask)
WebResult(com.biglybt.core.metasearch.impl.web.WebResult)
Example 2 with WebResult
use of com.biglybt.core.metasearch.impl.web.WebResult in project BiglyBT by BiglySoftware.
the class JSONEngine method searchSupport.
@Override
protected Result[] searchSupport(SearchParameter[] searchParameters, Map searchContext, int desired_max_matches, int absolute_max_matches, String headers, ResultListener listener) throws SearchException {
debugStart();
pageDetails page_details = super.getWebPageContent(searchParameters, searchContext, headers, false);
String page = page_details.getContent();
if (listener != null) {
listener.contentReceived(this, page);
}
String searchQuery = null;
for (int i = 0; i < searchParameters.length; i++) {
if (searchParameters[i].getMatchPattern().equals("s")) {
searchQuery = searchParameters[i].getValue();
}
}
FieldMapping[] mappings = getMappings();
try {
Object jsonObject;
try {
jsonObject = JSONValue.parse(page);
} catch (Throwable e) {
// fix a vaguely common error: trailing \ before end-of-string: - \",
String temp_page = page.replaceAll("\\\\\",", "\",");
try {
jsonObject = JSONValue.parse(temp_page);
} catch (Throwable f) {
throw (e);
}
}
if (rankDivisorPath != null) {
String[] split = rankDivisorPath.split("\\.");
try {
if (split.length > 0) {
Object jsonRankDivisor = jsonObject;
for (int i = 0; i < split.length - 1; i++) {
String key = split[i];
if (jsonRankDivisor instanceof JSONObject) {
jsonRankDivisor = ((JSONObject) jsonRankDivisor).get(key);
} else {
break;
}
}
if (jsonRankDivisor instanceof Map) {
jsonRankDivisor = ((Map) jsonRankDivisor).get(split[split.length - 1]);
}
if (jsonRankDivisor instanceof Number) {
rankDivisor = ((Number) jsonRankDivisor).floatValue();
}
}
} catch (Exception e) {
}
}
JSONArray resultArray = null;
if (resultsEntryPath != null && resultsEntryPath.length() > 0) {
String[] split = resultsEntryPath.split("\\.");
if (jsonObject instanceof JSONArray && split.length > 0 && !split[0].startsWith("[")) {
JSONArray array = (JSONArray) jsonObject;
if (array.size() == 1) {
jsonObject = array.get(0);
}
}
for (String pathEntry : split) {
if (jsonObject == null) {
throw new SearchException("Invalid entry path : " + resultsEntryPath);
}
try {
if (pathEntry.startsWith("[") && pathEntry.endsWith("]")) {
int idx = Integer.parseInt(pathEntry.substring(1, pathEntry.length() - 1));
jsonObject = ((JSONArray) jsonObject).get(idx);
} else {
jsonObject = ((JSONObject) jsonObject).get(pathEntry);
}
} catch (Throwable t) {
throw new SearchException("Invalid entry path : " + resultsEntryPath, t);
}
}
}
try {
resultArray = (JSONArray) jsonObject;
} catch (Throwable t) {
throw new SearchException("Object is not a result array. Check the JSON service and/or the entry path");
}
if (resultArray != null) {
List results = new ArrayList();
Throwable decode_failure = null;
for (int i = 0; i < resultArray.size(); i++) {
Object obj = resultArray.get(i);
if (obj instanceof JSONObject) {
JSONObject jsonEntry = (JSONObject) obj;
if (absolute_max_matches >= 0) {
if (--absolute_max_matches < 0) {
break;
}
}
if (listener != null) {
// sort for consistent order
Iterator it = new TreeMap(jsonEntry).entrySet().iterator();
String[] groups = new String[jsonEntry.size()];
int pos = 0;
while (it.hasNext()) {
Map.Entry entry = (Map.Entry) it.next();
Object key = entry.getKey();
Object value = entry.getValue();
if (key != null && value != null) {
groups[pos++] = key.toString() + "=" + UrlUtils.encode(value.toString());
} else {
groups[pos++] = "";
}
}
listener.matchFound(this, groups);
}
WebResult result = new WebResult(this, getRootPage(), getBasePage(), getDateParser(), searchQuery);
try {
for (int j = 0; j < mappings.length; j++) {
String fieldFrom = mappings[j].getName();
if (fieldFrom == null) {
continue;
}
int fieldTo = mappings[j].getField();
String fieldContent = null;
Matcher matcher = patternVariable.matcher(fieldFrom);
if (matcher.find()) {
fieldContent = fieldFrom;
do {
String key = matcher.group();
key = key.substring(2, key.length() - 1);
String[] keys = key.split(",", -1);
try {
Object replaceWithObject = jsonEntry.get(keys[0]);
String replaceWith = replaceWithObject == null ? "" : replaceWithObject.toString();
if (keys.length > 1) {
String[] commands = keys[1].split("\\+");
int keyPos = 2;
for (String command : commands) {
try {
if (DEBUG_MAPPINGS) {
System.out.println("command " + command);
}
if (command.equals("replace")) {
if (keyPos + 2 > keys.length) {
if (DEBUG_MAPPINGS) {
System.out.println("not enough keys. have " + keys.length + "; need " + (keyPos + 3));
}
break;
}
String simpleReplace = keys[keyPos];
keyPos++;
String simpleReplacement = keys[keyPos];
keyPos++;
replaceWith = replaceWith.replaceAll(simpleReplace, simpleReplacement);
} else if (command.equals("ucase")) {
replaceWith = replaceWith.toUpperCase();
} else if (command.equals("lcase")) {
replaceWith = replaceWith.toLowerCase();
} else if (command.equals("urldecode")) {
replaceWith = UrlUtils.decode(replaceWith);
}
if (DEBUG_MAPPINGS) {
System.out.println("replaceWith now " + replaceWith);
}
} catch (Exception e) {
if (DEBUG_MAPPINGS) {
System.out.println(e.toString());
}
}
}
}
fieldContent = fieldContent.replaceFirst(variablePattern, replaceWith);
} catch (Exception e) {
}
} while (matcher.find());
} else {
Object fieldContentObj = jsonEntry.get(fieldFrom);
fieldContent = fieldContentObj == null ? "" : fieldContentObj.toString();
}
if (fieldContent == null) {
continue;
}
switch(fieldTo) {
case FIELD_NAME:
result.setNameFromHTML(fieldContent);
break;
case FIELD_SIZE:
result.setSizeFromHTML(fieldContent);
break;
case FIELD_PEERS:
result.setNbPeersFromHTML(fieldContent);
break;
case FIELD_SEEDS:
result.setNbSeedsFromHTML(fieldContent);
break;
case FIELD_CATEGORY:
result.setCategoryFromHTML(fieldContent);
break;
case FIELD_DATE:
result.setPublishedDateFromHTML(fieldContent);
break;
case FIELD_COMMENTS:
result.setCommentsFromHTML(fieldContent);
break;
case FIELD_CDPLINK:
result.setCDPLink(fieldContent);
break;
case FIELD_TORRENTLINK:
result.setTorrentLink(fieldContent);
break;
case FIELD_PLAYLINK:
result.setPlayLink(fieldContent);
break;
case FIELD_DOWNLOADBTNLINK:
result.setDownloadButtonLink(fieldContent);
break;
case FIELD_VOTES:
result.setVotesFromHTML(fieldContent);
break;
case FIELD_SUPERSEEDS:
result.setNbSuperSeedsFromHTML(fieldContent);
break;
case FIELD_PRIVATE:
result.setPrivateFromHTML(fieldContent);
break;
case FIELD_DRMKEY:
result.setDrmKey(fieldContent);
break;
case FIELD_VOTES_DOWN:
result.setVotesDownFromHTML(fieldContent);
break;
case FIELD_HASH:
// seen a magnet being returned as hash!
if (fieldContent.startsWith("magnet:")) {
byte[] hash = UrlUtils.getHashFromMagnetURI(fieldContent);
if (hash != null) {
fieldContent = ByteFormatter.encodeString(hash);
} else {
fieldContent = null;
}
}
if (fieldContent != null) {
result.setHash(fieldContent);
}
break;
case FIELD_RANK:
{
result.setRankFromHTML(fieldContent, rankDivisor);
break;
}
default:
break;
}
}
if (result.getHash() == null) {
String downloadLink = result.getDownloadLink();
String possibleMagnet = UrlUtils.parseTextForMagnets(downloadLink);
byte[] hash = UrlUtils.getHashFromMagnetURI(possibleMagnet);
if (hash != null) {
result.setHash(ByteFormatter.nicePrint(hash, true));
}
}
results.add(result);
} catch (Throwable e) {
decode_failure = e;
}
}
}
if (results.size() == 0 && decode_failure != null) {
throw (decode_failure);
}
Result[] res = (Result[]) results.toArray(new Result[results.size()]);
debugLog("success: found " + res.length + " results");
return (res);
} else {
debugLog("success: no result array found so no results");
return (new Result[0]);
}
} catch (Throwable e) {
debugLog("failed: " + Debug.getNestedExceptionMessageAndStack(e));
if (e instanceof SearchException) {
throw ((SearchException) e);
}
String content_str = page;
if (content_str.length() > 256) {
content_str = content_str.substring(0, 256) + "...";
}
throw (new SearchException("JSON matching failed for " + getName() + ", content=" + content_str, e));
}
}
Also used :
Matcher(java.util.regex.Matcher)
FieldMapping(com.biglybt.core.metasearch.impl.web.FieldMapping)
WebResult(com.biglybt.core.metasearch.impl.web.WebResult)
WebResult(com.biglybt.core.metasearch.impl.web.WebResult)
JSONArray(org.json.simple.JSONArray)
IOException(java.io.IOException)
JSONObject(org.json.simple.JSONObject)
JSONObject(org.json.simple.JSONObject)
Example 3 with WebResult
use of com.biglybt.core.metasearch.impl.web.WebResult in project BiglyBT by BiglySoftware.
the class RSSEngine method searchSupport.
@Override
protected Result[] searchSupport(SearchParameter[] searchParameters, Map searchContext, int desired_max_matches, int absolute_max_matches, String headers, ResultListener listener) throws SearchException {
debugStart();
boolean only_if_mod = !searchContext.containsKey(Engine.SC_FORCE_FULL);
pageDetails page_details = super.getWebPageContent(searchParameters, searchContext, headers, only_if_mod, new pageDetailsVerifier() {
@Override
public void verify(pageDetails details) throws SearchException {
try {
String page = details.getContent();
if (page != null && page.length() > 0) {
ByteArrayInputStream bais = new ByteArrayInputStream(page.getBytes("UTF-8"));
RSSFeed rssFeed = StaticUtilities.getRSSFeed(details.getInitialURL(), bais);
details.setVerifiedState(rssFeed);
}
} catch (Throwable e) {
debugLog("failed: " + Debug.getNestedExceptionMessageAndStack(e));
if (e instanceof SearchException) {
throw ((SearchException) e);
}
throw (new SearchException("RSS matching failed", e));
}
}
});
String page = page_details.getContent();
if (listener != null) {
listener.contentReceived(this, page);
}
if (page == null || page.length() == 0) {
return (new Result[0]);
}
try {
RSSFeed rssFeed = (RSSFeed) page_details.getVerifiedState();
RSSChannel[] channels = rssFeed.getChannels();
List results = new ArrayList();
for (int i = 0; i < channels.length; i++) {
RSSChannel channel = channels[i];
SimpleXMLParserDocumentNode[] channel_kids = channel.getNode().getChildren();
int auto_dl_state = AUTO_DL_SUPPORTED_YES;
for (int j = 0; j < channel_kids.length; j++) {
SimpleXMLParserDocumentNode child = channel_kids[j];
String lc_full_child_name = child.getFullName().toLowerCase();
if (lc_full_child_name.equals("vuze:auto_dl_enabled")) {
if (!child.getValue().equalsIgnoreCase("true")) {
auto_dl_state = AUTO_DL_SUPPORTED_NO;
}
}
}
setLocalLong(LD_AUTO_DL_SUPPORTED, auto_dl_state);
RSSItem[] items = channel.getItems();
for (int j = 0; j < items.length; j++) {
RSSItem item = items[j];
WebResult result = new WebResult(this, getRootPage(), getBasePage(), getDateParser(), "");
result.setPublishedDate(item.getPublicationDate());
result.setNameFromHTML(item.getTitle());
URL cdp_link = item.getLink();
boolean cdp_set = false;
if (cdp_link != null) {
String link_url = cdp_link.toExternalForm();
String lc_url = link_url.toLowerCase(Locale.US);
if (lc_url.startsWith("http") || lc_url.startsWith("tor:http")) {
result.setCDPLink(link_url);
cdp_set = true;
}
}
String uid = item.getUID();
if (uid != null) {
result.setUID(uid);
if (!cdp_set) {
try {
String test_url = new URL(uid).toExternalForm();
if (test_url.toLowerCase().startsWith("http")) {
result.setCDPLink(test_url);
}
} catch (Throwable e) {
}
}
}
boolean got_seeds_peers = false;
int item_seeds = -1;
int item_peers = -1;
String item_hash = null;
String item_magnet = null;
String desc_size = null;
SimpleXMLParserDocumentNode node = item.getNode();
if (node != null) {
SimpleXMLParserDocumentNode[] children = node.getChildren();
boolean vuze_feed = false;
for (int k = 0; k < children.length; k++) {
SimpleXMLParserDocumentNode child = children[k];
String lc_full_child_name = child.getFullName().toLowerCase();
if (lc_full_child_name.startsWith("vuze:")) {
vuze_feed = true;
break;
}
}
for (int k = 0; k < children.length; k++) {
SimpleXMLParserDocumentNode child = children[k];
String lc_child_name = child.getName().toLowerCase();
String lc_full_child_name = child.getFullName().toLowerCase();
String value = child.getValue().trim();
if (lc_child_name.equals("enclosure")) {
SimpleXMLParserDocumentAttribute typeAtt = child.getAttribute("type");
if (typeAtt != null && typeAtt.getValue().equalsIgnoreCase("application/x-bittorrent")) {
SimpleXMLParserDocumentAttribute urlAtt = child.getAttribute("url");
if (urlAtt != null) {
result.setTorrentLink(urlAtt.getValue());
}
SimpleXMLParserDocumentAttribute lengthAtt = child.getAttribute("length");
if (lengthAtt != null) {
result.setSizeFromHTML(lengthAtt.getValue());
}
}
} else if (lc_child_name.equals("category")) {
result.setCategoryFromHTML(value);
} else if (lc_child_name.equals("comments")) {
result.setCommentsFromHTML(value);
} else if (lc_child_name.equals("link") || lc_child_name.equals("guid")) {
String lc_value = value.toLowerCase();
try {
URL url = new URL(value);
if (lc_value.endsWith(".torrent") || lc_value.startsWith("magnet:") || lc_value.startsWith("bc:") || lc_value.startsWith("bctp:") || lc_value.startsWith("dht:")) {
result.setTorrentLink(value);
} else if (lc_child_name.equals("link") && !vuze_feed) {
long test = getLocalLong(LD_LINK_IS_TORRENT, 0);
if (test == 1) {
result.setTorrentLink(value);
} else if (test == 0 || SystemTime.getCurrentTime() - test > 60 * 1000) {
if (linkIsToTorrent(url)) {
result.setTorrentLink(value);
setLocalLong(LD_LINK_IS_TORRENT, 1);
} else {
setLocalLong(LD_LINK_IS_TORRENT, SystemTime.getCurrentTime());
}
}
}
} catch (Throwable e) {
// see if this is an atom feed
// <link rel="alternate" type="application/x-bittorrent" href="http://asdasd/
SimpleXMLParserDocumentAttribute typeAtt = child.getAttribute("type");
if (typeAtt != null && typeAtt.getValue().equalsIgnoreCase("application/x-bittorrent")) {
SimpleXMLParserDocumentAttribute hrefAtt = child.getAttribute("href");
if (hrefAtt != null) {
String href = hrefAtt.getValue().trim();
try {
result.setTorrentLink(new URL(href).toExternalForm());
} catch (Throwable f) {
}
}
}
}
} else if (lc_child_name.equals("content") && rssFeed.isAtomFeed()) {
SimpleXMLParserDocumentAttribute srcAtt = child.getAttribute("src");
String src = srcAtt == null ? null : srcAtt.getValue();
if (src != null) {
boolean is_dl_link = false;
SimpleXMLParserDocumentAttribute typeAtt = child.getAttribute("type");
if (typeAtt != null && typeAtt.getValue().equalsIgnoreCase("application/x-bittorrent")) {
is_dl_link = true;
}
if (!is_dl_link) {
is_dl_link = src.toLowerCase().contains(".torrent");
}
if (is_dl_link) {
try {
new URL(src);
result.setTorrentLink(src);
} catch (Throwable e) {
}
}
}
} else if (lc_full_child_name.equals("vuze:size")) {
result.setSizeFromHTML(value);
} else if (lc_full_child_name.equals("vuze:seeds")) {
got_seeds_peers = true;
result.setNbSeedsFromHTML(value);
} else if (lc_full_child_name.equals("vuze:superseeds")) {
got_seeds_peers = true;
result.setNbSuperSeedsFromHTML(value);
} else if (lc_full_child_name.equals("vuze:peers")) {
got_seeds_peers = true;
result.setNbPeersFromHTML(value);
} else if (lc_full_child_name.equals("vuze:rank")) {
result.setRankFromHTML(value);
} else if (lc_full_child_name.equals("vuze:contenttype")) {
String type = value.toLowerCase();
if (type.startsWith("video")) {
type = Engine.CT_VIDEO;
} else if (type.startsWith("audio")) {
type = Engine.CT_AUDIO;
} else if (type.startsWith("games")) {
type = Engine.CT_GAME;
}
result.setContentType(type);
} else if (lc_full_child_name.equals("vuze:downloadurl")) {
result.setTorrentLink(value);
} else if (lc_full_child_name.equals("vuze:playurl")) {
result.setPlayLink(value);
} else if (lc_full_child_name.equals("vuze:drmkey")) {
result.setDrmKey(value);
} else if (lc_full_child_name.equals("vuze:assethash")) {
result.setHash(value);
} else if (lc_child_name.equals("seeds") || lc_child_name.equals("seeders")) {
try {
item_seeds = Integer.parseInt(value);
} catch (Throwable e) {
}
} else if (lc_child_name.equals("peers") || lc_child_name.equals("leechers")) {
try {
item_peers = Integer.parseInt(value);
} catch (Throwable e) {
}
} else if (lc_child_name.equals("infohash") || lc_child_name.equals("info_hash")) {
item_hash = value;
} else if (lc_child_name.equals("magneturi")) {
item_magnet = value;
}
}
}
if (!got_seeds_peers) {
if (item_peers >= 0 && item_seeds >= 0) {
result.setNbSeedsFromHTML(String.valueOf(item_seeds));
result.setNbPeersFromHTML(String.valueOf(item_peers));
got_seeds_peers = true;
}
}
if (!got_seeds_peers) {
try {
SimpleXMLParserDocumentNode desc_node = node.getChild("description");
if (desc_node != null) {
String desc = desc_node.getValue().trim();
// see if we can pull from description
desc = desc.replaceAll("\\(s\\)", "s");
desc = desc.replaceAll("seeders", "seeds");
Matcher m = seed_leecher_pat.matcher(desc);
while (m.find()) {
String num = m.group(1);
String type = m.group(2);
if (type.toLowerCase().charAt(0) == 's') {
result.setNbSeedsFromHTML(num);
} else {
result.setNbPeersFromHTML(num);
}
}
m = size_pat.matcher(desc);
if (m.find()) {
desc_size = m.group(1) + " " + m.group(2);
}
}
} catch (Throwable e) {
}
}
try {
SimpleXMLParserDocumentNode torrent_node = node.getChild("torrent");
if (torrent_node != null) {
if (result.getSize() <= 0) {
SimpleXMLParserDocumentNode n = torrent_node.getChild("contentLength");
if (n != null) {
try {
long l = Long.parseLong(n.getValue().trim());
result.setSizeFromHTML(l + " B");
} catch (Throwable e) {
}
}
}
String dlink = result.getDownloadLink();
if (dlink == null || dlink.length() == 0) {
SimpleXMLParserDocumentNode n = torrent_node.getChild("magnetURI");
if (n != null) {
dlink = n.getValue().trim();
result.setTorrentLink(dlink);
}
}
String hash = result.getHash();
if (hash == null || hash.length() == 0) {
SimpleXMLParserDocumentNode n = torrent_node.getChild("infoHash");
if (n != null) {
String h = n.getValue().trim();
result.setHash(h);
if (dlink == null || dlink.length() == 0) {
String uri = UrlUtils.normaliseMagnetURI(h);
if (uri != null) {
result.setTorrentLink(uri);
}
}
}
}
SimpleXMLParserDocumentNode trackers_node = torrent_node.getChild("trackers");
if (trackers_node != null && !got_seeds_peers) {
SimpleXMLParserDocumentNode[] groups = trackers_node.getChildren();
int max_total = -1;
int best_seeds = 0;
int best_leechers = 0;
for (SimpleXMLParserDocumentNode group : groups) {
SimpleXMLParserDocumentNode[] g_kids = group.getChildren();
for (SimpleXMLParserDocumentNode t : g_kids) {
if (t.getName().equalsIgnoreCase("tracker")) {
SimpleXMLParserDocumentAttribute a_seeds = t.getAttribute("seeds");
SimpleXMLParserDocumentAttribute a_leechers = t.getAttribute("peers");
int seeds = a_seeds == null ? -1 : Integer.parseInt(a_seeds.getValue().trim());
int leechers = a_leechers == null ? -1 : Integer.parseInt(a_leechers.getValue().trim());
int total = seeds + leechers;
if (total > max_total) {
max_total = total;
best_seeds = seeds;
best_leechers = leechers;
}
}
}
}
if (max_total >= 0) {
result.setNbSeedsFromHTML(String.valueOf(Math.max(0, best_seeds)));
result.setNbPeersFromHTML(String.valueOf(Math.max(0, best_leechers)));
}
}
}
} catch (Throwable e) {
e.printStackTrace();
}
if (result.getSize() <= 0) {
SimpleXMLParserDocumentNode n = node.getChild("size");
if (n != null) {
result.setSizeFromHTML(n.getValue().trim());
}
}
if (item_hash != null && result.getHash() == null) {
result.setHash(item_hash);
}
if (item_magnet != null) {
String existing = result.getTorrentLinkRaw();
if (existing == null || existing.length() == 0) {
result.setTorrentLink(item_magnet);
}
}
// if we still have no download link see if the magnet is in the title
String dlink = result.getDownloadLink();
if (dlink == null || dlink.length() == 0) {
String name = result.getName();
if (name != null) {
String magnet = UrlUtils.parseTextForMagnets(name);
if (magnet != null) {
result.setTorrentLink(magnet);
}
}
}
dlink = result.getDownloadLink();
if (dlink == null || dlink.length() == 0) {
// last ditch effort, sometimes the download link is the <link> so stuff it in and hope (could test download once I guess and then
// record the outcome if it is a torrent but, meh)
result.setTorrentLink(result.getCDPLink());
}
// some feeds have the cdp link as guid - grab this if cdp link is blank or same as dl link
String latest_cdp_link = result.getCDPLink();
String latest_torrent_link = result.getTorrentLink();
if (latest_cdp_link == null || (latest_cdp_link.length() > 0 && latest_cdp_link.equals(latest_torrent_link))) {
try {
String test_url = new URL(uid).toExternalForm();
if (test_url.toLowerCase().startsWith("http")) {
result.setCDPLink(test_url);
}
} catch (Throwable e) {
}
}
if (result.getSize() <= 0) {
if (desc_size != null) {
result.setSizeFromHTML(desc_size);
}
}
if (result.getHash() == null) {
if (dlink != null) {
String mag = UrlUtils.parseTextForMagnets(dlink);
if (mag == null) {
String tlink = result.getTorrentLinkRaw();
if (tlink != null) {
mag = UrlUtils.parseTextForMagnets(tlink);
}
}
if (mag != null) {
byte[] hash = UrlUtils.getHashFromMagnetURI(mag);
if (hash != null) {
result.setHash(ByteFormatter.encodeString(hash));
}
}
}
}
results.add(result);
if (absolute_max_matches >= 0 && results.size() == absolute_max_matches) {
break;
}
}
}
Result[] res = (Result[]) results.toArray(new Result[results.size()]);
debugLog("success: found " + res.length + " results");
return (res);
} catch (Throwable e) {
debugLog("failed: " + Debug.getNestedExceptionMessageAndStack(e));
if (e instanceof SearchException) {
throw ((SearchException) e);
}
throw (new SearchException("RSS matching failed", e));
}
}
Also used :
Matcher(java.util.regex.Matcher)
RSSChannel(com.biglybt.pif.utils.xml.rss.RSSChannel)
URL(java.net.URL)
WebResult(com.biglybt.core.metasearch.impl.web.WebResult)
WebResult(com.biglybt.core.metasearch.impl.web.WebResult)
RSSFeed(com.biglybt.pif.utils.xml.rss.RSSFeed)
SimpleXMLParserDocumentAttribute(com.biglybt.pif.utils.xml.simpleparser.SimpleXMLParserDocumentAttribute)
RSSItem(com.biglybt.pif.utils.xml.rss.RSSItem)
ByteArrayInputStream(java.io.ByteArrayInputStream)
SimpleXMLParserDocumentNode(com.biglybt.pif.utils.xml.simpleparser.SimpleXMLParserDocumentNode)
Aggregations
WebResult (com.biglybt.core.metasearch.impl.web.WebResult)3
Matcher (java.util.regex.Matcher)3
IOException (java.io.IOException)2
JSONObject (org.json.simple.JSONObject)2
FieldMapping (com.biglybt.core.metasearch.impl.web.FieldMapping)1
TimeLimitedTask (com.biglybt.core.util.TimeLimitedTask)1
RSSChannel (com.biglybt.pif.utils.xml.rss.RSSChannel)1
RSSFeed (com.biglybt.pif.utils.xml.rss.RSSFeed)1
RSSItem (com.biglybt.pif.utils.xml.rss.RSSItem)1
SimpleXMLParserDocumentAttribute (com.biglybt.pif.utils.xml.simpleparser.SimpleXMLParserDocumentAttribute)1
SimpleXMLParserDocumentNode (com.biglybt.pif.utils.xml.simpleparser.SimpleXMLParserDocumentNode)1
ByteArrayInputStream (java.io.ByteArrayInputStream)1
URL (java.net.URL)1
ArrayList (java.util.ArrayList)1
List (java.util.List)1
Pattern (java.util.regex.Pattern)1
JSONArray (org.json.simple.JSONArray)1
