Example 1 with DHTInterface
use of com.biglybt.plugin.dht.DHTPluginInterface.DHTInterface in project BiglyBT by BiglySoftware.
the class RelatedContentSearcher method searchRCM.
protected SearchInstance searchRCM(Map<String, Object> search_parameters, SearchObserver _observer) throws SearchException {
final String term = fixupTerm((String) search_parameters.get(SearchProvider.SP_SEARCH_TERM));
final boolean is_popular = isPopularity(term);
final int min_seeds = MapUtils.importInt(search_parameters, SearchProvider.SP_MIN_SEEDS, is_popular ? SEARCH_POP_MIN_SEEDS_DEFAULT : SEARCH_MIN_SEEDS_DEFAULT);
final int min_leechers = MapUtils.importInt(search_parameters, SearchProvider.SP_MIN_LEECHERS, is_popular ? SEARCH_POP_MIN_LEECHERS_DEFAULT : SEARCH_MIN_LEECHERS_DEFAULT);
final MySearchObserver observer = new MySearchObserver(_observer, min_seeds, min_leechers);
final SearchInstance si = new SearchInstance() {
@Override
public void cancel() {
Debug.out("Cancelled");
}
};
if (term == null) {
observer.complete();
} else {
new AEThread2("RCM:search", true) {
@Override
public void run() {
boolean search_cvs_only = SEARCH_CVS_ONLY_DEFAULT;
final Set<String> hashes_sync_me = new HashSet<>();
try {
List<RelatedContent> matches = matchContent(term, min_seeds, min_leechers, true, search_cvs_only);
for (final RelatedContent c : matches) {
final byte[] hash = c.getHash();
if (hash == null) {
continue;
}
hashes_sync_me.add(Base32.encode(hash));
SearchResult result = new SearchResult() {
@Override
public Object getProperty(int property_name) {
if (property_name == SearchResult.PR_VERSION) {
return (new Long(c.getVersion()));
} else if (property_name == SearchResult.PR_NAME) {
return (c.getTitle());
} else if (property_name == SearchResult.PR_SIZE) {
return (c.getSize());
} else if (property_name == SearchResult.PR_HASH) {
return (hash);
} else if (property_name == SearchResult.PR_RANK) {
return (new Long(c.getRank() / 4));
} else if (property_name == SearchResult.PR_SEED_COUNT) {
return (new Long(c.getSeeds()));
} else if (property_name == SearchResult.PR_LEECHER_COUNT) {
return (new Long(c.getLeechers()));
} else if (property_name == SearchResult.PR_SUPER_SEED_COUNT) {
return (new Long(0));
} else if (property_name == SearchResult.PR_PUB_DATE) {
long date = c.getPublishDate();
if (date <= 0) {
return (null);
}
return (new Date(date));
} else if (property_name == SearchResult.PR_TORRENT_LINK || property_name == SearchResult.PR_DOWNLOAD_LINK || property_name == SearchResult.PR_DOWNLOAD_BUTTON_LINK) {
byte[] hash = c.getHash();
if (hash != null) {
return (UrlUtils.getMagnetURI(hash, c.getTitle(), c.getNetworks()));
}
} else if (property_name == SearchResult.PR_CATEGORY) {
String[] tags = c.getTags();
if (tags != null) {
for (String tag : tags) {
if (!tag.startsWith("_")) {
return (tag);
}
}
}
} else if (property_name == RelatedContentManager.RCM_SEARCH_PROPERTY_TRACKER_KEYS) {
return (c.getTrackerKeys());
} else if (property_name == RelatedContentManager.RCM_SEARCH_PROPERTY_WEB_SEED_KEYS) {
return (c.getWebSeedKeys());
} else if (property_name == RelatedContentManager.RCM_SEARCH_PROPERTY_TAGS) {
return (c.getTags());
} else if (property_name == RelatedContentManager.RCM_SEARCH_PROPERTY_NETWORKS) {
return (c.getNetworks());
}
return (null);
}
};
observer.resultReceived(si, result);
}
} finally {
try {
final List<DistributedDatabaseContact> initial_hinted_contacts = searchForeignBlooms(term);
final Set<DistributedDatabaseContact> extra_hinted_contacts = new HashSet<>();
Collections.shuffle(initial_hinted_contacts);
// test injection of local
// hinted_contacts.add( 0, ddb.getLocalContact());
final LinkedList<DistributedDatabaseContact> contacts_to_search = new LinkedList<>();
final Map<InetSocketAddress, DistributedDatabaseContact> contact_map = new HashMap<>();
for (DistributedDatabaseContact c : initial_hinted_contacts) {
// stick in map so non-hinted get removed below, but interleave later
contact_map.put(c.getAddress(), c);
}
if (ddb != null) {
DHTInterface[] dhts = dht_plugin.getDHTInterfaces();
boolean public_dht = dht_plugin.getNetwork() == AENetworkClassifier.AT_PUBLIC;
for (DHTInterface dht : dhts) {
if (dht.isIPV6()) {
continue;
}
int network = dht.getNetwork();
if (public_dht && search_cvs_only && network != DHT.NW_AZ_CVS) {
logSearch("Search: ignoring main DHT");
continue;
}
DHTPluginContact[] contacts = dht.getReachableContacts();
Collections.shuffle(Arrays.asList(contacts));
for (DHTPluginContact dc : contacts) {
InetSocketAddress address = dc.getAddress();
if (!contact_map.containsKey(address)) {
try {
DistributedDatabaseContact c = importContact(dc, network);
contact_map.put(address, c);
contacts_to_search.add(c);
} catch (Throwable e) {
}
}
}
}
if (contact_map.size() < MAX_REMOTE_SEARCH_CONTACTS) {
for (DHTInterface dht : dhts) {
if (dht.isIPV6()) {
continue;
}
int network = dht.getNetwork();
if (public_dht && search_cvs_only && network != DHT.NW_AZ_CVS) {
logSearch("Search: ignoring main DHT");
continue;
}
DHTPluginContact[] contacts = dht.getRecentContacts();
for (DHTPluginContact dc : contacts) {
InetSocketAddress address = dc.getAddress();
if (!contact_map.containsKey(address)) {
try {
DistributedDatabaseContact c = importContact(dc, network);
contact_map.put(address, c);
contacts_to_search.add(c);
if (contact_map.size() >= MAX_REMOTE_SEARCH_CONTACTS) {
break;
}
} catch (Throwable e) {
}
}
}
if (contact_map.size() >= MAX_REMOTE_SEARCH_CONTACTS) {
break;
}
}
}
}
// interleave hinted ones so we get some variety
int desired_pos = 0;
for (DistributedDatabaseContact dc : initial_hinted_contacts) {
if (desired_pos < contacts_to_search.size()) {
contacts_to_search.add(desired_pos, dc);
desired_pos += 2;
} else {
contacts_to_search.addLast(dc);
}
}
long start = SystemTime.getMonotonousTime();
long max = MAX_REMOTE_SEARCH_MILLIS;
final AESemaphore sem = new AESemaphore("RCM:rems");
int sent = 0;
final int[] done = { 0 };
logSearch("Search starts: contacts=" + contacts_to_search.size() + ", hinted=" + initial_hinted_contacts.size());
while (true) {
if (observer.getResultCount() >= 200 || SystemTime.getMonotonousTime() - start >= max) {
logSearch("Hard limit exceeded");
return;
}
if (sent >= MAX_REMOTE_SEARCH_CONTACTS) {
logSearch("Max contacts searched");
break;
}
final DistributedDatabaseContact contact_to_search;
synchronized (contacts_to_search) {
if (contacts_to_search.isEmpty()) {
logSearch("Contacts exhausted");
break;
} else {
contact_to_search = contacts_to_search.removeFirst();
}
}
new AEThread2("RCM:rems", true) {
@Override
public void run() {
try {
logSearch("Searching " + contact_to_search.getAddress());
List<DistributedDatabaseContact> extra_contacts = sendRemoteSearch(si, hashes_sync_me, contact_to_search, term, min_seeds, min_leechers, observer);
if (extra_contacts == null) {
logSearch(" " + contact_to_search.getAddress() + " failed");
foreignBloomFailed(contact_to_search);
} else {
String type;
if (initial_hinted_contacts.contains(contact_to_search)) {
type = "i";
} else if (extra_hinted_contacts.contains(contact_to_search)) {
type = "e";
} else {
type = "n";
}
logSearch(" " + contact_to_search.getAddress() + " OK " + type + " - additional=" + extra_contacts.size());
synchronized (contacts_to_search) {
int insert_point = 0;
if (type.equals("i")) {
for (int i = 0; i < contacts_to_search.size(); i++) {
if (extra_hinted_contacts.contains(contacts_to_search.get(i))) {
insert_point = i + 1;
}
}
}
for (DistributedDatabaseContact c : extra_contacts) {
InetSocketAddress address = c.getAddress();
if (!contact_map.containsKey(address)) {
logSearch(" additional target: " + address);
extra_hinted_contacts.add(c);
contact_map.put(address, c);
contacts_to_search.add(insert_point, c);
}
}
}
}
} finally {
synchronized (done) {
done[0]++;
}
sem.release();
}
}
}.start();
sent++;
synchronized (done) {
if (done[0] >= MAX_REMOTE_SEARCH_CONTACTS / 2) {
logSearch("Switching to reduced time limit (1)");
// give another 5 secs for results to come in
start = SystemTime.getMonotonousTime();
max = REDUCED_REMOTE_SEARCH_MILLIS;
break;
}
}
if (sent > 10) {
try {
Thread.sleep(250);
} catch (Throwable e) {
}
}
}
logSearch("Request dispatch complete: sent=" + sent + ", done=" + done[0]);
for (int i = 0; i < sent; i++) {
if (done[0] > sent * 9 / 10) {
logSearch("9/10ths replied (" + done[0] + "/" + sent + "), done");
break;
}
long remaining = (start + max) - SystemTime.getMonotonousTime();
if (remaining > REDUCED_REMOTE_SEARCH_MILLIS && done[0] >= MAX_REMOTE_SEARCH_CONTACTS / 2) {
logSearch("Switching to reduced time limit (2)");
// give another 5 secs for results to come in
start = SystemTime.getMonotonousTime();
max = REDUCED_REMOTE_SEARCH_MILLIS;
}
if (remaining > 0) {
sem.reserve(250);
} else {
logSearch("Time exhausted");
break;
}
}
} finally {
logSearch("Search complete");
observer.complete();
}
}
}
}.start();
}
return (si);
}
Also used :
InetSocketAddress(java.net.InetSocketAddress)
DHTInterface(com.biglybt.plugin.dht.DHTPluginInterface.DHTInterface)
DHTPluginContact(com.biglybt.plugin.dht.DHTPluginContact)
Example 2 with DHTInterface
use of com.biglybt.plugin.dht.DHTPluginInterface.DHTInterface in project BiglyBT by BiglySoftware.
the class RelatedContentSearcher method harvestBlooms.
private void harvestBlooms() {
harvest_dispatcher.dispatch(new AERunnable() {
@Override
public void runSupport() {
if (harvest_dispatcher.getQueueSize() > 0) {
return;
}
ForeignBloom oldest = null;
synchronized (harvested_blooms) {
for (ForeignBloom bloom : harvested_blooms.values()) {
if (oldest == null || bloom.getLastUpdateTime() < oldest.getLastUpdateTime()) {
oldest = bloom;
}
}
}
long now = SystemTime.getMonotonousTime();
if (oldest != null) {
if (now - oldest.getLastUpdateTime() > HARVEST_BLOOM_UPDATE_MILLIS) {
DistributedDatabaseContact ddb_contact = oldest.getContact();
if (now - oldest.getCreateTime() > HARVEST_BLOOM_DISCARD_MILLIS && harvested_blooms.size() >= HARVEST_MAX_BLOOMS / 2) {
// don't want to stick with a stable one for too long otherwise the stabler
// nodes will end up in lots of other nodes' harvest set and receive
// undue attention (unless we don't have that many nodes...)
logSearch("Harvest: discarding " + ddb_contact.getAddress());
synchronized (harvested_blooms) {
harvested_blooms.remove(ddb_contact.getID());
}
} else {
BloomFilter updated_filter = sendRemoteUpdate(oldest);
if (updated_filter == null) {
synchronized (harvested_blooms) {
harvested_blooms.remove(ddb_contact.getID());
harvested_fails.put(ddb_contact.getID(), "");
}
} else {
oldest.updateFilter(updated_filter);
}
}
}
}
if (harvested_blooms.size() < HARVEST_MAX_BLOOMS) {
try {
int fail_count = 0;
DHTInterface[] dhts = dht_plugin.getDHTInterfaces();
outer: for (DHTInterface dht : dhts) {
if (dht.isIPV6()) {
continue;
}
int network = dht.getNetwork();
if (SEARCH_CVS_ONLY_DEFAULT && network != DHT.NW_AZ_CVS) {
logSearch("Harvest: ignoring main DHT");
continue;
}
DHTPluginContact[] contacts = dht.getReachableContacts();
byte[] dht_id = dht.getID();
for (DHTPluginContact contact : contacts) {
byte[] contact_id = contact.getID();
if (Arrays.equals(dht_id, contact_id)) {
continue;
}
DistributedDatabaseContact ddb_contact = importContact(contact, network);
synchronized (harvested_blooms) {
if (harvested_fails.containsKey(contact_id)) {
continue;
}
if (harvested_blooms.containsKey(contact_id)) {
continue;
}
}
BloomFilter filter = sendRemoteFetch(ddb_contact);
logSearch("harvest: " + contact.getString() + " -> " + (filter == null ? "null" : filter.getString()));
if (filter != null) {
synchronized (harvested_blooms) {
harvested_blooms.put(contact_id, new ForeignBloom(ddb_contact, filter));
}
break outer;
} else {
synchronized (harvested_blooms) {
harvested_fails.put(contact_id, "");
}
fail_count++;
if (fail_count > 5) {
break outer;
}
}
}
}
} catch (Throwable e) {
e.printStackTrace();
}
}
synchronized (harvested_blooms) {
if (harvested_fails.size() > HARVEST_MAX_FAILS_HISTORY) {
harvested_fails.clear();
}
}
}
});
}
Also used :
DHTInterface(com.biglybt.plugin.dht.DHTPluginInterface.DHTInterface)
BloomFilter(com.biglybt.core.util.bloom.BloomFilter)
DHTPluginContact(com.biglybt.plugin.dht.DHTPluginContact)
Example 3 with DHTInterface
use of com.biglybt.plugin.dht.DHTPluginInterface.DHTInterface in project BiglyBT by BiglySoftware.
the class RelatedContentSearcher method receiveRemoteRequest.
protected Map<String, Object> receiveRemoteRequest(DistributedDatabaseContact originator, Map<String, Object> request) {
Map<String, Object> response = new HashMap<>();
try {
boolean originator_is_neighbour = false;
DHTInterface[] dhts = dht_plugin.getDHTInterfaces();
byte[] originator_id = originator.getID();
byte[] originator_bytes = AddressUtils.getAddressBytes(originator.getAddress());
for (DHTInterface d : dhts) {
List<DHTPluginContact> contacts = d.getClosestContacts(d.getID(), true);
for (DHTPluginContact c : contacts) {
if (Arrays.equals(c.getID(), originator_id)) {
originator_is_neighbour = true;
break;
}
}
if (originator_is_neighbour) {
break;
}
}
String req_type = MapUtils.getMapString(request, "x", null);
if (req_type != null) {
boolean dup = harvest_op_requester_bloom.contains(originator_bytes);
logSearch("Received remote request: " + BDecoder.decodeStrings(request) + " from " + originator.getAddress() + "/" + originator.getNetwork() + ", dup=" + dup + ", bs=" + harvest_op_requester_bloom.getEntryCount());
if (!dup) {
harvest_op_requester_bloom.add(originator_bytes);
if (req_type.equals("f")) {
BloomFilter filter = getKeyBloom(!originator_is_neighbour);
if (filter != null) {
response.put("f", filter.serialiseToMap());
}
} else if (req_type.equals("u")) {
BloomFilter filter = getKeyBloom(!originator_is_neighbour);
if (filter != null) {
int existing_size = MapUtils.importInt(request, "s", 0);
if (existing_size != filter.getEntryCount()) {
response.put("f", filter.serialiseToMap());
} else {
response.put("s", new Long(existing_size));
}
}
}
}
} else {
// fallback to default handling
int hits = harvest_se_requester_bloom.count(originator_bytes);
String term = MapUtils.getMapString(request, "t", null);
term = fixupTerm(term);
String network = MapUtils.getMapString(request, "n", "");
boolean search_cvs_only = network.equals("c");
int min_seeds = MapUtils.importInt(request, "s", SEARCH_MIN_SEEDS_DEFAULT);
int min_leechers = MapUtils.importInt(request, "l", SEARCH_MIN_LEECHERS_DEFAULT);
// System.out.println( "Received remote search: '" + term + "' from " + originator.getAddress() + ", hits=" + hits + ", bs=" + harvest_se_requester_bloom.getEntryCount());
logSearch("Received remote search: '" + term + "' from " + originator.getAddress() + ", hits=" + hits + ", bs=" + harvest_se_requester_bloom.getEntryCount());
if (hits < 10) {
harvest_se_requester_bloom.add(originator_bytes);
if (term != null) {
List<RelatedContent> matches = matchContent(term, min_seeds, min_leechers, false, search_cvs_only);
List<Map<String, Object>> l_list = new ArrayList<>();
for (int i = 0; i < matches.size(); i++) {
RelatedContent c = matches.get(i);
Map<String, Object> map = new HashMap<>();
l_list.add(map);
MapUtils.exportLong(map, "v", c.getVersion());
MapUtils.setMapString(map, "n", c.getTitle());
MapUtils.exportLong(map, "s", c.getSize());
MapUtils.exportLong(map, "r", c.getRank());
MapUtils.exportLong(map, "d", c.getLastSeenSecs());
MapUtils.exportLong(map, "p", c.getPublishDate() / (60 * 60 * 1000));
MapUtils.exportLong(map, "l", c.getLeechers());
MapUtils.exportLong(map, "z", c.getSeeds());
byte[] hash = c.getHash();
if (hash != null) {
map.put("h", hash);
}
byte[] tracker_keys = c.getTrackerKeys();
if (tracker_keys != null) {
map.put("k", tracker_keys);
}
byte[] ws_keys = c.getWebSeedKeys();
if (ws_keys != null) {
map.put("w", ws_keys);
}
String[] tags = c.getTags();
if (tags != null) {
map.put("g", manager.encodeTags(tags));
}
byte nets = c.getNetworksInternal();
if (nets != RelatedContentManager.NET_NONE && nets != RelatedContentManager.NET_PUBLIC) {
map.put("o", new Long(nets & 0x00ff));
}
// don't bother with tracker as no use to caller really
}
response.put("l", l_list);
List<DistributedDatabaseContact> bloom_hits = searchForeignBlooms(term);
if (bloom_hits.size() > 0) {
List<Map> c_list = new ArrayList<>();
for (DistributedDatabaseContact c : bloom_hits) {
Map m = new HashMap();
c_list.add(m);
InetSocketAddress address = c.getAddress();
if (address.isUnresolved()) {
m.put("m", c.exportToMap());
} else {
m.put("a", address.getAddress().getHostAddress());
m.put("p", new Long(address.getPort()));
}
}
response.put("c", c_list);
}
}
}
}
} catch (Throwable e) {
}
return (response);
}
Also used :
InetSocketAddress(java.net.InetSocketAddress)
DHTInterface(com.biglybt.plugin.dht.DHTPluginInterface.DHTInterface)
DHTPluginContact(com.biglybt.plugin.dht.DHTPluginContact)
BloomFilter(com.biglybt.core.util.bloom.BloomFilter)
Aggregations
DHTPluginContact (com.biglybt.plugin.dht.DHTPluginContact)3
DHTInterface (com.biglybt.plugin.dht.DHTPluginInterface.DHTInterface)3
BloomFilter (com.biglybt.core.util.bloom.BloomFilter)2
InetSocketAddress (java.net.InetSocketAddress)2
