Examples with BloomFilter com.biglybt.core.util.bloom.BloomFilter used on opensource projects

Example 6 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class BloomFilterRotator method remove.

@Override
public int remove(byte[] value) {
    int res = 0;
    for (int i = 0; i < filters.length; i++) {
        BloomFilter filter = filters[i];
        int r = filter.remove(value);
        if (filter == current_filter) {
            res = r;
        }
    }
    return (res);
}

Example 7 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class RelatedContentSearcher method harvestBlooms.

private void harvestBlooms() {
    harvest_dispatcher.dispatch(new AERunnable() {

        @Override
        public void runSupport() {
            if (harvest_dispatcher.getQueueSize() > 0) {
                return;
            }
            ForeignBloom oldest = null;
            synchronized (harvested_blooms) {
                for (ForeignBloom bloom : harvested_blooms.values()) {
                    if (oldest == null || bloom.getLastUpdateTime() < oldest.getLastUpdateTime()) {
                        oldest = bloom;
                    }
                }
            }
            long now = SystemTime.getMonotonousTime();
            if (oldest != null) {
                if (now - oldest.getLastUpdateTime() > HARVEST_BLOOM_UPDATE_MILLIS) {
                    DistributedDatabaseContact ddb_contact = oldest.getContact();
                    if (now - oldest.getCreateTime() > HARVEST_BLOOM_DISCARD_MILLIS && harvested_blooms.size() >= HARVEST_MAX_BLOOMS / 2) {
                        // don't want to stick with a stable one for too long otherwise the stabler
                        // nodes will end up in lots of other nodes' harvest set and receive
                        // undue attention (unless we don't have that many nodes...)
                        logSearch("Harvest: discarding " + ddb_contact.getAddress());
                        synchronized (harvested_blooms) {
                            harvested_blooms.remove(ddb_contact.getID());
                        }
                    } else {
                        BloomFilter updated_filter = sendRemoteUpdate(oldest);
                        if (updated_filter == null) {
                            synchronized (harvested_blooms) {
                                harvested_blooms.remove(ddb_contact.getID());
                                harvested_fails.put(ddb_contact.getID(), "");
                            }
                        } else {
                            oldest.updateFilter(updated_filter);
                        }
                    }
                }
            }
            if (harvested_blooms.size() < HARVEST_MAX_BLOOMS) {
                try {
                    int fail_count = 0;
                    DHTInterface[] dhts = dht_plugin.getDHTInterfaces();
                    outer: for (DHTInterface dht : dhts) {
                        if (dht.isIPV6()) {
                            continue;
                        }
                        int network = dht.getNetwork();
                        if (SEARCH_CVS_ONLY_DEFAULT && network != DHT.NW_AZ_CVS) {
                            logSearch("Harvest: ignoring main DHT");
                            continue;
                        }
                        DHTPluginContact[] contacts = dht.getReachableContacts();
                        byte[] dht_id = dht.getID();
                        for (DHTPluginContact contact : contacts) {
                            byte[] contact_id = contact.getID();
                            if (Arrays.equals(dht_id, contact_id)) {
                                continue;
                            }
                            DistributedDatabaseContact ddb_contact = importContact(contact, network);
                            synchronized (harvested_blooms) {
                                if (harvested_fails.containsKey(contact_id)) {
                                    continue;
                                }
                                if (harvested_blooms.containsKey(contact_id)) {
                                    continue;
                                }
                            }
                            BloomFilter filter = sendRemoteFetch(ddb_contact);
                            logSearch("harvest: " + contact.getString() + " -> " + (filter == null ? "null" : filter.getString()));
                            if (filter != null) {
                                synchronized (harvested_blooms) {
                                    harvested_blooms.put(contact_id, new ForeignBloom(ddb_contact, filter));
                                }
                                break outer;
                            } else {
                                synchronized (harvested_blooms) {
                                    harvested_fails.put(contact_id, "");
                                }
                                fail_count++;
                                if (fail_count > 5) {
                                    break outer;
                                }
                            }
                        }
                    }
                } catch (Throwable e) {
                    e.printStackTrace();
                }
            }
            synchronized (harvested_blooms) {
                if (harvested_fails.size() > HARVEST_MAX_FAILS_HISTORY) {
                    harvested_fails.clear();
                }
            }
        }
    });
}

Example 8 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class RelatedContentSearcher method receiveRemoteRequest.

protected Map<String, Object> receiveRemoteRequest(DistributedDatabaseContact originator, Map<String, Object> request) {
    Map<String, Object> response = new HashMap<>();
    try {
        boolean originator_is_neighbour = false;
        DHTInterface[] dhts = dht_plugin.getDHTInterfaces();
        byte[] originator_id = originator.getID();
        byte[] originator_bytes = AddressUtils.getAddressBytes(originator.getAddress());
        for (DHTInterface d : dhts) {
            List<DHTPluginContact> contacts = d.getClosestContacts(d.getID(), true);
            for (DHTPluginContact c : contacts) {
                if (Arrays.equals(c.getID(), originator_id)) {
                    originator_is_neighbour = true;
                    break;
                }
            }
            if (originator_is_neighbour) {
                break;
            }
        }
        String req_type = MapUtils.getMapString(request, "x", null);
        if (req_type != null) {
            boolean dup = harvest_op_requester_bloom.contains(originator_bytes);
            logSearch("Received remote request: " + BDecoder.decodeStrings(request) + " from " + originator.getAddress() + "/" + originator.getNetwork() + ", dup=" + dup + ", bs=" + harvest_op_requester_bloom.getEntryCount());
            if (!dup) {
                harvest_op_requester_bloom.add(originator_bytes);
                if (req_type.equals("f")) {
                    BloomFilter filter = getKeyBloom(!originator_is_neighbour);
                    if (filter != null) {
                        response.put("f", filter.serialiseToMap());
                    }
                } else if (req_type.equals("u")) {
                    BloomFilter filter = getKeyBloom(!originator_is_neighbour);
                    if (filter != null) {
                        int existing_size = MapUtils.importInt(request, "s", 0);
                        if (existing_size != filter.getEntryCount()) {
                            response.put("f", filter.serialiseToMap());
                        } else {
                            response.put("s", new Long(existing_size));
                        }
                    }
                }
            }
        } else {
            // fallback to default handling
            int hits = harvest_se_requester_bloom.count(originator_bytes);
            String term = MapUtils.getMapString(request, "t", null);
            term = fixupTerm(term);
            String network = MapUtils.getMapString(request, "n", "");
            boolean search_cvs_only = network.equals("c");
            int min_seeds = MapUtils.importInt(request, "s", SEARCH_MIN_SEEDS_DEFAULT);
            int min_leechers = MapUtils.importInt(request, "l", SEARCH_MIN_LEECHERS_DEFAULT);
            // System.out.println( "Received remote search: '" + term + "' from " + originator.getAddress() + ", hits=" + hits + ", bs=" + harvest_se_requester_bloom.getEntryCount());
            logSearch("Received remote search: '" + term + "' from " + originator.getAddress() + ", hits=" + hits + ", bs=" + harvest_se_requester_bloom.getEntryCount());
            if (hits < 10) {
                harvest_se_requester_bloom.add(originator_bytes);
                if (term != null) {
                    List<RelatedContent> matches = matchContent(term, min_seeds, min_leechers, false, search_cvs_only);
                    List<Map<String, Object>> l_list = new ArrayList<>();
                    for (int i = 0; i < matches.size(); i++) {
                        RelatedContent c = matches.get(i);
                        Map<String, Object> map = new HashMap<>();
                        l_list.add(map);
                        MapUtils.exportLong(map, "v", c.getVersion());
                        MapUtils.setMapString(map, "n", c.getTitle());
                        MapUtils.exportLong(map, "s", c.getSize());
                        MapUtils.exportLong(map, "r", c.getRank());
                        MapUtils.exportLong(map, "d", c.getLastSeenSecs());
                        MapUtils.exportLong(map, "p", c.getPublishDate() / (60 * 60 * 1000));
                        MapUtils.exportLong(map, "l", c.getLeechers());
                        MapUtils.exportLong(map, "z", c.getSeeds());
                        byte[] hash = c.getHash();
                        if (hash != null) {
                            map.put("h", hash);
                        }
                        byte[] tracker_keys = c.getTrackerKeys();
                        if (tracker_keys != null) {
                            map.put("k", tracker_keys);
                        }
                        byte[] ws_keys = c.getWebSeedKeys();
                        if (ws_keys != null) {
                            map.put("w", ws_keys);
                        }
                        String[] tags = c.getTags();
                        if (tags != null) {
                            map.put("g", manager.encodeTags(tags));
                        }
                        byte nets = c.getNetworksInternal();
                        if (nets != RelatedContentManager.NET_NONE && nets != RelatedContentManager.NET_PUBLIC) {
                            map.put("o", new Long(nets & 0x00ff));
                        }
                    // don't bother with tracker as no use to caller really
                    }
                    response.put("l", l_list);
                    List<DistributedDatabaseContact> bloom_hits = searchForeignBlooms(term);
                    if (bloom_hits.size() > 0) {
                        List<Map> c_list = new ArrayList<>();
                        for (DistributedDatabaseContact c : bloom_hits) {
                            Map m = new HashMap();
                            c_list.add(m);
                            InetSocketAddress address = c.getAddress();
                            if (address.isUnresolved()) {
                                m.put("m", c.exportToMap());
                            } else {
                                m.put("a", address.getAddress().getHostAddress());
                                m.put("p", new Long(address.getPort()));
                            }
                        }
                        response.put("c", c_list);
                    }
                }
            }
        }
    } catch (Throwable e) {
    }
    return (response);
}

Example 9 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class RelatedContentSearcher method searchForeignBlooms.

List<DistributedDatabaseContact> searchForeignBlooms(String term) {
    boolean is_popularity = isPopularity(term);
    List<DistributedDatabaseContact> result = new ArrayList<>();
    try {
        String[] bits = Constants.PAT_SPLIT_SPACE.split(term.toLowerCase());
        // note that we don't need to unescape tags in this process as tags are escaped when
        // inserted into the blooms and include the 'tag:' prefix
        int[] bit_types = new int[bits.length];
        byte[][] bit_bytes = new byte[bit_types.length][];
        byte[][][] extras = new byte[bit_types.length][][];
        for (int i = 0; i < bits.length; i++) {
            String bit = bits[i].trim();
            if (bit.length() > 0) {
                char c = bit.charAt(0);
                if (c == '+') {
                    bit_types[i] = 1;
                    bit = bit.substring(1);
                } else if (c == '-') {
                    bit_types[i] = 2;
                    bit = bit.substring(1);
                }
                if (bit.startsWith("(") && bit.endsWith((")"))) {
                    // ignore
                    bit_types[i] = 3;
                } else if (bit.contains("|")) {
                    String[] parts = bit.split("\\|");
                    List<String> p = new ArrayList<>();
                    for (String part : parts) {
                        part = part.trim();
                        if (part.length() > 0) {
                            p.add(part);
                        }
                    }
                    if (p.size() == 0) {
                        bit_types[i] = 3;
                    } else {
                        bit_types[i] = 4;
                        extras[i] = new byte[p.size()][];
                        for (int j = 0; j < p.size(); j++) {
                            extras[i][j] = p.get(j).getBytes("UTF8");
                        }
                    }
                }
                bit_bytes[i] = bit.getBytes("UTF8");
            }
        }
        synchronized (harvested_blooms) {
            for (ForeignBloom fb : harvested_blooms.values()) {
                if (is_popularity) {
                    result.add(fb.getContact());
                } else {
                    BloomFilter filter = fb.getFilter();
                    boolean failed = false;
                    int matches = 0;
                    for (int i = 0; i < bit_bytes.length; i++) {
                        byte[] bit = bit_bytes[i];
                        if (bit == null || bit.length == 0) {
                            continue;
                        }
                        int type = bit_types[i];
                        if (type == 3) {
                            continue;
                        }
                        if (type == 0 || type == 1) {
                            if (filter.contains(bit)) {
                                matches++;
                            } else {
                                failed = true;
                                break;
                            }
                        } else if (type == 2) {
                            if (!filter.contains(bit)) {
                                matches++;
                            } else {
                                failed = true;
                                break;
                            }
                        } else if (type == 4) {
                            byte[][] parts = extras[i];
                            int old_matches = matches;
                            for (byte[] p : parts) {
                                if (filter.contains(p)) {
                                    matches++;
                                    break;
                                }
                            }
                            if (matches == old_matches) {
                                failed = true;
                                break;
                            }
                        }
                    }
                    if (matches > 0 && !failed) {
                        result.add(fb.getContact());
                    }
                }
            }
        }
    } catch (UnsupportedEncodingException e) {
        Debug.out(e);
    }
    return (result);
}

Example 10 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class RelatedContentSearcher method testKeyBloom.

private void testKeyBloom() {
    if (true) {
        return;
    }
    System.out.println("test key bloom");
    try {
        Map<String, int[]> all_words = new HashMap<>();
        synchronized (manager.rcm_lock) {
            ContentCache cache = manager.loadRelatedContent();
            List<DownloadInfo> dht_infos = getDHTInfos(false);
            Iterator<DownloadInfo> it_dht = dht_infos.iterator();
            Iterator<DownloadInfo> it_transient = RelatedContentManager.transient_info_cache.values().iterator();
            Iterator<DownloadInfo> it_rc = cache.related_content.values().iterator();
            updateKeyBloom(cache);
            int i = 0;
            for (Iterator _it : new Iterator[] { it_transient, it_rc, it_dht }) {
                Iterator<DownloadInfo> it = (Iterator<DownloadInfo>) _it;
                while (it.hasNext()) {
                    DownloadInfo di = it.next();
                    List<String> words = getDHTWords(di);
                    for (String word : words) {
                        int[] x = all_words.get(word);
                        if (x == null) {
                            x = new int[3];
                            all_words.put(word, x);
                        }
                        x[i] = 1;
                    }
                }
                i++;
            }
        }
        BloomFilter bloom = getKeyBloom(true);
        int total = 0;
        int clashes = 0;
        int misses = 0;
        int match_fails = 0;
        Random random = new Random();
        for (Map.Entry<String, int[]> entry : all_words.entrySet()) {
            String word = entry.getKey();
            int[] source = entry.getValue();
            boolean r1 = bloom.contains(word.getBytes("UTF-8"));
            boolean r2 = bloom.contains((word + random.nextLong()).getBytes("UTF-8"));
            System.out.println(word + " -> " + r1 + "/" + r2);
            total++;
            if (r1 && r2) {
                clashes++;
            }
            if (!r1) {
                misses++;
            }
            List<RelatedContent> hits = matchContent(word, SEARCH_MIN_SEEDS_DEFAULT, SEARCH_MIN_LEECHERS_DEFAULT, true, false);
            if (hits.size() == 0) {
                hits = matchContent(word, SEARCH_MIN_SEEDS_DEFAULT, SEARCH_MIN_LEECHERS_DEFAULT, true, false);
                match_fails++;
            }
        }
        System.out.println("total=" + total + ", clash=" + clashes + ", miss=" + misses + ", fails=" + match_fails + ", bloom=" + bloom.getString());
    } catch (Throwable e) {
        e.printStackTrace();
    }
}