Examples with BloomFilter com.biglybt.core.util.bloom.BloomFilter used on opensource projects

Example 1 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class DHTDBImpl method rebuildIPBloomFilter.

protected void rebuildIPBloomFilter(boolean increase_size) {
    BloomFilter new_filter;
    if (increase_size) {
        new_filter = BloomFilterFactory.createAddRemove8Bit(ip_count_bloom_filter.getSize() + IP_COUNT_BLOOM_SIZE_INCREASE_CHUNK);
    } else {
        new_filter = BloomFilterFactory.createAddRemove8Bit(ip_count_bloom_filter.getSize());
    }
    try {
        // Map		sender_map	= new HashMap();
        // List	senders		= new ArrayList();
        Iterator<DHTDBMapping> it = stored_values.values().iterator();
        int max_hits = 0;
        while (it.hasNext()) {
            DHTDBMapping mapping = it.next();
            mapping.rebuildIPBloomFilter(false);
            Iterator<DHTDBValueImpl> it2 = mapping.getDirectValues();
            while (it2.hasNext()) {
                DHTDBValueImpl val = it2.next();
                if (!val.isLocal()) {
                    // logger.log( "    adding " + val.getOriginator().getAddress());
                    byte[] bloom_key = val.getOriginator().getBloomKey();
                    int hits = new_filter.add(bloom_key);
                    if (hits > max_hits) {
                        max_hits = hits;
                    }
                }
            }
        // survey our neighbourhood
        /*
				 * its is non-trivial to do anything about nodes that get "close" to us and then
				 * spam us with crap. Ultimately, of course, to take a key out you "just" create
				 * the 20 closest nodes to the key and then run nodes that swallow all registrations
				 * and return nothing.
				 * Protecting against one or two such nodes that flood crap requires crap to be
				 * identified. Tracing shows a large disparity between number of values registered
				 * per neighbour (factors of 100), so an approach based on number of registrations
				 * is non-trivial (assuming future scaling of the DHT, what do we consider crap?)
				 * A further approach would be to query the claimed originators of values (obviously
				 * a low bandwith approach, e.g. query 3 values from the contact with highest number
				 * of forwarded values). This requires originators to support long term knowledge of
				 * what they've published (we don't want to blacklist a neighbour because an originator
				 * has deleted a value/been restarted). We also then have to consider how to deal with
				 * non-responses to queries (assuming an affirmative Yes -> value has been forwarded
				 * correnctly, No -> probably crap). We can't treat non-replies as No. Thus a bad
				 * neighbour only has to forward crap with originators that aren't AZ nodes (very
				 * easy to do!) to break this aproach.
				 *
				 *
				it2 = mapping.getIndirectValues();

				while( it2.hasNext()){

					DHTDBValueImpl	val = (DHTDBValueImpl)it2.next();

					DHTTransportContact sender = val.getSender();

					HashWrapper	hw = new HashWrapper( sender.getID());

					Integer	sender_count = (Integer)sender_map.get( hw );

					if ( sender_count == null ){

						sender_count = new Integer(1);

						senders.add( sender );

					}else{

						sender_count = new Integer( sender_count.intValue() + 1 );
					}

					sender_map.put( hw, sender_count );
				}
				*/
        }
        logger.log("Rebuilt global IP bloom filter, size=" + new_filter.getSize() + ", entries=" + new_filter.getEntryCount() + ", max hits=" + max_hits);
    /*
			senders = control.sortContactsByDistance( senders );

			for (int i=0;i<senders.size();i++){

				DHTTransportContact	sender = (DHTTransportContact)senders.get(i);

				System.out.println( i + ":" + sender.getString() + " -> " + sender_map.get(new HashWrapper(sender.getID())));
			}
			*/
    } finally {
        ip_count_bloom_filter = new_filter;
    }
}

Example 2 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class DownloadManagerController method activateRequest.

@Override
public boolean activateRequest(InetSocketAddress address) {
    if (getState() == DownloadManager.STATE_QUEUED) {
        BloomFilter bloom = activation_bloom;
        if (bloom == null) {
            activation_bloom = bloom = BloomFilterFactory.createAddRemove4Bit(BLOOM_SIZE);
        }
        byte[] address_bytes = AddressUtils.getAddressBytes(address);
        int hit_count = bloom.add(address_bytes);
        if (hit_count > 5) {
            Logger.log(new LogEvent(this, LogIDs.CORE, LogEvent.LT_WARNING, "Activate request for " + getDisplayName() + " from " + address + " denied as too many recently received"));
            return (false);
        }
        Logger.log(new LogEvent(this, LogIDs.CORE, "Activate request for " + getDisplayName() + " from " + address));
        long now = SystemTime.getCurrentTime();
        if (now < activation_bloom_create_time || now - activation_bloom_create_time > ACTIVATION_REBUILD_TIME) {
            activation_bloom = BloomFilterFactory.createAddRemove4Bit(BLOOM_SIZE);
            activation_bloom_create_time = now;
        }
        activation_count = bloom.getEntryCount();
        activation_count_time = now;
        return (download_manager.activateRequest(activation_count));
    }
    return (false);
}

Example 3 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class DownloadManagerController method deactivateRequest.

@Override
public void deactivateRequest(InetSocketAddress address) {
    BloomFilter bloom = activation_bloom;
    if (bloom != null) {
        byte[] address_bytes = AddressUtils.getAddressBytes(address);
        int count = bloom.count(address_bytes);
        for (int i = 0; i < count; i++) {
            bloom.remove(address_bytes);
        }
        activation_count = bloom.getEntryCount();
    }
}

Example 4 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class RelatedContentSearcher method updateKeyBloom.

protected void updateKeyBloom(RelatedContentManager.ContentCache cc) {
    synchronized (manager.rcm_lock) {
        Set<String> dht_only_words = new HashSet<>();
        Set<String> non_dht_words = new HashSet<>();
        List<DownloadInfo> dht_infos = getDHTInfos(SEARCH_CVS_ONLY_DEFAULT);
        Iterator<DownloadInfo> it_dht = dht_infos.iterator();
        Iterator<DownloadInfo> it_transient = RelatedContentManager.transient_info_cache.values().iterator();
        Iterator<DownloadInfo> it_rc = cc.related_content.values().iterator();
        for (Iterator<DownloadInfo> it : new Iterator[] { it_transient, it_rc, it_dht }) {
            while (it.hasNext()) {
                DownloadInfo di = it.next();
                List<String> words = getDHTWords(di);
                for (String word : words) {
                    if (it == it_dht) {
                        if (!non_dht_words.contains(word)) {
                            dht_only_words.add(word);
                        }
                    } else {
                        non_dht_words.add(word);
                    }
                }
            }
        }
        int all_desired_bits = (dht_only_words.size() + non_dht_words.size()) * KEY_BLOOM_LOAD_FACTOR;
        all_desired_bits = Math.max(all_desired_bits, KEY_BLOOM_MIN_BITS);
        all_desired_bits = Math.min(all_desired_bits, KEY_BLOOM_MAX_BITS);
        BloomFilter all_bloom = BloomFilterFactory.createAddOnly(all_desired_bits);
        int non_dht_desired_bits = non_dht_words.size() * KEY_BLOOM_LOAD_FACTOR;
        non_dht_desired_bits = Math.max(non_dht_desired_bits, KEY_BLOOM_MIN_BITS);
        non_dht_desired_bits = Math.min(non_dht_desired_bits, KEY_BLOOM_MAX_BITS);
        BloomFilter non_dht_bloom = BloomFilterFactory.createAddOnly(non_dht_desired_bits);
        List<String> non_dht_words_rand = new ArrayList<>(non_dht_words);
        Collections.shuffle(non_dht_words_rand);
        for (String word : non_dht_words_rand) {
            try {
                byte[] bytes = word.getBytes("UTF8");
                all_bloom.add(bytes);
                if (all_bloom.getEntryCount() >= KEY_BLOOM_MAX_ENTRIES) {
                    break;
                }
                if (non_dht_bloom.getEntryCount() < KEY_BLOOM_MAX_ENTRIES) {
                    non_dht_bloom.add(bytes);
                }
            } catch (Throwable e) {
            }
        }
        List<String> dht_only_words_rand = new ArrayList<>(dht_only_words);
        Collections.shuffle(dht_only_words_rand);
        for (String word : dht_only_words_rand) {
            try {
                byte[] bytes = word.getBytes("UTF8");
                all_bloom.add(bytes);
                if (all_bloom.getEntryCount() >= KEY_BLOOM_MAX_ENTRIES) {
                    break;
                }
            } catch (Throwable e) {
            }
        }
        logSearch("blooms=" + all_bloom.getSize() + "/" + all_bloom.getEntryCount() + ", " + non_dht_bloom.getSize() + "/" + non_dht_bloom.getEntryCount() + ": rcm=" + cc.related_content.size() + ", trans=" + RelatedContentManager.transient_info_cache.size() + ", dht=" + dht_infos.size());
        key_bloom_with_local = all_bloom;
        key_bloom_without_local = non_dht_bloom;
        last_key_bloom_update = SystemTime.getMonotonousTime();
    }
}

Example 5 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class BloomFilterImpl method main.

public static void main(String[] args) {
    Random rand = new Random();
    /*
		BloomFilter b1 = new BloomFilterAddRemove8Bit(10000);

		for (int i=0;i<260;i++){

			System.out.println( b1.add( "parp".getBytes()) + ", count = " + b1.count( "parp".getBytes()) + ", ent = " + b1.getEntryCount());

		}

		for (int i=0;i<260;i++){

			System.out.println( b1.remove( "parp".getBytes())+ ", count = " + b1.count( "parp".getBytes()) + ", ent = " + b1.getEntryCount());
		}
		*/
    /*

		BloomFilter b1 = new BloomFilterAddOnly(90*10/3);

		byte[]	key1 = new byte[4];

		for (int i=0;i<200;i++){


			if ( i%2==0){
				rand.nextBytes( key1 );
			}

			b1.add( key1 );

			System.out.println( "entries = " + b1.getEntryCount() + ", act = " + i );
		}

		System.exit(0);
		*/
    int fp_count = 0;
    for (int j = 0; j < 1000; j++) {
        long start = System.currentTimeMillis();
        BloomFilter b = new BloomFilterAddRemove8Bit(10000);
        // BloomFilter b = new BloomFilterAddOnly(10000);
        int fp = 0;
        for (int i = 0; i < 1000; i++) {
            // String	key = "" + rand.nextInt();
            byte[] key = new byte[6];
            rand.nextBytes(key);
            if (i % 2 == 0) {
                b.add(key);
                if (!b.contains(key)) {
                    System.out.println("false negative on add!!!!");
                }
            } else {
                if (b.contains(key)) {
                    fp++;
                }
            }
        /*
				if ( i%2 == 0 ){

					b.remove( key  );

					if ( b.contains( key )){

						System.out.println( "false positive" );
					}
				}
				*/
        }
        System.out.println("" + (System.currentTimeMillis() - start + ", fp = " + fp));
        if (fp > 0) {
            fp_count++;
        }
    }
    System.out.println(fp_count);
}