Examples with BloomFilter com.biglybt.core.util.bloom.BloomFilter used on opensource projects

Example 11 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class PEPeerControlImpl method doConnectionChecks.

private void doConnectionChecks() {
    // every 1 second
    if (mainloop_loop_count % MAINLOOP_ONE_SECOND_INTERVAL == 0) {
        // need to sync the rates periodically as when upload is disabled (for example) the we can end up with
        // nothing requesting the rate in order for a change to be noticed
        upload_limited_rate_group.getRateLimitBytesPerSecond();
        download_limited_rate_group.getRateLimitBytesPerSecond();
        final List<PEPeerTransport> peer_transports = peer_transports_cow;
        int num_waiting_establishments = 0;
        int udp_connections = 0;
        for (int i = 0; i < peer_transports.size(); i++) {
            final PEPeerTransport transport = peer_transports.get(i);
            // update waiting count
            final int state = transport.getConnectionState();
            if (state == PEPeerTransport.CONNECTION_PENDING || state == PEPeerTransport.CONNECTION_CONNECTING) {
                num_waiting_establishments++;
            }
            if (!transport.isTCP()) {
                udp_connections++;
            }
        }
        int[] allowed_seeds_info = getMaxSeedConnections();
        int base_allowed_seeds = allowed_seeds_info[0];
        if (base_allowed_seeds > 0) {
            int extra_seeds = allowed_seeds_info[1];
            int to_disconnect = _seeds - base_allowed_seeds;
            if (to_disconnect > 0) {
                // seeds are limited by people trying to get a reasonable upload by connecting
                // to leechers where possible. disconnect seeds from end of list to prevent
                // cycling of seeds
                Set<PEPeerTransport> to_retain = new HashSet<>();
                if (extra_seeds > 0) {
                    for (PEPeerTransport transport : peer_transports) {
                        if (transport.isSeed() && transport.getNetwork() != AENetworkClassifier.AT_PUBLIC) {
                            to_retain.add(transport);
                            if (to_retain.size() == extra_seeds) {
                                break;
                            }
                        }
                    }
                    to_disconnect -= to_retain.size();
                }
                for (int i = peer_transports.size() - 1; i >= 0 && to_disconnect > 0; i--) {
                    final PEPeerTransport transport = peer_transports.get(i);
                    if (transport.isSeed()) {
                        if (!to_retain.contains(transport)) {
                            closeAndRemovePeer(transport, "Too many seeds", false);
                            to_disconnect--;
                        }
                    }
                }
            }
        }
        int[] allowed_info = getMaxNewConnectionsAllowed();
        int allowed_base = allowed_info[0];
        if (allowed_base < 0 || allowed_base > 1000) {
            // ensure a very upper limit so it doesnt get out of control when using PEX
            allowed_base = 1000;
            allowed_info[0] = allowed_base;
        }
        if (adapter.isNATHealthy()) {
            // if unfirewalled, leave slots avail for remote connections
            // leave 5%
            int free = getMaxConnections()[0] / 20;
            allowed_base = allowed_base - free;
            allowed_info[0] = allowed_base;
        }
        for (int i = 0; i < allowed_info.length; i++) {
            int allowed = allowed_info[i];
            if (allowed > 0) {
                // try and connect only as many as necessary
                final int wanted = TCPConnectionManager.MAX_SIMULTANIOUS_CONNECT_ATTEMPTS - num_waiting_establishments;
                if (wanted > allowed) {
                    num_waiting_establishments += wanted - allowed;
                }
                int remaining = allowed;
                int tcp_remaining = TCPNetworkManager.getSingleton().getConnectDisconnectManager().getMaxOutboundPermitted();
                int udp_remaining = UDPNetworkManager.getSingleton().getConnectionManager().getMaxOutboundPermitted();
                while (num_waiting_establishments < TCPConnectionManager.MAX_SIMULTANIOUS_CONNECT_ATTEMPTS && (tcp_remaining > 0 || udp_remaining > 0)) {
                    if (!is_running)
                        break;
                    final PeerItem item = peer_database.getNextOptimisticConnectPeer(i == 1);
                    if (item == null || !is_running)
                        break;
                    final PeerItem self = peer_database.getSelfPeer();
                    if (self != null && self.equals(item)) {
                        continue;
                    }
                    if (!isAlreadyConnected(item)) {
                        final String source = PeerItem.convertSourceString(item.getSource());
                        final boolean use_crypto = item.getHandshakeType() == PeerItemFactory.HANDSHAKE_TYPE_CRYPTO;
                        int tcp_port = item.getTCPPort();
                        int udp_port = item.getUDPPort();
                        if (udp_port == 0 && udp_probe_enabled) {
                            // for probing we assume udp port same as tcp
                            udp_port = tcp_port;
                        }
                        boolean prefer_udp_overall = prefer_udp || prefer_udp_default;
                        if (prefer_udp_overall && udp_port == 0) {
                            // see if we have previous record of this address as udp connectable
                            byte[] address = item.getIP().getBytes();
                            BloomFilter bloom = prefer_udp_bloom;
                            if (bloom != null && bloom.contains(address)) {
                                udp_port = tcp_port;
                            }
                        }
                        boolean tcp_ok = TCPNetworkManager.TCP_OUTGOING_ENABLED && tcp_port > 0 && tcp_remaining > 0;
                        boolean udp_ok = UDPNetworkManager.UDP_OUTGOING_ENABLED && udp_port > 0 && udp_remaining > 0;
                        if (tcp_ok && !(prefer_udp_overall && udp_ok)) {
                            if (makeNewOutgoingConnection(source, item.getAddressString(), tcp_port, udp_port, true, use_crypto, item.getCryptoLevel(), null) == null) {
                                tcp_remaining--;
                                num_waiting_establishments++;
                                remaining--;
                            }
                        } else if (udp_ok) {
                            if (makeNewOutgoingConnection(source, item.getAddressString(), tcp_port, udp_port, false, use_crypto, item.getCryptoLevel(), null) == null) {
                                udp_remaining--;
                                num_waiting_establishments++;
                                remaining--;
                            }
                        }
                    }
                }
                if (i == 0) {
                    if (UDPNetworkManager.UDP_OUTGOING_ENABLED && remaining > 0 && udp_remaining > 0 && udp_connections < MAX_UDP_CONNECTIONS) {
                        doUDPConnectionChecks(remaining);
                    }
                }
            }
        }
    }
    // every 5 seconds
    if (mainloop_loop_count % MAINLOOP_FIVE_SECOND_INTERVAL == 0) {
        final List<PEPeerTransport> peer_transports = peer_transports_cow;
        for (int i = 0; i < peer_transports.size(); i++) {
            final PEPeerTransport transport = peer_transports.get(i);
            // check for timeouts
            if (transport.doTimeoutChecks())
                continue;
            // keep-alive check
            transport.doKeepAliveCheck();
            // speed tuning check
            transport.doPerformanceTuningCheck();
        }
    }
    // every 10 seconds check for connected + banned peers
    if (mainloop_loop_count % MAINLOOP_TEN_SECOND_INTERVAL == 0) {
        final long last_update = ip_filter.getLastUpdateTime();
        if (last_update != ip_filter_last_update_time) {
            ip_filter_last_update_time = last_update;
            checkForBannedConnections();
        }
    }
    // every 30 seconds
    if (mainloop_loop_count % MAINLOOP_THIRTY_SECOND_INTERVAL == 0) {
        // if we're at our connection limit, time out the least-useful
        // one so we can establish a possibly-better new connection
        optimisticDisconnectCount = 0;
        int[] allowed = getMaxNewConnectionsAllowed();
        if (allowed[0] + allowed[1] == 0) {
            // we've reached limit
            doOptimisticDisconnect(false, false, "");
        }
    }
    // sweep over all peers in a 60 second timespan
    float percentage = ((mainloop_loop_count % MAINLOOP_SIXTY_SECOND_INTERVAL) + 1F) / (1F * MAINLOOP_SIXTY_SECOND_INTERVAL);
    int goal;
    if (mainloop_loop_count % MAINLOOP_SIXTY_SECOND_INTERVAL == 0) {
        goal = 0;
        sweepList = peer_transports_cow;
    } else
        goal = (int) Math.floor(percentage * sweepList.size());
    for (int i = nextPEXSweepIndex; i < goal && i < sweepList.size(); i++) {
        // System.out.println(mainloop_loop_count+" %:"+percentage+" start:"+nextPEXSweepIndex+" current:"+i+" <"+goal+"/"+sweepList.size());
        final PEPeerTransport peer = sweepList.get(i);
        peer.updatePeerExchange();
    }
    nextPEXSweepIndex = goal;
    if (mainloop_loop_count % MAINLOOP_SIXTY_SECOND_INTERVAL == 0) {
        List<PEPeerTransport> peer_transports = peer_transports_cow;
        if (peer_transports.size() > 1) {
            Map<String, List<PEPeerTransport>> peer_map = new HashMap<>();
            for (PEPeerTransport peer : peer_transports) {
                if (peer.isIncoming()) {
                    continue;
                }
                if (peer.getPeerState() == PEPeer.CONNECTING && peer.getConnectionState() == PEPeerTransport.CONNECTION_CONNECTING && peer.getLastMessageSentTime() != 0) {
                    String key = peer.getIp() + ":" + peer.getPort();
                    List<PEPeerTransport> list = peer_map.get(key);
                    if (list == null) {
                        list = new ArrayList<>(1);
                        peer_map.put(key, list);
                    }
                    list.add(peer);
                }
            }
            for (List<PEPeerTransport> list : peer_map.values()) {
                if (list.size() >= 2) {
                    long newest_time = Long.MIN_VALUE;
                    PEPeerTransport newest_peer = null;
                    for (PEPeerTransport peer : list) {
                        long last_sent = peer.getLastMessageSentTime();
                        if (last_sent > newest_time) {
                            newest_time = last_sent;
                            newest_peer = peer;
                        }
                    }
                    for (PEPeerTransport peer : list) {
                        if (peer != newest_peer) {
                            if (peer.getPeerState() == PEPeer.CONNECTING && peer.getConnectionState() == PEPeerTransport.CONNECTION_CONNECTING) {
                                closeAndRemovePeer(peer, "Removing old duplicate connection", false);
                            }
                        }
                    }
                }
            }
        }
    }
}

Example 12 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class PEPeerControlImpl method peerRemoved.

@Override
public void peerRemoved(PEPeer pc) {
    if (is_running && !seeding_mode && (prefer_udp || prefer_udp_default)) {
        int udp = pc.getUDPListenPort();
        if (udp != 0 && udp == pc.getTCPListenPort()) {
            BloomFilter filter = prefer_udp_bloom;
            if (filter == null) {
                filter = prefer_udp_bloom = BloomFilterFactory.createAddOnly(PREFER_UDP_BLOOM_SIZE);
            }
            if (filter.getEntryCount() < PREFER_UDP_BLOOM_SIZE / 10) {
                filter.add(pc.getIp().getBytes());
            }
        }
    }
    final int piece = pc.getUniqueAnnounce();
    if (piece != -1 && superSeedMode) {
        superSeedModeNumberOfAnnounces--;
        superSeedPieces[piece].peerLeft();
    }
    int[] reserved_pieces = pc.getReservedPieceNumbers();
    if (reserved_pieces != null) {
        for (int reserved_piece : reserved_pieces) {
            PEPiece pe_piece = pePieces[reserved_piece];
            if (pe_piece != null) {
                String reserved_by = pe_piece.getReservedBy();
                if (reserved_by != null && reserved_by.equals(pc.getIp())) {
                    pe_piece.setReservedBy(null);
                }
            }
        }
    }
    if (pc.isSeed()) {
        last_seed_disconnect_time = SystemTime.getCurrentTime();
    }
    // async downloadmanager notification
    adapter.removePeer(pc);
    // sync peermanager notification
    final ArrayList peer_manager_listeners = peer_manager_listeners_cow;
    for (int i = 0; i < peer_manager_listeners.size(); i++) {
        ((PEPeerManagerListener) peer_manager_listeners.get(i)).peerRemoved(this, pc);
    }
}

Example 13 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class DHTDBMapping method removeFromBloom.

protected void removeFromBloom(DHTDBValueImpl value) {
    DHTTransportContact originator = value.getOriginator();
    if (ip_count_bloom_filter == null) {
        return;
    }
    byte[] bloom_key = originator.getBloomKey();
    if (ip_count_bloom_filter instanceof byte[]) {
        byte[] existing_address = (byte[]) ip_count_bloom_filter;
        if (Arrays.equals(bloom_key, existing_address)) {
            ip_count_bloom_filter = null;
        }
        return;
    }
    BloomFilter filter = (BloomFilter) ip_count_bloom_filter;
    int hit_count = filter.remove(bloom_key);
    if (DHTLog.LOCAL_BLOOM_TRACE) {
        System.out.println("direct local remove from " + originator.getAddress() + ", hit count = " + hit_count);
    }
}

Example 14 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class DHTDBMapping method addToBloom.

protected void addToBloom(DHTDBValueImpl value) {
    // we don't check for flooding on indirect stores as this could be used to force a
    // direct store to be bounced (flood a node with indirect stores before the direct
    // store occurs)
    DHTTransportContact originator = value.getOriginator();
    byte[] bloom_key = originator.getBloomKey();
    if (ip_count_bloom_filter == null) {
        ip_count_bloom_filter = bloom_key;
        return;
    }
    BloomFilter filter;
    if (ip_count_bloom_filter instanceof byte[]) {
        byte[] existing_address = (byte[]) ip_count_bloom_filter;
        ip_count_bloom_filter = filter = BloomFilterFactory.createAddRemove4Bit(IP_COUNT_BLOOM_SIZE_INCREASE_CHUNK);
        filter.add(existing_address);
    } else {
        filter = (BloomFilter) ip_count_bloom_filter;
    }
    int hit_count = filter.add(bloom_key);
    if (DHTLog.LOCAL_BLOOM_TRACE) {
        System.out.println("direct local add from " + originator.getAddress() + ", hit count = " + hit_count);
    }
    if (filter.getSize() / filter.getEntryCount() < 10) {
        rebuildIPBloomFilter(true);
    }
    if (hit_count >= 15) {
        db.banContact(originator, "local flood on '" + DHTLog.getFullString(key.getBytes()) + "'");
    }
}

Example 15 with BloomFilter

use of com.biglybt.core.util.bloom.BloomFilter in project BiglyBT by BiglySoftware.

the class DHTDBMapping method rebuildIPBloomFilter.

protected void rebuildIPBloomFilter(boolean increase_size) {
    BloomFilter new_filter;
    int old_size;
    if (ip_count_bloom_filter instanceof BloomFilter) {
        old_size = ((BloomFilter) ip_count_bloom_filter).getSize();
    } else {
        old_size = IP_COUNT_BLOOM_SIZE_INCREASE_CHUNK;
    }
    if (increase_size) {
        new_filter = BloomFilterFactory.createAddRemove4Bit(old_size + IP_COUNT_BLOOM_SIZE_INCREASE_CHUNK);
    } else {
        new_filter = BloomFilterFactory.createAddRemove4Bit(old_size);
    }
    try {
        // only do flood prevention on direct stores as we can't trust the originator
        // details for indirect and this can be used to DOS a direct store later
        Iterator<DHTDBValueImpl> it = getDirectValues();
        int max_hits = 0;
        while (it.hasNext()) {
            DHTDBValueImpl val = it.next();
            if (!val.isLocal()) {
                // logger.log( "    adding " + val.getOriginator().getAddress());
                int hits = new_filter.add(val.getOriginator().getBloomKey());
                if (hits > max_hits) {
                    max_hits = hits;
                }
            }
        }
        if (DHTLog.LOCAL_BLOOM_TRACE) {
            db.log("Rebuilt local IP bloom filter, size = " + new_filter.getSize() + ", entries =" + new_filter.getEntryCount() + ", max hits = " + max_hits);
        }
    } finally {
        ip_count_bloom_filter = new_filter;
    }
}