Examples with NetworkInformation org.deeplearning4j.spark.models.sequencevectors.primitives.NetworkInformation used on opensource projects

Example 6 with NetworkInformation

use of org.deeplearning4j.spark.models.sequencevectors.primitives.NetworkInformation in project deeplearning4j by deeplearning4j.

the class NetworkOrganizer method getIntersections.

/**
     * This method returns specified numbers of IP's by parsing original list of trees into some form of binary tree
     *
     * @param numShards
     * @param primary
     * @return
     */
protected List<String> getIntersections(int numShards, Collection<String> primary) {
    /**
         * Since each ip address can be represented in 4-byte sequence, 1 byte per value, with leading order - we'll use that to build tree
         */
    if (primary == null) {
        for (NetworkInformation information : informationCollection) {
            for (String ip : information.getIpAddresses()) {
                // first we get binary representation for each IP
                String octet = convertIpToOctets(ip);
                // then we map each of them into virtual "tree", to find most popular networks within cluster
                tree.map(octet);
            }
        }
        // we get most "popular" A network from tree now
        String octetA = tree.getHottestNetworkA();
        List<String> candidates = new ArrayList<>();
        AtomicInteger matchCount = new AtomicInteger(0);
        for (NetworkInformation node : informationCollection) {
            for (String ip : node.getIpAddresses()) {
                String octet = convertIpToOctets(ip);
                // calculating matches
                if (octet.startsWith(octetA)) {
                    matchCount.incrementAndGet();
                    candidates.add(ip);
                    break;
                }
            }
        }
        /**
             * TODO: improve this. we just need to iterate over popular networks instead of single top A network
             */
        if (matchCount.get() != informationCollection.size())
            throw new ND4JIllegalStateException("Mismatching A class");
        Collections.shuffle(candidates);
        return new ArrayList<>(candidates.subList(0, Math.min(numShards, candidates.size())));
    } else {
        // if primary isn't null, we expect network to be already filtered
        String octetA = tree.getHottestNetworkA();
        List<String> candidates = new ArrayList<>();
        for (NetworkInformation node : informationCollection) {
            for (String ip : node.getIpAddresses()) {
                String octet = convertIpToOctets(ip);
                // calculating matches
                if (octet.startsWith(octetA) && !primary.contains(ip)) {
                    candidates.add(ip);
                    break;
                }
            }
        }
        Collections.shuffle(candidates);
        return new ArrayList<>(candidates.subList(0, Math.min(numShards, candidates.size())));
    }
}

Example 7 with NetworkInformation

use of org.deeplearning4j.spark.models.sequencevectors.primitives.NetworkInformation in project deeplearning4j by deeplearning4j.

the class NetworkOrganizerTest method testSelectionUniformNetworkC1.

@Test
public void testSelectionUniformNetworkC1() {
    List<NetworkInformation> collection = new ArrayList<>();
    for (int i = 1; i < 128; i++) {
        NetworkInformation information = new NetworkInformation();
        information.addIpAddress("192.168.0." + i);
        information.addIpAddress(getRandomIp());
        collection.add(information);
    }
    NetworkOrganizer discoverer = new NetworkOrganizer(collection, "192.168.0.0/24");
    // check for primary subset (aka Shards)
    List<String> shards = discoverer.getSubset(10);
    assertEquals(10, shards.size());
    for (String ip : shards) {
        assertNotEquals(null, ip);
        assertTrue(ip.startsWith("192.168.0"));
    }
    // check for secondary subset (aka Backup)
    List<String> backup = discoverer.getSubset(10, shards);
    assertEquals(10, backup.size());
    for (String ip : backup) {
        assertNotEquals(null, ip);
        assertTrue(ip.startsWith("192.168.0"));
        assertFalse(shards.contains(ip));
    }
}

Example 8 with NetworkInformation

use of org.deeplearning4j.spark.models.sequencevectors.primitives.NetworkInformation in project deeplearning4j by deeplearning4j.

the class NetworkOrganizerTest method testSelectionWithoutMaskB2.

/**
     * In this test we check for environment which has AWS-like setup:
     *  1) Each box has IP address from 172.16.0.0/12 range
     *  2) Within original homogenous network, we have 3 separate networks:
     *      A) 192.168.0.X
     *      B) 10.0.12.X
     *      C) 10.172.12.X
     *
     * @throws Exception
     */
@Test
public void testSelectionWithoutMaskB2() throws Exception {
    List<NetworkInformation> collection = new ArrayList<>();
    // we imitiate 512 cluster nodes here
    for (int i = 0; i < 512; i++) {
        NetworkInformation information = new NetworkInformation();
        information.addIpAddress(getRandomAwsIp());
        if (i < 30) {
            information.addIpAddress("192.168.0." + i);
        } else if (i < 95) {
            information.addIpAddress("10.0.12." + i);
        } else if (i < 255) {
            information.addIpAddress("10.172.12." + i);
        }
        collection.add(information);
    }
    NetworkOrganizer organizer = new NetworkOrganizer(collection);
    List<String> shards = organizer.getSubset(15);
    assertEquals(15, shards.size());
    for (String ip : shards) {
        assertNotEquals(null, ip);
        assertTrue(ip.startsWith("172."));
    }
    List<String> backup = organizer.getSubset(15, shards);
    for (String ip : backup) {
        assertNotEquals(null, ip);
        assertTrue(ip.startsWith("172."));
        assertFalse(shards.contains(ip));
    }
}

Example 9 with NetworkInformation

use of org.deeplearning4j.spark.models.sequencevectors.primitives.NetworkInformation in project deeplearning4j by deeplearning4j.

the class NetworkOrganizerTest method testSelectionDisjointNetworkC1.

@Test
public void testSelectionDisjointNetworkC1() {
    List<NetworkInformation> collection = new ArrayList<>();
    for (int i = 1; i < 128; i++) {
        NetworkInformation information = new NetworkInformation();
        if (i < 20)
            information.addIpAddress("172.12.0." + i);
        information.addIpAddress(getRandomIp());
        collection.add(information);
    }
    NetworkOrganizer discoverer = new NetworkOrganizer(collection, "172.12.0.0/24");
    // check for primary subset (aka Shards)
    List<String> shards = discoverer.getSubset(10);
    assertEquals(10, shards.size());
    List<String> backup = discoverer.getSubset(10, shards);
    // we expect 9 here, thus backups will be either incomplete or complex sharding will be used for them
    assertEquals(9, backup.size());
    for (String ip : backup) {
        assertNotEquals(null, ip);
        assertTrue(ip.startsWith("172.12.0"));
        assertFalse(shards.contains(ip));
    }
}