Examples with Multimap com.google.common.collect.Multimap used on opensource projects

Example 11 with Multimap

use of com.google.common.collect.Multimap in project presto by prestodb.

the class SqlStageExecution method scheduleTask.

private synchronized RemoteTask scheduleTask(Node node, TaskId taskId, Multimap<PlanNodeId, Split> sourceSplits) {
    checkArgument(!allTasks.contains(taskId), "A task with id %s already exists", taskId);
    ImmutableMultimap.Builder<PlanNodeId, Split> initialSplits = ImmutableMultimap.builder();
    initialSplits.putAll(sourceSplits);
    for (Entry<PlanNodeId, URI> entry : exchangeLocations.entries()) {
        initialSplits.put(entry.getKey(), createRemoteSplitFor(taskId, entry.getValue()));
    }
    OutputBuffers outputBuffers = this.outputBuffers.get();
    checkState(outputBuffers != null, "Initial output buffers must be set before a task can be scheduled");
    RemoteTask task = remoteTaskFactory.createRemoteTask(stateMachine.getSession(), taskId, node, stateMachine.getFragment(), initialSplits.build(), outputBuffers, nodeTaskMap.createPartitionedSplitCountTracker(node, taskId), summarizeTaskInfo);
    completeSources.forEach(task::noMoreSplits);
    allTasks.add(taskId);
    tasks.computeIfAbsent(node, key -> newConcurrentHashSet()).add(task);
    nodeTaskMap.addTask(node, task);
    task.addStateChangeListener(new StageTaskListener());
    if (!stateMachine.getState().isDone()) {
        task.start();
    } else {
        // stage finished while we were scheduling this task
        task.abort();
    }
    return task;
}

Example 12 with Multimap

use of com.google.common.collect.Multimap in project cassandra by apache.

the class NetworkTopologyStrategyTest method calculateNaturalEndpoints.

// Copy of older endpoints calculation algorithm for comparison
public static List<InetAddress> calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata, Map<String, Integer> datacenters, IEndpointSnitch snitch) {
    // we want to preserve insertion order so that the first added endpoint becomes primary
    Set<InetAddress> replicas = new LinkedHashSet<>();
    // replicas we have found in each DC
    Map<String, Set<InetAddress>> dcReplicas = new HashMap<>(datacenters.size());
    for (Map.Entry<String, Integer> dc : datacenters.entrySet()) dcReplicas.put(dc.getKey(), new HashSet<InetAddress>(dc.getValue()));
    Topology topology = tokenMetadata.getTopology();
    // all endpoints in each DC, so we can check when we have exhausted all the members of a DC
    Multimap<String, InetAddress> allEndpoints = topology.getDatacenterEndpoints();
    // all racks in a DC so we can check when we have exhausted all racks in a DC
    Map<String, Multimap<String, InetAddress>> racks = topology.getDatacenterRacks();
    assert !allEndpoints.isEmpty() && !racks.isEmpty() : "not aware of any cluster members";
    // tracks the racks we have already placed replicas in
    Map<String, Set<String>> seenRacks = new HashMap<>(datacenters.size());
    for (Map.Entry<String, Integer> dc : datacenters.entrySet()) seenRacks.put(dc.getKey(), new HashSet<String>());
    // tracks the endpoints that we skipped over while looking for unique racks
    // when we relax the rack uniqueness we can append this to the current result so we don't have to wind back the iterator
    Map<String, Set<InetAddress>> skippedDcEndpoints = new HashMap<>(datacenters.size());
    for (Map.Entry<String, Integer> dc : datacenters.entrySet()) skippedDcEndpoints.put(dc.getKey(), new LinkedHashSet<InetAddress>());
    Iterator<Token> tokenIter = TokenMetadata.ringIterator(tokenMetadata.sortedTokens(), searchToken, false);
    while (tokenIter.hasNext() && !hasSufficientReplicas(dcReplicas, allEndpoints, datacenters)) {
        Token next = tokenIter.next();
        InetAddress ep = tokenMetadata.getEndpoint(next);
        String dc = snitch.getDatacenter(ep);
        // have we already found all replicas for this dc?
        if (!datacenters.containsKey(dc) || hasSufficientReplicas(dc, dcReplicas, allEndpoints, datacenters))
            continue;
        // can we skip checking the rack?
        if (seenRacks.get(dc).size() == racks.get(dc).keySet().size()) {
            dcReplicas.get(dc).add(ep);
            replicas.add(ep);
        } else {
            String rack = snitch.getRack(ep);
            // is this a new rack?
            if (seenRacks.get(dc).contains(rack)) {
                skippedDcEndpoints.get(dc).add(ep);
            } else {
                dcReplicas.get(dc).add(ep);
                replicas.add(ep);
                seenRacks.get(dc).add(rack);
                // if we've run out of distinct racks, add the hosts we skipped past already (up to RF)
                if (seenRacks.get(dc).size() == racks.get(dc).keySet().size()) {
                    Iterator<InetAddress> skippedIt = skippedDcEndpoints.get(dc).iterator();
                    while (skippedIt.hasNext() && !hasSufficientReplicas(dc, dcReplicas, allEndpoints, datacenters)) {
                        InetAddress nextSkipped = skippedIt.next();
                        dcReplicas.get(dc).add(nextSkipped);
                        replicas.add(nextSkipped);
                    }
                }
            }
        }
    }
    return new ArrayList<InetAddress>(replicas);
}

Example 13 with Multimap

use of com.google.common.collect.Multimap in project cassandra by apache.

the class TokenMetadataTest method testTopologyUpdate_RackConsolidation.

@Test
public void testTopologyUpdate_RackConsolidation() throws UnknownHostException {
    final InetAddress first = InetAddress.getByName("127.0.0.1");
    final InetAddress second = InetAddress.getByName("127.0.0.6");
    final String DATA_CENTER = "datacenter1";
    final String RACK1 = "rack1";
    final String RACK2 = "rack2";
    DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch() {

        @Override
        public String getRack(InetAddress endpoint) {
            return endpoint.equals(first) ? RACK1 : RACK2;
        }

        @Override
        public String getDatacenter(InetAddress endpoint) {
            return DATA_CENTER;
        }

        @Override
        public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2) {
            return 0;
        }
    });
    tmd.updateNormalToken(token(ONE), first);
    tmd.updateNormalToken(token(SIX), second);
    TokenMetadata tokenMetadata = tmd.cloneOnlyTokenMap();
    assertNotNull(tokenMetadata);
    TokenMetadata.Topology topology = tokenMetadata.getTopology();
    assertNotNull(topology);
    Multimap<String, InetAddress> allEndpoints = topology.getDatacenterEndpoints();
    assertNotNull(allEndpoints);
    assertTrue(allEndpoints.size() == 2);
    assertTrue(allEndpoints.containsKey(DATA_CENTER));
    assertTrue(allEndpoints.get(DATA_CENTER).contains(first));
    assertTrue(allEndpoints.get(DATA_CENTER).contains(second));
    Map<String, Multimap<String, InetAddress>> racks = topology.getDatacenterRacks();
    assertNotNull(racks);
    assertTrue(racks.size() == 1);
    assertTrue(racks.containsKey(DATA_CENTER));
    assertTrue(racks.get(DATA_CENTER).size() == 2);
    assertTrue(racks.get(DATA_CENTER).containsKey(RACK1));
    assertTrue(racks.get(DATA_CENTER).containsKey(RACK2));
    assertTrue(racks.get(DATA_CENTER).get(RACK1).contains(first));
    assertTrue(racks.get(DATA_CENTER).get(RACK2).contains(second));
    DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch() {

        @Override
        public String getRack(InetAddress endpoint) {
            return RACK1;
        }

        @Override
        public String getDatacenter(InetAddress endpoint) {
            return DATA_CENTER;
        }

        @Override
        public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2) {
            return 0;
        }
    });
    tokenMetadata.updateTopology(first);
    tokenMetadata.updateTopology(second);
    allEndpoints = topology.getDatacenterEndpoints();
    assertNotNull(allEndpoints);
    assertTrue(allEndpoints.size() == 2);
    assertTrue(allEndpoints.containsKey(DATA_CENTER));
    assertTrue(allEndpoints.get(DATA_CENTER).contains(first));
    assertTrue(allEndpoints.get(DATA_CENTER).contains(second));
    racks = topology.getDatacenterRacks();
    assertNotNull(racks);
    assertTrue(racks.size() == 1);
    assertTrue(racks.containsKey(DATA_CENTER));
    assertTrue(racks.get(DATA_CENTER).size() == 2);
    assertTrue(racks.get(DATA_CENTER).containsKey(RACK1));
    assertFalse(racks.get(DATA_CENTER).containsKey(RACK2));
    assertTrue(racks.get(DATA_CENTER).get(RACK1).contains(first));
    assertTrue(racks.get(DATA_CENTER).get(RACK1).contains(second));
}

Example 14 with Multimap

use of com.google.common.collect.Multimap in project cassandra by apache.

the class TokenMetadataTest method testTopologyUpdate_RackExpansion.

@Test
public void testTopologyUpdate_RackExpansion() throws UnknownHostException {
    final InetAddress first = InetAddress.getByName("127.0.0.1");
    final InetAddress second = InetAddress.getByName("127.0.0.6");
    final String DATA_CENTER = "datacenter1";
    final String RACK1 = "rack1";
    final String RACK2 = "rack2";
    DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch() {

        @Override
        public String getRack(InetAddress endpoint) {
            return RACK1;
        }

        @Override
        public String getDatacenter(InetAddress endpoint) {
            return DATA_CENTER;
        }

        @Override
        public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2) {
            return 0;
        }
    });
    tmd.updateNormalToken(token(ONE), first);
    tmd.updateNormalToken(token(SIX), second);
    TokenMetadata tokenMetadata = tmd.cloneOnlyTokenMap();
    assertNotNull(tokenMetadata);
    TokenMetadata.Topology topology = tokenMetadata.getTopology();
    assertNotNull(topology);
    Multimap<String, InetAddress> allEndpoints = topology.getDatacenterEndpoints();
    assertNotNull(allEndpoints);
    assertTrue(allEndpoints.size() == 2);
    assertTrue(allEndpoints.containsKey(DATA_CENTER));
    assertTrue(allEndpoints.get(DATA_CENTER).contains(first));
    assertTrue(allEndpoints.get(DATA_CENTER).contains(second));
    Map<String, Multimap<String, InetAddress>> racks = topology.getDatacenterRacks();
    assertNotNull(racks);
    assertTrue(racks.size() == 1);
    assertTrue(racks.containsKey(DATA_CENTER));
    assertTrue(racks.get(DATA_CENTER).size() == 2);
    assertTrue(racks.get(DATA_CENTER).containsKey(RACK1));
    assertFalse(racks.get(DATA_CENTER).containsKey(RACK2));
    assertTrue(racks.get(DATA_CENTER).get(RACK1).contains(first));
    assertTrue(racks.get(DATA_CENTER).get(RACK1).contains(second));
    DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch() {

        @Override
        public String getRack(InetAddress endpoint) {
            return endpoint.equals(first) ? RACK1 : RACK2;
        }

        @Override
        public String getDatacenter(InetAddress endpoint) {
            return DATA_CENTER;
        }

        @Override
        public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2) {
            return 0;
        }
    });
    tokenMetadata.updateTopology();
    allEndpoints = topology.getDatacenterEndpoints();
    assertNotNull(allEndpoints);
    assertTrue(allEndpoints.size() == 2);
    assertTrue(allEndpoints.containsKey(DATA_CENTER));
    assertTrue(allEndpoints.get(DATA_CENTER).contains(first));
    assertTrue(allEndpoints.get(DATA_CENTER).contains(second));
    racks = topology.getDatacenterRacks();
    assertNotNull(racks);
    assertTrue(racks.size() == 1);
    assertTrue(racks.containsKey(DATA_CENTER));
    assertTrue(racks.get(DATA_CENTER).size() == 2);
    assertTrue(racks.get(DATA_CENTER).containsKey(RACK1));
    assertTrue(racks.get(DATA_CENTER).containsKey(RACK2));
    assertTrue(racks.get(DATA_CENTER).get(RACK1).contains(first));
    assertTrue(racks.get(DATA_CENTER).get(RACK2).contains(second));
}

Example 15 with Multimap

use of com.google.common.collect.Multimap in project cassandra by apache.

the class LeaveAndBootstrapTest method testSimultaneousMove.

/**
     * Test pending ranges and write endpoints when multiple nodes are on the move
     * simultaneously
     */
@Test
public void testSimultaneousMove() throws UnknownHostException {
    StorageService ss = StorageService.instance;
    final int RING_SIZE = 10;
    TokenMetadata tmd = ss.getTokenMetadata();
    tmd.clearUnsafe();
    IPartitioner partitioner = RandomPartitioner.instance;
    VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);
    ArrayList<Token> endpointTokens = new ArrayList<Token>();
    ArrayList<Token> keyTokens = new ArrayList<Token>();
    List<InetAddress> hosts = new ArrayList<InetAddress>();
    List<UUID> hostIds = new ArrayList<UUID>();
    // create a ring or 10 nodes
    Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, RING_SIZE);
    // nodes 6, 8 and 9 leave
    final int[] LEAVING = new int[] { 6, 8, 9 };
    for (int leaving : LEAVING) ss.onChange(hosts.get(leaving), ApplicationState.STATUS, valueFactory.leaving(Collections.singleton(endpointTokens.get(leaving))));
    // boot two new nodes with keyTokens.get(5) and keyTokens.get(7)
    InetAddress boot1 = InetAddress.getByName("127.0.1.1");
    Gossiper.instance.initializeNodeUnsafe(boot1, UUID.randomUUID(), 1);
    Gossiper.instance.injectApplicationState(boot1, ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(keyTokens.get(5))));
    ss.onChange(boot1, ApplicationState.STATUS, valueFactory.bootstrapping(Collections.<Token>singleton(keyTokens.get(5))));
    InetAddress boot2 = InetAddress.getByName("127.0.1.2");
    Gossiper.instance.initializeNodeUnsafe(boot2, UUID.randomUUID(), 1);
    Gossiper.instance.injectApplicationState(boot2, ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(keyTokens.get(7))));
    ss.onChange(boot2, ApplicationState.STATUS, valueFactory.bootstrapping(Collections.<Token>singleton(keyTokens.get(7))));
    Collection<InetAddress> endpoints = null;
    /* don't require test update every time a new keyspace is added to test/conf/cassandra.yaml */
    Map<String, AbstractReplicationStrategy> keyspaceStrategyMap = new HashMap<String, AbstractReplicationStrategy>();
    for (int i = 1; i <= 4; i++) {
        keyspaceStrategyMap.put("LeaveAndBootstrapTestKeyspace" + i, getStrategy("LeaveAndBootstrapTestKeyspace" + i, tmd));
    }
    // pre-calculate the results.
    Map<String, Multimap<Token, InetAddress>> expectedEndpoints = new HashMap<String, Multimap<Token, InetAddress>>();
    expectedEndpoints.put(KEYSPACE1, HashMultimap.<Token, InetAddress>create());
    expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2"));
    expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3"));
    expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4"));
    expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("35"), makeAddrs("127.0.0.5"));
    expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("45"), makeAddrs("127.0.0.6"));
    expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("55"), makeAddrs("127.0.0.7", "127.0.0.8", "127.0.1.1"));
    expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("65"), makeAddrs("127.0.0.8"));
    expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("75"), makeAddrs("127.0.0.9", "127.0.1.2", "127.0.0.1"));
    expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("85"), makeAddrs("127.0.0.10", "127.0.0.1"));
    expectedEndpoints.get(KEYSPACE1).putAll(new BigIntegerToken("95"), makeAddrs("127.0.0.1"));
    expectedEndpoints.put(KEYSPACE2, HashMultimap.<Token, InetAddress>create());
    expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2"));
    expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3"));
    expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4"));
    expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("35"), makeAddrs("127.0.0.5"));
    expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("45"), makeAddrs("127.0.0.6"));
    expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("55"), makeAddrs("127.0.0.7", "127.0.0.8", "127.0.1.1"));
    expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("65"), makeAddrs("127.0.0.8"));
    expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("75"), makeAddrs("127.0.0.9", "127.0.1.2", "127.0.0.1"));
    expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("85"), makeAddrs("127.0.0.10", "127.0.0.1"));
    expectedEndpoints.get(KEYSPACE2).putAll(new BigIntegerToken("95"), makeAddrs("127.0.0.1"));
    expectedEndpoints.put(KEYSPACE3, HashMultimap.<Token, InetAddress>create());
    expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5", "127.0.0.6"));
    expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3", "127.0.0.4", "127.0.0.5", "127.0.0.6", "127.0.0.7", "127.0.1.1", "127.0.0.8"));
    expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4", "127.0.0.5", "127.0.0.6", "127.0.0.7", "127.0.0.8", "127.0.1.2", "127.0.0.1", "127.0.1.1"));
    expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("35"), makeAddrs("127.0.0.5", "127.0.0.6", "127.0.0.7", "127.0.0.8", "127.0.0.9", "127.0.1.2", "127.0.0.1", "127.0.0.2", "127.0.1.1"));
    expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("45"), makeAddrs("127.0.0.6", "127.0.0.7", "127.0.0.8", "127.0.0.9", "127.0.0.10", "127.0.1.2", "127.0.0.1", "127.0.0.2", "127.0.1.1", "127.0.0.3"));
    expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("55"), makeAddrs("127.0.0.7", "127.0.0.8", "127.0.0.9", "127.0.0.10", "127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.1.1", "127.0.1.2"));
    expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("65"), makeAddrs("127.0.0.8", "127.0.0.9", "127.0.0.10", "127.0.0.1", "127.0.0.2", "127.0.1.2", "127.0.0.3", "127.0.0.4"));
    expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("75"), makeAddrs("127.0.0.9", "127.0.0.10", "127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.1.2", "127.0.0.4", "127.0.0.5"));
    expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("85"), makeAddrs("127.0.0.10", "127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5"));
    expectedEndpoints.get(KEYSPACE3).putAll(new BigIntegerToken("95"), makeAddrs("127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5"));
    expectedEndpoints.put(KEYSPACE4, HashMultimap.<Token, InetAddress>create());
    expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("5"), makeAddrs("127.0.0.2", "127.0.0.3", "127.0.0.4"));
    expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("15"), makeAddrs("127.0.0.3", "127.0.0.4", "127.0.0.5"));
    expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("25"), makeAddrs("127.0.0.4", "127.0.0.5", "127.0.0.6"));
    expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("35"), makeAddrs("127.0.0.5", "127.0.0.6", "127.0.0.7", "127.0.1.1", "127.0.0.8"));
    expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("45"), makeAddrs("127.0.0.6", "127.0.0.7", "127.0.0.8", "127.0.1.2", "127.0.0.1", "127.0.1.1"));
    expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("55"), makeAddrs("127.0.0.7", "127.0.0.8", "127.0.0.9", "127.0.0.1", "127.0.0.2", "127.0.1.1", "127.0.1.2"));
    expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("65"), makeAddrs("127.0.0.8", "127.0.0.9", "127.0.0.10", "127.0.1.2", "127.0.0.1", "127.0.0.2"));
    expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("75"), makeAddrs("127.0.0.9", "127.0.0.10", "127.0.0.1", "127.0.1.2", "127.0.0.2", "127.0.0.3"));
    expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("85"), makeAddrs("127.0.0.10", "127.0.0.1", "127.0.0.2", "127.0.0.3"));
    expectedEndpoints.get(KEYSPACE4).putAll(new BigIntegerToken("95"), makeAddrs("127.0.0.1", "127.0.0.2", "127.0.0.3"));
    PendingRangeCalculatorService.instance.blockUntilFinished();
    for (Map.Entry<String, AbstractReplicationStrategy> keyspaceStrategy : keyspaceStrategyMap.entrySet()) {
        String keyspaceName = keyspaceStrategy.getKey();
        AbstractReplicationStrategy strategy = keyspaceStrategy.getValue();
        for (int i = 0; i < keyTokens.size(); i++) {
            endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i)));
            assertEquals(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).size(), endpoints.size());
            assertTrue(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).containsAll(endpoints));
        }
        // just to be sure that things still work according to the old tests, run them:
        if (strategy.getReplicationFactor() != 3)
            continue;
        // tokens 5, 15 and 25 should go three nodes
        for (int i = 0; i < 3; ++i) {
            endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i)));
            assertEquals(3, endpoints.size());
            assertTrue(endpoints.contains(hosts.get(i + 1)));
            assertTrue(endpoints.contains(hosts.get(i + 2)));
            assertTrue(endpoints.contains(hosts.get(i + 3)));
        }
        // token 35 should go to nodes 4, 5, 6, 7 and boot1
        endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(3)));
        assertEquals(5, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(4)));
        assertTrue(endpoints.contains(hosts.get(5)));
        assertTrue(endpoints.contains(hosts.get(6)));
        assertTrue(endpoints.contains(hosts.get(7)));
        assertTrue(endpoints.contains(boot1));
        // token 45 should go to nodes 5, 6, 7, 0, boot1 and boot2
        endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(4)));
        assertEquals(6, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(5)));
        assertTrue(endpoints.contains(hosts.get(6)));
        assertTrue(endpoints.contains(hosts.get(7)));
        assertTrue(endpoints.contains(hosts.get(0)));
        assertTrue(endpoints.contains(boot1));
        assertTrue(endpoints.contains(boot2));
        // token 55 should go to nodes 6, 7, 8, 0, 1, boot1 and boot2
        endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(5)));
        assertEquals(7, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(6)));
        assertTrue(endpoints.contains(hosts.get(7)));
        assertTrue(endpoints.contains(hosts.get(8)));
        assertTrue(endpoints.contains(hosts.get(0)));
        assertTrue(endpoints.contains(hosts.get(1)));
        assertTrue(endpoints.contains(boot1));
        assertTrue(endpoints.contains(boot2));
        // token 65 should go to nodes 7, 8, 9, 0, 1 and boot2
        endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(6)));
        assertEquals(6, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(7)));
        assertTrue(endpoints.contains(hosts.get(8)));
        assertTrue(endpoints.contains(hosts.get(9)));
        assertTrue(endpoints.contains(hosts.get(0)));
        assertTrue(endpoints.contains(hosts.get(1)));
        assertTrue(endpoints.contains(boot2));
        // token 75 should to go nodes 8, 9, 0, 1, 2 and boot2
        endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(7)));
        assertEquals(6, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(8)));
        assertTrue(endpoints.contains(hosts.get(9)));
        assertTrue(endpoints.contains(hosts.get(0)));
        assertTrue(endpoints.contains(hosts.get(1)));
        assertTrue(endpoints.contains(hosts.get(2)));
        assertTrue(endpoints.contains(boot2));
        // token 85 should go to nodes 9, 0, 1 and 2
        endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(8)));
        assertEquals(4, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(9)));
        assertTrue(endpoints.contains(hosts.get(0)));
        assertTrue(endpoints.contains(hosts.get(1)));
        assertTrue(endpoints.contains(hosts.get(2)));
        // token 95 should go to nodes 0, 1 and 2
        endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(9)));
        assertEquals(3, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(0)));
        assertTrue(endpoints.contains(hosts.get(1)));
        assertTrue(endpoints.contains(hosts.get(2)));
    }
    // Now finish node 6 and node 9 leaving, as well as boot1 (after this node 8 is still
    // leaving and boot2 in progress
    ss.onChange(hosts.get(LEAVING[0]), ApplicationState.STATUS, valueFactory.left(Collections.singleton(endpointTokens.get(LEAVING[0])), Gossiper.computeExpireTime()));
    ss.onChange(hosts.get(LEAVING[2]), ApplicationState.STATUS, valueFactory.left(Collections.singleton(endpointTokens.get(LEAVING[2])), Gossiper.computeExpireTime()));
    ss.onChange(boot1, ApplicationState.STATUS, valueFactory.normal(Collections.singleton(keyTokens.get(5))));
    // adjust precalcuated results.  this changes what the epected endpoints are.
    expectedEndpoints.get(KEYSPACE1).get(new BigIntegerToken("55")).removeAll(makeAddrs("127.0.0.7", "127.0.0.8"));
    expectedEndpoints.get(KEYSPACE1).get(new BigIntegerToken("85")).removeAll(makeAddrs("127.0.0.10"));
    expectedEndpoints.get(KEYSPACE2).get(new BigIntegerToken("55")).removeAll(makeAddrs("127.0.0.7", "127.0.0.8"));
    expectedEndpoints.get(KEYSPACE2).get(new BigIntegerToken("85")).removeAll(makeAddrs("127.0.0.10"));
    expectedEndpoints.get(KEYSPACE3).get(new BigIntegerToken("15")).removeAll(makeAddrs("127.0.0.7", "127.0.0.8"));
    expectedEndpoints.get(KEYSPACE3).get(new BigIntegerToken("25")).removeAll(makeAddrs("127.0.0.7", "127.0.1.2", "127.0.0.1"));
    expectedEndpoints.get(KEYSPACE3).get(new BigIntegerToken("35")).removeAll(makeAddrs("127.0.0.7", "127.0.0.2"));
    expectedEndpoints.get(KEYSPACE3).get(new BigIntegerToken("45")).removeAll(makeAddrs("127.0.0.7", "127.0.0.10", "127.0.0.3"));
    expectedEndpoints.get(KEYSPACE3).get(new BigIntegerToken("55")).removeAll(makeAddrs("127.0.0.7", "127.0.0.10", "127.0.0.4"));
    expectedEndpoints.get(KEYSPACE3).get(new BigIntegerToken("65")).removeAll(makeAddrs("127.0.0.10"));
    expectedEndpoints.get(KEYSPACE3).get(new BigIntegerToken("75")).removeAll(makeAddrs("127.0.0.10"));
    expectedEndpoints.get(KEYSPACE3).get(new BigIntegerToken("85")).removeAll(makeAddrs("127.0.0.10"));
    expectedEndpoints.get(KEYSPACE4).get(new BigIntegerToken("35")).removeAll(makeAddrs("127.0.0.7", "127.0.0.8"));
    expectedEndpoints.get(KEYSPACE4).get(new BigIntegerToken("45")).removeAll(makeAddrs("127.0.0.7", "127.0.1.2", "127.0.0.1"));
    expectedEndpoints.get(KEYSPACE4).get(new BigIntegerToken("55")).removeAll(makeAddrs("127.0.0.2", "127.0.0.7"));
    expectedEndpoints.get(KEYSPACE4).get(new BigIntegerToken("65")).removeAll(makeAddrs("127.0.0.10"));
    expectedEndpoints.get(KEYSPACE4).get(new BigIntegerToken("75")).removeAll(makeAddrs("127.0.0.10"));
    expectedEndpoints.get(KEYSPACE4).get(new BigIntegerToken("85")).removeAll(makeAddrs("127.0.0.10"));
    PendingRangeCalculatorService.instance.blockUntilFinished();
    for (Map.Entry<String, AbstractReplicationStrategy> keyspaceStrategy : keyspaceStrategyMap.entrySet()) {
        String keyspaceName = keyspaceStrategy.getKey();
        AbstractReplicationStrategy strategy = keyspaceStrategy.getValue();
        for (int i = 0; i < keyTokens.size(); i++) {
            endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i)));
            assertEquals(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).size(), endpoints.size());
            assertTrue(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).containsAll(endpoints));
        }
        if (strategy.getReplicationFactor() != 3)
            continue;
        // tokens 5, 15 and 25 should go three nodes
        for (int i = 0; i < 3; ++i) {
            endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i)));
            assertEquals(3, endpoints.size());
            assertTrue(endpoints.contains(hosts.get(i + 1)));
            assertTrue(endpoints.contains(hosts.get(i + 2)));
            assertTrue(endpoints.contains(hosts.get(i + 3)));
        }
        // token 35 goes to nodes 4, 5 and boot1
        endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(3)));
        assertEquals(3, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(4)));
        assertTrue(endpoints.contains(hosts.get(5)));
        assertTrue(endpoints.contains(boot1));
        // token 45 goes to nodes 5, boot1 and node7
        endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(4)));
        assertEquals(3, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(5)));
        assertTrue(endpoints.contains(boot1));
        assertTrue(endpoints.contains(hosts.get(7)));
        // token 55 goes to boot1, 7, boot2, 8 and 0
        endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(5)));
        assertEquals(5, endpoints.size());
        assertTrue(endpoints.contains(boot1));
        assertTrue(endpoints.contains(hosts.get(7)));
        assertTrue(endpoints.contains(boot2));
        assertTrue(endpoints.contains(hosts.get(8)));
        assertTrue(endpoints.contains(hosts.get(0)));
        // token 65 goes to nodes 7, boot2, 8, 0 and 1
        endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(6)));
        assertEquals(5, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(7)));
        assertTrue(endpoints.contains(boot2));
        assertTrue(endpoints.contains(hosts.get(8)));
        assertTrue(endpoints.contains(hosts.get(0)));
        assertTrue(endpoints.contains(hosts.get(1)));
        // token 75 goes to nodes boot2, 8, 0, 1 and 2
        endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(7)));
        assertEquals(5, endpoints.size());
        assertTrue(endpoints.contains(boot2));
        assertTrue(endpoints.contains(hosts.get(8)));
        assertTrue(endpoints.contains(hosts.get(0)));
        assertTrue(endpoints.contains(hosts.get(1)));
        assertTrue(endpoints.contains(hosts.get(2)));
        // token 85 goes to nodes 0, 1 and 2
        endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(8)));
        assertEquals(3, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(0)));
        assertTrue(endpoints.contains(hosts.get(1)));
        assertTrue(endpoints.contains(hosts.get(2)));
        // token 95 goes to nodes 0, 1 and 2
        endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(9)));
        assertEquals(3, endpoints.size());
        assertTrue(endpoints.contains(hosts.get(0)));
        assertTrue(endpoints.contains(hosts.get(1)));
        assertTrue(endpoints.contains(hosts.get(2)));
    }
}