Examples with Zone voldemort.cluster.Zone used on opensource projects

Example 21 with Zone

use of voldemort.cluster.Zone in project voldemort by voldemort.

the class StreamingSlopPusherJob method run.

public void run() {
    // load the metadata before each run, in case the cluster is changed
    loadMetadata();
    // don't try to run slop pusher job when rebalancing
    if (metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
        logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
        return;
    }
    boolean terminatedEarly = false;
    Date startTime = new Date();
    logger.info("Started streaming slop pusher job at " + startTime);
    SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
    ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
    if (adminClient == null) {
        adminClient = new AdminClient(cluster, new AdminClientConfig().setMaxConnectionsPerNode(1));
    }
    if (voldemortConfig.getSlopZonesDownToTerminate() > 0) {
        // Populating the zone mapping for early termination
        zoneMapping.clear();
        for (Node n : cluster.getNodes()) {
            if (failureDetector.isAvailable(n)) {
                Set<Integer> nodes = zoneMapping.get(n.getZoneId());
                if (nodes == null) {
                    nodes = Sets.newHashSet();
                    zoneMapping.put(n.getZoneId(), nodes);
                }
                nodes.add(n.getId());
            }
        }
        // Check how many zones are down
        int zonesDown = 0;
        for (Zone zone : cluster.getZones()) {
            if (zoneMapping.get(zone.getId()) == null || zoneMapping.get(zone.getId()).size() == 0)
                zonesDown++;
        }
        // Terminate early
        if (voldemortConfig.getSlopZonesDownToTerminate() <= zoneMapping.size() && zonesDown >= voldemortConfig.getSlopZonesDownToTerminate()) {
            logger.info("Completed streaming slop pusher job at " + startTime + " early because " + zonesDown + " zones are down");
            stopAdminClient();
            return;
        }
    }
    // Clearing the statistics
    AtomicLong attemptedPushes = new AtomicLong(0);
    for (Node node : cluster.getNodes()) {
        attemptedByNode.put(node.getId(), 0L);
        succeededByNode.put(node.getId(), 0L);
    }
    Set<String> storeNames = StoreDefinitionUtils.getStoreNamesSet(metadataStore.getStoreDefList());
    acquireRepairPermit();
    try {
        StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
        iterator = slopStore.entries();
        while (iterator.hasNext()) {
            Pair<ByteArray, Versioned<Slop>> keyAndVal;
            try {
                keyAndVal = iterator.next();
                Versioned<Slop> versioned = keyAndVal.getSecond();
                // Track the scan progress
                if (this.streamStats != null) {
                    this.streamStats.reportStreamingSlopScan();
                }
                // Retrieve the node
                int nodeId = versioned.getValue().getNodeId();
                // check for dead slops
                if (isSlopDead(cluster, storeNames, versioned.getValue())) {
                    handleDeadSlop(slopStorageEngine, keyAndVal);
                    // ignore it.
                    continue;
                }
                Node node = cluster.getNodeById(nodeId);
                attemptedPushes.incrementAndGet();
                Long attempted = attemptedByNode.get(nodeId);
                attemptedByNode.put(nodeId, attempted + 1L);
                if (attemptedPushes.get() % 10000 == 0)
                    logger.info("Attempted pushing " + attemptedPushes + " slops");
                if (logger.isTraceEnabled())
                    logger.trace("Pushing slop for " + versioned.getValue().getNodeId() + " and store  " + versioned.getValue().getStoreName() + " of key: " + versioned.getValue().getKey());
                if (failureDetector.isAvailable(node)) {
                    SynchronousQueue<Versioned<Slop>> slopQueue = slopQueues.get(nodeId);
                    if (slopQueue == null) {
                        // No previous slop queue, add one
                        slopQueue = new SynchronousQueue<Versioned<Slop>>();
                        slopQueues.put(nodeId, slopQueue);
                        consumerResults.add(consumerExecutor.submit(new SlopConsumer(nodeId, slopQueue, slopStorageEngine)));
                    }
                    boolean offered = slopQueue.offer(versioned, voldemortConfig.getClientRoutingTimeoutMs(), TimeUnit.MILLISECONDS);
                    if (!offered) {
                        if (logger.isDebugEnabled())
                            logger.debug("No consumer appeared for slop in " + voldemortConfig.getClientConnectionTimeoutMs() + " ms");
                    }
                    readThrottler.maybeThrottle(nBytesRead(keyAndVal));
                } else {
                    logger.trace(node + " declared down, won't push slop");
                }
            } catch (RejectedExecutionException e) {
                throw new VoldemortException("Ran out of threads in executor", e);
            }
        }
    } catch (InterruptedException e) {
        logger.warn("Interrupted exception", e);
        terminatedEarly = true;
    } catch (Exception e) {
        logger.error(e, e);
        terminatedEarly = true;
    } finally {
        try {
            if (iterator != null)
                iterator.close();
        } catch (Exception e) {
            logger.warn("Failed to close iterator cleanly as database might be closed", e);
        }
        // Adding the poison pill
        for (SynchronousQueue<Versioned<Slop>> slopQueue : slopQueues.values()) {
            try {
                slopQueue.put(END);
            } catch (InterruptedException e) {
                logger.warn("Error putting poison pill", e);
            }
        }
        for (Future result : consumerResults) {
            try {
                result.get();
            } catch (Exception e) {
                logger.warn("Exception in consumer", e);
            }
        }
        // Only if exception didn't take place do we update the counts
        if (!terminatedEarly) {
            Map<Integer, Long> outstanding = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
            for (int nodeId : succeededByNode.keySet()) {
                logger.info("Slops to node " + nodeId + " - Succeeded - " + succeededByNode.get(nodeId) + " - Attempted - " + attemptedByNode.get(nodeId));
                outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
            }
            slopStorageEngine.resetStats(outstanding);
            logger.info("Completed streaming slop pusher job which started at " + startTime);
        } else {
            for (int nodeId : succeededByNode.keySet()) {
                logger.info("Slops to node " + nodeId + " - Succeeded - " + succeededByNode.get(nodeId) + " - Attempted - " + attemptedByNode.get(nodeId));
            }
            logger.info("Completed early streaming slop pusher job which started at " + startTime);
        }
        // Shut down admin client as not to waste connections
        consumerResults.clear();
        slopQueues.clear();
        stopAdminClient();
        this.repairPermits.release(this.getClass().getCanonicalName());
    }
}

Example 22 with Zone

use of voldemort.cluster.Zone in project voldemort by voldemort.

the class AdminServiceBasicTest method testReplicationMappingWithZonePreferenceWithNonContiguousZones.

@Test
public void testReplicationMappingWithZonePreferenceWithNonContiguousZones() {
    int[] zoneIds = { 1, 3 };
    List<Zone> zones = ServerTestUtils.getZonesFromZoneIds(zoneIds);
    List<Node> nodes = Lists.newArrayList();
    nodes.add(new Node(3, "localhost", 1, 2, 3, 1, Lists.newArrayList(0, 4, 8)));
    nodes.add(new Node(4, "localhost", 1, 2, 3, 1, Lists.newArrayList(1, 5, 9)));
    nodes.add(new Node(5, "localhost", 1, 2, 3, 3, Lists.newArrayList(2, 6, 10)));
    nodes.add(new Node(6, "localhost", 1, 2, 3, 3, Lists.newArrayList(3, 7, 11)));
    // Node 3 - With rep-factor 1; zone 1
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("consistent", 1, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    Cluster newCluster = new Cluster("single_zone_cluster", nodes, zones);
    try {
        adminClient.replicaOps.getReplicationMapping(3, newCluster, storeDef, 1);
        fail("Should have thrown an exception since rep-factor = 1");
    } catch (VoldemortException e) {
    }
    // With rep-factor 1; zone 1
    storeDef = ServerTestUtils.getStoreDef("consistent", 1, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    newCluster = new Cluster("single_zone_cluster", nodes, zones);
    try {
        adminClient.replicaOps.getReplicationMapping(3, newCluster, storeDef, 1);
        fail("Should have thrown an exception since rep-factor = 1");
    } catch (VoldemortException e) {
    }
    // Node 1 - With consistent routing strategy
    storeDef = ServerTestUtils.getStoreDef("consistent", 4, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    // On node 3; zone id 1
    Map<Integer, List<Integer>> replicationMapping = adminClient.replicaOps.getReplicationMapping(3, newCluster, storeDef, 1);
    {
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(4, Lists.newArrayList(0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    // On node 3; zone id 1
    replicationMapping = adminClient.replicaOps.getReplicationMapping(3, newCluster, storeDef, 1);
    {
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(4, Lists.newArrayList(0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    // Test 2 - With zone routing strategy, and zone replication factor 1
    HashMap<Integer, Integer> zoneReplicationFactors = Maps.newHashMap();
    for (int index = 0; index < zoneIds.length; index++) {
        zoneReplicationFactors.put(zoneIds[index], 1);
    }
    storeDef = ServerTestUtils.getStoreDef("zone", 2, 1, 1, 1, 0, 0, zoneReplicationFactors, HintedHandoffStrategyType.PROXIMITY_STRATEGY, RoutingStrategyType.ZONE_STRATEGY);
    newCluster = new Cluster("multi_zone_cluster", nodes, zones);
    {
        try {
            replicationMapping = adminClient.replicaOps.getReplicationMapping(3, newCluster, storeDef, 1);
            fail("Should have thrown an exception since  zoneReplicationFactor is 1");
        } catch (VoldemortException e) {
        }
    }
    {
        // On node 3, zone 3
        replicationMapping = adminClient.replicaOps.getReplicationMapping(3, newCluster, storeDef, 3);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(5, Lists.newArrayList(0, 4, 8, 2, 6, 10));
        expectedMapping.put(6, Lists.newArrayList(3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 4, zone 3
        replicationMapping = adminClient.replicaOps.getReplicationMapping(4, newCluster, storeDef, 3);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(5, Lists.newArrayList(1, 5, 9));
        assertEquals(expectedMapping, replicationMapping);
    }
}

Example 23 with Zone

use of voldemort.cluster.Zone in project voldemort by voldemort.

the class AdminServiceBasicTest method testReplicationMappingWithNonContiguousZones.

@Test
public void testReplicationMappingWithNonContiguousZones() {
    int[] zoneIds = { 1, 3 };
    List<Zone> zones = ServerTestUtils.getZonesFromZoneIds(zoneIds);
    List<Node> nodes = Lists.newArrayList();
    nodes.add(new Node(3, "localhost", 1, 2, 3, 1, Lists.newArrayList(0, 4, 8)));
    nodes.add(new Node(4, "localhost", 1, 2, 3, 1, Lists.newArrayList(1, 5, 9)));
    nodes.add(new Node(5, "localhost", 1, 2, 3, 3, Lists.newArrayList(2, 6, 10)));
    nodes.add(new Node(6, "localhost", 1, 2, 3, 3, Lists.newArrayList(3, 7, 11)));
    // Node 3 - With rep-factor 1
    StoreDefinition storeDef = ServerTestUtils.getStoreDef("consistent", 1, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    Cluster newCluster = new Cluster("single_zone_cluster", nodes, zones);
    try {
        adminClient.replicaOps.getReplicationMapping(3, newCluster, storeDef);
        fail("Should have thrown an exception since rep-factor = 1");
    } catch (VoldemortException e) {
    }
    // Test 1 - With consistent routing strategy
    storeDef = ServerTestUtils.getStoreDef("consistent", 2, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    // On node 3
    Map<Integer, List<Integer>> replicationMapping = adminClient.replicaOps.getReplicationMapping(3, newCluster, storeDef);
    {
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(4, Lists.newArrayList(0, 4, 8));
        expectedMapping.put(6, Lists.newArrayList(3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 4
        replicationMapping = adminClient.replicaOps.getReplicationMapping(4, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(3, Lists.newArrayList(0, 4, 8));
        expectedMapping.put(5, Lists.newArrayList(1, 5, 9));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 5
        replicationMapping = adminClient.replicaOps.getReplicationMapping(5, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(4, Lists.newArrayList(1, 5, 9));
        expectedMapping.put(6, Lists.newArrayList(2, 6, 10));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 6
        replicationMapping = adminClient.replicaOps.getReplicationMapping(6, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(3, Lists.newArrayList(3, 7, 11));
        expectedMapping.put(5, Lists.newArrayList(2, 6, 10));
        assertEquals(expectedMapping, replicationMapping);
    }
    // Test 2 - With zone routing strategy
    HashMap<Integer, Integer> zoneReplicationFactors = Maps.newHashMap();
    for (int index = 0; index < zoneIds.length; index++) {
        zoneReplicationFactors.put(zoneIds[index], 1);
    }
    storeDef = ServerTestUtils.getStoreDef("zone", 2, 1, 1, 1, 0, 0, zoneReplicationFactors, HintedHandoffStrategyType.PROXIMITY_STRATEGY, RoutingStrategyType.ZONE_STRATEGY);
    newCluster = new Cluster("multi_zone_cluster", nodes, zones);
    {
        // On node 3
        replicationMapping = adminClient.replicaOps.getReplicationMapping(3, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(5, Lists.newArrayList(0, 4, 8, 2, 6, 10));
        expectedMapping.put(6, Lists.newArrayList(3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 4
        replicationMapping = adminClient.replicaOps.getReplicationMapping(4, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(5, Lists.newArrayList(1, 5, 9));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 5
        replicationMapping = adminClient.replicaOps.getReplicationMapping(5, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(3, Lists.newArrayList(0, 4, 8, 2, 6, 10));
        expectedMapping.put(4, Lists.newArrayList(1, 5, 9));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 6
        replicationMapping = adminClient.replicaOps.getReplicationMapping(6, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(3, Lists.newArrayList(3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    // Test 3 - Consistent with rep factor 3
    storeDef = ServerTestUtils.getStoreDef("consistent", 3, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    newCluster = new Cluster("single_zone_cluster", nodes, zones);
    {
        // On node 3
        replicationMapping = adminClient.replicaOps.getReplicationMapping(3, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(4, Lists.newArrayList(0, 4, 8));
        expectedMapping.put(6, Lists.newArrayList(3, 7, 11));
        expectedMapping.put(5, Lists.newArrayList(2, 6, 10));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 4
        replicationMapping = adminClient.replicaOps.getReplicationMapping(4, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(3, Lists.newArrayList(0, 4, 8));
        expectedMapping.put(6, Lists.newArrayList(3, 7, 11));
        expectedMapping.put(5, Lists.newArrayList(1, 5, 9));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 5
        replicationMapping = adminClient.replicaOps.getReplicationMapping(5, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(3, Lists.newArrayList(0, 4, 8));
        expectedMapping.put(4, Lists.newArrayList(1, 5, 9));
        expectedMapping.put(6, Lists.newArrayList(2, 6, 10));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 6
        replicationMapping = adminClient.replicaOps.getReplicationMapping(6, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(3, Lists.newArrayList(3, 7, 11));
        expectedMapping.put(4, Lists.newArrayList(1, 5, 9));
        expectedMapping.put(5, Lists.newArrayList(2, 6, 10));
        assertEquals(expectedMapping, replicationMapping);
    }
    zoneReplicationFactors = Maps.newHashMap();
    for (int index = 0; index < zoneIds.length; index++) {
        zoneReplicationFactors.put(zoneIds[index], 2);
    }
    storeDef = ServerTestUtils.getStoreDef("zone", 1, 1, 1, 1, 0, 0, zoneReplicationFactors, HintedHandoffStrategyType.PROXIMITY_STRATEGY, RoutingStrategyType.ZONE_STRATEGY);
    newCluster = new Cluster("multi_zone_cluster", nodes, zones);
    {
        // On node 3
        replicationMapping = adminClient.replicaOps.getReplicationMapping(3, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(4, Lists.newArrayList(0, 4, 8, 1, 5, 9));
        expectedMapping.put(5, Lists.newArrayList(2, 6, 10));
        expectedMapping.put(6, Lists.newArrayList(3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 4
        replicationMapping = adminClient.replicaOps.getReplicationMapping(4, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(3, Lists.newArrayList(0, 4, 8));
        expectedMapping.put(5, Lists.newArrayList(1, 5, 9, 2, 6, 10));
        expectedMapping.put(6, Lists.newArrayList(3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 5
        replicationMapping = adminClient.replicaOps.getReplicationMapping(5, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(3, Lists.newArrayList(0, 4, 8));
        expectedMapping.put(4, Lists.newArrayList(1, 5, 9));
        expectedMapping.put(6, Lists.newArrayList(2, 6, 10, 3, 7, 11));
        assertEquals(expectedMapping, replicationMapping);
    }
    {
        // On node 6
        replicationMapping = adminClient.replicaOps.getReplicationMapping(6, newCluster, storeDef);
        HashMap<Integer, List<Integer>> expectedMapping = Maps.newHashMap();
        expectedMapping.put(3, Lists.newArrayList(0, 4, 8, 3, 7, 11));
        expectedMapping.put(4, Lists.newArrayList(1, 5, 9));
        expectedMapping.put(5, Lists.newArrayList(2, 6, 10));
        assertEquals(expectedMapping, replicationMapping);
    }
}