Examples with RoutingStrategy voldemort.routing.RoutingStrategy used on opensource projects

Example 31 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ReadOnlyReplicationHelperCLI method getReadOnlyReplicationInfo.

/**
     * Analyze read-only storage file replication info, for replicating
     * the partitions on the given nodeId.  Note that because that node
     * is being replicated, we cannot rely on the node being up or
     * consistent, so we carefully avoid talking to it.
     *
     * @param cluster
     * @param nodeId
     * @return List of read-only replication info in format of:
     *         store_name,src_node_id,src_file_name,dest_file_name
     */
public static List<String> getReadOnlyReplicationInfo(AdminClient adminClient, Integer nodeId, Boolean local) {
    List<String> infoList = Lists.newArrayList();
    Cluster cluster = adminClient.getAdminClientCluster();
    List<StoreDefinition> storeDefs = adminClient.rebalanceOps.getCurrentStoreDefinitionsExcept(cluster, nodeId);
    for (StoreDefinition storeDef : storeDefs) {
        String storeName = storeDef.getName();
        // Skip a store without replication, or has wrong ro storage format.
        String storageFormat = null;
        if (storeDef.getReplicationFactor() <= 1) {
            logger.error("Store " + storeName + " cannot be restored, as it has replication factor = " + storeDef.getReplicationFactor());
            continue;
        }
        if (!storeDef.getType().equals(ReadOnlyStorageConfiguration.TYPE_NAME)) {
            logger.error("Store " + storeName + " cannot be restored, as it has type = " + storeDef.getType() + " instead of " + ReadOnlyStorageConfiguration.TYPE_NAME);
            continue;
        }
        logger.info("Processing store " + storeName);
        RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
        // belong in the replication partition list
        for (int masterPartitionId = 0; masterPartitionId < cluster.getNumberOfPartitions(); ++masterPartitionId) {
            List<Integer> naryPartitionIds = strategy.getReplicatingPartitionList(masterPartitionId);
            int nary = 0;
            Boolean hasPartition = false;
            for (int naryPartitionId : naryPartitionIds) {
                Node naryNode = cluster.getNodeForPartitionId(naryPartitionId);
                if (naryNode.getId() == nodeId) {
                    // Test if destination node has the partition of
                    // masterPartitionId, and record what kind of replica it
                    // is and break out of the loop
                    hasPartition = true;
                    naryPartitionIds.remove(nary);
                    break;
                }
                nary++;
            }
            if (!hasPartition) {
                logger.trace("Node " + nodeId + " doesn't have partition " + masterPartitionId);
                continue;
            }
            // Now find out which node hosts one of these replicas
            Node sourceNode = cluster.getNodeForPartitionId(naryPartitionIds.get(0));
            Integer sourceNodeId = sourceNode.getId();
            Long version = adminClient.readonlyOps.getROCurrentVersion(sourceNodeId, Arrays.asList(storeName)).get(storeName);
            // Now get all the file names from this node.
            List<String> fileNames = null;
            if (local) {
                List<Integer> srcPartitionIds = Lists.newArrayList();
                srcPartitionIds.addAll(sourceNode.getPartitionIds());
                fileNames = getROStorageFileListLocally(srcPartitionIds, strategy);
            } else {
                fileNames = adminClient.readonlyOps.getROStorageFileList(sourceNode.getId(), storeName);
            }
            List<String> sourceFileNames = parseAndCompare(fileNames, masterPartitionId);
            // masterPartitionId
            if (sourceFileNames.size() > 0) {
                for (String sourceFileName : sourceFileNames) {
                    String[] partitionIdReplicaChunk = sourceFileName.split(SPLIT_LITERAL);
                    // At the destination node the replicaId will be
                    // different, so only change it to nary
                    String partitionId = partitionIdReplicaChunk[0];
                    String replicaId = String.valueOf(nary);
                    String chunkId = partitionIdReplicaChunk[2];
                    // Now concat the parts together to create the file name
                    // on the destination node
                    String destFileName = partitionId.concat(SPLIT_LITERAL).concat(replicaId).concat(SPLIT_LITERAL).concat(chunkId);
                    String sourceRelPath = storeName + "/version-" + version + "/" + sourceFileName;
                    String destRelPath = storeName + "/version-" + version + "/" + destFileName;
                    infoList.add(sourceNode.getHost() + "," + sourceNode.getId() + "," + sourceRelPath + "," + destRelPath);
                }
            } else {
                logger.warn("Cannot find file for partition " + masterPartitionId + " on source node " + sourceNode.getId());
            }
        }
    }
    return infoList;
}

Example 32 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class MetadataStore method getRoutingStrategy.

/*
     * First check in the map of regular stores. If not present, check in the
     * system stores map.
     */
@SuppressWarnings("unchecked")
public RoutingStrategy getRoutingStrategy(String storeName) {
    // acquire read lock
    readLock.lock();
    try {
        Map<String, RoutingStrategy> routingStrategyMap = (Map<String, RoutingStrategy>) metadataCache.get(ROUTING_STRATEGY_KEY).getValue();
        RoutingStrategy strategy = routingStrategyMap.get(storeName);
        if (strategy == null) {
            Map<String, RoutingStrategy> systemRoutingStrategyMap = (Map<String, RoutingStrategy>) metadataCache.get(SYSTEM_ROUTING_STRATEGY_KEY).getValue();
            strategy = systemRoutingStrategyMap.get(storeName);
        }
        return strategy;
    } finally {
        readLock.unlock();
    }
}

Example 33 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class MetadataStore method updateRoutingStrategies.

/**
     * Changes to cluster OR store definition metadata results in routing
     * strategies changing. These changes need to be propagated to all the
     * listeners.
     * 
     * @param cluster The updated cluster metadata
     * @param storeDefs The updated list of store definition
     */
private void updateRoutingStrategies(Cluster cluster, List<StoreDefinition> storeDefs) {
    // acquire write lock
    writeLock.lock();
    try {
        VectorClock clock = new VectorClock();
        if (metadataCache.containsKey(ROUTING_STRATEGY_KEY))
            clock = (VectorClock) metadataCache.get(ROUTING_STRATEGY_KEY).getVersion();
        logger.info("Updating routing strategy for all stores");
        HashMap<String, StoreDefinition> storeDefMap = makeStoreDefinitionMap(storeDefs);
        HashMap<String, RoutingStrategy> routingStrategyMap = createRoutingStrategyMap(cluster, storeDefMap);
        this.metadataCache.put(ROUTING_STRATEGY_KEY, new Versioned<Object>(routingStrategyMap, clock.incremented(getNodeId(), System.currentTimeMillis())));
        for (String storeName : storeNameTolisteners.keySet()) {
            RoutingStrategy updatedRoutingStrategy = routingStrategyMap.get(storeName);
            if (updatedRoutingStrategy != null) {
                try {
                    for (MetadataStoreListener listener : storeNameTolisteners.get(storeName)) {
                        listener.updateRoutingStrategy(updatedRoutingStrategy);
                        listener.updateStoreDefinition(storeDefMap.get(storeName));
                    }
                } catch (Exception e) {
                    if (logger.isEnabledFor(Level.WARN))
                        logger.warn(e, e);
                }
            }
        }
    } finally {
        writeLock.unlock();
    }
}

Example 34 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class PartitionPrefixedBdbStorageEngineTest method testHashConsistencyAcrossRoutingStrategies.

@Test
public void testHashConsistencyAcrossRoutingStrategies() {
    // check that as long as the cluster.xml is the same, a key will hash to
    // the same partition, immaterial of whether it is zone or consistent
    // routing strategy
    StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
    List<StoreDefinition> storeDefs = mapper.readStoreList(new StringReader(VoldemortTestConstants.getTwoStoresWithZonesXml()));
    StoreDefinition consistentStore = storeDefs.get(0);
    StoreDefinition zoneStore = storeDefs.get(1);
    assertEquals(consistentStore.getName(), "cstore");
    assertEquals(zoneStore.getName(), "zstore");
    Cluster cluster = VoldemortTestConstants.getEightNodeClusterWithZones();
    RoutingStrategy cStrategy = new RoutingStrategyFactory().updateRoutingStrategy(consistentStore, cluster);
    RoutingStrategy zStrategy = new RoutingStrategyFactory().updateRoutingStrategy(zoneStore, cluster);
    BdbStorageEngine cPrefixedBdbStore = (BdbStorageEngine) bdbStorage.getStore(consistentStore, cStrategy);
    BdbStorageEngine zPrefixedBdbStore = (BdbStorageEngine) bdbStorage.getStore(zoneStore, zStrategy);
    HashMap<ByteArray, byte[]> kvpairs = ServerTestUtils.createRandomKeyValuePairs(10000);
    for (ByteArray key : kvpairs.keySet()) {
        assertEquals(cStrategy.getPartitionList(key.get()).get(0), zStrategy.getPartitionList(key.get()).get(0));
        cPrefixedBdbStore.put(key, new Versioned<byte[]>(kvpairs.get(key)), null);
        zPrefixedBdbStore.put(key, new Versioned<byte[]>(kvpairs.get(key)), null);
    }
    for (ByteArray key : kvpairs.keySet()) {
        assertEquals("Values read back does not match up", 0, ByteUtils.compare(cPrefixedBdbStore.get(key, null).get(0).getValue(), zPrefixedBdbStore.get(key, null).get(0).getValue()));
    }
    cPrefixedBdbStore.close();
    zPrefixedBdbStore.close();
}

Example 35 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class PartitionPrefixedBdbStorageEngineTest method testPartitionScan.

@Test
public void testPartitionScan() {
    StoreDefinition storedef = TestUtils.makeStoreDefinition("storeA");
    RoutingStrategy strategy = TestUtils.makeSingleNodeRoutingStrategy();
    BdbStorageEngine prefixedBdbStore = (BdbStorageEngine) bdbStorage.getStore(storedef, strategy);
    try {
        // insert a bunch of records
        HashMap<Integer, Set<String>> partitionToKeysMap = new HashMap<Integer, Set<String>>();
        for (int i = 0; i < 10000; i++) {
            String key = "key" + i;
            byte[] bkey = key.getBytes();
            int partition = strategy.getPartitionList(bkey).get(0);
            if (!partitionToKeysMap.containsKey(partition))
                partitionToKeysMap.put(partition, new HashSet<String>());
            partitionToKeysMap.get(partition).add(key);
            prefixedBdbStore.put(new ByteArray(bkey), new Versioned<byte[]>(("value" + i).getBytes()), null);
        }
        // check if they are properly retrieved by that partition id
        for (int p = 0; p < strategy.getNumReplicas(); p++) {
            // verify keys
            Set<String> keys = getKeys(prefixedBdbStore.keys(p));
            assertEquals(partitionToKeysMap.get(p).size(), keys.size());
            assertEquals(partitionToKeysMap.get(p), keys);
            // verify values
            keys = getEntries(prefixedBdbStore.entries(p));
            assertEquals(partitionToKeysMap.get(p).size(), keys.size());
            assertEquals(partitionToKeysMap.get(p), keys);
        }
        // make sure the entries() code path does not break.
        HashSet<String> allKeys = new HashSet<String>();
        for (Integer p : partitionToKeysMap.keySet()) {
            Set<String> pkeys = partitionToKeysMap.get(p);
            int originalSize = allKeys.size();
            allKeys.removeAll(pkeys);
            // this is to make sure the pkeys have 0 overlap
            assertEquals(allKeys.size(), originalSize);
            allKeys.addAll(pkeys);
        }
        // this makes sure all the data we put in is what you get out, not a
        // byte less or more
        Set<String> keys = getKeys(prefixedBdbStore.keys());
        assertEquals(allKeys.size(), keys.size());
        assertEquals(allKeys, keys);
        keys = getEntries(prefixedBdbStore.entries());
        assertEquals(allKeys.size(), keys.size());
        assertEquals(allKeys, keys);
    } catch (Exception e) {
        fail("Should not have thrown any exceptions" + e.getMessage());
    } finally {
        prefixedBdbStore.close();
    }
}