Examples with RoutingStrategy voldemort.routing.RoutingStrategy used on opensource projects

Example 36 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class InvalidMetadataCheckingStoreTest method doOperations.

private void doOperations(int nodeId, Store<ByteArray, byte[], byte[]> store, MetadataStore metadata, StoreDefinition storeDef) {
    for (int i = 0; i < LOOP_COUNT; ) {
        ByteArray key = new ByteArray(ByteUtils.md5(Integer.toString((int) (Math.random() * Integer.MAX_VALUE)).getBytes()));
        byte[] value = "value".getBytes();
        RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, metadata.getCluster());
        if (containsNodeId(routingStrategy.routeRequest(key.get()), nodeId)) {
            // increment count
            i++;
            switch(i % 4) {
                case 0:
                    store.get(key, null);
                    break;
                case 1:
                    store.delete(key, null);
                    break;
                case 2:
                    store.put(key, new Versioned<byte[]>(value), null);
                    break;
                case 3:
                    store.getAll(ImmutableList.of(key), null);
                    break;
            }
        }
    }
}

Example 37 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ServerSideRoutingTest method checkServerSideRouting.

private void checkServerSideRouting(VoldemortServer server0, VoldemortServer server1) {
    // create bunch of key-value pairs
    HashMap<ByteArray, byte[]> entryMap = ServerTestUtils.createRandomKeyValuePairs(TEST_VALUES_SIZE);
    // populate all entries in server1
    Store<ByteArray, byte[], byte[]> store = server1.getStoreRepository().getRoutedStore(testStoreName);
    for (Entry<ByteArray, byte[]> entry : entryMap.entrySet()) {
        store.put(entry.getKey(), Versioned.value(entry.getValue(), new VectorClock().incremented(0, System.currentTimeMillis())), null);
    }
    // try fetching them from server0
    store = server0.getStoreRepository().getLocalStore(testStoreName);
    RoutingStrategy routing = server0.getMetadataStore().getRoutingStrategy(testStoreName);
    for (Entry<ByteArray, byte[]> entry : entryMap.entrySet()) {
        List<Node> nodes = routing.routeRequest(entry.getKey().get());
        if (hasNode(nodes, 0)) {
            assertTrue("ServerSideRouting should return keys from other nodes.", ByteUtils.compare(entry.getValue(), store.get(entry.getKey(), null).get(0).getValue()) == 0);
        }
    }
}

Example 38 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class TestDistribution method main.

public static void main(String[] args) throws Exception {
    if (args.length < 2)
        Utils.croak("USAGE: java TestDistribution cluster.xml replication_factor max_id");
    long start = System.currentTimeMillis();
    File file = new File(args[0]);
    int repFactor = Integer.parseInt(args[1]);
    int maxVal = Integer.parseInt(args[2]);
    ClusterMapper mapper = new ClusterMapper();
    Cluster cluster = mapper.readCluster(file);
    RoutingStrategy strategy = new ConsistentRoutingStrategy(cluster, repFactor);
    JsonTypeSerializer serializer = new JsonTypeSerializer(JsonTypeDefinition.INT32);
    Map<Integer, Integer> counts = new HashMap<Integer, Integer>();
    for (int i = 0; i < maxVal; i++) {
        for (Node node : strategy.routeRequest(serializer.toBytes(i))) {
            int newCount = 1;
            if (counts.get(node.getId()) != null) {
                newCount = counts.get(node.getId()) + 1;
            }
            counts.put(node.getId(), newCount);
        }
    }
    int sum = 0;
    int totalCounts = 0;
    for (int countVal : counts.values()) {
        sum += countVal;
        totalCounts++;
    }
    int avg = sum / totalCounts;
    NumberFormat percent = NumberFormat.getPercentInstance();
    percent.setMaximumFractionDigits(2);
    System.out.println("Node\tKeys\tPercent\tVariation");
    for (int nodeId : counts.keySet()) {
        System.out.println(nodeId + "\t" + counts.get(nodeId) + "\t" + percent.format(counts.get(nodeId) / (double) sum) + "\t" + percent.format((counts.get(nodeId) - avg) / (double) counts.get(nodeId)));
    }
    double msPerHash = (System.currentTimeMillis() - start) / ((double) repFactor * maxVal);
    NumberFormat nf = NumberFormat.getNumberInstance();
    nf.setMaximumFractionDigits(10);
    System.out.println(nf.format(msPerHash) + " ms per hash");
}

Example 39 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class StreamingClientTest method verifyKeysExist.

/*
     * Checks if each node has the keys it is reponsible for returns false
     * otherwise
     */
public boolean verifyKeysExist(int nodeIdToVerifyOn) {
    RoutingStrategyFactory factory = new RoutingStrategyFactory();
    RoutingStrategy storeRoutingStrategy = factory.updateRoutingStrategy(storeDef, adminClient.getAdminClientCluster());
    HashMap<Integer, ArrayList<String>> expectedNodeIdToKeys;
    expectedNodeIdToKeys = new HashMap();
    Collection<Node> nodesInCluster = adminClient.getAdminClientCluster().getNodes();
    for (Node node : nodesInCluster) {
        ArrayList<String> keysForNode = new ArrayList();
        expectedNodeIdToKeys.put(node.getId(), keysForNode);
    }
    for (int i = 0; i < NUM_KEYS_1; i++) {
        String key = i + "";
        String value = key;
        List<Node> nodeList = storeRoutingStrategy.routeRequest(key.getBytes());
        for (Node node : nodeList) {
            ArrayList<String> keysForNode = expectedNodeIdToKeys.get(node.getId());
            keysForNode.add(key);
        }
    }
    ArrayList<String> fetchedKeysForNode = new ArrayList();
    for (Node node : nodesInCluster) {
        List<Integer> partitionIdList = Lists.newArrayList();
        partitionIdList.addAll(node.getPartitionIds());
        Iterator<ByteArray> keyIteratorRef = null;
        keyIteratorRef = adminClient.bulkFetchOps.fetchKeys(node.getId(), TEST_STORE_NAME, partitionIdList, null, false);
        final SerializerDefinition serializerDef = storeDef.getKeySerializer();
        final SerializerFactory serializerFactory = new DefaultSerializerFactory();
        @SuppressWarnings("unchecked") final Serializer<Object> serializer = (Serializer<Object>) serializerFactory.getSerializer(serializerDef);
        final CompressionStrategy keysCompressionStrategy;
        if (serializerDef != null && serializerDef.hasCompression()) {
            keysCompressionStrategy = new CompressionStrategyFactory().get(serializerDef.getCompression());
        } else {
            keysCompressionStrategy = null;
        }
        final Iterator<ByteArray> keyIterator = keyIteratorRef;
        while (keyIterator.hasNext()) {
            byte[] keyBytes = keyIterator.next().get();
            try {
                Object keyObject = serializer.toObject((null == keysCompressionStrategy) ? keyBytes : keysCompressionStrategy.inflate(keyBytes));
                fetchedKeysForNode.add((String) keyObject);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    ArrayList<String> keysForNode = expectedNodeIdToKeys.get(nodeIdToVerifyOn);
    if (!fetchedKeysForNode.containsAll(keysForNode))
        return false;
    else
        return true;
}

Example 40 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class AdminServiceBasicTest method testFetchAndUpdateRW.

/**
     * Tests the basic RW fetch and update
     */
@Test
public void testFetchAndUpdateRW() {
    HashMap<ByteArray, byte[]> entrySet = ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    List<Integer> primaryMoved = Arrays.asList(0, 2);
    List<Integer> secondaryMoved = Arrays.asList(1, 4);
    List<Integer> combinedLists = Arrays.asList(0, 1, 2, 4);
    Cluster targetCluster = UpdateClusterUtils.createUpdatedCluster(cluster, 1, primaryMoved);
    HashMap<ByteArray, byte[]> keysMovedWith0AsSecondary = Maps.newHashMap();
    // insert it into server-0 store
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs, "test-recovery-data"), cluster);
    Store<ByteArray, byte[], byte[]> store0 = getStore(0, "test-recovery-data");
    Store<ByteArray, byte[], byte[]> store1 = getStore(1, "test-recovery-data");
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
        store0.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()), null);
        List<Integer> partitions = strategy.getPartitionList(entry.getKey().get());
        if (primaryMoved.contains(partitions.get(0)) || (secondaryMoved.contains(partitions.get(0)) && cluster.getNodeById(0).getPartitionIds().contains(partitions.get(1)))) {
            keysMovedWith0AsSecondary.put(entry.getKey(), entry.getValue());
        }
    }
    // Assert that server1 is empty.
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) assertEquals("server1 should be empty at start.", 0, store1.get(entry.getKey(), null).size());
    // Set some other metadata, so as to pick the right up later
    getServer(0).getMetadataStore().put(MetadataStore.CLUSTER_KEY, targetCluster);
    // Migrate the partition
    AdminClient client = getAdminClient();
    int id = client.storeMntOps.migratePartitions(0, 1, "test-recovery-data", combinedLists, null, cluster);
    client.rpcOps.waitForCompletion(1, id, 120, TimeUnit.SECONDS);
    // Check the values
    for (Entry<ByteArray, byte[]> entry : keysMovedWith0AsSecondary.entrySet()) {
        assertEquals("server1 store should contain fetchAndupdated partitions.", 1, store1.get(entry.getKey(), null).size());
        assertEquals("entry value should match", new String(entry.getValue()), new String(store1.get(entry.getKey(), null).get(0).getValue()));
    }
}