Examples with RoutingStrategy voldemort.routing.RoutingStrategy used on opensource projects

Example 1 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class MetadataStore method createRoutingStrategyMap.

private HashMap<String, RoutingStrategy> createRoutingStrategyMap(Cluster cluster, HashMap<String, StoreDefinition> storeDefs) {
    HashMap<String, RoutingStrategy> map = new HashMap<String, RoutingStrategy>();
    for (StoreDefinition store : storeDefs.values()) {
        map.put(store.getName(), routingFactory.updateRoutingStrategy(store, cluster));
    }
    // add metadata Store route to ALL routing strategy.
    map.put(METADATA_STORE_NAME, new RouteToAllStrategy(getCluster().getNodesShuffled()));
    return map;
}

Example 2 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class InvalidMetadataCheckingStore method getAll.

@Override
public Map<ByteArray, List<Versioned<byte[]>>> getAll(Iterable<ByteArray> keys, Map<ByteArray, byte[]> transforms) throws VoldemortException {
    StoreUtils.assertValidKeys(keys);
    StoreUtils.assertValidNode(metadata, nodeId);
    RoutingStrategy routingStrategy = metadata.getRoutingStrategy(getName());
    Node node = metadata.getCluster().getNodeById(nodeId);
    for (ByteArray key : keys) StoreUtils.assertValidMetadata(key, routingStrategy, node);
    return getInnerStore().getAll(keys, transforms);
}

Example 3 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class RequestFileFilter method main.

/**
     * Filter requests specified in a file, generating a new file containing
     * only requests destined for a specific node.
     * 
     * @param args See usage for more information
     * @throws Exception In case of I/O or Voldemort-specific errors
     */
public static void main(String[] args) throws Exception {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.accepts("node", "[REQUIRED] node id").withRequiredArg().ofType(Integer.class).describedAs("node id");
    parser.accepts("store-name", "[REQUIRED] store name").withRequiredArg().describedAs("store name");
    parser.accepts("url", "[REQUIRED] bootstrap URL").withRequiredArg().describedAs("bootstrap-url");
    parser.accepts("input", "[REQUIRED] input request file").withRequiredArg().describedAs("input-file");
    parser.accepts("output", "[REQUIRED] output file").withRequiredArg().describedAs("output-file");
    parser.accepts("string-keys");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    Set<String> missing = CmdUtils.missing(options, "node", "store-name", "url", "input", "output");
    if (missing.size() > 0) {
        System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing));
        parser.printHelpOn(System.err);
        System.exit(1);
    }
    int nodeId = (Integer) options.valueOf("node");
    String storeName = (String) options.valueOf("store-name");
    String bootstrapURL = (String) options.valueOf("url");
    String inputFile = (String) options.valueOf("input");
    String outputFile = (String) options.valueOf("output");
    boolean stringKeys = options.has("string-keys");
    AdminClient adminClient = new AdminClient(bootstrapURL);
    List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId).getValue();
    StoreDefinition storeDefinition = null;
    for (StoreDefinition def : storeDefinitionList) {
        if (storeName.equals(def.getName())) {
            storeDefinition = def;
        }
    }
    if (storeDefinition == null) {
        Utils.croak("No store found with name\"" + storeName + "\"");
    }
    Cluster cluster = adminClient.metadataMgmtOps.getRemoteCluster(nodeId).getValue();
    Node node = null;
    try {
        node = cluster.getNodeById(nodeId);
    } catch (VoldemortException e) {
        Utils.croak("Can't find a node with id " + nodeId);
    }
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
    try {
        new RequestFileFilter(storeDefinition, routingStrategy, inputFile, outputFile, node, stringKeys).filter();
    } catch (FileNotFoundException e) {
        Utils.croak(e.getMessage());
    }
}

Example 4 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class AdminServiceBasicTest method testQuery.

@Test
public void testQuery() {
    HashMap<ByteArray, byte[]> belongToAndInsideServer0 = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> belongToAndInsideServer1 = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> notBelongServer0ButInsideServer0 = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> belongToServer0ButOutsideBoth = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> notBelongToServer0AndOutsideBoth = new HashMap<ByteArray, byte[]>();
    Store<ByteArray, byte[], byte[]> store0 = getStore(0, testStoreName);
    Store<ByteArray, byte[], byte[]> store1 = getStore(1, testStoreName);
    HashMap<ByteArray, byte[]> entrySet = null;
    Iterator<ByteArray> keys = null;
    RoutingStrategy strategy = servers[0].getMetadataStore().getRoutingStrategy(testStoreName);
    while (true) {
        ByteArray key;
        byte[] value;
        if (keys == null || !keys.hasNext()) {
            entrySet = ServerTestUtils.createRandomKeyValuePairs(100);
            keys = entrySet.keySet().iterator();
        }
        key = keys.next();
        value = entrySet.get(key);
        List<Node> routedNodes = strategy.routeRequest(key.get());
        boolean keyShouldBeInNode0 = false;
        boolean keyShouldBeInNode1 = false;
        for (Node node : routedNodes) {
            keyShouldBeInNode0 = keyShouldBeInNode0 || (node.getId() == 0);
            keyShouldBeInNode1 = keyShouldBeInNode1 || (node.getId() == 1);
        }
        if (belongToAndInsideServer0.size() < 10) {
            if (keyShouldBeInNode0) {
                belongToAndInsideServer0.put(key, value);
                store0.put(key, new Versioned<byte[]>(value), null);
            }
        } else if (belongToAndInsideServer1.size() < 10) {
            if (keyShouldBeInNode1) {
                belongToAndInsideServer1.put(key, value);
                store1.put(key, new Versioned<byte[]>(value), null);
            }
        } else if (notBelongServer0ButInsideServer0.size() < 5) {
            if (!keyShouldBeInNode0) {
                notBelongServer0ButInsideServer0.put(key, value);
                store0.put(key, new Versioned<byte[]>(value), null);
            }
        } else if (belongToServer0ButOutsideBoth.size() < 5) {
            if (keyShouldBeInNode0) {
                belongToServer0ButOutsideBoth.put(key, value);
            }
        } else if (notBelongToServer0AndOutsideBoth.size() < 5) {
            if (!keyShouldBeInNode0) {
                notBelongToServer0AndOutsideBoth.put(key, value);
            }
        } else {
            break;
        }
    }
    ArrayList<ByteArray> belongToAndInsideServer0Keys = new ArrayList<ByteArray>(belongToAndInsideServer0.keySet());
    ArrayList<ByteArray> belongToAndInsideServer1Keys = new ArrayList<ByteArray>(belongToAndInsideServer1.keySet());
    ArrayList<ByteArray> notBelongServer0ButInsideServer0Keys = new ArrayList<ByteArray>(notBelongServer0ButInsideServer0.keySet());
    ArrayList<ByteArray> belongToServer0ButOutsideBothKeys = new ArrayList<ByteArray>(belongToServer0ButOutsideBoth.keySet());
    ArrayList<ByteArray> notBelongToServer0AndOutsideBothKeys = new ArrayList<ByteArray>(notBelongToServer0AndOutsideBoth.keySet());
    List<ByteArray> queryKeys;
    Iterator<QueryKeyResult> results;
    QueryKeyResult entry;
    // test one key on store 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer0Keys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertEquals(queryKeys.get(0), entry.getKey());
    assertNull("There should not be exception in response", entry.getException());
    assertEquals("There should be only 1 value in versioned list", 1, entry.getValues().size());
    assertEquals("Two byte[] should be equal", 0, ByteUtils.compare(belongToAndInsideServer0.get(queryKeys.get(0)), entry.getValues().get(0).getValue()));
    assertFalse("There should be only one result", results.hasNext());
    // test one key belongs to but not exists in server 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToServer0ButOutsideBothKeys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertFalse("There should not be exception", entry.hasException());
    assertTrue("There should be values", entry.hasValues());
    assertNotNull("Response should be non-null", entry.getValues());
    assertEquals("Value should be empty list", 0, entry.getValues().size());
    assertNull("There should not be exception", entry.getException());
    // test one key not exist and does not belong to server 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(notBelongToServer0AndOutsideBothKeys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertTrue("There should be exception", entry.hasException());
    assertFalse("There should not be values", entry.hasValues());
    assertNull("Value should be null", entry.getValues());
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // test one key that exists on server 0 but does not belong to server 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(notBelongServer0ButInsideServer0Keys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertTrue("There should be exception", entry.hasException());
    assertFalse("There should not be values", entry.hasValues());
    assertNull("Value should be null", entry.getValues());
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // test one key deleted
    store0.delete(belongToAndInsideServer0Keys.get(4), null);
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer0Keys.get(4));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertFalse("There should not be exception", entry.hasException());
    assertTrue("There should be values", entry.hasValues());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertEquals("Value should be empty list", 0, entry.getValues().size());
    // test empty request
    queryKeys = new ArrayList<ByteArray>();
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertFalse("Results should be empty", results.hasNext());
    // test null key
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(null);
    assertEquals(1, queryKeys.size());
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertTrue("There should be exception", entry.hasException());
    assertFalse("There should not be values", entry.hasValues());
    assertNull("Value should be null", entry.getValues());
    assertTrue("There should be IllegalArgumentException exception", entry.getException() instanceof IllegalArgumentException);
    // test multiple keys (3) on store 1
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer1Keys.get(0));
    queryKeys.add(belongToAndInsideServer1Keys.get(1));
    queryKeys.add(belongToAndInsideServer1Keys.get(2));
    results = getAdminClient().streamingOps.queryKeys(1, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    Map<ByteArray, List<Versioned<byte[]>>> entries = new HashMap<ByteArray, List<Versioned<byte[]>>>();
    int resultCount = 0;
    while (results.hasNext()) {
        resultCount++;
        entry = results.next();
        assertNull("There should not be exception in response", entry.getException());
        assertNotNull("Value should not be null for Key: ", entry.getValues());
        entries.put(entry.getKey(), entry.getValues());
    }
    assertEquals("There should 3 and only 3 results", 3, resultCount);
    for (ByteArray key : queryKeys) {
        // this loop and the count ensure one-to-one mapping
        assertNotNull("This key should exist in the results: " + key, entries.get(key));
        assertEquals("Two byte[] should be equal for key: " + key, 0, ByteUtils.compare(belongToAndInsideServer1.get(key), entries.get(key).get(0).getValue()));
    }
    // test multiple keys, mixed situation
    // key 0: Exists and belongs to
    // key 1: Exists but does not belong to
    // key 2: Does not exist but belongs to
    // key 3: Does not belong and not exist
    // key 4: Same situation with key0
    // key 5: Deleted
    // key 6: Same situation with key2
    store0.delete(belongToAndInsideServer0Keys.get(5), null);
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer0Keys.get(2));
    queryKeys.add(notBelongServer0ButInsideServer0Keys.get(1));
    queryKeys.add(belongToServer0ButOutsideBothKeys.get(1));
    queryKeys.add(notBelongToServer0AndOutsideBothKeys.get(1));
    queryKeys.add(belongToAndInsideServer0Keys.get(3));
    queryKeys.add(belongToAndInsideServer0Keys.get(5));
    queryKeys.add(notBelongServer0ButInsideServer0Keys.get(2));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    // key 0
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(0).get(), entry.getKey().get()));
    assertEquals(0, ByteUtils.compare(belongToAndInsideServer0.get(queryKeys.get(0)), entry.getValues().get(0).getValue()));
    assertNull(entry.getException());
    // key 1
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(1).get(), entry.getKey().get()));
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // key 2
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(2).get(), entry.getKey().get()));
    assertEquals(0, entry.getValues().size());
    assertNull(entry.getException());
    // key 3
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(3).get(), entry.getKey().get()));
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // key 4
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(4).get(), entry.getKey().get()));
    assertEquals(0, ByteUtils.compare(belongToAndInsideServer0.get(queryKeys.get(4)), entry.getValues().get(0).getValue()));
    assertNull(entry.getException());
    // key 5
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(5).get(), entry.getKey().get()));
    assertEquals(0, entry.getValues().size());
    assertNull(entry.getException());
    // key 6
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(6).get(), entry.getKey().get()));
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // no more keys
    assertFalse(results.hasNext());
}

Example 5 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class AbstractAdminServiceFilterTest method testDeleteStreamWithFilter.

@Test
public void testDeleteStreamWithFilter() {
    // user store should be present
    Store<ByteArray, byte[], byte[]> store = getStore(0, testStoreName);
    assertNotSame("Store '" + testStoreName + "' should not be null", null, store);
    Set<Pair<ByteArray, Versioned<byte[]>>> entrySet = createEntries();
    VoldemortFilter filter = new VoldemortFilterImpl();
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(getStoreDef(), getCluster());
    for (Pair<ByteArray, Versioned<byte[]>> pair : entrySet) {
        if (Utils.nodeListToNodeIdList(strategy.routeRequest(pair.getFirst().get())).contains(0))
            store.put(pair.getFirst(), pair.getSecond(), null);
    }
    // make delete stream call with filter
    getAdminClient().storeMntOps.deletePartitions(0, testStoreName, Lists.newArrayList(0, 1), filter);
    // assert none of the filtered entries are returned.
    for (Pair<ByteArray, Versioned<byte[]>> entry : entrySet) {
        if (Utils.nodeListToNodeIdList(strategy.routeRequest(entry.getFirst().get())).contains(0)) {
            if (filter.accept(entry.getFirst(), entry.getSecond())) {
                assertEquals("All entries should be deleted except the filtered ones.", 0, store.get(entry.getFirst(), null).size());
            } else {
                assertNotSame("filtered entry should be still present.", 0, store.get(entry.getFirst(), null).size());
                assertEquals("values should match", new String(entry.getSecond().getValue()), new String(store.get(entry.getFirst(), null).get(0).getValue()));
            }
        }
    }
}